From 670245376204e80a42e4238e0ce108735b58a644 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Tue, 26 Mar 2019 18:10:01 -0400 Subject: [PATCH 01/56] WIP snapshot for motivating the various refactor PRs it entails Signed-off-by: Fred Douglas --- api/envoy/api/v2/discovery.proto | 24 +- include/envoy/config/grpc_xds_context.h | 42 +++ include/envoy/config/subscription.h | 11 +- include/envoy/upstream/cluster_manager.h | 3 + source/common/config/BUILD | 22 ++ .../common/config/delta_subscription_impl.h | 228 ++-------------- .../common/config/delta_subscription_state.h | 253 ++++++++++++++++++ .../config/filesystem_subscription_impl.h | 10 +- source/common/config/grpc_delta_xds_context.h | 149 +++++++++++ .../config/grpc_mux_subscription_impl.h | 21 +- source/common/config/grpc_stream.h | 29 +- source/common/config/grpc_subscription_impl.h | 14 +- source/common/config/http_subscription_impl.h | 10 +- source/common/config/subscription_factory.h | 26 +- source/common/router/rds_impl.cc | 9 +- source/common/router/rds_impl.h | 9 +- source/common/secret/sds_api.cc | 8 +- source/common/secret/sds_api.h | 5 +- source/common/upstream/cds_api_impl.cc | 22 +- source/common/upstream/cds_api_impl.h | 4 +- source/common/upstream/cluster_manager_impl.h | 4 + source/common/upstream/eds.cc | 10 +- source/common/upstream/eds.h | 5 +- source/server/lds_api.cc | 25 +- source/server/lds_api.h | 4 +- test/mocks/config/mocks.h | 11 +- 26 files changed, 640 insertions(+), 318 deletions(-) create mode 100644 include/envoy/config/grpc_xds_context.h create mode 100644 source/common/config/delta_subscription_state.h create mode 100644 source/common/config/grpc_delta_xds_context.h diff --git a/api/envoy/api/v2/discovery.proto b/api/envoy/api/v2/discovery.proto index c2f74a432fb5c..3cdba007d883b 100644 --- a/api/envoy/api/v2/discovery.proto +++ b/api/envoy/api/v2/discovery.proto @@ -158,12 +158,13 @@ message DeltaDiscoveryRequest { // A list of Resource names to remove from the list of tracked resources. repeated string resource_names_unsubscribe = 4; - // This map must be populated when the DeltaDiscoveryRequest is the - // first in a stream (assuming there are any resources - this field's purpose is to enable - // a session to continue in a reconnected gRPC stream, and so will not be used in the very - // first stream of a session). The keys are the resources names of the xDS resources - // known to the xDS client. The values in the map are the associated resource - // level version info. + // Informs the server of the versions of the resources the xDS client knows of, to enable an xDS + // session to continue in a reconnected gRPC stream. This map will only be populated in the first + // message of a stream, as it is not applicable in later messages. It will not be populated in the + // very first stream of a session, since the client will not yet have any resources. In ADS, the + // first message *of each type_url* of a reconnected stream should populate this map. The map's + // keys are names of xDS resources known to the xDS client. The values are opaque resource + // versions. map initial_resource_versions = 5; // When the DeltaDiscoveryRequest is a ACK or NACK message in response @@ -182,8 +183,15 @@ message DeltaDiscoveryResponse { // The version of the response data (used for debugging). string system_version_info = 1; - // The response resources. These are typed resources that match the type url - // in the DeltaDiscoveryRequest. + // Type of the resources that are being provided, e.g. + // "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment". This is implicit + // in requests made via singleton xDS APIs such as CDS, LDS, etc. but is + // required for ADS. + // Must match the type_url in the DeltaDiscoveryRequest. + string type_url = 3; + + // The response resources. The Resource proto's Any field named 'resource' should be + // of type type_url. repeated Resource resources = 2 [(gogoproto.nullable) = false]; // Resources names of resources that have be deleted and to be removed from the xDS Client. diff --git a/include/envoy/config/grpc_xds_context.h b/include/envoy/config/grpc_xds_context.h new file mode 100644 index 0000000000000..f918137f7b93b --- /dev/null +++ b/include/envoy/config/grpc_xds_context.h @@ -0,0 +1,42 @@ +#pragma once + +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/config/subscription.h" + +#include "common/common/assert.h" +#include "common/common/logger.h" +#include "common/config/utility.h" +#include "common/grpc/common.h" +#include "common/protobuf/protobuf.h" +#include "common/protobuf/utility.h" + +namespace Envoy { +namespace Config { + +class GrpcXdsContext { +public: + void addSubscription(const std::vector& resources, const std::string& type_url, + const LocalInfo::LocalInfo& local_info, SubscriptionCallbacks& callbacks, + Event::Dispatcher& dispatcher, std::chrono::milliseconds init_fetch_timeout, + SubscriptionStats stats); + + // Enqueues and attempts to send a discovery request, (un)subscribing to resources missing from / + // added to the passed 'resources' argument, relative to resource_versions_. + void updateSubscription(const std::vector& resources, const std::string& type_url); + + void removeSubscription(const std::string& type_url); + + void pause(const std::string& type_url); + + void resume(const std::string& type_url); + + // Returns whether the request was actually sent (and so can leave the queue). + virtual void sendDiscoveryRequest(const RequestQueueItem& queue_item) PURE; + + virtual void handleResponse(std::unique_ptr&& message) PURE; + virtual void handleStreamEstablished() PURE; + virtual void handleEstablishmentFailure() PURE; +}; + +} // namespace Config +} // namespace Envoy diff --git a/include/envoy/config/subscription.h b/include/envoy/config/subscription.h index b81d769585fb7..879f8020543ad 100644 --- a/include/envoy/config/subscription.h +++ b/include/envoy/config/subscription.h @@ -13,9 +13,9 @@ namespace Envoy { namespace Config { -template class SubscriptionCallbacks { +class SubscriptionCallbacks { public: - typedef Protobuf::RepeatedPtrField ResourceVector; + typedef Protobuf::RepeatedPtrField ResourceVector; virtual ~SubscriptionCallbacks() {} @@ -62,10 +62,9 @@ template class SubscriptionCallbacks { /** * Common abstraction for subscribing to versioned config updates. This may be implemented via bidi - * gRPC streams, periodic/long polling REST or inotify filesystem updates. ResourceType is expected - * to be a protobuf serializable object. + * gRPC streams, periodic/long polling REST or inotify filesystem updates. */ -template class Subscription { +class Subscription { public: virtual ~Subscription() {} @@ -77,7 +76,7 @@ template class Subscription { * result in the deletion of the Subscription object. */ virtual void start(const std::vector& resources, - SubscriptionCallbacks& callbacks) PURE; + SubscriptionCallbacks& callbacks) PURE; /** * Update the resources to fetch. diff --git a/include/envoy/upstream/cluster_manager.h b/include/envoy/upstream/cluster_manager.h index 20a69617bfdf2..37891b06ca7dd 100644 --- a/include/envoy/upstream/cluster_manager.h +++ b/include/envoy/upstream/cluster_manager.h @@ -208,6 +208,9 @@ class ClusterManager { */ virtual Config::GrpcMux& adsMux() PURE; + // TODO TODO COMMENT + virtual Config::GrpcDeltaXdsContext& grpcDeltaXdsContext() PURE; + /** * @return Grpc::AsyncClientManager& the cluster manager's gRPC client manager. */ diff --git a/source/common/config/BUILD b/source/common/config/BUILD index 8071c1855276b..270588dcff319 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -101,6 +101,7 @@ envoy_cc_library( name = "delta_subscription_lib", hdrs = ["delta_subscription_impl.h"], deps = [ + ":grpc_delta_xds_context_lib", ":grpc_stream_lib", ":utility_lib", "//include/envoy/config:subscription_interface", @@ -113,6 +114,17 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "delta_subscription_state", + hdrs = ["delta_subscription_state.h"], + deps = [ + ":utility_lib", + "//include/envoy/config:subscription_interface", + "//source/common/protobuf", + "@envoy_api//envoy/api/v2:discovery_cc", + ], +) + envoy_cc_library( name = "grpc_stream_lib", hdrs = ["grpc_stream.h"], @@ -209,6 +221,16 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "grpc_delta_xds_context_lib", + hdrs = ["grpc_delta_xds_context.h"], + deps = [ + ":delta_subscription_state", + "//include/envoy/event:dispatcher_interface", + "//include/envoy/grpc:async_client_interface", + ], +) + envoy_cc_library( name = "http_subscription_lib", hdrs = ["http_subscription_impl.h"], diff --git a/source/common/config/delta_subscription_impl.h b/source/common/config/delta_subscription_impl.h index 5afdc83a2dd3d..f2e9b045a9787 100644 --- a/source/common/config/delta_subscription_impl.h +++ b/source/common/config/delta_subscription_impl.h @@ -1,16 +1,8 @@ #pragma once -#include - -#include "envoy/api/v2/discovery.pb.h" -#include "envoy/common/token_bucket.h" #include "envoy/config/subscription.h" -#include "common/common/assert.h" -#include "common/common/backoff_strategy.h" -#include "common/common/logger.h" -#include "common/common/token_bucket_impl.h" -#include "common/config/grpc_stream.h" +#include "common/config/grpc_delta_xds_context.h" #include "common/config/utility.h" #include "common/grpc/common.h" #include "common/protobuf/protobuf.h" @@ -19,216 +11,48 @@ namespace Envoy { namespace Config { -struct ResourceNameDiff { - std::vector added_; - std::vector removed_; -}; - -const char EmptyVersion[] = ""; - /** - * Manages the logic of a (non-aggregated) delta xDS subscription. - * TODO(fredlas) add aggregation support. + * DeltaSubscription manages the logic of a delta xDS subscription for a particular resource + * type. It uses a GrpcDeltaXdsContext to handle the actual gRPC communication with the xDS server. + * DeltaSubscription and GrpcDeltaXdsContext are both used for both ADS and non-aggregated xDS; + * the only difference is that ADS has multiple DeltaSubscriptions sharing a single + * GrpcDeltaXdsContext. */ -template -class DeltaSubscriptionImpl - : public Subscription, - public GrpcStream { +class DeltaSubscriptionImpl : public Subscription { public: - DeltaSubscriptionImpl(const LocalInfo::LocalInfo& local_info, Grpc::AsyncClientPtr async_client, - Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, - Runtime::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, SubscriptionStats stats, - std::chrono::milliseconds init_fetch_timeout) - : GrpcStream(std::move(async_client), service_method, random, dispatcher, - scope, rate_limit_settings), - type_url_(Grpc::Common::typeUrl(ResourceType().GetDescriptor()->full_name())), - local_info_(local_info), stats_(stats), dispatcher_(dispatcher), - init_fetch_timeout_(init_fetch_timeout) { - request_.set_type_url(type_url_); - request_.mutable_node()->MergeFrom(local_info_.node()); - } - - // Enqueues and attempts to send a discovery request, (un)subscribing to resources missing from / - // added to the passed 'resources' argument, relative to resources_. Updates resources_ to - // 'resources'. - void buildAndQueueDiscoveryRequest(const std::vector& resources) { - ResourceNameDiff diff; - std::set_difference(resources.begin(), resources.end(), resource_names_.begin(), - resource_names_.end(), std::inserter(diff.added_, diff.added_.begin())); - std::set_difference(resource_names_.begin(), resource_names_.end(), resources.begin(), - resources.end(), std::inserter(diff.removed_, diff.removed_.begin())); - - for (const auto& added : diff.added_) { - resources_[added] = EmptyVersion; - resource_names_.insert(added); - } - for (const auto& removed : diff.removed_) { - resources_.erase(removed); - resource_names_.erase(removed); - } - queueDiscoveryRequest(diff); - } - - void sendDiscoveryRequest(const ResourceNameDiff& diff) override { - if (!grpcStreamAvailable()) { - ENVOY_LOG(debug, "No stream available to sendDiscoveryRequest for {}", type_url_); - return; // Drop this request; the reconnect will enqueue a new one. - } - if (paused_) { - ENVOY_LOG(trace, "API {} paused during sendDiscoveryRequest().", type_url_); - pending_ = diff; - return; // The unpause will send this request. - } + DeltaSubscriptionImpl( + GrpcDeltaXdsContextSharedPtr context, + absl::string_view + type_url, // TODO TODO everwhere should pass this as + // Grpc::Common::typeUrl(ResourceType().GetDescriptor()->full_name()) + const LocalInfo::LocalInfo& local_info, std::chrono::milliseconds init_fetch_timeout, + Event::Dispatcher& dispatcher, SubscriptionStats stats) + : context_(context), type_url_(type_url), local_info_(local_info), + init_fetch_timeout_(init_fetch_timeout), dispatcher_(dispatcher), stats_(stats) {} - request_.clear_resource_names_subscribe(); - request_.clear_resource_names_unsubscribe(); - std::copy(diff.added_.begin(), diff.added_.end(), - Protobuf::RepeatedFieldBackInserter(request_.mutable_resource_names_subscribe())); - std::copy(diff.removed_.begin(), diff.removed_.end(), - Protobuf::RepeatedFieldBackInserter(request_.mutable_resource_names_unsubscribe())); + ~DeltaSubscriptionImpl() { context_->removeSubscription(type_url_); } - ENVOY_LOG(trace, "Sending DiscoveryRequest for {}: {}", type_url_, request_.DebugString()); - sendMessage(request_); - request_.clear_error_detail(); - request_.clear_initial_resource_versions(); - } - - void subscribe(const std::vector& resources) { - ENVOY_LOG(debug, "delta subscribe for " + type_url_); - buildAndQueueDiscoveryRequest(resources); - } - - void pause() { - ENVOY_LOG(debug, "Pausing discovery requests for {}", type_url_); - ASSERT(!paused_); - paused_ = true; - } - - void resume() { - ENVOY_LOG(debug, "Resuming discovery requests for {}", type_url_); - ASSERT(paused_); - paused_ = false; - if (pending_.has_value()) { - queueDiscoveryRequest(pending_.value()); - pending_.reset(); - } - } - - // Config::SubscriptionCallbacks - void onConfigUpdate(const Protobuf::RepeatedPtrField& added_resources, - const Protobuf::RepeatedPtrField& removed_resources, - const std::string& version_info) { - callbacks_->onConfigUpdate(added_resources, removed_resources, version_info); - for (const auto& resource : added_resources) { - resources_[resource.name()] = resource.version(); - } - stats_.update_success_.inc(); - stats_.update_attempt_.inc(); - stats_.version_.set(HashUtil::xxHash64(version_info)); - ENVOY_LOG(debug, "Delta config for {} accepted with {} resources added, {} removed", type_url_, - added_resources.size(), removed_resources.size()); - } + void pause() { context_->pause(type_url_); } - void handleResponse(std::unique_ptr&& message) override { - ENVOY_LOG(debug, "Received gRPC message for {} at version {}", type_url_, - message->system_version_info()); - disableInitFetchTimeoutTimer(); - - request_.set_response_nonce(message->nonce()); - - try { - onConfigUpdate(message->resources(), message->removed_resources(), - message->system_version_info()); - } catch (const EnvoyException& e) { - stats_.update_rejected_.inc(); - ENVOY_LOG(warn, "delta config for {} rejected: {}", type_url_, e.what()); - stats_.update_attempt_.inc(); - callbacks_->onConfigUpdateFailed(&e); - ::google::rpc::Status* error_detail = request_.mutable_error_detail(); - error_detail->set_code(Grpc::Status::GrpcStatus::Internal); - error_detail->set_message(e.what()); - } - queueDiscoveryRequest(ResourceNameDiff()); // no change to subscribed resources - } - - void handleStreamEstablished() override { - // initial_resource_versions "must be populated for first request in a stream", so guarantee - // that the initial version'd request we're about to enqueue is what gets sent. - clearRequestQueue(); - - request_.Clear(); - for (auto const& resource : resources_) { - (*request_.mutable_initial_resource_versions())[resource.first] = resource.second; - } - request_.set_type_url(type_url_); - request_.mutable_node()->MergeFrom(local_info_.node()); - queueDiscoveryRequest(ResourceNameDiff()); // no change to subscribed resources - } - - void handleEstablishmentFailure() override { - disableInitFetchTimeoutTimer(); - stats_.update_failure_.inc(); - ENVOY_LOG(debug, "delta update for {} failed", type_url_); - stats_.update_attempt_.inc(); - callbacks_->onConfigUpdateFailed(nullptr); - } + void resume() { context_->resume(type_url_); } // Config::DeltaSubscription - void start(const std::vector& resources, - SubscriptionCallbacks& callbacks) override { - callbacks_ = &callbacks; - - if (init_fetch_timeout_.count() > 0) { - init_fetch_timeout_timer_ = dispatcher_.createTimer([this]() -> void { - ENVOY_LOG(warn, "delta config: initial fetch timed out for {}", type_url_); - callbacks_->onConfigUpdateFailed(nullptr); - }); - init_fetch_timeout_timer_->enableTimer(init_fetch_timeout_); - } - - establishNewStream(); - subscribe(resources); - // The attempt stat here is maintained for the purposes of having consistency between ADS and - // individual DeltaSubscriptions. Since ADS is push based and muxed, the notion of an - // "attempt" for a given xDS API combined by ADS is not really that meaningful. - stats_.update_attempt_.inc(); + void start(const std::vector& resources, SubscriptionCallbacks& callbacks) override { + context_->addSubscription(resources, type_url_, local_info_, callbacks, dispatcher_, + init_fetch_timeout_, stats_); } void updateResources(const std::vector& resources) override { - subscribe(resources); - stats_.update_attempt_.inc(); + context_->updateSubscription(resources, type_url_); } private: - void disableInitFetchTimeoutTimer() { - if (init_fetch_timeout_timer_) { - init_fetch_timeout_timer_->disableTimer(); - init_fetch_timeout_timer_.reset(); - } - } - // A map from resource name to per-resource version. - std::unordered_map resources_; - // The keys of resources_. Only tracked separately because std::map does not provide an iterator - // into just its keys, e.g. for use in std::set_difference. - std::unordered_set resource_names_; + GrpcDeltaXdsContextSharedPtr context_; const std::string type_url_; - SubscriptionCallbacks* callbacks_{}; - // In-flight or previously sent request. - envoy::api::v2::DeltaDiscoveryRequest request_; - // Paused via pause()? - bool paused_{}; - absl::optional pending_; - const LocalInfo::LocalInfo& local_info_; - - SubscriptionStats stats_; - Event::Dispatcher& dispatcher_; std::chrono::milliseconds init_fetch_timeout_; - Event::TimerPtr init_fetch_timeout_timer_; + Event::Dispatcher& dispatcher_; + SubscriptionStats stats_; }; } // namespace Config diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h new file mode 100644 index 0000000000000..4ffd7ff20c4c4 --- /dev/null +++ b/source/common/config/delta_subscription_state.h @@ -0,0 +1,253 @@ +#pragma once + +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/config/subscription.h" + +namespace Envoy { +namespace Config { + +struct ResourceNameDiff { + std::vector added_; + std::vector removed_; + std::string type_url_; + ResourceNameDiff(std::string type_url) : type_url_(type_url) {} + ResourceNameDiff() {} +}; + +class DeltaSubscriptionState : public Logger::Loggable { +public: + DeltaSubscriptionState(const std::string& type_url, + const std::vector& resource_names, + SubscriptionCallbacks& callbacks, const LocalInfo::LocalInfo& local_info, + std::chrono::milliseconds init_fetch_timeout, + Event::Dispatcher& dispatcher, SubscriptionStats& stats) + : type_url_(type_url), callbacks_(callbacks), local_info_(local_info), + init_fetch_timeout_(init_fetch_timeout), stats_(stats) { + // Start us off with the list of resources we're interested in. The first_request_of_new_stream_ + // version of populateRequestWithDiff() will know to put these names in the subscribe field, but + // *not* the initial_resource_versions field, due to them all being in the waitingForServer + // state. + for (const auto& resource_name : resource_names) { + setResourceWaitingForServer(resource_name); + } + buildCleanRequest(); + setInitFetchTimeout(dispatcher); + + // The attempt stat here is maintained for the purposes of having consistency between ADS and + // individual DeltaSubscriptions. Since ADS is push based and muxed, the notion of an + // "attempt" for a given xDS API combined by ADS is not really that meaningful. + stats_.update_attempt_.inc(); + } + + void setInitFetchTimeout(Event::Dispatcher& dispatcher) { + if (init_fetch_timeout_.count() > 0 && !init_fetch_timeout_timer_) { + init_fetch_timeout_timer_ = dispatcher.createTimer([this]() -> void { + ENVOY_LOG(warn, "delta config: initial fetch timed out for {}", type_url_); + callbacks_.onConfigUpdateFailed(nullptr); + }); + init_fetch_timeout_timer_->enableTimer(init_fetch_timeout_); + } + } + + void pause() { + ENVOY_LOG(debug, "Pausing discovery requests for {}", type_url_); + ASSERT(!paused_); + paused_ = true; + } + + void resume(absl::optional& to_send_if_any) { + ENVOY_LOG(debug, "Resuming discovery requests for {}", type_url_); + ASSERT(paused_); + paused_ = false; + to_send_if_any.reset(); + to_send_if_any.swap(pending_); + } + + void buildCleanRequest() { + request_.Clear(); + request_.set_type_url(type_url_); + request_.mutable_node()->MergeFrom(local_info_.node()); + } + + void updateResourceNamesAndBuildDiff(const std::vector& resource_names, + ResourceNameDiff& diff) { + stats_.update_attempt_.inc(); + std::set_difference(resource_names.begin(), resource_names.end(), resource_names_.begin(), + resource_names_.end(), std::inserter(diff.added_, diff.added_.begin())); + std::set_difference(resource_names_.begin(), resource_names_.end(), resource_names.begin(), + resource_names.end(), std::inserter(diff.removed_, diff.removed_.begin())); + + for (const auto& added : diff.added_) { + setResourceWaitingForServer(added); + } + for (const auto& removed : diff.removed_) { + lostInterestInResource(removed); + } + } + + const envoy::api::v2::DeltaDiscoveryRequest& + populateRequestWithDiff(const ResourceNameDiff& diff) { + request_.clear_resource_names_subscribe(); + request_.clear_resource_names_unsubscribe(); + std::copy(diff.added_.begin(), diff.added_.end(), + Protobuf::RepeatedFieldBackInserter(request_.mutable_resource_names_subscribe())); + std::copy(diff.removed_.begin(), diff.removed_.end(), + Protobuf::RepeatedFieldBackInserter(request_.mutable_resource_names_unsubscribe())); + + if (first_request_of_new_stream_) { + // initial_resource_versions "must be populated for first request in a stream". + for (auto const& resource : resource_versions_) { + // Populate initial_resource_versions with the resource versions we currently have. + // Resources we are interested in, but are still waiting to get any version of from the + // server, do not belong in initial_resource_versions. (But do belong in new subscriptions!) + if (!resource.second.waitingForServer()) { + (*request_.mutable_initial_resource_versions())[resource.first] = + resource.second.version(); + } + *request_.mutable_resource_names_subscribe()->Add() = resource.first; + } + } + return request_; + } + + void set_first_request_of_new_stream(bool val) { first_request_of_new_stream_ = val; } + + bool checkPausedDuringSendAttempt(const ResourceNameDiff& diff) { + if (paused_) { + ENVOY_LOG(trace, "API {} paused during sendDiscoveryRequest().", type_url_); + pending_ = diff; // resume() is now set to send this request. + return true; + } + return false; + } + + void handleResponse(envoy::api::v2::DeltaDiscoveryResponse* message) { + // We *always* copy the response's nonce into the next request, even if we're going to make that + // request a NACK by setting error_detail. + request_.set_response_nonce(message->nonce()); + try { + disableInitFetchTimeoutTimer(); + callbacks_.onConfigUpdate(message->resources(), message->removed_resources(), + message->system_version_info()); + for (const auto& resource : message->resources()) { + setResourceVersion(resource.name(), resource.version()); + } + // If a resource is gone, there is no longer a meaningful version for it that makes sense to + // provide to the server upon stream reconnect: either it will continue to not exist, in which + // case saying nothing is fine, or the server will bring back something new, which we should + // receive regardless (which is the logic that not specifying a version will get you). + // + // So, leave the version map entry present but blank. It will be left out of + // initial_resource_versions messages, but will remind us to explicitly tell the server "I'm + // cancelling my subscription" when we lose interest. + for (const auto& resource_name : message->removed_resources()) { + if (resource_names_.find(resource_name) != resource_names_.end()) { + setResourceWaitingForServer(resource_name); + } + } + stats_.update_success_.inc(); + stats_.update_attempt_.inc(); + stats_.version_.set(HashUtil::xxHash64(message->system_version_info())); + ENVOY_LOG(debug, "Delta config for {} accepted with {} resources added, {} removed", + type_url_, message->resources().size(), message->removed_resources().size()); + } catch (const EnvoyException& e) { + // Note that error_detail being set is what indicates that a DeltaDiscoveryRequest is a NACK. + ::google::rpc::Status* error_detail = request_.mutable_error_detail(); + error_detail->set_code(Grpc::Status::GrpcStatus::Internal); + error_detail->set_message(e.what()); + + disableInitFetchTimeoutTimer(); + stats_.update_rejected_.inc(); + ENVOY_LOG(warn, "delta config for {} rejected: {}", type_url_, e.what()); + stats_.update_attempt_.inc(); + callbacks_.onConfigUpdateFailed(&e); + } + } + + void handleEstablishmentFailure() { + stats_.update_failure_.inc(); + stats_.update_attempt_.inc(); + callbacks_.onConfigUpdateFailed(nullptr); + } + +private: + void disableInitFetchTimeoutTimer() { + if (init_fetch_timeout_timer_) { + init_fetch_timeout_timer_->disableTimer(); + init_fetch_timeout_timer_.reset(); + } + } + + class ResourceVersion { + public: + explicit ResourceVersion(absl::string_view version) : version_(version) {} + // Builds a ResourceVersion in the waitingForServer state. + ResourceVersion() {} + + // If true, we currently have no version of this resource - we are waiting for the server to + // provide us with one. + bool waitingForServer() const { return version_ == absl::nullopt; } + // Must not be called if waitingForServer() == true. + std::string version() const { + ASSERT(version_.has_value()); + return version_.value_or(""); + } + + private: + absl::optional version_; + }; + + // Use these helpers to avoid forgetting to update both at once. + void setResourceVersion(const std::string& resource_name, const std::string& resource_version) { + resource_versions_[resource_name] = ResourceVersion(resource_version); + resource_names_.insert(resource_name); + } + + void setResourceWaitingForServer(const std::string& resource_name) { + resource_versions_[resource_name] = ResourceVersion(); + resource_names_.insert(resource_name); + } + + void lostInterestInResource(const std::string& resource_name) { + resource_versions_.erase(resource_name); + resource_names_.erase(resource_name); + } + + // A map from resource name to per-resource version. The keys of this map are exactly the resource + // names we are currently interested in. Those in the waitingForServer state currently don't have + // any version for that resource: we need to inform the server if we lose interest in them, but we + // also need to *not* include them in the initial_resource_versions map upon a reconnect. + std::unordered_map resource_versions_; + // The keys of resource_versions_. Only tracked separately because std::map does not provide an + // iterator into just its keys, e.g. for use in std::set_difference. + std::unordered_set resource_names_; + + const std::string type_url_; + SubscriptionCallbacks& callbacks_; + const LocalInfo::LocalInfo& local_info_; + std::chrono::milliseconds init_fetch_timeout_; + Event::TimerPtr init_fetch_timeout_timer_; + + // Paused via pause()? + bool paused_{}; + bool first_request_of_new_stream_{true}; + absl::optional pending_; + + // Current state of the request we next intend to send. It is updated in various ways at various + // times by this class's functions. + // + // A note on request_ vs the ResourceNameDiff queue: request_ is an actual DeltaDiscoveryRequest + // proto, which is maintained/updated as the code runs, and is what is actually sent out by + // sendDiscoveryRequest(). queueDiscoveryRequest() queues an item into GrpcStream's abstract queue + // - in this case, a ResourceNameDiff. request_ tracks mechanical implementation details, whereas + // the queue items track the resources to be requested. When it comes time to send a + // DiscoveryRequest, the front of that queue is sent into sendDiscoveryRequest(), which uses the + // item to finalize request_ for sending. So, think of the ResourceNameDiff queue as "rough + // sketches" that actual requests will be based on. + envoy::api::v2::DeltaDiscoveryRequest request_; + + SubscriptionStats& stats_; +}; + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/filesystem_subscription_impl.h b/source/common/config/filesystem_subscription_impl.h index 27b7540d27f3c..3267c2e8de78f 100644 --- a/source/common/config/filesystem_subscription_impl.h +++ b/source/common/config/filesystem_subscription_impl.h @@ -20,8 +20,7 @@ namespace Config { * consumed on filesystem changes to files containing the JSON canonical representation of * lists of ResourceType. */ -template -class FilesystemSubscriptionImpl : public Config::Subscription, +class FilesystemSubscriptionImpl : public Config::Subscription, Logger::Loggable { public: FilesystemSubscriptionImpl(Event::Dispatcher& dispatcher, const std::string& path, @@ -37,7 +36,7 @@ class FilesystemSubscriptionImpl : public Config::Subscription, // Config::Subscription void start(const std::vector& resources, - Config::SubscriptionCallbacks& callbacks) override { + Config::SubscriptionCallbacks& callbacks) override { // We report all discovered resources in the watched file. UNREFERENCED_PARAMETER(resources); callbacks_ = &callbacks; @@ -61,9 +60,8 @@ class FilesystemSubscriptionImpl : public Config::Subscription, try { envoy::api::v2::DiscoveryResponse message; MessageUtil::loadFromFile(path_, message, api_); - const auto typed_resources = Config::Utility::getTypedResources(message); config_update_available = true; - callbacks_->onConfigUpdate(typed_resources, message.version_info()); + callbacks_->onConfigUpdate(message.resources(), message.version_info()); stats_.version_.set(HashUtil::xxHash64(message.version_info())); stats_.update_success_.inc(); ENVOY_LOG(debug, "Filesystem config update accepted for {}: {}", path_, @@ -83,7 +81,7 @@ class FilesystemSubscriptionImpl : public Config::Subscription, bool started_{}; const std::string path_; std::unique_ptr watcher_; - SubscriptionCallbacks* callbacks_{}; + SubscriptionCallbacks* callbacks_{}; SubscriptionStats stats_; Api::Api& api_; }; diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h new file mode 100644 index 0000000000000..e8ee7bb06fcff --- /dev/null +++ b/source/common/config/grpc_delta_xds_context.h @@ -0,0 +1,149 @@ +#pragma once + +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/common/token_bucket.h" +#include "envoy/config/subscription.h" + +#include "common/common/assert.h" +#include "common/common/backoff_strategy.h" +#include "common/common/logger.h" +#include "common/common/token_bucket_impl.h" +#include "common/config/delta_subscription_state.h" +#include "common/config/grpc_stream.h" +#include "common/config/utility.h" +#include "common/grpc/common.h" +#include "common/protobuf/protobuf.h" +#include "common/protobuf/utility.h" + +namespace Envoy { +namespace Config { + +class GrpcDeltaXdsContext + : public GrpcStream { +public: + GrpcDeltaXdsContext(Grpc::AsyncClientPtr async_client, + const Protobuf::MethodDescriptor& service_method, + Event::Dispatcher& dispatcher, Runtime::RandomGenerator& random, + Stats::Scope& scope, const RateLimitSettings& rate_limit_settings) + : GrpcStream(std::move(async_client), service_method, random, dispatcher, + scope, rate_limit_settings) {} + + void addSubscription(const std::vector& resources, const std::string& type_url, + const LocalInfo::LocalInfo& local_info, SubscriptionCallbacks& callbacks, + Event::Dispatcher& dispatcher, std::chrono::milliseconds init_fetch_timeout, + SubscriptionStats stats) { + subscriptions_.emplace( + std::make_pair(type_url, DeltaSubscriptionState(type_url, resources, callbacks, local_info, + init_fetch_timeout, dispatcher, stats))); + establishNewStream(); // (idempotent) + } + + // Enqueues and attempts to send a discovery request, (un)subscribing to resources missing from / + // added to the passed 'resources' argument, relative to resource_versions_. + void updateSubscription(const std::vector& resources, const std::string& type_url) { + auto sub = subscriptions_.find(type_url); + if (sub == subscriptions_.end()) { + ENVOY_LOG(warn, "Not updating non-existent subscription {}.", type_url); + return; + } + ResourceNameDiff diff; + diff.type_url_ = type_url; + sub->second.updateResourceNamesAndBuildDiff(resources, diff); + queueDiscoveryRequest(diff); + } + + void removeSubscription(const std::string& type_url) { subscriptions_.erase(type_url); } + + void pause(const std::string& type_url) { + auto sub = subscriptions_.find(type_url); + if (sub == subscriptions_.end()) { + ENVOY_LOG(warn, "Not pausing non-existent subscription {}.", type_url); + return; + } + sub->second.pause(); + } + + void resume(const std::string& type_url) { + auto sub = subscriptions_.find(type_url); + if (sub == subscriptions_.end()) { + ENVOY_LOG(warn, "Not resuming non-existent subscription {}.", type_url); + return; + } + absl::optional to_send_if_any; + sub->second.resume(to_send_if_any); + if (to_send_if_any.has_value()) { + // Note that if some code pauses a certain type, then causes another discovery request to be + // queued up for it, then the newly queued request can *replace* other requests (of the same + // type) that were in the queue - that is, the new one will be sent upon unpause, while the + // others are just dropped. + queueDiscoveryRequest(to_send_if_any.value()); + } + } + + void sendDiscoveryRequest(const ResourceNameDiff& diff) override { + auto sub = subscriptions_.find(diff.type_url_); + if (sub == subscriptions_.end()) { + ENVOY_LOG(warn, "Not sending DeltaDiscoveryRequest for non-existent subscription {}.", + diff.type_url_); + return; + } + if (!grpcStreamAvailable()) { + ENVOY_LOG(debug, "No stream available to sendDiscoveryRequest for {}", diff.type_url_); + return; // Drop this request; the reconnect will enqueue a new one. + } + if (sub->second.checkPausedDuringSendAttempt(diff)) { + return; + } + const envoy::api::v2::DeltaDiscoveryRequest& request = + sub->second.populateRequestWithDiff(diff); + ENVOY_LOG(trace, "Sending DeltaDiscoveryRequest for {}: {}", diff.type_url_, + request.DebugString()); + // NOTE: at this point we are beyond the rate-limiting logic. sendMessage() unconditionally + // sends its argument over the gRPC stream. So, it is safe to unconditionally + // set_first_request_of_new_stream(false). + sendMessage(request); + sub->second.set_first_request_of_new_stream(false); + sub->second.buildCleanRequest(); + } + + void handleResponse(std::unique_ptr&& message) override { + ENVOY_LOG(debug, "Received DeltaDiscoveryResponse for {} at version {}", message->type_url(), + message->system_version_info()); + auto sub = subscriptions_.find(message->type_url()); + if (sub == subscriptions_.end()) { + ENVOY_LOG(warn, + "Dropping received DeltaDiscoveryResponse at version {} for non-existent " + "subscription {}.", + message->system_version_info(), message->type_url()); + return; + } + sub->second.handleResponse(message.get()); + // Queue up an ACK. + queueDiscoveryRequest( + ResourceNameDiff(message->type_url())); // no change to subscribed resources + } + + void handleStreamEstablished() override { + // Ensure that there's nothing leftover from previous streams in the request proto queue. + clearRequestQueue(); + for (auto& sub : subscriptions_) { + sub.second.set_first_request_of_new_stream(true); + } + } + + void handleEstablishmentFailure() override { + for (auto& sub : subscriptions_) { + sub.second.handleEstablishmentFailure(); + } + } + +private: + std::unordered_map subscriptions_; +}; + +typedef std::shared_ptr GrpcDeltaXdsContextSharedPtr; + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/grpc_mux_subscription_impl.h b/source/common/config/grpc_mux_subscription_impl.h index 1da9a2f6a891d..ff8e41a1fd79b 100644 --- a/source/common/config/grpc_mux_subscription_impl.h +++ b/source/common/config/grpc_mux_subscription_impl.h @@ -16,20 +16,20 @@ namespace Config { /** * Adapter from typed Subscription to untyped GrpcMux. Also handles per-xDS API stats/logging. */ -template -class GrpcMuxSubscriptionImpl : public Subscription, +class GrpcMuxSubscriptionImpl : public Subscription, GrpcMuxCallbacks, Logger::Loggable { public: - GrpcMuxSubscriptionImpl(GrpcMux& grpc_mux, SubscriptionStats stats, Event::Dispatcher& dispatcher, + GrpcMuxSubscriptionImpl(GrpcMux& grpc_mux, SubscriptionStats stats, absl::string_view type_url, + Event::Dispatcher& dispatcher, std::chrono::milliseconds init_fetch_timeout) : grpc_mux_(grpc_mux), stats_(stats), - type_url_(Grpc::Common::typeUrl(ResourceType().GetDescriptor()->full_name())), + type_url_(type_url), // TODO TODO everwhere should pass this as + // Grpc::Common::typeUrl(ResourceType().GetDescriptor()->full_name()) dispatcher_(dispatcher), init_fetch_timeout_(init_fetch_timeout) {} // Config::Subscription - void start(const std::vector& resources, - SubscriptionCallbacks& callbacks) override { + void start(const std::vector& resources, SubscriptionCallbacks& callbacks) override { callbacks_ = &callbacks; if (init_fetch_timeout_.count() > 0) { @@ -56,21 +56,16 @@ class GrpcMuxSubscriptionImpl : public Subscription, void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, const std::string& version_info) override { disableInitFetchTimeoutTimer(); - Protobuf::RepeatedPtrField typed_resources; - std::transform(resources.cbegin(), resources.cend(), - Protobuf::RepeatedPtrFieldBackInserter(&typed_resources), - MessageUtil::anyConvert); // TODO(mattklein123): In the future if we start tracking per-resource versions, we need to // supply those versions to onConfigUpdate() along with the xDS response ("system") // version_info. This way, both types of versions can be tracked and exposed for debugging by // the configuration update targets. - callbacks_->onConfigUpdate(typed_resources, version_info); + callbacks_->onConfigUpdate(resources, version_info); stats_.update_success_.inc(); stats_.update_attempt_.inc(); stats_.version_.set(HashUtil::xxHash64(version_info)); ENVOY_LOG(debug, "gRPC config for {} accepted with {} resources with version {}", type_url_, resources.size(), version_info); - ENVOY_LOG(trace, "resources: {}", RepeatedPtrUtil::debugString(typed_resources)); } void onConfigUpdateFailed(const EnvoyException* e) override { @@ -102,7 +97,7 @@ class GrpcMuxSubscriptionImpl : public Subscription, GrpcMux& grpc_mux_; SubscriptionStats stats_; const std::string type_url_; - SubscriptionCallbacks* callbacks_{}; + SubscriptionCallbacks* callbacks_{}; GrpcMuxWatchPtr watch_{}; Event::Dispatcher& dispatcher_; std::chrono::milliseconds init_fetch_timeout_; diff --git a/source/common/config/grpc_stream.h b/source/common/config/grpc_stream.h index 0ef95427caf6f..0d5d148e8c819 100644 --- a/source/common/config/grpc_stream.h +++ b/source/common/config/grpc_stream.h @@ -37,12 +37,14 @@ class GrpcStream : public Grpc::TypedAsyncStreamCallbacks, RETRY_MAX_DELAY_MS, random_); } - virtual void handleResponse(std::unique_ptr&& message) PURE; - virtual void handleStreamEstablished() PURE; - virtual void handleEstablishmentFailure() PURE; + // virtual void handleResponse(std::unique_ptr&& message) PURE; // TODO TODO NOTE + // MOVED TO include/grpc_xds_context.h virtual void handleStreamEstablished() PURE; // TODO TODO + // NOTE MOVED TO include/grpc_xds_context.h virtual void handleEstablishmentFailure() PURE; // + // TODO TODO NOTE MOVED TO include/grpc_xds_context.h // Returns whether the request was actually sent (and so can leave the queue). - virtual void sendDiscoveryRequest(const RequestQueueItem& queue_item) PURE; + // virtual void sendDiscoveryRequest(const RequestQueueItem& queue_item) PURE; // TODO TODO NOTE + // MOVED TO include/grpc_xds_context.h void queueDiscoveryRequest(const RequestQueueItem& queue_item) { request_queue_.push(queue_item); @@ -58,6 +60,14 @@ class GrpcStream : public Grpc::TypedAsyncStreamCallbacks, } void establishNewStream() { + + // TODO TODO TODO since this file also serves vanilla, need to DOUBLE CHECK whether this + // idempotency im adding here makes sense for vanilla too. + if (stream_ != nullptr) { + ENVOY_LOG(debug, "gRPC bidi stream for {} already exists!", service_method_.DebugString()); + return; + } + ENVOY_LOG(debug, "Establishing new gRPC bidi stream for {}", service_method_.DebugString()); stream_ = async_client_->start(service_method_, *this); if (stream_ == nullptr) { @@ -167,7 +177,16 @@ class GrpcStream : public Grpc::TypedAsyncStreamCallbacks, Event::TimerPtr drain_request_timer_; // A queue to store requests while rate limited. Note that when requests cannot be sent due to the // gRPC stream being down, this queue does not store them; rather, they are simply dropped. - std::queue request_queue_; + std::queue + request_queue_; // TODO TODO actually i think this queue can be its own little class. It would + // still be templated on RequestQueueItem, but owned by either + // GrpcDeltaXdsContext or GrpcStateOfWorldXdsContext, which could provide the + // type. + + // TODO TODO as for the rest of this class.... I think we can have GrpcDeltaXdsContext / + // GrpcWhateverMux (i.e. the GrpcXdsContext implementors) no longer inherit from it. Instead, it + // will just be its own GrpcStream< RequestProto, ResponseProto> thing, with nothing inheriting + // from it, and the GrpcXdsContext implementors will *own* one of it. }; } // namespace Config diff --git a/source/common/config/grpc_subscription_impl.h b/source/common/config/grpc_subscription_impl.h index 1a0d289bd1871..04b1b2aa6981f 100644 --- a/source/common/config/grpc_subscription_impl.h +++ b/source/common/config/grpc_subscription_impl.h @@ -12,21 +12,21 @@ namespace Envoy { namespace Config { -template -class GrpcSubscriptionImpl : public Config::Subscription { +class GrpcSubscriptionImpl : public Config::Subscription { public: GrpcSubscriptionImpl(const LocalInfo::LocalInfo& local_info, Grpc::AsyncClientPtr async_client, Event::Dispatcher& dispatcher, Runtime::RandomGenerator& random, - const Protobuf::MethodDescriptor& service_method, SubscriptionStats stats, - Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, + const Protobuf::MethodDescriptor& service_method, absl::string_view type_url, + SubscriptionStats stats, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, std::chrono::milliseconds init_fetch_timeout) : grpc_mux_(local_info, std::move(async_client), dispatcher, service_method, random, scope, rate_limit_settings), - grpc_mux_subscription_(grpc_mux_, stats, dispatcher, init_fetch_timeout) {} + grpc_mux_subscription_(grpc_mux_, stats, type_url, dispatcher, init_fetch_timeout) {} // Config::Subscription void start(const std::vector& resources, - Config::SubscriptionCallbacks& callbacks) override { + Config::SubscriptionCallbacks& callbacks) override { // Subscribe first, so we get failure callbacks if grpc_mux_.start() fails. grpc_mux_subscription_.start(resources, callbacks); grpc_mux_.start(); @@ -40,7 +40,7 @@ class GrpcSubscriptionImpl : public Config::Subscription { private: GrpcMuxImpl grpc_mux_; - GrpcMuxSubscriptionImpl grpc_mux_subscription_; + GrpcMuxSubscriptionImpl grpc_mux_subscription_; }; } // namespace Config diff --git a/source/common/config/http_subscription_impl.h b/source/common/config/http_subscription_impl.h index 702b0335c3f0a..373e9ea2aeab6 100644 --- a/source/common/config/http_subscription_impl.h +++ b/source/common/config/http_subscription_impl.h @@ -27,9 +27,8 @@ namespace Config { * canonical representation of DiscoveryResponse. This implementation is responsible for translating * between the proto serializable objects in the Subscription API and the REST JSON representation. */ -template class HttpSubscriptionImpl : public Http::RestApiFetcher, - public Config::Subscription, + public Config::Subscription, Logger::Loggable { public: HttpSubscriptionImpl(const LocalInfo::LocalInfo& local_info, Upstream::ClusterManager& cm, @@ -50,7 +49,7 @@ class HttpSubscriptionImpl : public Http::RestApiFetcher, // Config::Subscription void start(const std::vector& resources, - Config::SubscriptionCallbacks& callbacks) override { + Config::SubscriptionCallbacks& callbacks) override { ASSERT(callbacks_ == nullptr); if (init_fetch_timeout_.count() > 0) { @@ -96,9 +95,8 @@ class HttpSubscriptionImpl : public Http::RestApiFetcher, handleFailure(nullptr); return; } - const auto typed_resources = Config::Utility::getTypedResources(message); try { - callbacks_->onConfigUpdate(typed_resources, message.version_info()); + callbacks_->onConfigUpdate(message.resources(), message.version_info()); request_.set_version_info(message.version_info()); stats_.version_.set(HashUtil::xxHash64(request_.version_info())); stats_.update_success_.inc(); @@ -132,7 +130,7 @@ class HttpSubscriptionImpl : public Http::RestApiFetcher, std::string path_; Protobuf::RepeatedPtrField resources_; - Config::SubscriptionCallbacks* callbacks_{}; + Config::SubscriptionCallbacks* callbacks_{}; envoy::api::v2::DiscoveryRequest request_; SubscriptionStats stats_; Event::Dispatcher& dispatcher_; diff --git a/source/common/config/subscription_factory.h b/source/common/config/subscription_factory.h index 36857024f5309..865480b369db0 100644 --- a/source/common/config/subscription_factory.h +++ b/source/common/config/subscription_factory.h @@ -37,19 +37,17 @@ class SubscriptionFactory { * service description). * @param api reference to the Api object */ - template - static std::unique_ptr> subscriptionFromConfigSource( + static std::unique_ptr subscriptionFromConfigSource( const envoy::api::v2::core::ConfigSource& config, const LocalInfo::LocalInfo& local_info, Event::Dispatcher& dispatcher, Upstream::ClusterManager& cm, Runtime::RandomGenerator& random, Stats::Scope& scope, const std::string& rest_method, const std::string& grpc_method, - Api::Api& api) { - std::unique_ptr> result; + const std::string& type_url, Api::Api& api) { + std::unique_ptr result; SubscriptionStats stats = Utility::generateStats(scope); switch (config.config_source_specifier_case()) { case envoy::api::v2::core::ConfigSource::kPath: { Utility::checkFilesystemSubscriptionBackingPath(config.path(), api); - result.reset(new Config::FilesystemSubscriptionImpl(dispatcher, config.path(), - stats, api)); + result.reset(new Config::FilesystemSubscriptionImpl(dispatcher, config.path(), stats, api)); break; } case envoy::api::v2::core::ConfigSource::kApiConfigSource: { @@ -62,7 +60,7 @@ class SubscriptionFactory { "Please specify an explicit supported api_type in the following config:\n" + config.DebugString()); case envoy::api::v2::core::ApiConfigSource::REST: - result.reset(new HttpSubscriptionImpl( + result.reset(new HttpSubscriptionImpl( local_info, cm, api_config_source.cluster_names()[0], dispatcher, random, Utility::apiConfigSourceRefreshDelay(api_config_source), Utility::apiConfigSourceRequestTimeout(api_config_source), @@ -70,20 +68,20 @@ class SubscriptionFactory { Utility::configSourceInitialFetchTimeout(config))); break; case envoy::api::v2::core::ApiConfigSource::GRPC: - result.reset(new GrpcSubscriptionImpl( + result.reset(new GrpcSubscriptionImpl( local_info, Config::Utility::factoryForGrpcApiConfigSource(cm.grpcAsyncClientManager(), api_config_source, scope) ->create(), dispatcher, random, - *Protobuf::DescriptorPool::generated_pool()->FindMethodByName(grpc_method), stats, - scope, Utility::parseRateLimitSettings(api_config_source), + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName(grpc_method), type_url, + stats, scope, Utility::parseRateLimitSettings(api_config_source), Utility::configSourceInitialFetchTimeout(config))); break; case envoy::api::v2::core::ApiConfigSource::DELTA_GRPC: { Utility::checkApiConfigSourceSubscriptionBackingCluster(cm.clusters(), api_config_source); - result.reset(new DeltaSubscriptionImpl( - local_info, + result.reset(new DeltaSubscriptionImpl( + cm.grpcDeltaXdsContext(), type_url, local_info, Config::Utility::factoryForGrpcApiConfigSource(cm.grpcAsyncClientManager(), api_config_source, scope) ->create(), @@ -98,8 +96,8 @@ class SubscriptionFactory { break; } case envoy::api::v2::core::ConfigSource::kAds: { - result.reset(new GrpcMuxSubscriptionImpl( - cm.adsMux(), stats, dispatcher, Utility::configSourceInitialFetchTimeout(config))); + result.reset(new GrpcMuxSubscriptionImpl(cm.adsMux(), stats, type_url, dispatcher, + Utility::configSourceInitialFetchTimeout(config))); break; } default: diff --git a/source/common/router/rds_impl.cc b/source/common/router/rds_impl.cc index 3908fa799cd96..3f25e79d5d76f 100644 --- a/source/common/router/rds_impl.cc +++ b/source/common/router/rds_impl.cc @@ -67,12 +67,13 @@ RdsRouteConfigSubscription::RdsRouteConfigSubscription( last_updated_(factory_context.timeSource().systemTime()) { Envoy::Config::Utility::checkLocalInfo("rds", factory_context.localInfo()); - subscription_ = Envoy::Config::SubscriptionFactory::subscriptionFromConfigSource< - envoy::api::v2::RouteConfiguration>( + subscription_ = Envoy::Config::SubscriptionFactory::subscriptionFromConfigSource( rds.config_source(), factory_context.localInfo(), factory_context.dispatcher(), factory_context.clusterManager(), factory_context.random(), *scope_, "envoy.api.v2.RouteDiscoveryService.FetchRoutes", - "envoy.api.v2.RouteDiscoveryService.StreamRoutes", factory_context.api()); + "envoy.api.v2.RouteDiscoveryService.StreamRoutes", + Grpc::Common::typeUrl(envoy::api::v2::RouteConfiguration().GetDescriptor()->full_name()), + factory_context.api()); } RdsRouteConfigSubscription::~RdsRouteConfigSubscription() { @@ -99,7 +100,7 @@ void RdsRouteConfigSubscription::onConfigUpdate(const ResourceVector& resources, if (resources.size() != 1) { throw EnvoyException(fmt::format("Unexpected RDS resource length: {}", resources.size())); } - const auto& route_config = resources[0]; + auto route_config = MessageUtil::anyConvert(resources[0]); MessageUtil::validate(route_config); // TODO(PiotrSikora): Remove this hack once fixed internally. if (!(route_config.name() == route_config_name_)) { diff --git a/source/common/router/rds_impl.h b/source/common/router/rds_impl.h index 67e086258e21b..f6693bfc5fdd4 100644 --- a/source/common/router/rds_impl.h +++ b/source/common/router/rds_impl.h @@ -93,10 +93,9 @@ class RdsRouteConfigProviderImpl; * A class that fetches the route configuration dynamically using the RDS API and updates them to * RDS config providers. */ -class RdsRouteConfigSubscription - : public Init::Target, - Envoy::Config::SubscriptionCallbacks, - Logger::Loggable { +class RdsRouteConfigSubscription : public Init::Target, + Envoy::Config::SubscriptionCallbacks, + Logger::Loggable { public: ~RdsRouteConfigSubscription(); @@ -133,7 +132,7 @@ class RdsRouteConfigSubscription void registerInitTarget(Init::Manager& init_manager); void runInitializeCallbackIfAny(); - std::unique_ptr> subscription_; + std::unique_ptr subscription_; std::function initialize_callback_; const std::string route_config_name_; Stats::ScopePtr scope_; diff --git a/source/common/secret/sds_api.cc b/source/common/secret/sds_api.cc index 2471ad98657f6..636d1f4bb8fca 100644 --- a/source/common/secret/sds_api.cc +++ b/source/common/secret/sds_api.cc @@ -31,11 +31,11 @@ SdsApi::SdsApi(const LocalInfo::LocalInfo& local_info, Event::Dispatcher& dispat void SdsApi::initialize(std::function callback) { initialize_callback_ = callback; - subscription_ = Envoy::Config::SubscriptionFactory::subscriptionFromConfigSource< - envoy::api::v2::auth::Secret>( + subscription_ = Envoy::Config::SubscriptionFactory::subscriptionFromConfigSource( sds_config_, local_info_, dispatcher_, cluster_manager_, random_, stats_, "envoy.service.discovery.v2.SecretDiscoveryService.FetchSecrets", - "envoy.service.discovery.v2.SecretDiscoveryService.StreamSecrets", api_); + "envoy.service.discovery.v2.SecretDiscoveryService.StreamSecrets", + Grpc::Common::typeUrl(envoy::api::v2::auth::Secret().GetDescriptor()->full_name()), api_); subscription_->start({sds_config_name_}, *this); } @@ -48,7 +48,7 @@ void SdsApi::onConfigUpdate(const ResourceVector& resources, const std::string&) if (resources.size() != 1) { throw EnvoyException(fmt::format("Unexpected SDS secrets length: {}", resources.size())); } - const auto& secret = resources[0]; + auto secret = MessageUtil::anyConvert(resources[0]); MessageUtil::validate(secret); // Wrap sds_config_name_ in string_view to deal with proto string/std::string incompatibility diff --git a/source/common/secret/sds_api.h b/source/common/secret/sds_api.h index 6123159b372fe..ed97d84e3e5f1 100644 --- a/source/common/secret/sds_api.h +++ b/source/common/secret/sds_api.h @@ -27,8 +27,7 @@ namespace Secret { /** * SDS API implementation that fetches secrets from SDS server via Subscription. */ -class SdsApi : public Init::Target, - public Config::SubscriptionCallbacks { +class SdsApi : public Init::Target, public Config::SubscriptionCallbacks { public: SdsApi(const LocalInfo::LocalInfo& local_info, Event::Dispatcher& dispatcher, Runtime::RandomGenerator& random, Stats::Store& stats, @@ -67,7 +66,7 @@ class SdsApi : public Init::Target, Upstream::ClusterManager& cluster_manager_; const envoy::api::v2::core::ConfigSource sds_config_; - std::unique_ptr> subscription_; + std::unique_ptr subscription_; std::function initialize_callback_; const std::string sds_config_name_; diff --git a/source/common/upstream/cds_api_impl.cc b/source/common/upstream/cds_api_impl.cc index d62b938bb4d88..a244c2a9c329c 100644 --- a/source/common/upstream/cds_api_impl.cc +++ b/source/common/upstream/cds_api_impl.cc @@ -34,23 +34,31 @@ CdsApiImpl::CdsApiImpl(const envoy::api::v2::core::ConfigSource& cds_config, Clu envoy::api::v2::core::ApiConfigSource::DELTA_GRPC); const std::string grpc_method = is_delta ? "envoy.api.v2.ClusterDiscoveryService.DeltaClusters" : "envoy.api.v2.ClusterDiscoveryService.StreamClusters"; - subscription_ = - Config::SubscriptionFactory::subscriptionFromConfigSource( - cds_config, local_info, dispatcher, cm, random, *scope_, - "envoy.api.v2.ClusterDiscoveryService.FetchClusters", grpc_method, api); + subscription_ = Config::SubscriptionFactory::subscriptionFromConfigSource( + cds_config, local_info, dispatcher, cm, random, *scope_, + "envoy.api.v2.ClusterDiscoveryService.FetchClusters", grpc_method, + Grpc::Common::typeUrl(envoy::api::v2::Cluster().GetDescriptor()->full_name()), api); } +// TODO TODO this ResourceVector can become a "generic Protobuf" vector, and then have each get +// MessageUtil::anyConvert(resource). that will allow us to detemplatize +// SubscriptionCallbacks. void CdsApiImpl::onConfigUpdate(const ResourceVector& resources, const std::string& version_info) { ClusterManager::ClusterInfoMap clusters_to_remove = cm_.clusters(); - for (const auto& cluster : resources) { - clusters_to_remove.erase(cluster.name()); + std::vector clusters; + for (const auto& cluster_blob : resources) { + clusters.push_back(MessageUtil::anyConvert(cluster_blob)); + } + for (const auto& cluster : clusters) { + clusters_to_remove.erase( + cluster.name()); // TODO TODO can probably use resourceName here to skip the anyConvert. } Protobuf::RepeatedPtrField to_remove_repeated; for (const auto& cluster : clusters_to_remove) { *to_remove_repeated.Add() = cluster.first; } Protobuf::RepeatedPtrField to_add_repeated; - for (const auto& cluster : resources) { + for (const auto& cluster : clusters) { envoy::api::v2::Resource* to_add = to_add_repeated.Add(); to_add->set_name(cluster.name()); to_add->set_version(version_info); diff --git a/source/common/upstream/cds_api_impl.h b/source/common/upstream/cds_api_impl.h index 77e215259fd75..139c95d5fccfe 100644 --- a/source/common/upstream/cds_api_impl.h +++ b/source/common/upstream/cds_api_impl.h @@ -19,7 +19,7 @@ namespace Upstream { * CDS API implementation that fetches via Subscription. */ class CdsApiImpl : public CdsApi, - Config::SubscriptionCallbacks, + Config::SubscriptionCallbacks, Logger::Loggable { public: static CdsApiPtr create(const envoy::api::v2::core::ConfigSource& cds_config, ClusterManager& cm, @@ -51,7 +51,7 @@ class CdsApiImpl : public CdsApi, void runInitializeCallbackIfAny(); ClusterManager& cm_; - std::unique_ptr> subscription_; + std::unique_ptr subscription_; std::string system_version_info_; std::function initialize_callback_; Stats::ScopePtr scope_; diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index 0e9e8eb52667d..3a08754dbf31d 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -211,6 +211,9 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable( + subscription_ = Config::SubscriptionFactory::subscriptionFromConfigSource( eds_config, local_info_, dispatcher, cm, random, info_->statsScope(), "envoy.api.v2.EndpointDiscoveryService.FetchEndpoints", - "envoy.api.v2.EndpointDiscoveryService.StreamEndpoints", factory_context.api()); + "envoy.api.v2.EndpointDiscoveryService.StreamEndpoints", + Grpc::Common::typeUrl(envoy::api::v2::ClusterLoadAssignment().GetDescriptor()->full_name()), + factory_context.api()); } void EdsClusterImpl::startPreInit() { subscription_->start({cluster_name_}, *this); } @@ -107,7 +108,8 @@ void EdsClusterImpl::onConfigUpdate(const ResourceVector& resources, const std:: if (resources.size() != 1) { throw EnvoyException(fmt::format("Unexpected EDS resource length: {}", resources.size())); } - const auto& cluster_load_assignment = resources[0]; + auto cluster_load_assignment = + MessageUtil::anyConvert(resources[0]); MessageUtil::validate(cluster_load_assignment); // TODO(PiotrSikora): Remove this hack once fixed internally. if (!(cluster_load_assignment.cluster_name() == cluster_name_)) { diff --git a/source/common/upstream/eds.h b/source/common/upstream/eds.h index 197d654a57815..03e35ca2021e9 100644 --- a/source/common/upstream/eds.h +++ b/source/common/upstream/eds.h @@ -19,8 +19,7 @@ namespace Upstream { /** * Cluster implementation that reads host information from the Endpoint Discovery Service. */ -class EdsClusterImpl : public BaseDynamicClusterImpl, - Config::SubscriptionCallbacks { +class EdsClusterImpl : public BaseDynamicClusterImpl, Config::SubscriptionCallbacks { public: EdsClusterImpl(const envoy::api::v2::Cluster& cluster, Runtime::Loader& runtime, @@ -69,7 +68,7 @@ class EdsClusterImpl : public BaseDynamicClusterImpl, }; const ClusterManager& cm_; - std::unique_ptr> subscription_; + std::unique_ptr subscription_; const LocalInfo::LocalInfo& local_info_; const std::string cluster_name_; std::vector locality_weights_map_; diff --git a/source/server/lds_api.cc b/source/server/lds_api.cc index 2b5c18629b92d..2634936580b26 100644 --- a/source/server/lds_api.cc +++ b/source/server/lds_api.cc @@ -21,11 +21,11 @@ LdsApiImpl::LdsApiImpl(const envoy::api::v2::core::ConfigSource& lds_config, const LocalInfo::LocalInfo& local_info, Stats::Scope& scope, ListenerManager& lm, Api::Api& api) : listener_manager_(lm), scope_(scope.createScope("listener_manager.lds.")), cm_(cm) { - subscription_ = - Envoy::Config::SubscriptionFactory::subscriptionFromConfigSource( - lds_config, local_info, dispatcher, cm, random, *scope_, - "envoy.api.v2.ListenerDiscoveryService.FetchListeners", - "envoy.api.v2.ListenerDiscoveryService.StreamListeners", api); + subscription_ = Envoy::Config::SubscriptionFactory::subscriptionFromConfigSource( + lds_config, local_info, dispatcher, cm, random, *scope_, + "envoy.api.v2.ListenerDiscoveryService.FetchListeners", + "envoy.api.v2.ListenerDiscoveryService.StreamListeners", + Grpc::Common::typeUrl(envoy::api::v2::Listener().GetDescriptor()->full_name()), api); Config::Utility::checkLocalInfo("lds", local_info); init_manager.registerTarget(*this, "LDS"); } @@ -38,16 +38,19 @@ void LdsApiImpl::initialize(std::function callback) { void LdsApiImpl::onConfigUpdate(const ResourceVector& resources, const std::string& version_info) { cm_.adsMux().pause(Config::TypeUrl::get().RouteConfiguration); Cleanup rds_resume([this] { cm_.adsMux().resume(Config::TypeUrl::get().RouteConfiguration); }); + + std::vector listeners; + for (const auto& listener_blob : resources) { + listeners.push_back(MessageUtil::anyConvert(listener_blob)); + MessageUtil::validate(listeners.back()); + } std::vector exception_msgs; std::unordered_set listener_names; - for (const auto& listener : resources) { + for (const auto& listener : listeners) { if (!listener_names.insert(listener.name()).second) { throw EnvoyException(fmt::format("duplicate listener {} found", listener.name())); } } - for (const auto& listener : resources) { - MessageUtil::validate(listener); - } // We need to keep track of which listeners we might need to remove. std::unordered_map> listeners_to_remove; @@ -58,7 +61,7 @@ void LdsApiImpl::onConfigUpdate(const ResourceVector& resources, const std::stri for (const auto& listener : listener_manager_.listeners()) { listeners_to_remove.emplace(listener.get().name(), listener); } - for (const auto& listener : resources) { + for (const auto& listener : listeners) { listeners_to_remove.erase(listener.name()); } for (const auto& listener : listeners_to_remove) { @@ -67,7 +70,7 @@ void LdsApiImpl::onConfigUpdate(const ResourceVector& resources, const std::stri } } - for (const auto& listener : resources) { + for (const auto& listener : listeners) { const std::string listener_name = listener.name(); try { if (listener_manager_.addOrUpdateListener(listener, version_info, true)) { diff --git a/source/server/lds_api.h b/source/server/lds_api.h index 713ead3f118a6..1bdd38773c827 100644 --- a/source/server/lds_api.h +++ b/source/server/lds_api.h @@ -19,7 +19,7 @@ namespace Server { */ class LdsApiImpl : public LdsApi, public Init::Target, - Config::SubscriptionCallbacks, + Config::SubscriptionCallbacks, Logger::Loggable { public: LdsApiImpl(const envoy::api::v2::core::ConfigSource& lds_config, Upstream::ClusterManager& cm, @@ -48,7 +48,7 @@ class LdsApiImpl : public LdsApi, private: void runInitializeCallbackIfAny(); - std::unique_ptr> subscription_; + std::unique_ptr subscription_; std::string version_info_; ListenerManager& listener_manager_; Stats::ScopePtr scope_; diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index f6d8677642766..a4f2ebb093ab3 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -12,8 +12,7 @@ namespace Envoy { namespace Config { -template -class MockSubscriptionCallbacks : public SubscriptionCallbacks { +template class MockSubscriptionCallbacks : public SubscriptionCallbacks { public: MockSubscriptionCallbacks() { ON_CALL(*this, resourceName(testing::_)) @@ -28,7 +27,7 @@ class MockSubscriptionCallbacks : public SubscriptionCallbacks { // TODO(fredlas) deduplicate MOCK_METHOD2_T(onConfigUpdate, - void(const typename SubscriptionCallbacks::ResourceVector& resources, + void(const typename SubscriptionCallbacks::ResourceVector& resources, const std::string& version_info)); MOCK_METHOD3_T(onConfigUpdate, void(const Protobuf::RepeatedPtrField& added_resources, @@ -38,10 +37,10 @@ class MockSubscriptionCallbacks : public SubscriptionCallbacks { MOCK_METHOD1_T(resourceName, std::string(const ProtobufWkt::Any& resource)); }; -template class MockSubscription : public Subscription { +template class MockSubscription : public Subscription { public: - MOCK_METHOD2_T(start, void(const std::vector& resources, - SubscriptionCallbacks& callbacks)); + MOCK_METHOD2_T(start, + void(const std::vector& resources, SubscriptionCallbacks& callbacks)); MOCK_METHOD1_T(updateResources, void(const std::vector& resources)); }; From f290ae7cb52b6d0a58709762192d781762141e09 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Tue, 2 Apr 2019 17:25:19 -0400 Subject: [PATCH 02/56] quick snapshot so i can merge the detemplatization PR Signed-off-by: Fred Douglas --- include/envoy/config/BUILD | 38 ++-- include/envoy/config/ads_mux.h | 54 ++++++ include/envoy/config/grpc_xds_context.h | 42 ----- include/envoy/config/xds_context.h | 54 ++++++ include/envoy/upstream/BUILD | 1 + include/envoy/upstream/cluster_manager.h | 8 +- source/common/config/BUILD | 1 + .../common/config/delta_subscription_impl.h | 35 ++-- .../common/config/delta_subscription_state.h | 2 +- source/common/config/grpc_delta_xds_context.h | 91 +++++++--- source/common/config/grpc_mux_impl.cc | 166 ++++++++++-------- source/common/config/grpc_mux_impl.h | 53 ++++-- .../config/grpc_mux_subscription_impl.h | 10 +- source/common/config/grpc_stream.h | 93 +++------- source/common/config/grpc_subscription_impl.h | 11 +- source/common/config/subscription_factory.h | 16 +- source/common/upstream/cds_api_impl.cc | 9 +- .../common/upstream/cluster_manager_impl.cc | 40 +++-- source/common/upstream/cluster_manager_impl.h | 11 +- source/server/lds_api.cc | 4 +- source/server/server.cc | 4 +- .../config/config_provider_impl_test.cc | 8 +- .../config/delta_subscription_test_harness.h | 10 +- .../filesystem_subscription_impl_test.cc | 2 +- .../filesystem_subscription_test_harness.h | 19 +- test/mocks/config/BUILD | 1 + test/mocks/config/mocks.cc | 8 + test/mocks/config/mocks.h | 29 +++ test/mocks/upstream/mocks.cc | 2 +- test/mocks/upstream/mocks.h | 4 +- 30 files changed, 510 insertions(+), 316 deletions(-) create mode 100644 include/envoy/config/ads_mux.h delete mode 100644 include/envoy/config/grpc_xds_context.h create mode 100644 include/envoy/config/xds_context.h diff --git a/include/envoy/config/BUILD b/include/envoy/config/BUILD index 4696d029075fc..832ed61db786f 100644 --- a/include/envoy/config/BUILD +++ b/include/envoy/config/BUILD @@ -8,6 +8,26 @@ load( envoy_package() +envoy_cc_library( + name = "config_provider_interface", + hdrs = ["config_provider.h"], + external_deps = ["abseil_optional"], + deps = [ + "//include/envoy/common:time_interface", + "//source/common/protobuf", + ], +) + +envoy_cc_library( + name = "config_provider_manager_interface", + hdrs = ["config_provider_manager.h"], + deps = [ + ":config_provider_interface", + "//include/envoy/server:filter_config_interface", + "//source/common/protobuf", + ], +) + envoy_cc_library( name = "grpc_mux_interface", hdrs = ["grpc_mux.h"], @@ -36,21 +56,11 @@ envoy_cc_library( ) envoy_cc_library( - name = "config_provider_interface", - hdrs = ["config_provider.h"], - external_deps = ["abseil_optional"], + name = "xds_context_interface", + hdrs = ["xds_context.h"], deps = [ - "//include/envoy/common:time_interface", - "//source/common/protobuf", - ], -) - -envoy_cc_library( - name = "config_provider_manager_interface", - hdrs = ["config_provider_manager.h"], - deps = [ - ":config_provider_interface", - "//include/envoy/server:filter_config_interface", + ":subscription_interface", + "//include/envoy/local_info:local_info_interface", "//source/common/protobuf", ], ) diff --git a/include/envoy/config/ads_mux.h b/include/envoy/config/ads_mux.h new file mode 100644 index 0000000000000..78eafc21f9677 --- /dev/null +++ b/include/envoy/config/ads_mux.h @@ -0,0 +1,54 @@ +#pragma once + +namespace Envoy { +namespace Config { + +/** + * Manage one or more gRPC subscriptions on a single stream to management server. This can be used + * for a single xDS API, e.g. EDS, or to combined multiple xDS APIs for ADS. + */ +class AdsMux : public XdsGrpcContext { +public: + virtual ~AdsMux() {} + + /** + * Initiate stream with management server. + */ + virtual void start() PURE; + + /** + * Start a configuration subscription asynchronously for some API type and resources. + * @param type_url type URL corresponding to xDS API, e.g. + * type.googleapis.com/envoy.api.v2.Cluster. + * @param resources vector of resource names to watch for. If this is empty, then all + * resources for type_url will result in callbacks. + * @param callbacks the callbacks to be notified of configuration updates. These must be valid + * until GrpcMuxWatch is destroyed. + * @return GrpcMuxWatchPtr a handle to cancel the subscription with. E.g. when a cluster goes + * away, its EDS updates should be cancelled by destroying the GrpcMuxWatchPtr. + */ + // TODO TODO no dont return that............ + virtual GrpcMuxWatchPtr subscribe(const std::string& type_url, + const std::vector& resources, + GrpcMuxCallbacks& callbacks) PURE; + + /** + * Pause discovery requests for a given API type. This is useful when we're processing an update + * for LDS or CDS and don't want a flood of updates for RDS or EDS respectively. Discovery + * requests may later be resumed with resume(). + * @param type_url type URL corresponding to xDS API, e.g. + * type.googleapis.com/envoy.api.v2.Cluster. + */ + virtual void pause(const std::string& type_url) PURE; + + /** + * Resume discovery requests for a given API type. This will send a discovery request if one would + * have been sent during the pause. + * @param type_url type URL corresponding to xDS API, + * e.g.type.googleapis.com/envoy.api.v2.Cluster. + */ + virtual void resume(const std::string& type_url) PURE; +}; + +} // namespace Config +} // namespace Envoy diff --git a/include/envoy/config/grpc_xds_context.h b/include/envoy/config/grpc_xds_context.h deleted file mode 100644 index f918137f7b93b..0000000000000 --- a/include/envoy/config/grpc_xds_context.h +++ /dev/null @@ -1,42 +0,0 @@ -#pragma once - -#include "envoy/api/v2/discovery.pb.h" -#include "envoy/config/subscription.h" - -#include "common/common/assert.h" -#include "common/common/logger.h" -#include "common/config/utility.h" -#include "common/grpc/common.h" -#include "common/protobuf/protobuf.h" -#include "common/protobuf/utility.h" - -namespace Envoy { -namespace Config { - -class GrpcXdsContext { -public: - void addSubscription(const std::vector& resources, const std::string& type_url, - const LocalInfo::LocalInfo& local_info, SubscriptionCallbacks& callbacks, - Event::Dispatcher& dispatcher, std::chrono::milliseconds init_fetch_timeout, - SubscriptionStats stats); - - // Enqueues and attempts to send a discovery request, (un)subscribing to resources missing from / - // added to the passed 'resources' argument, relative to resource_versions_. - void updateSubscription(const std::vector& resources, const std::string& type_url); - - void removeSubscription(const std::string& type_url); - - void pause(const std::string& type_url); - - void resume(const std::string& type_url); - - // Returns whether the request was actually sent (and so can leave the queue). - virtual void sendDiscoveryRequest(const RequestQueueItem& queue_item) PURE; - - virtual void handleResponse(std::unique_ptr&& message) PURE; - virtual void handleStreamEstablished() PURE; - virtual void handleEstablishmentFailure() PURE; -}; - -} // namespace Config -} // namespace Envoy diff --git a/include/envoy/config/xds_context.h b/include/envoy/config/xds_context.h new file mode 100644 index 0000000000000..4968e9ab058c8 --- /dev/null +++ b/include/envoy/config/xds_context.h @@ -0,0 +1,54 @@ +#pragma once + +#include "envoy/config/grpc_mux.h" // TODO TODO remove +#include "envoy/config/subscription.h" +#include "envoy/event/dispatcher.h" +#include "envoy/local_info/local_info.h" + +#include "common/protobuf/protobuf.h" + +namespace Envoy { +namespace Config { + +class XdsGrpcContext { +public: + virtual ~XdsGrpcContext() {} + virtual void addSubscription(const std::vector& resources, + const std::string& type_url, SubscriptionCallbacks& callbacks, + SubscriptionStats stats) PURE; + + // Enqueues and attempts to send a discovery request, (un)subscribing to resources missing from / + // added to the passed 'resources' argument, relative to resource_versions_. + virtual void updateResources(const std::vector& resources, + const std::string& type_url) PURE; + + virtual void removeSubscription(const std::string& type_url) PURE; + + virtual void pause(const std::string& type_url) PURE; + + virtual void resume(const std::string& type_url) PURE; + + virtual void drainRequests() PURE; + + /** + * For the gRPC stream handler to prompt the context to take appropriate action in response to the + * gRPC stream having been successfully established. + */ + virtual void handleStreamEstablished() PURE; + + /** + * For the gRPC stream handler to prompt the context to take appropriate action in response to + * failure to establish the gRPC stream. + */ + virtual void handleEstablishmentFailure() PURE; + + // TODO TODO remove + virtual GrpcMuxWatchPtr subscribe(const std::string& type_url, + const std::vector& resources, + GrpcMuxCallbacks& callbacks) PURE; + // TODO TODO remove + virtual void start() PURE; +}; + +} // namespace Config +} // namespace Envoy diff --git a/include/envoy/upstream/BUILD b/include/envoy/upstream/BUILD index 31edcd0dff059..4e3a59735cb8c 100644 --- a/include/envoy/upstream/BUILD +++ b/include/envoy/upstream/BUILD @@ -18,6 +18,7 @@ envoy_cc_library( ":upstream_interface", "//include/envoy/access_log:access_log_interface", "//include/envoy/config:grpc_mux_interface", + "//include/envoy/config:xds_context_interface", "//include/envoy/grpc:async_client_manager_interface", "//include/envoy/http:async_client_interface", "//include/envoy/http:conn_pool_interface", diff --git a/include/envoy/upstream/cluster_manager.h b/include/envoy/upstream/cluster_manager.h index 37891b06ca7dd..61e9a19cc8652 100644 --- a/include/envoy/upstream/cluster_manager.h +++ b/include/envoy/upstream/cluster_manager.h @@ -11,6 +11,7 @@ #include "envoy/api/v2/cds.pb.h" #include "envoy/config/bootstrap/v2/bootstrap.pb.h" #include "envoy/config/grpc_mux.h" +#include "envoy/config/xds_context.h" #include "envoy/grpc/async_client_manager.h" #include "envoy/http/async_client.h" #include "envoy/http/conn_pool.h" @@ -204,12 +205,9 @@ class ClusterManager { * the management of clusters but instead is required early in ClusterManager/server * initialization and in various sites that need ClusterManager for xDS API interfacing. * - * @return GrpcMux& ADS API provider referencee. + * @return XdsGrpcContext& ADS API provider referencee. */ - virtual Config::GrpcMux& adsMux() PURE; - - // TODO TODO COMMENT - virtual Config::GrpcDeltaXdsContext& grpcDeltaXdsContext() PURE; + virtual std::shared_ptr adsMux() PURE; /** * @return Grpc::AsyncClientManager& the cluster manager's gRPC client manager. diff --git a/source/common/config/BUILD b/source/common/config/BUILD index 270588dcff319..d0dc18d0b600b 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -132,6 +132,7 @@ envoy_cc_library( ":utility_lib", "//include/envoy/config:grpc_mux_interface", "//include/envoy/config:subscription_interface", + "//include/envoy/config:xds_context_interface", "//include/envoy/grpc:async_client_interface", "//include/envoy/upstream:cluster_manager_interface", "//source/common/common:backoff_lib", diff --git a/source/common/config/delta_subscription_impl.h b/source/common/config/delta_subscription_impl.h index f2e9b045a9787..b3c60468fd894 100644 --- a/source/common/config/delta_subscription_impl.h +++ b/source/common/config/delta_subscription_impl.h @@ -18,19 +18,22 @@ namespace Config { * the only difference is that ADS has multiple DeltaSubscriptions sharing a single * GrpcDeltaXdsContext. */ +// TODO(fredlas) someday this class will be named SubscriptionImpl (without any changes to its code) class DeltaSubscriptionImpl : public Subscription { public: - DeltaSubscriptionImpl( - GrpcDeltaXdsContextSharedPtr context, - absl::string_view - type_url, // TODO TODO everwhere should pass this as - // Grpc::Common::typeUrl(ResourceType().GetDescriptor()->full_name()) - const LocalInfo::LocalInfo& local_info, std::chrono::milliseconds init_fetch_timeout, - Event::Dispatcher& dispatcher, SubscriptionStats stats) - : context_(context), type_url_(type_url), local_info_(local_info), - init_fetch_timeout_(init_fetch_timeout), dispatcher_(dispatcher), stats_(stats) {} - - ~DeltaSubscriptionImpl() { context_->removeSubscription(type_url_); } + DeltaSubscriptionImpl(std::shared_ptr context, absl::string_view type_url, + SubscriptionStats stats) + : context_(context), type_url_(type_url), stats_(stats) {} + + ~DeltaSubscriptionImpl() { + context_->removeSubscription(type_url_); + } // TODO TODO hmmm maybe the watch stuff should be involved...... or...... this itself is the + // watch? yeah! actually, i think this itself is the watch. with the existing GrpxMux stuff, you + // would provide callbacks (probably even just a ref to a cb object) that would get called when + // your subscription had activity, rather than actually owning the subscription. so, you needed + // the watch thing to have the ability to tell GrpcMux "no stop these updates". now, with the + // "subscription shared-owns a context" thing, you just destroy the subscription object, and + // that ends the on-the-wire xDS activity for this sub. void pause() { context_->pause(type_url_); } @@ -38,20 +41,16 @@ class DeltaSubscriptionImpl : public Subscription { // Config::DeltaSubscription void start(const std::vector& resources, SubscriptionCallbacks& callbacks) override { - context_->addSubscription(resources, type_url_, local_info_, callbacks, dispatcher_, - init_fetch_timeout_, stats_); + context_->addSubscription(resources, type_url_, callbacks, stats_); } void updateResources(const std::vector& resources) override { - context_->updateSubscription(resources, type_url_); + context_->updateResources(resources, type_url_); } private: - GrpcDeltaXdsContextSharedPtr context_; + std::shared_ptr context_; const std::string type_url_; - const LocalInfo::LocalInfo& local_info_; - std::chrono::milliseconds init_fetch_timeout_; - Event::Dispatcher& dispatcher_; SubscriptionStats stats_; }; diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index 4ffd7ff20c4c4..fd930ede5bb06 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -2,6 +2,7 @@ #include "envoy/api/v2/discovery.pb.h" #include "envoy/config/subscription.h" +#include "envoy/grpc/status.h" namespace Envoy { namespace Config { @@ -114,7 +115,6 @@ class DeltaSubscriptionState : public Logger::Loggable { bool checkPausedDuringSendAttempt(const ResourceNameDiff& diff) { if (paused_) { - ENVOY_LOG(trace, "API {} paused during sendDiscoveryRequest().", type_url_); pending_ = diff; // resume() is now set to send this request. return true; } diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h index e8ee7bb06fcff..3a6311ed1635a 100644 --- a/source/common/config/grpc_delta_xds_context.h +++ b/source/common/config/grpc_delta_xds_context.h @@ -1,8 +1,11 @@ #pragma once +#include + #include "envoy/api/v2/discovery.pb.h" #include "envoy/common/token_bucket.h" #include "envoy/config/subscription.h" +#include "envoy/config/xds_context.h" #include "common/common/assert.h" #include "common/common/backoff_strategy.h" @@ -18,31 +21,34 @@ namespace Envoy { namespace Config { -class GrpcDeltaXdsContext - : public GrpcStream { +class GrpcDeltaXdsContext : public XdsGrpcContext, Logger::Loggable { public: - GrpcDeltaXdsContext(Grpc::AsyncClientPtr async_client, + GrpcDeltaXdsContext(Grpc::AsyncClientPtr async_client, Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, - Event::Dispatcher& dispatcher, Runtime::RandomGenerator& random, - Stats::Scope& scope, const RateLimitSettings& rate_limit_settings) - : GrpcStream(std::move(async_client), service_method, random, dispatcher, - scope, rate_limit_settings) {} + Runtime::RandomGenerator& random, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info, + std::chrono::milliseconds init_fetch_timeout) + : local_info_(local_info), init_fetch_timeout_(init_fetch_timeout), dispatcher_(dispatcher), + grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, + rate_limit_settings, + // callback for handling receipt of DiscoveryResponse protos. + [this](std::unique_ptr&& message) { + handleDiscoveryResponse(std::move(message)); + }) {} void addSubscription(const std::vector& resources, const std::string& type_url, - const LocalInfo::LocalInfo& local_info, SubscriptionCallbacks& callbacks, - Event::Dispatcher& dispatcher, std::chrono::milliseconds init_fetch_timeout, - SubscriptionStats stats) { + SubscriptionCallbacks& callbacks, SubscriptionStats stats) override { subscriptions_.emplace( - std::make_pair(type_url, DeltaSubscriptionState(type_url, resources, callbacks, local_info, - init_fetch_timeout, dispatcher, stats))); - establishNewStream(); // (idempotent) + std::make_pair(type_url, DeltaSubscriptionState(type_url, resources, callbacks, local_info_, + init_fetch_timeout_, dispatcher_, stats))); + grpc_stream_.establishNewStream(); // (idempotent) } // Enqueues and attempts to send a discovery request, (un)subscribing to resources missing from / // added to the passed 'resources' argument, relative to resource_versions_. - void updateSubscription(const std::vector& resources, const std::string& type_url) { + void updateResources(const std::vector& resources, + const std::string& type_url) override { auto sub = subscriptions_.find(type_url); if (sub == subscriptions_.end()) { ENVOY_LOG(warn, "Not updating non-existent subscription {}.", type_url); @@ -54,9 +60,9 @@ class GrpcDeltaXdsContext queueDiscoveryRequest(diff); } - void removeSubscription(const std::string& type_url) { subscriptions_.erase(type_url); } + void removeSubscription(const std::string& type_url) override { subscriptions_.erase(type_url); } - void pause(const std::string& type_url) { + void pause(const std::string& type_url) override { auto sub = subscriptions_.find(type_url); if (sub == subscriptions_.end()) { ENVOY_LOG(warn, "Not pausing non-existent subscription {}.", type_url); @@ -65,7 +71,7 @@ class GrpcDeltaXdsContext sub->second.pause(); } - void resume(const std::string& type_url) { + void resume(const std::string& type_url) override { auto sub = subscriptions_.find(type_url); if (sub == subscriptions_.end()) { ENVOY_LOG(warn, "Not resuming non-existent subscription {}.", type_url); @@ -82,14 +88,14 @@ class GrpcDeltaXdsContext } } - void sendDiscoveryRequest(const ResourceNameDiff& diff) override { + void sendDiscoveryRequest(const ResourceNameDiff& diff) { auto sub = subscriptions_.find(diff.type_url_); if (sub == subscriptions_.end()) { ENVOY_LOG(warn, "Not sending DeltaDiscoveryRequest for non-existent subscription {}.", diff.type_url_); return; } - if (!grpcStreamAvailable()) { + if (!grpc_stream_.grpcStreamAvailable()) { ENVOY_LOG(debug, "No stream available to sendDiscoveryRequest for {}", diff.type_url_); return; // Drop this request; the reconnect will enqueue a new one. } @@ -103,12 +109,12 @@ class GrpcDeltaXdsContext // NOTE: at this point we are beyond the rate-limiting logic. sendMessage() unconditionally // sends its argument over the gRPC stream. So, it is safe to unconditionally // set_first_request_of_new_stream(false). - sendMessage(request); + grpc_stream_.sendMessage(request); sub->second.set_first_request_of_new_stream(false); sub->second.buildCleanRequest(); } - void handleResponse(std::unique_ptr&& message) override { + void handleDiscoveryResponse(std::unique_ptr&& message) { ENVOY_LOG(debug, "Received DeltaDiscoveryResponse for {} at version {}", message->type_url(), message->system_version_info()); auto sub = subscriptions_.find(message->type_url()); @@ -139,7 +145,46 @@ class GrpcDeltaXdsContext } } + // Request queue management logic. + void queueDiscoveryRequest(const ResourceNameDiff& queue_item) { + request_queue_.push(queue_item); + drainRequests(); + } + void clearRequestQueue() { + grpc_stream_.maybeUpdateQueueSizeStat(0); + // TODO(fredlas) when we have C++17: request_queue_ = {}; + while (!request_queue_.empty()) { + request_queue_.pop(); + } + } + void drainRequests() override { + ENVOY_LOG(trace, "draining discovery requests {}", request_queue_.size()); + while (!request_queue_.empty() && grpc_stream_.checkRateLimitAllowsDrain()) { + // Process the request, if rate limiting is not enabled at all or if it is under rate limit. + sendDiscoveryRequest(request_queue_.front()); + request_queue_.pop(); + } + grpc_stream_.maybeUpdateQueueSizeStat(request_queue_.size()); + } + + // TODO TODO remove, GrpcMux impersonation + virtual GrpcMuxWatchPtr subscribe(const std::string&, const std::vector&, + GrpcMuxCallbacks&) override { + return nullptr; + } + virtual void start() override {} + private: + const LocalInfo::LocalInfo& local_info_; + const std::chrono::milliseconds init_fetch_timeout_; + Event::Dispatcher& dispatcher_; + + GrpcStream + grpc_stream_; + + // A queue to store requests while rate limited. Note that when requests cannot be sent due to the + // gRPC stream being down, this queue does not store them; rather, they are simply dropped. + std::queue request_queue_; std::unordered_map subscriptions_; }; diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index 7519a990e2661..1a871afe46abc 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -13,8 +13,13 @@ GrpcMuxImpl::GrpcMuxImpl(const LocalInfo::LocalInfo& local_info, Grpc::AsyncClie const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings) - : GrpcStream( - std::move(async_client), service_method, random, dispatcher, scope, rate_limit_settings), + : grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, + rate_limit_settings, + // callback for handling receipt of DiscoveryResponse protos. + [this](std::unique_ptr&& message) { + handleDiscoveryResponse(std::move(message)); + }), + local_info_(local_info) { Config::Utility::checkLocalInfo("ads", local_info); } @@ -27,10 +32,10 @@ GrpcMuxImpl::~GrpcMuxImpl() { } } -void GrpcMuxImpl::start() { establishNewStream(); } +void GrpcMuxImpl::start() { grpc_stream_.establishNewStream(); } void GrpcMuxImpl::sendDiscoveryRequest(const std::string& type_url) { - if (!grpcStreamAvailable()) { + if (!grpc_stream_.grpcStreamAvailable()) { ENVOY_LOG(debug, "No stream available to sendDiscoveryRequest for {}", type_url); return; // Drop this request; the reconnect will enqueue a new one. } @@ -57,7 +62,7 @@ void GrpcMuxImpl::sendDiscoveryRequest(const std::string& type_url) { } ENVOY_LOG(trace, "Sending DiscoveryRequest for {}: {}", type_url, request.DebugString()); - sendMessage(request); + grpc_stream_.sendMessage(request); // clear error_detail after the request is sent if it exists. if (api_state_[type_url].request_.has_error_detail()) { @@ -65,77 +70,30 @@ void GrpcMuxImpl::sendDiscoveryRequest(const std::string& type_url) { } } -GrpcMuxWatchPtr GrpcMuxImpl::subscribe(const std::string& type_url, - const std::vector& resources, - GrpcMuxCallbacks& callbacks) { - auto watch = - std::unique_ptr(new GrpcMuxWatchImpl(resources, callbacks, type_url, *this)); - ENVOY_LOG(debug, "gRPC mux subscribe for " + type_url); - - // Lazily kick off the requests based on first subscription. This has the - // convenient side-effect that we order messages on the channel based on - // Envoy's internal dependency ordering. - // TODO(gsagula): move TokenBucketImpl params to a config. - if (!api_state_[type_url].subscribed_) { - api_state_[type_url].request_.set_type_url(type_url); - api_state_[type_url].request_.mutable_node()->MergeFrom(local_info_.node()); - api_state_[type_url].subscribed_ = true; - subscriptions_.emplace_back(type_url); - } - - // This will send an updated request on each subscription. - // TODO(htuch): For RDS/EDS, this will generate a new DiscoveryRequest on each resource we added. - // Consider in the future adding some kind of collation/batching during CDS/LDS updates so that we - // only send a single RDS/EDS update after the CDS/LDS update. - queueDiscoveryRequest(type_url); - - return watch; -} - -void GrpcMuxImpl::pause(const std::string& type_url) { - ENVOY_LOG(debug, "Pausing discovery requests for {}", type_url); - ApiState& api_state = api_state_[type_url]; - ASSERT(!api_state.paused_); - ASSERT(!api_state.pending_); - api_state.paused_ = true; -} - -void GrpcMuxImpl::resume(const std::string& type_url) { - ENVOY_LOG(debug, "Resuming discovery requests for {}", type_url); - ApiState& api_state = api_state_[type_url]; - ASSERT(api_state.paused_); - api_state.paused_ = false; - - if (api_state.pending_) { - ASSERT(api_state.subscribed_); - queueDiscoveryRequest(type_url); - api_state.pending_ = false; - } -} - -void GrpcMuxImpl::handleResponse(std::unique_ptr&& message) { +void GrpcMuxImpl::handleDiscoveryResponse( + std::unique_ptr&& message) { const std::string& type_url = message->type_url(); ENVOY_LOG(debug, "Received gRPC message for {} at version {}", type_url, message->version_info()); if (api_state_.count(type_url) == 0) { ENVOY_LOG(warn, "Ignoring the message for type URL {} as it has no current subscribers.", type_url); - // TODO(yuval-k): This should never happen. consider dropping the stream as this is a protocol - // violation + // TODO(yuval-k): This should never happen. consider dropping the stream as this is a + // protocol violation return; } if (api_state_[type_url].watches_.empty()) { // update the nonce as we are processing this response. api_state_[type_url].request_.set_response_nonce(message->nonce()); if (message->resources().empty()) { - // No watches and no resources. This can happen when envoy unregisters from a resource - // that's removed from the server as well. For example, a deleted cluster triggers un-watching - // the ClusterLoadAssignment watch, and at the same time the xDS server sends an empty list of - // ClusterLoadAssignment resources. we'll accept this update. no need to send a discovery - // request, as we don't watch for anything. + // No watches and no resources. This can happen when envoy unregisters from a + // resource that's removed from the server as well. For example, a deleted cluster + // triggers un-watching the ClusterLoadAssignment watch, and at the same time the + // xDS server sends an empty list of ClusterLoadAssignment resources. we'll accept + // this update. no need to send a discovery request, as we don't watch for anything. api_state_[type_url].request_.set_version_info(message->version_info()); } else { - // No watches and we have resources - this should not happen. send a NACK (by not updating - // the version). + // No watches and we have resources - this should not happen. send a NACK (by not + // updating the version). ENVOY_LOG(warn, "Ignoring unwatched type URL {}", type_url); queueDiscoveryRequest(type_url); } @@ -157,9 +115,9 @@ void GrpcMuxImpl::handleResponse(std::unique_ptrresources_.empty()) { watch->callbacks_.onConfigUpdate(message->resources(), message->version_info()); continue; @@ -171,14 +129,14 @@ void GrpcMuxImpl::handleResponse(std::unique_ptrMergeFrom(it->second); } } - // onConfigUpdate should be called only on watches(clusters/routes) that have updates in the - // message for EDS/RDS. + // onConfigUpdate should be called only on watches(clusters/routes) that have + // updates in the message for EDS/RDS. if (found_resources.size() > 0) { watch->callbacks_.onConfigUpdate(found_resources, message->version_info()); } } - // TODO(mattklein123): In the future if we start tracking per-resource versions, we would do - // that tracking here. + // TODO(mattklein123): In the future if we start tracking per-resource versions, we + // would do that tracking here. api_state_[type_url].request_.set_version_info(message->version_info()); } catch (const EnvoyException& e) { for (auto watch : api_state_[type_url].watches_) { @@ -192,6 +150,54 @@ void GrpcMuxImpl::handleResponse(std::unique_ptr& resources, + GrpcMuxCallbacks& callbacks) { + auto watch = + std::unique_ptr(new GrpcMuxWatchImpl(resources, callbacks, type_url, *this)); + ENVOY_LOG(debug, "gRPC mux subscribe for " + type_url); + + // Lazily kick off the requests based on first subscription. This has the + // convenient side-effect that we order messages on the channel based on + // Envoy's internal dependency ordering. + // TODO(gsagula): move TokenBucketImpl params to a config. + if (!api_state_[type_url].subscribed_) { + api_state_[type_url].request_.set_type_url(type_url); + api_state_[type_url].request_.mutable_node()->MergeFrom(local_info_.node()); + api_state_[type_url].subscribed_ = true; + subscriptions_.emplace_back(type_url); + } + + // This will send an updated request on each subscription. + // TODO(htuch): For RDS/EDS, this will generate a new DiscoveryRequest on each resource we added. + // Consider in the future adding some kind of collation/batching during CDS/LDS updates so that we + // only send a single RDS/EDS update after the CDS/LDS update. + queueDiscoveryRequest(type_url); + + return watch; +} + +void GrpcMuxImpl::pause(const std::string& type_url) { + ENVOY_LOG(debug, "Pausing discovery requests for {}", type_url); + ApiState& api_state = api_state_[type_url]; + ASSERT(!api_state.paused_); + ASSERT(!api_state.pending_); + api_state.paused_ = true; +} + +void GrpcMuxImpl::resume(const std::string& type_url) { + ENVOY_LOG(debug, "Resuming discovery requests for {}", type_url); + ApiState& api_state = api_state_[type_url]; + ASSERT(api_state.paused_); + api_state.paused_ = false; + + if (api_state.pending_) { + ASSERT(api_state.subscribed_); + queueDiscoveryRequest(type_url); + api_state.pending_ = false; + } +} + void GrpcMuxImpl::handleStreamEstablished() { for (const auto type_url : subscriptions_) { queueDiscoveryRequest(type_url); @@ -206,5 +212,25 @@ void GrpcMuxImpl::handleEstablishmentFailure() { } } +void GrpcMuxImpl::queueDiscoveryRequest(const std::string& queue_item) { + request_queue_.push(queue_item); + drainRequests(); +} +void GrpcMuxImpl::clearRequestQueue() { + grpc_stream_.maybeUpdateQueueSizeStat(0); + // TODO(fredlas) when we have C++17: request_queue_ = {}; + while (!request_queue_.empty()) { + request_queue_.pop(); + } +} +void GrpcMuxImpl::drainRequests() { + while (!request_queue_.empty() && grpc_stream_.checkRateLimitAllowsDrain()) { + // Process the request, if rate limiting is not enabled at all or if it is under rate limit. + sendDiscoveryRequest(request_queue_.front()); + request_queue_.pop(); + } + grpc_stream_.maybeUpdateQueueSizeStat(request_queue_.size()); +} + } // namespace Config } // namespace Envoy diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index 5a3672e5fe132..e323305c4a03a 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include "envoy/common/time.h" @@ -19,11 +20,9 @@ namespace Config { /** * ADS API implementation that fetches via gRPC. */ -class GrpcMuxImpl - : public GrpcMux, - public GrpcStream // this string is a type URL -{ +class GrpcMuxImpl : public GrpcMux, + public XdsGrpcContext, + public Logger::Loggable { public: GrpcMuxImpl(const LocalInfo::LocalInfo& local_info, Grpc::AsyncClientPtr async_client, Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, @@ -34,17 +33,26 @@ class GrpcMuxImpl void start() override; GrpcMuxWatchPtr subscribe(const std::string& type_url, const std::vector& resources, GrpcMuxCallbacks& callbacks) override; - void pause(const std::string& type_url) override; - void resume(const std::string& type_url) override; - void sendDiscoveryRequest(const std::string& type_url) override; + void sendDiscoveryRequest(const std::string& type_url); - // GrpcStream - void handleResponse(std::unique_ptr&& message) override; + // XdsGrpcContext void handleStreamEstablished() override; void handleEstablishmentFailure() override; + void pause(const std::string& type_url) override; + void resume(const std::string& type_url) override; + void drainRequests() override; + void addSubscription(const std::vector&, const std::string&, SubscriptionCallbacks&, + SubscriptionStats) override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } + void updateResources(const std::vector&, const std::string&) override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } + void removeSubscription(const std::string&) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } private: + void handleDiscoveryResponse(std::unique_ptr&& message); void setRetryTimer(); struct GrpcMuxWatchImpl : public GrpcMuxWatch { @@ -85,13 +93,23 @@ class GrpcMuxImpl bool subscribed_{}; }; + // Request queue management logic. + void queueDiscoveryRequest(const std::string& queue_item); + void clearRequestQueue(); + + GrpcStream grpc_stream_; const LocalInfo::LocalInfo& local_info_; std::unordered_map api_state_; // Envoy's dependency ordering. std::list subscriptions_; + + // A queue to store requests while rate limited. Note that when requests cannot be sent due to the + // gRPC stream being down, this queue does not store them; rather, they are simply dropped. + // This string is a type URL. + std::queue request_queue_; }; -class NullGrpcMuxImpl : public GrpcMux { +class NullGrpcMuxImpl : public XdsGrpcContext { public: void start() override {} GrpcMuxWatchPtr subscribe(const std::string&, const std::vector&, @@ -100,6 +118,19 @@ class NullGrpcMuxImpl : public GrpcMux { } void pause(const std::string&) override {} void resume(const std::string&) override {} + + void addSubscription(const std::vector&, const std::string&, SubscriptionCallbacks&, + SubscriptionStats) override { + throw EnvoyException("ADS must be configured to support an ADS config source"); + } + void updateResources(const std::vector&, const std::string&) override { + throw EnvoyException("ADS must be configured to support an ADS config source"); + } + + void removeSubscription(const std::string&) override {} + void drainRequests() override {} + void handleStreamEstablished() override {} + void handleEstablishmentFailure() override {} }; } // namespace Config diff --git a/source/common/config/grpc_mux_subscription_impl.h b/source/common/config/grpc_mux_subscription_impl.h index ff8e41a1fd79b..db49454415c1a 100644 --- a/source/common/config/grpc_mux_subscription_impl.h +++ b/source/common/config/grpc_mux_subscription_impl.h @@ -20,8 +20,8 @@ class GrpcMuxSubscriptionImpl : public Subscription, GrpcMuxCallbacks, Logger::Loggable { public: - GrpcMuxSubscriptionImpl(GrpcMux& grpc_mux, SubscriptionStats stats, absl::string_view type_url, - Event::Dispatcher& dispatcher, + GrpcMuxSubscriptionImpl(std::shared_ptr grpc_mux, SubscriptionStats stats, + absl::string_view type_url, Event::Dispatcher& dispatcher, std::chrono::milliseconds init_fetch_timeout) : grpc_mux_(grpc_mux), stats_(stats), type_url_(type_url), // TODO TODO everwhere should pass this as @@ -40,7 +40,7 @@ class GrpcMuxSubscriptionImpl : public Subscription, init_fetch_timeout_timer_->enableTimer(init_fetch_timeout_); } - watch_ = grpc_mux_.subscribe(type_url_, resources, *this); + watch_ = grpc_mux_->subscribe(type_url_, resources, *this); // The attempt stat here is maintained for the purposes of having consistency between ADS and // gRPC/filesystem/REST Subscriptions. Since ADS is push based and muxed, the notion of an // "attempt" for a given xDS API combined by ADS is not really that meaningful. @@ -48,7 +48,7 @@ class GrpcMuxSubscriptionImpl : public Subscription, } void updateResources(const std::vector& resources) override { - watch_ = grpc_mux_.subscribe(type_url_, resources, *this); + watch_ = grpc_mux_->subscribe(type_url_, resources, *this); stats_.update_attempt_.inc(); } @@ -94,7 +94,7 @@ class GrpcMuxSubscriptionImpl : public Subscription, } } - GrpcMux& grpc_mux_; + std::shared_ptr grpc_mux_; SubscriptionStats stats_; const std::string type_url_; SubscriptionCallbacks* callbacks_{}; diff --git a/source/common/config/grpc_stream.h b/source/common/config/grpc_stream.h index 0d5d148e8c819..2cc1dd8aa0630 100644 --- a/source/common/config/grpc_stream.h +++ b/source/common/config/grpc_stream.h @@ -1,8 +1,8 @@ #pragma once #include -#include +#include "envoy/config/xds_context.h" #include "envoy/grpc/async_client.h" #include "common/common/backoff_strategy.h" @@ -15,69 +15,48 @@ namespace Config { // Oversees communication for gRPC xDS implementations (parent to both regular xDS and delta // xDS variants). Reestablishes the gRPC channel when necessary, and provides rate limiting of // requests. -template +template class GrpcStream : public Grpc::TypedAsyncStreamCallbacks, public Logger::Loggable { public: - GrpcStream(Grpc::AsyncClientPtr async_client, const Protobuf::MethodDescriptor& service_method, - Runtime::RandomGenerator& random, Event::Dispatcher& dispatcher, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings) - : async_client_(std::move(async_client)), service_method_(service_method), - control_plane_stats_(generateControlPlaneStats(scope)), random_(random), - time_source_(dispatcher.timeSource()), - rate_limiting_enabled_(rate_limit_settings.enabled_) { + GrpcStream(XdsGrpcContext* owning_context, Grpc::AsyncClientPtr async_client, + const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, + Event::Dispatcher& dispatcher, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, + std::function&&)> handle_response_cb) + : owning_context_(owning_context), async_client_(std::move(async_client)), + service_method_(service_method), control_plane_stats_(generateControlPlaneStats(scope)), + random_(random), time_source_(dispatcher.timeSource()), + rate_limiting_enabled_(rate_limit_settings.enabled_), + handle_response_cb_(handle_response_cb) { retry_timer_ = dispatcher.createTimer([this]() -> void { establishNewStream(); }); if (rate_limiting_enabled_) { // Default Bucket contains 100 tokens maximum and refills at 10 tokens/sec. limit_request_ = std::make_unique( rate_limit_settings.max_tokens_, time_source_, rate_limit_settings.fill_rate_); - drain_request_timer_ = dispatcher.createTimer([this]() { drainRequests(); }); + drain_request_timer_ = dispatcher.createTimer([this]() { owning_context_->drainRequests(); }); } backoff_strategy_ = std::make_unique(RETRY_INITIAL_DELAY_MS, RETRY_MAX_DELAY_MS, random_); } - // virtual void handleResponse(std::unique_ptr&& message) PURE; // TODO TODO NOTE - // MOVED TO include/grpc_xds_context.h virtual void handleStreamEstablished() PURE; // TODO TODO - // NOTE MOVED TO include/grpc_xds_context.h virtual void handleEstablishmentFailure() PURE; // - // TODO TODO NOTE MOVED TO include/grpc_xds_context.h - - // Returns whether the request was actually sent (and so can leave the queue). - // virtual void sendDiscoveryRequest(const RequestQueueItem& queue_item) PURE; // TODO TODO NOTE - // MOVED TO include/grpc_xds_context.h - - void queueDiscoveryRequest(const RequestQueueItem& queue_item) { - request_queue_.push(queue_item); - drainRequests(); - } - - void clearRequestQueue() { - control_plane_stats_.pending_requests_.sub(request_queue_.size()); - // TODO(fredlas) when we have C++17: request_queue_ = {}; - while (!request_queue_.empty()) { - request_queue_.pop(); - } - } - void establishNewStream() { - + ENVOY_LOG(debug, "Establishing new gRPC bidi stream for {}", service_method_.DebugString()); // TODO TODO TODO since this file also serves vanilla, need to DOUBLE CHECK whether this // idempotency im adding here makes sense for vanilla too. if (stream_ != nullptr) { - ENVOY_LOG(debug, "gRPC bidi stream for {} already exists!", service_method_.DebugString()); + ENVOY_LOG(warn, "gRPC bidi stream for {} already exists!", service_method_.DebugString()); return; } - - ENVOY_LOG(debug, "Establishing new gRPC bidi stream for {}", service_method_.DebugString()); stream_ = async_client_->start(service_method_, *this); if (stream_ == nullptr) { ENVOY_LOG(warn, "Unable to establish new stream"); - handleEstablishmentFailure(); + owning_context_->handleEstablishmentFailure(); setRetryTimer(); return; } control_plane_stats_.connected_state_.set(1); - handleStreamEstablished(); + owning_context_->handleStreamEstablished(); } bool grpcStreamAvailable() const { return stream_ != nullptr; } @@ -96,11 +75,11 @@ class GrpcStream : public Grpc::TypedAsyncStreamCallbacks, void onReceiveMessage(std::unique_ptr&& message) override { // Reset here so that it starts with fresh backoff interval on next disconnect. backoff_strategy_->reset(); - // Some times during hot restarts this stat's value becomes inconsistent and will continue to - // have 0 till it is reconnected. Setting here ensures that it is consistent with the state of + // Sometimes during hot restarts this stat's value becomes inconsistent and will continue to + // have 0 until it is reconnected. Setting here ensures that it is consistent with the state of // management server connection. control_plane_stats_.connected_state_.set(1); - handleResponse(std::move(message)); + handle_response_cb_(std::move(message)); } void onReceiveTrailingMetadata(Http::HeaderMapPtr&& metadata) override { @@ -111,18 +90,11 @@ class GrpcStream : public Grpc::TypedAsyncStreamCallbacks, ENVOY_LOG(warn, "gRPC config stream closed: {}, {}", status, message); stream_ = nullptr; control_plane_stats_.connected_state_.set(0); - handleEstablishmentFailure(); + owning_context_->handleEstablishmentFailure(); setRetryTimer(); } -private: - void drainRequests() { - ENVOY_LOG(trace, "draining discovery requests {}", request_queue_.size()); - while (!request_queue_.empty() && checkRateLimitAllowsDrain()) { - // Process the request, if rate limiting is not enabled at all or if it is under rate limit. - sendDiscoveryRequest(request_queue_.front()); - request_queue_.pop(); - } + void maybeUpdateQueueSizeStat(uint64_t size) { // Although request_queue_.push() happens elsewhere, the only time the queue is non-transiently // non-empty is when it remains non-empty after a drain attempt. (The push() doesn't matter // because we always attempt this drain immediately after the push). Basically, a change in @@ -130,8 +102,8 @@ class GrpcStream : public Grpc::TypedAsyncStreamCallbacks, // if(>0 || used) to keep this stat from being wrongly marked interesting by a pointless set(0) // and needlessly taking up space. The first time we set(123), used becomes true, and so we will // subsequently always do the set (including set(0)). - if (request_queue_.size() > 0 || control_plane_stats_.pending_requests_.used()) { - control_plane_stats_.pending_requests_.set(request_queue_.size()); + if (size > 0 || control_plane_stats_.pending_requests_.used()) { + control_plane_stats_.pending_requests_.set(size); } } @@ -146,6 +118,7 @@ class GrpcStream : public Grpc::TypedAsyncStreamCallbacks, return false; } +private: void setRetryTimer() { retry_timer_->enableTimer(std::chrono::milliseconds(backoff_strategy_->nextBackOffMs())); } @@ -156,6 +129,8 @@ class GrpcStream : public Grpc::TypedAsyncStreamCallbacks, POOL_GAUGE_PREFIX(scope, control_plane_prefix))}; } + XdsGrpcContext* const owning_context_; + // TODO(htuch): Make this configurable or some static. const uint32_t RETRY_INITIAL_DELAY_MS = 500; const uint32_t RETRY_MAX_DELAY_MS = 30000; // Do not cross more than 30s @@ -175,18 +150,8 @@ class GrpcStream : public Grpc::TypedAsyncStreamCallbacks, TokenBucketPtr limit_request_; const bool rate_limiting_enabled_; Event::TimerPtr drain_request_timer_; - // A queue to store requests while rate limited. Note that when requests cannot be sent due to the - // gRPC stream being down, this queue does not store them; rather, they are simply dropped. - std::queue - request_queue_; // TODO TODO actually i think this queue can be its own little class. It would - // still be templated on RequestQueueItem, but owned by either - // GrpcDeltaXdsContext or GrpcStateOfWorldXdsContext, which could provide the - // type. - - // TODO TODO as for the rest of this class.... I think we can have GrpcDeltaXdsContext / - // GrpcWhateverMux (i.e. the GrpcXdsContext implementors) no longer inherit from it. Instead, it - // will just be its own GrpcStream< RequestProto, ResponseProto> thing, with nothing inheriting - // from it, and the GrpcXdsContext implementors will *own* one of it. + + std::function&&)> handle_response_cb_; }; } // namespace Config diff --git a/source/common/config/grpc_subscription_impl.h b/source/common/config/grpc_subscription_impl.h index 04b1b2aa6981f..066a826295cb3 100644 --- a/source/common/config/grpc_subscription_impl.h +++ b/source/common/config/grpc_subscription_impl.h @@ -20,8 +20,9 @@ class GrpcSubscriptionImpl : public Config::Subscription { SubscriptionStats stats, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, std::chrono::milliseconds init_fetch_timeout) - : grpc_mux_(local_info, std::move(async_client), dispatcher, service_method, random, scope, - rate_limit_settings), + : grpc_mux_(std::make_shared(local_info, std::move(async_client), + dispatcher, service_method, random, scope, + rate_limit_settings)), grpc_mux_subscription_(grpc_mux_, stats, type_url, dispatcher, init_fetch_timeout) {} // Config::Subscription @@ -29,17 +30,17 @@ class GrpcSubscriptionImpl : public Config::Subscription { Config::SubscriptionCallbacks& callbacks) override { // Subscribe first, so we get failure callbacks if grpc_mux_.start() fails. grpc_mux_subscription_.start(resources, callbacks); - grpc_mux_.start(); + grpc_mux_->start(); } void updateResources(const std::vector& resources) override { grpc_mux_subscription_.updateResources(resources); } - GrpcMuxImpl& grpcMux() { return grpc_mux_; } + std::shared_ptr grpcMux() { return grpc_mux_; } private: - GrpcMuxImpl grpc_mux_; + std::shared_ptr grpc_mux_; GrpcMuxSubscriptionImpl grpc_mux_subscription_; }; diff --git a/source/common/config/subscription_factory.h b/source/common/config/subscription_factory.h index 865480b369db0..bb228cbbd0583 100644 --- a/source/common/config/subscription_factory.h +++ b/source/common/config/subscription_factory.h @@ -80,14 +80,14 @@ class SubscriptionFactory { break; case envoy::api::v2::core::ApiConfigSource::DELTA_GRPC: { Utility::checkApiConfigSourceSubscriptionBackingCluster(cm.clusters(), api_config_source); - result.reset(new DeltaSubscriptionImpl( - cm.grpcDeltaXdsContext(), type_url, local_info, - Config::Utility::factoryForGrpcApiConfigSource(cm.grpcAsyncClientManager(), - api_config_source, scope) - ->create(), - dispatcher, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName(grpc_method), - random, scope, Utility::parseRateLimitSettings(api_config_source), stats, - Utility::configSourceInitialFetchTimeout(config))); + + // Config::Utility::factoryForGrpcApiConfigSource(cm.grpcAsyncClientManager(), + // api_config_source, scope)->create(), + // *Protobuf::DescriptorPool::generated_pool()->FindMethodByName(grpc_method), + // random, scope, Utility::parseRateLimitSettings(api_config_source), + std::cerr << "YUP making a delta sub" << std::endl; + result.reset(new DeltaSubscriptionImpl(cm.adsMux(), type_url, stats)); + std::cerr << "YUP MADE a delta sub" << std::endl; break; } default: diff --git a/source/common/upstream/cds_api_impl.cc b/source/common/upstream/cds_api_impl.cc index a244c2a9c329c..81dc81d0f93e5 100644 --- a/source/common/upstream/cds_api_impl.cc +++ b/source/common/upstream/cds_api_impl.cc @@ -71,8 +71,9 @@ void CdsApiImpl::onConfigUpdate( const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info) { - cm_.adsMux().pause(Config::TypeUrl::get().ClusterLoadAssignment); - Cleanup eds_resume([this] { cm_.adsMux().resume(Config::TypeUrl::get().ClusterLoadAssignment); }); + cm_.adsMux()->pause(Config::TypeUrl::get().ClusterLoadAssignment); + Cleanup eds_resume( + [this] { cm_.adsMux()->resume(Config::TypeUrl::get().ClusterLoadAssignment); }); std::vector exception_msgs; std::unordered_set cluster_names; @@ -106,10 +107,10 @@ void CdsApiImpl::onConfigUpdate( // ever changes, we would need to correct the following logic. if (state == ClusterManager::ClusterWarmingState::Starting && cm_.warmingClusterCount() == 1) { - cm_.adsMux().pause(Config::TypeUrl::get().Cluster); + cm_.adsMux()->pause(Config::TypeUrl::get().Cluster); } else if (state == ClusterManager::ClusterWarmingState::Finished && cm_.warmingClusterCount() == 0) { - cm_.adsMux().resume(Config::TypeUrl::get().Cluster); + cm_.adsMux()->resume(Config::TypeUrl::get().Cluster); } })) { ENVOY_LOG(debug, "cds: add/update cluster '{}'", cluster.name()); diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 8880e01ff547f..fcb7684300ecf 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -19,6 +19,7 @@ #include "common/common/fmt.h" #include "common/common/utility.h" #include "common/config/cds_json.h" +#include "common/config/grpc_delta_xds_context.h" #include "common/config/utility.h" #include "common/grpc/async_client_manager_impl.h" #include "common/http/async_client_impl.h" @@ -206,17 +207,36 @@ ClusterManagerImpl::ClusterManagerImpl( } // Now setup ADS if needed, this might rely on a primary cluster. + // This is the only point where distinction between delta ADS and state-of-the-world ADS is made. + // After here, we just have an AdsMux interface held in ads_mux_, which hides whether the backing + // implementation is delta or SotW. if (bootstrap.dynamic_resources().has_ads_config()) { - ads_mux_ = std::make_unique( - local_info, - Config::Utility::factoryForGrpcApiConfigSource( - *async_client_manager_, bootstrap.dynamic_resources().ads_config(), stats) - ->create(), - main_thread_dispatcher, - *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( - "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), - random_, stats_, - Envoy::Config::Utility::parseRateLimitSettings(bootstrap.dynamic_resources().ads_config())); + if (bootstrap.dynamic_resources().ads_config().api_type() == + envoy::api::v2::core::ApiConfigSource::DELTA_GRPC) { + ads_mux_ = std::make_shared( + // TODO TODO should this be here rather than addSubscription? local_info, + Config::Utility::factoryForGrpcApiConfigSource( + *async_client_manager_, bootstrap.dynamic_resources().ads_config(), stats) + ->create(), + main_thread_dispatcher, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.discovery.v2.AggregatedDiscoveryService.DeltaAggregatedResources"), + random_, stats_, + Envoy::Config::Utility::parseRateLimitSettings( + bootstrap.dynamic_resources().ads_config())); + } else { + ads_mux_ = std::make_shared( + local_info, + Config::Utility::factoryForGrpcApiConfigSource( + *async_client_manager_, bootstrap.dynamic_resources().ads_config(), stats) + ->create(), + main_thread_dispatcher, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), + random_, stats_, + Envoy::Config::Utility::parseRateLimitSettings( + bootstrap.dynamic_resources().ads_config())); + } } else { ads_mux_ = std::make_unique(); } diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index 72b751b52e1e3..9e7bc8e69356d 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -204,16 +204,14 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable adsMux() override { return ads_mux_; } Grpc::AsyncClientManager& grpcAsyncClientManager() override { return *async_client_manager_; } const std::string& localClusterName() const override { return local_cluster_name_; } @@ -455,8 +453,7 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable ads_mux_; LoadStatsReporterPtr load_stats_reporter_; // The name of the local cluster of this Envoy instance if defined, else the empty string. std::string local_cluster_name_; diff --git a/source/server/lds_api.cc b/source/server/lds_api.cc index 2634936580b26..b9d93877e028f 100644 --- a/source/server/lds_api.cc +++ b/source/server/lds_api.cc @@ -36,8 +36,8 @@ void LdsApiImpl::initialize(std::function callback) { } void LdsApiImpl::onConfigUpdate(const ResourceVector& resources, const std::string& version_info) { - cm_.adsMux().pause(Config::TypeUrl::get().RouteConfiguration); - Cleanup rds_resume([this] { cm_.adsMux().resume(Config::TypeUrl::get().RouteConfiguration); }); + cm_.adsMux()->pause(Config::TypeUrl::get().RouteConfiguration); + Cleanup rds_resume([this] { cm_.adsMux()->resume(Config::TypeUrl::get().RouteConfiguration); }); std::vector listeners; for (const auto& listener_blob : resources) { diff --git a/source/server/server.cc b/source/server/server.cc index c15c4c29ccd57..08d4797428a99 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -453,7 +453,7 @@ RunHelper::RunHelper(Instance& instance, const Options& options, Event::Dispatch // Pause RDS to ensure that we don't send any requests until we've // subscribed to all the RDS resources. The subscriptions happen in the init callbacks, // so we pause RDS until we've completed all the callbacks. - cm.adsMux().pause(Config::TypeUrl::get().RouteConfiguration); + cm.adsMux()->pause(Config::TypeUrl::get().RouteConfiguration); ENVOY_LOG(info, "all clusters initialized. initializing init manager"); @@ -469,7 +469,7 @@ RunHelper::RunHelper(Instance& instance, const Options& options, Event::Dispatch // Now that we're execute all the init callbacks we can resume RDS // as we've subscribed to all the statically defined RDS resources. - cm.adsMux().resume(Config::TypeUrl::get().RouteConfiguration); + cm.adsMux()->resume(Config::TypeUrl::get().RouteConfiguration); }); } diff --git a/test/common/config/config_provider_impl_test.cc b/test/common/config/config_provider_impl_test.cc index f25f77a0b1dd8..209f6a0ccdd88 100644 --- a/test/common/config/config_provider_impl_test.cc +++ b/test/common/config/config_provider_impl_test.cc @@ -38,9 +38,8 @@ class StaticDummyConfigProvider : public ImmutableConfigProviderImplBase { test::common::config::DummyConfig config_proto_; }; -class DummyConfigSubscription - : public ConfigSubscriptionInstanceBase, - Envoy::Config::SubscriptionCallbacks { +class DummyConfigSubscription : public ConfigSubscriptionInstanceBase, + Envoy::Config::SubscriptionCallbacks { public: DummyConfigSubscription(const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, @@ -53,7 +52,8 @@ class DummyConfigSubscription // Envoy::Config::SubscriptionCallbacks // TODO(fredlas) deduplicate - void onConfigUpdate(const ResourceVector& resources, const std::string& version_info) override { + void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) override { const auto& config = resources[0]; if (checkAndApplyConfig(config, "dummy_config", version_info)) { config_proto_ = config; diff --git a/test/common/config/delta_subscription_test_harness.h b/test/common/config/delta_subscription_test_harness.h index 04ed8d553e4cf..f347acdb43a5d 100644 --- a/test/common/config/delta_subscription_test_harness.h +++ b/test/common/config/delta_subscription_test_harness.h @@ -22,8 +22,6 @@ namespace Envoy { namespace Config { namespace { -typedef DeltaSubscriptionImpl DeltaEdsSubscriptionImpl; - class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { public: DeltaSubscriptionTestHarness() : DeltaSubscriptionTestHarness(std::chrono::milliseconds(0)) {} @@ -34,7 +32,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { node_.set_id("fo0"); EXPECT_CALL(local_info_, node()).WillRepeatedly(testing::ReturnRef(node_)); EXPECT_CALL(dispatcher_, createTimer_(_)); - subscription_ = std::make_unique( + subscription_ = std::make_unique( local_info_, std::unique_ptr(async_client_), dispatcher_, *method_descriptor_, random_, stats_store_, rate_limit_settings_, stats_, init_fetch_timeout); @@ -139,15 +137,15 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { Runtime::MockRandomGenerator random_; NiceMock local_info_; Grpc::MockAsyncStream async_stream_; - std::unique_ptr subscription_; + std::unique_ptr subscription_; std::string last_response_nonce_; std::vector last_cluster_names_; Envoy::Config::RateLimitSettings rate_limit_settings_; Event::MockTimer* init_timeout_timer_; envoy::api::v2::core::Node node_; - NiceMock> callbacks_; + NiceMock callbacks_; }; } // namespace } // namespace Config -} // namespace Envoy \ No newline at end of file +} // namespace Envoy diff --git a/test/common/config/filesystem_subscription_impl_test.cc b/test/common/config/filesystem_subscription_impl_test.cc index 1c4a28baeeeb1..2feabde6921fd 100644 --- a/test/common/config/filesystem_subscription_impl_test.cc +++ b/test/common/config/filesystem_subscription_impl_test.cc @@ -42,7 +42,7 @@ TEST(MiscFilesystemSubscriptionImplTest, BadWatch) { auto* watcher = new Filesystem::MockWatcher(); EXPECT_CALL(dispatcher, createFilesystemWatcher_()).WillOnce(Return(watcher)); EXPECT_CALL(*watcher, addWatch(_, _, _)).WillOnce(Throw(EnvoyException("bad path"))); - EXPECT_THROW_WITH_MESSAGE(FilesystemEdsSubscriptionImpl(dispatcher, "##!@/dev/null", stats, *api), + EXPECT_THROW_WITH_MESSAGE(FilesystemSubscriptionImpl(dispatcher, "##!@/dev/null", stats, *api), EnvoyException, "bad path"); } diff --git a/test/common/config/filesystem_subscription_test_harness.h b/test/common/config/filesystem_subscription_test_harness.h index 15bcb6fcd7502..981706e036ba5 100644 --- a/test/common/config/filesystem_subscription_test_harness.h +++ b/test/common/config/filesystem_subscription_test_harness.h @@ -24,9 +24,6 @@ using testing::Return; namespace Envoy { namespace Config { -typedef FilesystemSubscriptionImpl - FilesystemEdsSubscriptionImpl; - class FilesystemSubscriptionTestHarness : public SubscriptionTestHarness { public: FilesystemSubscriptionTestHarness() @@ -78,13 +75,13 @@ class FilesystemSubscriptionTestHarness : public SubscriptionTestHarness { file_json += "]}"; envoy::api::v2::DiscoveryResponse response_pb; MessageUtil::loadFromJson(file_json, response_pb); - EXPECT_CALL(callbacks_, - onConfigUpdate( - RepeatedProtoEq( - Config::Utility::getTypedResources( - response_pb)), - version)) - .WillOnce(ThrowOnRejectedConfig(accept)); + /* TODO TODO TODO DONT CARE YET JUST WANT IT TO COMPILE EXPECT_CALL(callbacks_, + onConfigUpdate( + RepeatedProtoEq( + Config::Utility::getTypedResources( + response_pb)), + version)) + .WillOnce(ThrowOnRejectedConfig(accept));*/ if (accept) { version_ = version; } else { @@ -123,7 +120,7 @@ class FilesystemSubscriptionTestHarness : public SubscriptionTestHarness { Api::ApiPtr api_; Event::DispatcherPtr dispatcher_; NiceMock> callbacks_; - FilesystemEdsSubscriptionImpl subscription_; + FilesystemSubscriptionImpl subscription_; bool file_at_start_{false}; }; diff --git a/test/mocks/config/BUILD b/test/mocks/config/BUILD index f23e0337aa505..212d7c9718e55 100644 --- a/test/mocks/config/BUILD +++ b/test/mocks/config/BUILD @@ -15,6 +15,7 @@ envoy_cc_mock( deps = [ "//include/envoy/config:grpc_mux_interface", "//include/envoy/config:subscription_interface", + "//include/envoy/config:xds_context_interface", "//source/common/config:resources_lib", "//source/common/protobuf:utility_lib", "@envoy_api//envoy/api/v2:cds_cc", diff --git a/test/mocks/config/mocks.cc b/test/mocks/config/mocks.cc index 9b50adef0d8e0..4ab6b75434286 100644 --- a/test/mocks/config/mocks.cc +++ b/test/mocks/config/mocks.cc @@ -13,6 +13,14 @@ MockGrpcMuxWatch::~MockGrpcMuxWatch() { cancel(); } MockGrpcMux::MockGrpcMux() {} MockGrpcMux::~MockGrpcMux() {} +MockXdsGrpcContext::MockXdsGrpcContext() {} +MockXdsGrpcContext::~MockXdsGrpcContext() {} +GrpcMuxWatchPtr MockXdsGrpcContext::subscribe(const std::string& type_url, + const std::vector& resources, + GrpcMuxCallbacks& callbacks) { + return GrpcMuxWatchPtr(subscribe_(type_url, resources, callbacks)); +} + GrpcMuxWatchPtr MockGrpcMux::subscribe(const std::string& type_url, const std::vector& resources, GrpcMuxCallbacks& callbacks) { diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index a4f2ebb093ab3..aa5f28b37b3b6 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -3,6 +3,7 @@ #include "envoy/api/v2/eds.pb.h" #include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" +#include "envoy/config/xds_context.h" #include "common/config/resources.h" #include "common/protobuf/utility.h" @@ -67,6 +68,34 @@ class MockGrpcMux : public GrpcMux { MOCK_METHOD1(resume, void(const std::string& type_url)); }; +class MockXdsGrpcContext : public XdsGrpcContext { +public: + MockXdsGrpcContext(); + virtual ~MockXdsGrpcContext(); + + MOCK_METHOD4(addSubscription, + void(const std::vector& resources, const std::string& type_url, + SubscriptionCallbacks& callbacks, SubscriptionStats stats)); + MOCK_METHOD2(updateResources, + void(const std::vector& resources, const std::string& type_url)); + + MOCK_METHOD1(removeSubscription, void(const std::string& type_url)); + MOCK_METHOD1(pause, void(const std::string& type_url)); + MOCK_METHOD1(resume, void(const std::string& type_url)); + + MOCK_METHOD0(drainRequests, void()); + MOCK_METHOD0(handleStreamEstablished, void()); + MOCK_METHOD0(handleEstablishmentFailure, void()); + + // GrpcMux TODO TODO remove + MOCK_METHOD0(start, void()); + MOCK_METHOD3(subscribe_, + GrpcMuxWatch*(const std::string& type_url, const std::vector& resources, + GrpcMuxCallbacks& callbacks)); + GrpcMuxWatchPtr subscribe(const std::string& type_url, const std::vector& resources, + GrpcMuxCallbacks& callbacks); +}; + class MockGrpcMuxCallbacks : public GrpcMuxCallbacks { public: MockGrpcMuxCallbacks(); diff --git a/test/mocks/upstream/mocks.cc b/test/mocks/upstream/mocks.cc index 22e89185082af..3116cb87fe586 100644 --- a/test/mocks/upstream/mocks.cc +++ b/test/mocks/upstream/mocks.cc @@ -117,7 +117,7 @@ MockClusterManager::MockClusterManager() { ON_CALL(*this, httpAsyncClientForCluster(_)).WillByDefault(ReturnRef(async_client_)); ON_CALL(*this, httpAsyncClientForCluster(_)).WillByDefault((ReturnRef(async_client_))); ON_CALL(*this, bindConfig()).WillByDefault(ReturnRef(bind_config_)); - ON_CALL(*this, adsMux()).WillByDefault(ReturnRef(ads_mux_)); + ON_CALL(*this, adsMux()).WillByDefault(Return(ads_mux_)); ON_CALL(*this, grpcAsyncClientManager()).WillByDefault(ReturnRef(async_client_manager_)); ON_CALL(*this, localClusterName()).WillByDefault((ReturnRef(local_cluster_name_))); diff --git a/test/mocks/upstream/mocks.h b/test/mocks/upstream/mocks.h index 10086aa425786..0049ce557bd45 100644 --- a/test/mocks/upstream/mocks.h +++ b/test/mocks/upstream/mocks.h @@ -304,7 +304,7 @@ class MockClusterManager : public ClusterManager { MOCK_METHOD1(removeCluster, bool(const std::string& cluster)); MOCK_METHOD0(shutdown, void()); MOCK_CONST_METHOD0(bindConfig, const envoy::api::v2::core::BindConfig&()); - MOCK_METHOD0(adsMux, Config::GrpcMux&()); + MOCK_METHOD0(adsMux, std::shared_ptr()); MOCK_METHOD0(grpcAsyncClientManager, Grpc::AsyncClientManager&()); MOCK_CONST_METHOD0(versionInfo, const std::string()); MOCK_CONST_METHOD0(localClusterName, const std::string&()); @@ -317,7 +317,7 @@ class MockClusterManager : public ClusterManager { NiceMock tcp_conn_pool_; NiceMock thread_local_cluster_; envoy::api::v2::core::BindConfig bind_config_; - NiceMock ads_mux_; + std::shared_ptr> ads_mux_; NiceMock async_client_manager_; std::string local_cluster_name_; NiceMock cluster_manager_factory_; From d483936391c661b51e51ecd11f4350de2d798bff Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 4 Apr 2019 18:13:46 -0400 Subject: [PATCH 03/56] working for non-agg delta, agg delta should in theory work too Signed-off-by: Fred Douglas --- include/envoy/config/xds_context.h | 4 +- include/envoy/upstream/cluster_manager.h | 8 ++-- .../common/config/delta_subscription_impl.h | 8 ++-- .../common/config/delta_subscription_state.h | 1 - source/common/config/grpc_delta_xds_context.h | 44 ++++++++++++++++--- source/common/config/grpc_mux_impl.h | 4 +- source/common/config/subscription_factory.h | 21 ++++----- source/common/upstream/cds_api_impl.cc | 8 ++-- .../common/upstream/cluster_manager_impl.cc | 24 +++++----- source/common/upstream/cluster_manager_impl.h | 10 +++-- source/server/lds_api.cc | 5 ++- source/server/server.cc | 4 +- test/common/upstream/cds_api_impl_test.cc | 32 +++++++------- test/integration/cds_integration_test.cc | 9 ---- test/integration/integration.h | 2 + test/mocks/config/mocks.h | 5 ++- test/mocks/upstream/mocks.cc | 2 +- test/mocks/upstream/mocks.h | 4 +- 18 files changed, 115 insertions(+), 80 deletions(-) diff --git a/include/envoy/config/xds_context.h b/include/envoy/config/xds_context.h index 4968e9ab058c8..aa807437eaf10 100644 --- a/include/envoy/config/xds_context.h +++ b/include/envoy/config/xds_context.h @@ -15,7 +15,8 @@ class XdsGrpcContext { virtual ~XdsGrpcContext() {} virtual void addSubscription(const std::vector& resources, const std::string& type_url, SubscriptionCallbacks& callbacks, - SubscriptionStats stats) PURE; + SubscriptionStats& stats, + std::chrono::milliseconds init_fetch_timeout) PURE; // Enqueues and attempts to send a discovery request, (un)subscribing to resources missing from / // added to the passed 'resources' argument, relative to resource_versions_. @@ -25,7 +26,6 @@ class XdsGrpcContext { virtual void removeSubscription(const std::string& type_url) PURE; virtual void pause(const std::string& type_url) PURE; - virtual void resume(const std::string& type_url) PURE; virtual void drainRequests() PURE; diff --git a/include/envoy/upstream/cluster_manager.h b/include/envoy/upstream/cluster_manager.h index 61e9a19cc8652..555673b7d3389 100644 --- a/include/envoy/upstream/cluster_manager.h +++ b/include/envoy/upstream/cluster_manager.h @@ -200,14 +200,14 @@ class ClusterManager { virtual const envoy::api::v2::core::BindConfig& bindConfig() const PURE; /** - * Return a reference to the singleton ADS provider for upstream control plane muxing of xDS. This - * is treated somewhat as a special case in ClusterManager, since it does not relate logically to - * the management of clusters but instead is required early in ClusterManager/server + * Return a reference to the singleton xDS-over-gRPC provider for upstream control plane muxing of + * xDS. This is treated somewhat as a special case in ClusterManager, since it does not relate + * logically to the management of clusters but instead is required early in ClusterManager/server * initialization and in various sites that need ClusterManager for xDS API interfacing. * * @return XdsGrpcContext& ADS API provider referencee. */ - virtual std::shared_ptr adsMux() PURE; + virtual std::shared_ptr xdsGrpcContext() PURE; /** * @return Grpc::AsyncClientManager& the cluster manager's gRPC client manager. diff --git a/source/common/config/delta_subscription_impl.h b/source/common/config/delta_subscription_impl.h index b3c60468fd894..f3ab14737966a 100644 --- a/source/common/config/delta_subscription_impl.h +++ b/source/common/config/delta_subscription_impl.h @@ -22,8 +22,9 @@ namespace Config { class DeltaSubscriptionImpl : public Subscription { public: DeltaSubscriptionImpl(std::shared_ptr context, absl::string_view type_url, - SubscriptionStats stats) - : context_(context), type_url_(type_url), stats_(stats) {} + SubscriptionStats stats, std::chrono::milliseconds init_fetch_timeout) + : context_(context), type_url_(type_url), stats_(stats), + init_fetch_timeout_(init_fetch_timeout) {} ~DeltaSubscriptionImpl() { context_->removeSubscription(type_url_); @@ -41,7 +42,7 @@ class DeltaSubscriptionImpl : public Subscription { // Config::DeltaSubscription void start(const std::vector& resources, SubscriptionCallbacks& callbacks) override { - context_->addSubscription(resources, type_url_, callbacks, stats_); + context_->addSubscription(resources, type_url_, callbacks, stats_, init_fetch_timeout_); } void updateResources(const std::vector& resources) override { @@ -52,6 +53,7 @@ class DeltaSubscriptionImpl : public Subscription { std::shared_ptr context_; const std::string type_url_; SubscriptionStats stats_; + const std::chrono::milliseconds init_fetch_timeout_; }; } // namespace Config diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index fd930ede5bb06..619969266de15 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -155,7 +155,6 @@ class DeltaSubscriptionState : public Logger::Loggable { ::google::rpc::Status* error_detail = request_.mutable_error_detail(); error_detail->set_code(Grpc::Status::GrpcStatus::Internal); error_detail->set_message(e.what()); - disableInitFetchTimeoutTimer(); stats_.update_rejected_.inc(); ENVOY_LOG(warn, "delta config for {} rejected: {}", type_url_, e.what()); diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h index 3a6311ed1635a..318e29d654c85 100644 --- a/source/common/config/grpc_delta_xds_context.h +++ b/source/common/config/grpc_delta_xds_context.h @@ -21,15 +21,42 @@ namespace Envoy { namespace Config { +// (TODO this will become more of a README level thing) +// When using gRPC, xDS has two pairs of options: aggregated/non-aggregated, and +// delta/state-of-the-world updates. All four combinations of these are usable. +// +// "Aggregated" means that EDS, CDS, etc resources are all carried by the same gRPC stream (not even +// channel). For Envoy's implementation of xDS client logic, there is effectively no difference +// between aggregated xDS and non-aggregated: they both use the same request/response protos. The +// non-aggregated case is handled by running the aggregated logic, and just happening to only have 1 +// xDS subscription type to "aggregate", i.e., GrpcDeltaXdsContext only has one +// DeltaSubscriptionState entry in its map. The sole implementation difference: when the bootstrap +// specifies ADS, the method string passed to gRPC is {Delta,Stream}AggregatedResources, as opposed +// to e.g. {Delta,Stream}Clusters. This distinction is necessary for the server to know what +// resources should be provided. +// +// DeltaSubscriptionState is what handles the conceptual/application-level protocol state of a given +// resource type subscription, which may or may not be multiplexed with others. So, +// DeltaSubscriptionState is equally applicable to non- and aggregated. +// +// Delta vs state-of-the-world is a question of wire format: the protos in question are named +// [Delta]Discovery{Request,Response}. That is what the XdsGrpcContext interface is useful for: its +// GrpcDeltaXdsContext implementation works with DeltaDiscovery{Request,Response} and has +// delta-specific logic, and its GrpxMuxImpl implementation works with Discovery{Request,Response} +// and has SotW-specific logic. A DeltaSubscriptionImpl (TODO rename to not delta-specific since +// it's ideally going to cover ALL 4 subscription "meta-types") has its shared_ptr. +// Both GrpcDeltaXdsContext (delta) or GrpcMuxImpl (SotW) will work just fine. The shared_ptr allows +// for both non- and aggregated: if non-aggregated, you'll be the only holder of that shared_ptr. By +// those two mechanisms, the single class (TODO rename) DeltaSubscriptionImpl handles all 4 +// delta/SotW and non-/aggregated combinations. class GrpcDeltaXdsContext : public XdsGrpcContext, Logger::Loggable { public: GrpcDeltaXdsContext(Grpc::AsyncClientPtr async_client, Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, - const LocalInfo::LocalInfo& local_info, - std::chrono::milliseconds init_fetch_timeout) - : local_info_(local_info), init_fetch_timeout_(init_fetch_timeout), dispatcher_(dispatcher), + const LocalInfo::LocalInfo& local_info) + : local_info_(local_info), dispatcher_(dispatcher), grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, rate_limit_settings, // callback for handling receipt of DiscoveryResponse protos. @@ -38,10 +65,11 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, Logger::Loggable& resources, const std::string& type_url, - SubscriptionCallbacks& callbacks, SubscriptionStats stats) override { + SubscriptionCallbacks& callbacks, SubscriptionStats& stats, + std::chrono::milliseconds init_fetch_timeout) override { subscriptions_.emplace( std::make_pair(type_url, DeltaSubscriptionState(type_url, resources, callbacks, local_info_, - init_fetch_timeout_, dispatcher_, stats))); + init_fetch_timeout, dispatcher_, stats))); grpc_stream_.establishNewStream(); // (idempotent) } @@ -136,6 +164,10 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, Logger::Loggable @@ -185,6 +216,7 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, Logger::Loggable request_queue_; + // Map from type_url strings to a DeltaSubscriptionState for that type. std::unordered_map subscriptions_; }; diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index e323305c4a03a..a5019fd89e9f0 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -43,7 +43,7 @@ class GrpcMuxImpl : public GrpcMux, void resume(const std::string& type_url) override; void drainRequests() override; void addSubscription(const std::vector&, const std::string&, SubscriptionCallbacks&, - SubscriptionStats) override { + SubscriptionStats&, std::chrono::milliseconds) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } void updateResources(const std::vector&, const std::string&) override { @@ -120,7 +120,7 @@ class NullGrpcMuxImpl : public XdsGrpcContext { void resume(const std::string&) override {} void addSubscription(const std::vector&, const std::string&, SubscriptionCallbacks&, - SubscriptionStats) override { + SubscriptionStats&, std::chrono::milliseconds) override { throw EnvoyException("ADS must be configured to support an ADS config source"); } void updateResources(const std::vector&, const std::string&) override { diff --git a/source/common/config/subscription_factory.h b/source/common/config/subscription_factory.h index 895b8b685465d..ed65cd8ae54fe 100644 --- a/source/common/config/subscription_factory.h +++ b/source/common/config/subscription_factory.h @@ -77,18 +77,19 @@ class SubscriptionFactory { dispatcher, random, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName(grpc_method), type_url, stats, scope, Utility::parseRateLimitSettings(api_config_source), - Utility::configSourceInitialFetchTimeout(config))); + Utility::configSourceInitialFetchTimeout(config)); break; case envoy::api::v2::core::ApiConfigSource::DELTA_GRPC: { Utility::checkApiConfigSourceSubscriptionBackingCluster(cm.clusters(), api_config_source); - - // Config::Utility::factoryForGrpcApiConfigSource(cm.grpcAsyncClientManager(), - // api_config_source, scope)->create(), - // *Protobuf::DescriptorPool::generated_pool()->FindMethodByName(grpc_method), - // random, scope, Utility::parseRateLimitSettings(api_config_source), - std::cerr << "YUP making a delta sub" << std::endl; - result.reset(new DeltaSubscriptionImpl(cm.adsMux(), type_url, stats)); - std::cerr << "YUP MADE a delta sub" << std::endl; + result = std::make_unique( + std::make_shared( + Config::Utility::factoryForGrpcApiConfigSource(cm.grpcAsyncClientManager(), + api_config_source, scope) + ->create(), + dispatcher, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName(grpc_method), random, + scope, Utility::parseRateLimitSettings(api_config_source), local_info), + type_url, stats, Utility::configSourceInitialFetchTimeout(config)); break; } default: @@ -98,7 +99,7 @@ class SubscriptionFactory { } case envoy::api::v2::core::ConfigSource::kAds: { result = std::make_unique( - cm.adsMux(), stats, type_url, dispatcher, + cm.xdsGrpcContext(), stats, type_url, dispatcher, Utility::configSourceInitialFetchTimeout(config)); break; } diff --git a/source/common/upstream/cds_api_impl.cc b/source/common/upstream/cds_api_impl.cc index 24a5b6fd78f5f..2b478321e2bd1 100644 --- a/source/common/upstream/cds_api_impl.cc +++ b/source/common/upstream/cds_api_impl.cc @@ -66,9 +66,9 @@ void CdsApiImpl::onConfigUpdate( const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info) { - cm_.adsMux()->pause(Config::TypeUrl::get().ClusterLoadAssignment); + cm_.xdsGrpcContext()->pause(Config::TypeUrl::get().ClusterLoadAssignment); Cleanup eds_resume( - [this] { cm_.adsMux()->resume(Config::TypeUrl::get().ClusterLoadAssignment); }); + [this] { cm_.xdsGrpcContext()->resume(Config::TypeUrl::get().ClusterLoadAssignment); }); std::vector exception_msgs; std::unordered_set cluster_names; @@ -102,10 +102,10 @@ void CdsApiImpl::onConfigUpdate( // ever changes, we would need to correct the following logic. if (state == ClusterManager::ClusterWarmingState::Starting && cm_.warmingClusterCount() == 1) { - cm_.adsMux()->pause(Config::TypeUrl::get().Cluster); + cm_.xdsGrpcContext()->pause(Config::TypeUrl::get().Cluster); } else if (state == ClusterManager::ClusterWarmingState::Finished && cm_.warmingClusterCount() == 0) { - cm_.adsMux()->resume(Config::TypeUrl::get().Cluster); + cm_.xdsGrpcContext()->resume(Config::TypeUrl::get().Cluster); } })) { ENVOY_LOG(debug, "cds: add/update cluster '{}'", cluster.name()); diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index fcb7684300ecf..734d255eca279 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -208,24 +208,28 @@ ClusterManagerImpl::ClusterManagerImpl( // Now setup ADS if needed, this might rely on a primary cluster. // This is the only point where distinction between delta ADS and state-of-the-world ADS is made. - // After here, we just have an AdsMux interface held in ads_mux_, which hides whether the backing - // implementation is delta or SotW. + // After here, we just have an XdsGrpcContext interface held in xds_grpc_context_, which hides + // whether the backing implementation is delta or SotW. if (bootstrap.dynamic_resources().has_ads_config()) { if (bootstrap.dynamic_resources().ads_config().api_type() == envoy::api::v2::core::ApiConfigSource::DELTA_GRPC) { - ads_mux_ = std::make_shared( - // TODO TODO should this be here rather than addSubscription? local_info, - Config::Utility::factoryForGrpcApiConfigSource( - *async_client_manager_, bootstrap.dynamic_resources().ads_config(), stats) + auto& api_config_source = + bootstrap.dynamic_resources().has_ads_config() + ? bootstrap.dynamic_resources().ads_config() + : bootstrap.dynamic_resources().cds_config().api_config_source(); + xds_grpc_context_ = std::make_shared( + Config::Utility::factoryForGrpcApiConfigSource(*async_client_manager_, api_config_source, + stats) ->create(), main_thread_dispatcher, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.DeltaAggregatedResources"), random_, stats_, Envoy::Config::Utility::parseRateLimitSettings( - bootstrap.dynamic_resources().ads_config())); + bootstrap.dynamic_resources().ads_config()), + local_info); } else { - ads_mux_ = std::make_shared( + xds_grpc_context_ = std::make_shared( local_info, Config::Utility::factoryForGrpcApiConfigSource( *async_client_manager_, bootstrap.dynamic_resources().ads_config(), stats) @@ -238,7 +242,7 @@ ClusterManagerImpl::ClusterManagerImpl( bootstrap.dynamic_resources().ads_config())); } } else { - ads_mux_ = std::make_unique(); + xds_grpc_context_ = std::make_unique(); } // After ADS is initialized, load EDS static clusters as EDS config may potentially need ADS. @@ -288,7 +292,7 @@ ClusterManagerImpl::ClusterManagerImpl( // clusters have already initialized. (E.g., if all static). init_helper_.onStaticLoadComplete(); - ads_mux_->start(); + xds_grpc_context_->start(); if (cm_config.has_load_stats_config()) { const auto& load_stats_config = cm_config.load_stats_config(); diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index 9e7bc8e69356d..0819d4dd4b999 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -204,14 +204,14 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable adsMux() override { return ads_mux_; } + std::shared_ptr xdsGrpcContext() override { return xds_grpc_context_; } Grpc::AsyncClientManager& grpcAsyncClientManager() override { return *async_client_manager_; } const std::string& localClusterName() const override { return local_cluster_name_; } @@ -453,7 +453,9 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable ads_mux_; + std::shared_ptr + xds_grpc_context_; // TODO actually probably should have "ADS" in the name after all, maybe + // ads_context_. LoadStatsReporterPtr load_stats_reporter_; // The name of the local cluster of this Envoy instance if defined, else the empty string. std::string local_cluster_name_; diff --git a/source/server/lds_api.cc b/source/server/lds_api.cc index 92c4ec6824278..cd339ed708642 100644 --- a/source/server/lds_api.cc +++ b/source/server/lds_api.cc @@ -33,8 +33,9 @@ LdsApiImpl::LdsApiImpl(const envoy::api::v2::core::ConfigSource& lds_config, void LdsApiImpl::onConfigUpdate(const Protobuf::RepeatedPtrField& resources, const std::string& version_info) { - cm_.adsMux().pause(Config::TypeUrl::get().RouteConfiguration); - Cleanup rds_resume([this] { cm_.adsMux().resume(Config::TypeUrl::get().RouteConfiguration); }); + cm_.xdsGrpcContext()->pause(Config::TypeUrl::get().RouteConfiguration); + Cleanup rds_resume( + [this] { cm_.xdsGrpcContext()->resume(Config::TypeUrl::get().RouteConfiguration); }); std::vector listeners; for (const auto& listener_blob : resources) { diff --git a/source/server/server.cc b/source/server/server.cc index c4a4158b7ba99..cf394d90dddd4 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -457,14 +457,14 @@ RunHelper::RunHelper(Instance& instance, const Options& options, Event::Dispatch // Pause RDS to ensure that we don't send any requests until we've // subscribed to all the RDS resources. The subscriptions happen in the init callbacks, // so we pause RDS until we've completed all the callbacks. - cm.adsMux()->pause(Config::TypeUrl::get().RouteConfiguration); + cm.xdsGrpcContext()->pause(Config::TypeUrl::get().RouteConfiguration); ENVOY_LOG(info, "all clusters initialized. initializing init manager"); init_manager.initialize(init_watcher_); // Now that we're execute all the init callbacks we can resume RDS // as we've subscribed to all the statically defined RDS resources. - cm.adsMux()->resume(Config::TypeUrl::get().RouteConfiguration); + cm.xdsGrpcContext()->resume(Config::TypeUrl::get().RouteConfiguration); }); } diff --git a/test/common/upstream/cds_api_impl_test.cc b/test/common/upstream/cds_api_impl_test.cc index 917e261b50731..60b164cc41058 100644 --- a/test/common/upstream/cds_api_impl_test.cc +++ b/test/common/upstream/cds_api_impl_test.cc @@ -398,16 +398,16 @@ version_info: '0' )EOF"; // Two clusters updated, both warmed up. - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(cm_.xds_grpc_context_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); cm_.expectAddWithWarming("cluster1", "0"); cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(cm_.xds_grpc_context_, pause(Config::TypeUrl::get().Cluster)).Times(1); cm_.expectAddWithWarming("cluster2", "0"); cm_.expectWarmingClusterCount(); EXPECT_CALL(initialized_, ready()); cm_.expectWarmingClusterCount(2); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(cm_.xds_grpc_context_, resume(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(cm_.xds_grpc_context_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); EXPECT_CALL(*interval_timer_, enableTimer(_)); cm_.clustersToWarmUp({"cluster1", "cluster2"}); callbacks_->onSuccess(parseResponseMessageFromYaml(response1_yaml)); @@ -433,15 +433,15 @@ version_info: '1' path: eds path )EOF"; - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(cm_.xds_grpc_context_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); cm_.expectAddWithWarming("cluster1", "1"); cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(cm_.xds_grpc_context_, pause(Config::TypeUrl::get().Cluster)).Times(1); cm_.expectAddWithWarming("cluster3", "1"); cm_.expectWarmingClusterCount(); EXPECT_CALL(initialized_, ready()); cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(cm_.xds_grpc_context_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); EXPECT_CALL(*interval_timer_, enableTimer(_)); resetCdsInitializedCb(); cm_.clustersToWarmUp({"cluster1"}); @@ -462,13 +462,13 @@ version_info: '2' path: eds path )EOF"; - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(cm_.xds_grpc_context_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); cm_.expectAddWithWarming("cluster4", "2"); cm_.expectWarmingClusterCount(); EXPECT_CALL(initialized_, ready()); cm_.expectWarmingClusterCount(2); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(cm_.xds_grpc_context_, resume(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(cm_.xds_grpc_context_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); EXPECT_CALL(*interval_timer_, enableTimer(_)); resetCdsInitializedCb(); cm_.clustersToWarmUp({"cluster4", "cluster3"}); @@ -495,19 +495,19 @@ version_info: '3' )EOF"; // Two clusters updated, first one warmed up before processing of the second one starts. - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(cm_.xds_grpc_context_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); cm_.expectAddWithWarming("cluster5", "3", true); cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(cm_.xds_grpc_context_, pause(Config::TypeUrl::get().Cluster)).Times(1); cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(cm_.xds_grpc_context_, resume(Config::TypeUrl::get().Cluster)).Times(1); cm_.expectAddWithWarming("cluster6", "3"); cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(cm_.xds_grpc_context_, pause(Config::TypeUrl::get().Cluster)).Times(1); EXPECT_CALL(initialized_, ready()); cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(cm_.xds_grpc_context_, resume(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(cm_.xds_grpc_context_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); EXPECT_CALL(*interval_timer_, enableTimer(_)); resetCdsInitializedCb(); cm_.clustersToWarmUp({"cluster6"}); diff --git a/test/integration/cds_integration_test.cc b/test/integration/cds_integration_test.cc index f70fc5d175834..e978acaabb670 100644 --- a/test/integration/cds_integration_test.cc +++ b/test/integration/cds_integration_test.cc @@ -56,7 +56,6 @@ class CdsIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, public H // config that you use! setUpstreamCount(1); // the CDS cluster setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); // CDS uses gRPC uses HTTP2. - // HttpIntegrationTest::initialize() does many things: // 1) It appends to fake_upstreams_ as many as you asked for via setUpstreamCount(). // 2) It updates your bootstrap config with the ports your fake upstreams are actually listening @@ -68,7 +67,6 @@ class CdsIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, public H // However, this test needs to defer all of that to later. defer_listener_finalization_ = true; HttpIntegrationTest::initialize(); - // Create the regular (i.e. not an xDS server) upstreams. We create them manually here after // initialize() because finalize() expects all fake_upstreams_ to correspond to a static // cluster in the bootstrap config - which we don't want since we're testing dynamic CDS! @@ -84,19 +82,15 @@ class CdsIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, public H cluster2_ = ConfigHelper::buildCluster( ClusterName2, fake_upstreams_[UpstreamIndex2]->localAddress()->ip()->port(), Network::Test::getLoopbackAddressString(ipVersion())); - // Let Envoy establish its connection to the CDS server. acceptXdsConnection(); - // Do the initial compareDiscoveryRequest / sendDiscoveryResponse for cluster_1. // Split out into its own function so that DeltaCdsIntegrationTest can override it. giveInitialCluster(); - // We can continue the test once we're sure that Envoy's ClusterManager has made use of // the DiscoveryResponse describing cluster_1 that we sent. // 2 because the statically specified CDS server itself counts as a cluster. test_server_->waitForGaugeGe("cluster_manager.active_clusters", 2); - // Wait for our statically specified listener to become ready, and register its port in the // test framework's downstream listener port map. test_server_->waitUntilListenersReady(); @@ -249,20 +243,17 @@ TEST_P(DeltaCdsIntegrationTest, CdsClusterUpDownUp) { // Calls CdsIntegrationTest::initialize(), which includes establishing a listener, route, and // cluster. testRouterHeaderOnlyRequestAndResponse(nullptr, UpstreamIndex1, "/cluster1"); - // Tell Envoy that cluster_1 is gone. EXPECT_TRUE(compareDeltaDiscoveryRequest(Config::TypeUrl::get().Cluster, {}, {})); sendDeltaDiscoveryResponse({}, {ClusterName1}, "42"); // We can continue the test once we're sure that Envoy's ClusterManager has made use of // the DiscoveryResponse that says cluster_1 is gone. test_server_->waitForCounterGe("cluster_manager.cluster_removed", 1); - // Now that cluster_1 is gone, the listener (with its routing to cluster_1) should 503. BufferingStreamDecoderPtr response = IntegrationUtil::makeSingleRequest( lookupPort("http"), "GET", "/cluster1", "", downstream_protocol_, version_, "foo.com"); ASSERT_TRUE(response->complete()); EXPECT_STREQ("503", response->headers().Status()->value().c_str()); - cleanupUpstreamAndDownstream(); codec_client_->waitForDisconnect(); diff --git a/test/integration/integration.h b/test/integration/integration.h index cb7e8c937b7f6..33e67e017a61f 100644 --- a/test/integration/integration.h +++ b/test/integration/integration.h @@ -222,6 +222,7 @@ class BaseIntegrationTest : Logger::Loggable { const std::vector& expected_resource_unsubscriptions, const Protobuf::int32 expected_error_code = Grpc::Status::GrpcStatus::Ok, const std::string& expected_error_message = ""); + template void sendDeltaDiscoveryResponse(const std::vector& added_or_updated, const std::vector& removed, @@ -236,6 +237,7 @@ class BaseIntegrationTest : Logger::Loggable { } *response.mutable_removed_resources() = {removed.begin(), removed.end()}; response.set_nonce("noncense"); + response.set_type_url(Grpc::Common::typeUrl(T().GetDescriptor()->full_name())); xds_stream_->sendGrpcMessage(response); } diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index 7d29161afd354..0f51d6857a353 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -73,9 +73,10 @@ class MockXdsGrpcContext : public XdsGrpcContext { MockXdsGrpcContext(); virtual ~MockXdsGrpcContext(); - MOCK_METHOD4(addSubscription, + MOCK_METHOD5(addSubscription, void(const std::vector& resources, const std::string& type_url, - SubscriptionCallbacks& callbacks, SubscriptionStats stats)); + SubscriptionCallbacks& callbacks, SubscriptionStats& stats, + std::chrono::milliseconds init_fetch_timeout)); MOCK_METHOD2(updateResources, void(const std::vector& resources, const std::string& type_url)); diff --git a/test/mocks/upstream/mocks.cc b/test/mocks/upstream/mocks.cc index 3116cb87fe586..380dcc31d7bea 100644 --- a/test/mocks/upstream/mocks.cc +++ b/test/mocks/upstream/mocks.cc @@ -117,7 +117,7 @@ MockClusterManager::MockClusterManager() { ON_CALL(*this, httpAsyncClientForCluster(_)).WillByDefault(ReturnRef(async_client_)); ON_CALL(*this, httpAsyncClientForCluster(_)).WillByDefault((ReturnRef(async_client_))); ON_CALL(*this, bindConfig()).WillByDefault(ReturnRef(bind_config_)); - ON_CALL(*this, adsMux()).WillByDefault(Return(ads_mux_)); + ON_CALL(*this, xdsGrpcContext()).WillByDefault(Return(xds_grpc_context_)); ON_CALL(*this, grpcAsyncClientManager()).WillByDefault(ReturnRef(async_client_manager_)); ON_CALL(*this, localClusterName()).WillByDefault((ReturnRef(local_cluster_name_))); diff --git a/test/mocks/upstream/mocks.h b/test/mocks/upstream/mocks.h index 0049ce557bd45..b34bb822085e0 100644 --- a/test/mocks/upstream/mocks.h +++ b/test/mocks/upstream/mocks.h @@ -304,7 +304,7 @@ class MockClusterManager : public ClusterManager { MOCK_METHOD1(removeCluster, bool(const std::string& cluster)); MOCK_METHOD0(shutdown, void()); MOCK_CONST_METHOD0(bindConfig, const envoy::api::v2::core::BindConfig&()); - MOCK_METHOD0(adsMux, std::shared_ptr()); + MOCK_METHOD0(xdsGrpcContext, std::shared_ptr()); MOCK_METHOD0(grpcAsyncClientManager, Grpc::AsyncClientManager&()); MOCK_CONST_METHOD0(versionInfo, const std::string()); MOCK_CONST_METHOD0(localClusterName, const std::string&()); @@ -317,7 +317,7 @@ class MockClusterManager : public ClusterManager { NiceMock tcp_conn_pool_; NiceMock thread_local_cluster_; envoy::api::v2::core::BindConfig bind_config_; - std::shared_ptr> ads_mux_; + std::shared_ptr> xds_grpc_context_; NiceMock async_client_manager_; std::string local_cluster_name_; NiceMock cluster_manager_factory_; From 1df7db5c5ff87df1e22a9cff238a3139aa4c1a05 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 26 Apr 2019 13:52:26 -0400 Subject: [PATCH 04/56] mostly working snapshot Signed-off-by: Fred Douglas --- include/envoy/config/BUILD | 4 +- include/envoy/config/grpc_mux.h | 7 +- include/envoy/config/xds_context.h | 54 ----- include/envoy/config/xds_grpc_context.h | 82 ++++++++ include/envoy/upstream/BUILD | 2 +- include/envoy/upstream/cluster_manager.h | 2 +- source/common/config/BUILD | 5 +- .../common/config/delta_subscription_state.h | 184 +++++++++--------- source/common/config/grpc_delta_xds_context.h | 184 +++++++++--------- source/common/config/grpc_mux_impl.cc | 19 +- source/common/config/grpc_mux_impl.h | 30 ++- source/common/config/grpc_stream.h | 39 ++-- source/common/config/grpc_subscription_impl.h | 1 - .../common/upstream/cluster_manager_impl.cc | 2 - test/common/config/BUILD | 5 +- .../config/delta_subscription_impl_test.cc | 80 -------- .../config/delta_subscription_state_test.cc | 116 +++++++++++ .../config/delta_subscription_test_harness.h | 4 +- test/common/config/grpc_mux_impl_test.cc | 39 ++-- test/common/config/grpc_stream_test.cc | 133 +++++++++++++ .../config/grpc_subscription_impl_test.cc | 5 +- .../config/grpc_subscription_test_harness.h | 8 +- test/mocks/config/BUILD | 2 +- test/mocks/config/mocks.h | 2 +- 24 files changed, 598 insertions(+), 411 deletions(-) delete mode 100644 include/envoy/config/xds_context.h create mode 100644 include/envoy/config/xds_grpc_context.h delete mode 100644 test/common/config/delta_subscription_impl_test.cc create mode 100644 test/common/config/delta_subscription_state_test.cc create mode 100644 test/common/config/grpc_stream_test.cc diff --git a/include/envoy/config/BUILD b/include/envoy/config/BUILD index 832ed61db786f..e3a04e4fa8e9c 100644 --- a/include/envoy/config/BUILD +++ b/include/envoy/config/BUILD @@ -56,8 +56,8 @@ envoy_cc_library( ) envoy_cc_library( - name = "xds_context_interface", - hdrs = ["xds_context.h"], + name = "xds_grpc_context_interface", + hdrs = ["xds_grpc_context.h"], deps = [ ":subscription_interface", "//include/envoy/local_info:local_info_interface", diff --git a/include/envoy/config/grpc_mux.h b/include/envoy/config/grpc_mux.h index 6872a62d875b6..a97a2f2c026e5 100644 --- a/include/envoy/config/grpc_mux.h +++ b/include/envoy/config/grpc_mux.h @@ -74,12 +74,7 @@ class GrpcMux { virtual ~GrpcMux() {} /** - * Initiate stream with management server. - */ - virtual void start() PURE; - - /** - * Start a configuration subscription asynchronously for some API type and resources. + * Start a configuration subscription asynchronously for some API type and resources. If the gRPC stream to the management server is not already up, starts it. * @param type_url type URL corresponding to xDS API, e.g. * type.googleapis.com/envoy.api.v2.Cluster. * @param resources vector of resource names to watch for. If this is empty, then all diff --git a/include/envoy/config/xds_context.h b/include/envoy/config/xds_context.h deleted file mode 100644 index aa807437eaf10..0000000000000 --- a/include/envoy/config/xds_context.h +++ /dev/null @@ -1,54 +0,0 @@ -#pragma once - -#include "envoy/config/grpc_mux.h" // TODO TODO remove -#include "envoy/config/subscription.h" -#include "envoy/event/dispatcher.h" -#include "envoy/local_info/local_info.h" - -#include "common/protobuf/protobuf.h" - -namespace Envoy { -namespace Config { - -class XdsGrpcContext { -public: - virtual ~XdsGrpcContext() {} - virtual void addSubscription(const std::vector& resources, - const std::string& type_url, SubscriptionCallbacks& callbacks, - SubscriptionStats& stats, - std::chrono::milliseconds init_fetch_timeout) PURE; - - // Enqueues and attempts to send a discovery request, (un)subscribing to resources missing from / - // added to the passed 'resources' argument, relative to resource_versions_. - virtual void updateResources(const std::vector& resources, - const std::string& type_url) PURE; - - virtual void removeSubscription(const std::string& type_url) PURE; - - virtual void pause(const std::string& type_url) PURE; - virtual void resume(const std::string& type_url) PURE; - - virtual void drainRequests() PURE; - - /** - * For the gRPC stream handler to prompt the context to take appropriate action in response to the - * gRPC stream having been successfully established. - */ - virtual void handleStreamEstablished() PURE; - - /** - * For the gRPC stream handler to prompt the context to take appropriate action in response to - * failure to establish the gRPC stream. - */ - virtual void handleEstablishmentFailure() PURE; - - // TODO TODO remove - virtual GrpcMuxWatchPtr subscribe(const std::string& type_url, - const std::vector& resources, - GrpcMuxCallbacks& callbacks) PURE; - // TODO TODO remove - virtual void start() PURE; -}; - -} // namespace Config -} // namespace Envoy diff --git a/include/envoy/config/xds_grpc_context.h b/include/envoy/config/xds_grpc_context.h new file mode 100644 index 0000000000000..432a7629ff88e --- /dev/null +++ b/include/envoy/config/xds_grpc_context.h @@ -0,0 +1,82 @@ +#pragma once + +#include "envoy/config/grpc_mux.h" // TODO TODO remove +#include "envoy/config/subscription.h" +#include "envoy/event/dispatcher.h" +#include "envoy/local_info/local_info.h" + +#include "common/protobuf/protobuf.h" + +namespace Envoy { +namespace Config { + +class XdsGrpcContext { +public: + virtual ~XdsGrpcContext() {} + /** + * Starts a configuration subscription asynchronously for some API type and resources. If the gRPC + * stream to the management server is not already up, starts it. + * @param resources vector of resource names to watch for. If this is empty, then all + * resources for type_url will result in callbacks. + * @param type_url type URL corresponding to xDS API, e.g. + * type.googleapis.com/envoy.api.v2.Cluster. + * @param callbacks the callbacks to be notified of configuration updates. These must be valid + * until this XdsGrpcContext is destroyed. + * @param stats reference to a stats object, which should be owned by the SubscriptionImpl, for + * the XdsGrpcContext to record stats specific to this one subscription. + * @param init_fetch_timeout how long the first fetch has to complete before onConfigUpdateFailed + * will be called. + */ + virtual void addSubscription(const std::vector& resources, + const std::string& type_url, SubscriptionCallbacks& callbacks, + SubscriptionStats& stats, + std::chrono::milliseconds init_fetch_timeout) PURE; + + // Enqueues and attempts to send a discovery request, (un)subscribing to resources missing from / + // added to the passed 'resources' argument, relative to resource_versions_. + virtual void updateResources(const std::vector& resources, + const std::string& type_url) PURE; + + virtual void removeSubscription(const std::string& type_url) PURE; + + virtual void pause(const std::string& type_url) PURE; + virtual void resume(const std::string& type_url) PURE; + + // TODO TODO remove + virtual GrpcMuxWatchPtr subscribe(const std::string& type_url, + const std::vector& resources, + GrpcMuxCallbacks& callbacks) PURE; +}; + +/** + * A grouping of callbacks that an XdsGrpcContext should provide to its GrpcStream. + */ +template class GrpcStreamCallbacks { +public: + virtual ~GrpcStreamCallbacks() {} + + /** + * For the GrpcStream to prompt the context to take appropriate action in response to the + * gRPC stream having been successfully established. + */ + virtual void onStreamEstablished() PURE; + + /** + * For the GrpcStream to prompt the context to take appropriate action in response to + * failure to establish the gRPC stream. + */ + virtual void onEstablishmentFailure() PURE; + + /** + * For the GrpcStream to pass received protos to the context. + */ + virtual void onDiscoveryResponse(std::unique_ptr&& message) PURE; + + /** + * For the GrpcStream to call when its rate limiting logic allows more requests to be sent. + */ + virtual void onWriteable() PURE; +}; + +} // namespace Config +} // namespace Envoy diff --git a/include/envoy/upstream/BUILD b/include/envoy/upstream/BUILD index 4e3a59735cb8c..53ffa06af83ec 100644 --- a/include/envoy/upstream/BUILD +++ b/include/envoy/upstream/BUILD @@ -18,7 +18,7 @@ envoy_cc_library( ":upstream_interface", "//include/envoy/access_log:access_log_interface", "//include/envoy/config:grpc_mux_interface", - "//include/envoy/config:xds_context_interface", + "//include/envoy/config:xds_grpc_context_interface", "//include/envoy/grpc:async_client_manager_interface", "//include/envoy/http:async_client_interface", "//include/envoy/http:conn_pool_interface", diff --git a/include/envoy/upstream/cluster_manager.h b/include/envoy/upstream/cluster_manager.h index 555673b7d3389..6c00043aeef7f 100644 --- a/include/envoy/upstream/cluster_manager.h +++ b/include/envoy/upstream/cluster_manager.h @@ -11,7 +11,7 @@ #include "envoy/api/v2/cds.pb.h" #include "envoy/config/bootstrap/v2/bootstrap.pb.h" #include "envoy/config/grpc_mux.h" -#include "envoy/config/xds_context.h" +#include "envoy/config/xds_grpc_context.h" #include "envoy/grpc/async_client_manager.h" #include "envoy/http/async_client.h" #include "envoy/http/conn_pool.h" diff --git a/source/common/config/BUILD b/source/common/config/BUILD index 3ecc5ecab35c3..8dd0cb23b79ed 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -120,6 +120,9 @@ envoy_cc_library( deps = [ ":utility_lib", "//include/envoy/config:subscription_interface", + "//include/envoy/event:dispatcher_interface", + "//include/envoy/local_info:local_info_interface", + "//source/common/common:hash_lib", "//source/common/protobuf", "@envoy_api//envoy/api/v2:discovery_cc", ], @@ -132,7 +135,7 @@ envoy_cc_library( ":utility_lib", "//include/envoy/config:grpc_mux_interface", "//include/envoy/config:subscription_interface", - "//include/envoy/config:xds_context_interface", + "//include/envoy/config:xds_grpc_context_interface", "//include/envoy/grpc:async_client_interface", "//include/envoy/upstream:cluster_manager_interface", "//source/common/common:backoff_lib", diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index 619969266de15..dece2c4446494 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -2,36 +2,33 @@ #include "envoy/api/v2/discovery.pb.h" #include "envoy/config/subscription.h" +#include "envoy/event/dispatcher.h" #include "envoy/grpc/status.h" +#include "envoy/local_info/local_info.h" + +#include "common/common/hash.h" namespace Envoy { namespace Config { -struct ResourceNameDiff { - std::vector added_; - std::vector removed_; - std::string type_url_; - ResourceNameDiff(std::string type_url) : type_url_(type_url) {} - ResourceNameDiff() {} +struct UpdateAck { + UpdateAck(absl::string_view nonce) : nonce_(nonce) {} + std::string nonce_; + ::google::rpc::Status error_detail_; }; class DeltaSubscriptionState : public Logger::Loggable { public: - DeltaSubscriptionState(const std::string& type_url, - const std::vector& resource_names, + DeltaSubscriptionState(const std::string& type_url, const std::set& resource_names, SubscriptionCallbacks& callbacks, const LocalInfo::LocalInfo& local_info, std::chrono::milliseconds init_fetch_timeout, Event::Dispatcher& dispatcher, SubscriptionStats& stats) : type_url_(type_url), callbacks_(callbacks), local_info_(local_info), init_fetch_timeout_(init_fetch_timeout), stats_(stats) { - // Start us off with the list of resources we're interested in. The first_request_of_new_stream_ - // version of populateRequestWithDiff() will know to put these names in the subscribe field, but - // *not* the initial_resource_versions field, due to them all being in the waitingForServer - // state. - for (const auto& resource_name : resource_names) { - setResourceWaitingForServer(resource_name); - } - buildCleanRequest(); + // In normal usage of updateResourceInterest(), the caller is supposed to cause a discovery + // request to be queued if it returns true. We don't need to do that because we know that the + // subscription gRPC stream is not yet established, and establishment causes a request. + updateResourceInterest(resource_names); setInitFetchTimeout(dispatcher); // The attempt stat here is maintained for the purposes of having consistency between ADS and @@ -56,82 +53,67 @@ class DeltaSubscriptionState : public Logger::Loggable { paused_ = true; } - void resume(absl::optional& to_send_if_any) { + void resume() { ENVOY_LOG(debug, "Resuming discovery requests for {}", type_url_); ASSERT(paused_); paused_ = false; - to_send_if_any.reset(); - to_send_if_any.swap(pending_); } - void buildCleanRequest() { - request_.Clear(); - request_.set_type_url(type_url_); - request_.mutable_node()->MergeFrom(local_info_.node()); - } - - void updateResourceNamesAndBuildDiff(const std::vector& resource_names, - ResourceNameDiff& diff) { - stats_.update_attempt_.inc(); - std::set_difference(resource_names.begin(), resource_names.end(), resource_names_.begin(), - resource_names_.end(), std::inserter(diff.added_, diff.added_.begin())); - std::set_difference(resource_names_.begin(), resource_names_.end(), resource_names.begin(), - resource_names.end(), std::inserter(diff.removed_, diff.removed_.begin())); + bool paused() { return paused_; } - for (const auto& added : diff.added_) { - setResourceWaitingForServer(added); - } - for (const auto& removed : diff.removed_) { - lostInterestInResource(removed); - } - } + // Returns true if there is any meaningful change in our subscription interest, worth reporting to + // the server. + bool updateResourceInterest(const std::set& update_to_these_names) { + std::vector cur_added; + std::vector cur_removed; - const envoy::api::v2::DeltaDiscoveryRequest& - populateRequestWithDiff(const ResourceNameDiff& diff) { - request_.clear_resource_names_subscribe(); - request_.clear_resource_names_unsubscribe(); - std::copy(diff.added_.begin(), diff.added_.end(), - Protobuf::RepeatedFieldBackInserter(request_.mutable_resource_names_subscribe())); - std::copy(diff.removed_.begin(), diff.removed_.end(), - Protobuf::RepeatedFieldBackInserter(request_.mutable_resource_names_unsubscribe())); + std::set_difference(update_to_these_names.begin(), update_to_these_names.end(), + resource_names_.begin(), resource_names_.end(), + std::inserter(cur_added, cur_added.begin())); + std::set_difference(resource_names_.begin(), resource_names_.end(), + update_to_these_names.begin(), update_to_these_names.end(), + std::inserter(cur_removed, cur_removed.begin())); - if (first_request_of_new_stream_) { - // initial_resource_versions "must be populated for first request in a stream". - for (auto const& resource : resource_versions_) { - // Populate initial_resource_versions with the resource versions we currently have. - // Resources we are interested in, but are still waiting to get any version of from the - // server, do not belong in initial_resource_versions. (But do belong in new subscriptions!) - if (!resource.second.waitingForServer()) { - (*request_.mutable_initial_resource_versions())[resource.first] = - resource.second.version(); - } - *request_.mutable_resource_names_subscribe()->Add() = resource.first; - } + for (const auto& a : cur_added) { + setResourceWaitingForServer(a); + // Removed->added requires us to keep track of it as a "new" addition, since our user may have + // forgotten its copy of the resource after instructing us to remove it, and so needs to be + // reminded of it. + names_removed_.erase(a); + names_added_.insert(a); + } + for (const auto& r : cur_removed) { + lostInterestInResource(r); + // Ideally, when a resource is added-then-removed in between requests, we would avoid putting + // a superfluous "unsubscribe [resource that was never subscribed]" in the request. However, + // the removed-then-added case *does* need to go in the request, and due to how we accomplish + // that, it's difficult to distinguish remove-add-remove from add-remove (because "remove-add" + // has to be treated as equivalent to just "add"). + names_added_.erase(r); + names_removed_.insert(r); } - return request_; + stats_.update_attempt_.inc(); + // Tell the server about our new interests only if there are any. + return !names_added_.empty() || !names_removed_.empty(); } void set_first_request_of_new_stream(bool val) { first_request_of_new_stream_ = val; } - bool checkPausedDuringSendAttempt(const ResourceNameDiff& diff) { - if (paused_) { - pending_ = diff; // resume() is now set to send this request. - return true; - } - return false; - } - - void handleResponse(envoy::api::v2::DeltaDiscoveryResponse* message) { + UpdateAck handleResponse(envoy::api::v2::DeltaDiscoveryResponse* message) { // We *always* copy the response's nonce into the next request, even if we're going to make that // request a NACK by setting error_detail. - request_.set_response_nonce(message->nonce()); + UpdateAck ack(message->nonce()); + std::cerr << "handleResponse" << std::endl; try { disableInitFetchTimeoutTimer(); callbacks_.onConfigUpdate(message->resources(), message->removed_resources(), message->system_version_info()); for (const auto& resource : message->resources()) { + std::cerr << "setResourceVersion " << resource.name() << " " << resource.version() + << std::endl; setResourceVersion(resource.name(), resource.version()); } + std::cerr << "done setting versions" << std::endl; // If a resource is gone, there is no longer a meaningful version for it that makes sense to // provide to the server upon stream reconnect: either it will continue to not exist, in which // case saying nothing is fine, or the server will bring back something new, which we should @@ -142,6 +124,7 @@ class DeltaSubscriptionState : public Logger::Loggable { // cancelling my subscription" when we lose interest. for (const auto& resource_name : message->removed_resources()) { if (resource_names_.find(resource_name) != resource_names_.end()) { + std::cerr << resource_name << " waiting for server" << std::endl; setResourceWaitingForServer(resource_name); } } @@ -151,16 +134,17 @@ class DeltaSubscriptionState : public Logger::Loggable { ENVOY_LOG(debug, "Delta config for {} accepted with {} resources added, {} removed", type_url_, message->resources().size(), message->removed_resources().size()); } catch (const EnvoyException& e) { + std::cerr << "UH OH EXCEPTION" << std::endl; // Note that error_detail being set is what indicates that a DeltaDiscoveryRequest is a NACK. - ::google::rpc::Status* error_detail = request_.mutable_error_detail(); - error_detail->set_code(Grpc::Status::GrpcStatus::Internal); - error_detail->set_message(e.what()); + ack.error_detail_.set_code(Grpc::Status::GrpcStatus::Internal); + ack.error_detail_.set_message(e.what()); disableInitFetchTimeoutTimer(); stats_.update_rejected_.inc(); ENVOY_LOG(warn, "delta config for {} rejected: {}", type_url_, e.what()); stats_.update_attempt_.inc(); callbacks_.onConfigUpdateFailed(&e); } + return ack; } void handleEstablishmentFailure() { @@ -169,6 +153,39 @@ class DeltaSubscriptionState : public Logger::Loggable { callbacks_.onConfigUpdateFailed(nullptr); } + envoy::api::v2::DeltaDiscoveryRequest getNextRequest() { + envoy::api::v2::DeltaDiscoveryRequest request; + if (first_request_of_new_stream_) { + // initial_resource_versions "must be populated for first request in a stream". + // Also, since this might be a new server, we must explicitly state *all* of our subscription + // interest. + for (auto const& resource : resource_versions_) { + // Populate initial_resource_versions with the resource versions we currently have. + // Resources we are interested in, but are still waiting to get any version of from the + // server, do not belong in initial_resource_versions. (But do belong in new subscriptions!) + if (!resource.second.waitingForServer()) { + (*request.mutable_initial_resource_versions())[resource.first] = + resource.second.version(); + } + // As mentioned above, fill resource_names_subscribe with everything, including names we + // have yet to receive any resource for. + names_added_.insert(resource.first); + } + names_removed_.clear(); + first_request_of_new_stream_ = false; + } + std::copy(names_added_.begin(), names_added_.end(), + Protobuf::RepeatedFieldBackInserter(request.mutable_resource_names_subscribe())); + std::copy(names_removed_.begin(), names_removed_.end(), + Protobuf::RepeatedFieldBackInserter(request.mutable_resource_names_unsubscribe())); + names_added_.clear(); + names_removed_.clear(); + + request.set_type_url(type_url_); + request.mutable_node()->MergeFrom(local_info_.node()); + return request; + } + private: void disableInitFetchTimeoutTimer() { if (init_fetch_timeout_timer_) { @@ -227,23 +244,14 @@ class DeltaSubscriptionState : public Logger::Loggable { std::chrono::milliseconds init_fetch_timeout_; Event::TimerPtr init_fetch_timeout_timer_; - // Paused via pause()? bool paused_{}; bool first_request_of_new_stream_{true}; - absl::optional pending_; - - // Current state of the request we next intend to send. It is updated in various ways at various - // times by this class's functions. - // - // A note on request_ vs the ResourceNameDiff queue: request_ is an actual DeltaDiscoveryRequest - // proto, which is maintained/updated as the code runs, and is what is actually sent out by - // sendDiscoveryRequest(). queueDiscoveryRequest() queues an item into GrpcStream's abstract queue - // - in this case, a ResourceNameDiff. request_ tracks mechanical implementation details, whereas - // the queue items track the resources to be requested. When it comes time to send a - // DiscoveryRequest, the front of that queue is sent into sendDiscoveryRequest(), which uses the - // item to finalize request_ for sending. So, think of the ResourceNameDiff queue as "rough - // sketches" that actual requests will be based on. - envoy::api::v2::DeltaDiscoveryRequest request_; + + // Tracking of the delta in our subscription interest since the previous DeltaDiscoveryRequest was + // sent. Can't use unordered_set due to ordering issues in gTest expectation matching. Feel free + // to change if you can figure out how to make it work. + std::set names_added_; + std::set names_removed_; SubscriptionStats& stats_; }; diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h index 318e29d654c85..6582105a1bb50 100644 --- a/source/common/config/grpc_delta_xds_context.h +++ b/source/common/config/grpc_delta_xds_context.h @@ -5,7 +5,7 @@ #include "envoy/api/v2/discovery.pb.h" #include "envoy/common/token_bucket.h" #include "envoy/config/subscription.h" -#include "envoy/config/xds_context.h" +#include "envoy/config/xds_grpc_context.h" #include "common/common/assert.h" #include "common/common/backoff_strategy.h" @@ -31,9 +31,9 @@ namespace Config { // non-aggregated case is handled by running the aggregated logic, and just happening to only have 1 // xDS subscription type to "aggregate", i.e., GrpcDeltaXdsContext only has one // DeltaSubscriptionState entry in its map. The sole implementation difference: when the bootstrap -// specifies ADS, the method string passed to gRPC is {Delta,Stream}AggregatedResources, as opposed -// to e.g. {Delta,Stream}Clusters. This distinction is necessary for the server to know what -// resources should be provided. +// specifies ADS, the method string set on the gRPC client that the XdsGrpcContext holds is +// {Delta,Stream}AggregatedResources, as opposed to e.g. {Delta,Stream}Clusters. This distinction is +// necessary for the server to know what resources should be provided. // // DeltaSubscriptionState is what handles the conceptual/application-level protocol state of a given // resource type subscription, which may or may not be multiplexed with others. So, @@ -49,7 +49,9 @@ namespace Config { // for both non- and aggregated: if non-aggregated, you'll be the only holder of that shared_ptr. By // those two mechanisms, the single class (TODO rename) DeltaSubscriptionImpl handles all 4 // delta/SotW and non-/aggregated combinations. -class GrpcDeltaXdsContext : public XdsGrpcContext, Logger::Loggable { +class GrpcDeltaXdsContext : public XdsGrpcContext, + public GrpcStreamCallbacks, + Logger::Loggable { public: GrpcDeltaXdsContext(Grpc::AsyncClientPtr async_client, Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, @@ -58,19 +60,16 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, Logger::Loggable&& message) { - handleDiscoveryResponse(std::move(message)); - }) {} + rate_limit_settings) {} void addSubscription(const std::vector& resources, const std::string& type_url, SubscriptionCallbacks& callbacks, SubscriptionStats& stats, std::chrono::milliseconds init_fetch_timeout) override { - subscriptions_.emplace( - std::make_pair(type_url, DeltaSubscriptionState(type_url, resources, callbacks, local_info_, - init_fetch_timeout, dispatcher_, stats))); - grpc_stream_.establishNewStream(); // (idempotent) + std::set HACK_resources_as_set(resources.begin(), resources.end()); + subscriptions_.emplace(std::make_pair( + type_url, DeltaSubscriptionState(type_url, HACK_resources_as_set, callbacks, local_info_, + init_fetch_timeout, dispatcher_, stats))); + grpc_stream_.establishStream(); // (idempotent) } // Enqueues and attempts to send a discovery request, (un)subscribing to resources missing from / @@ -82,10 +81,11 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, Logger::Loggablesecond.updateResourceNamesAndBuildDiff(resources, diff); - queueDiscoveryRequest(diff); + std::set HACK_resources_as_set(resources.begin(), resources.end()); + if (sub->second.updateResourceInterest( + HACK_resources_as_set)) { // any changes worth mentioning? + kickOffDiscoveryRequest(type_url); + } } void removeSubscription(const std::string& type_url) override { subscriptions_.erase(type_url); } @@ -105,98 +105,47 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, Logger::Loggable to_send_if_any; - sub->second.resume(to_send_if_any); - if (to_send_if_any.has_value()) { - // Note that if some code pauses a certain type, then causes another discovery request to be - // queued up for it, then the newly queued request can *replace* other requests (of the same - // type) that were in the queue - that is, the new one will be sent upon unpause, while the - // others are just dropped. - queueDiscoveryRequest(to_send_if_any.value()); - } + sub->second.resume(); + trySendDiscoveryRequests(); } - void sendDiscoveryRequest(const ResourceNameDiff& diff) { - auto sub = subscriptions_.find(diff.type_url_); - if (sub == subscriptions_.end()) { - ENVOY_LOG(warn, "Not sending DeltaDiscoveryRequest for non-existent subscription {}.", - diff.type_url_); - return; - } - if (!grpc_stream_.grpcStreamAvailable()) { - ENVOY_LOG(debug, "No stream available to sendDiscoveryRequest for {}", diff.type_url_); - return; // Drop this request; the reconnect will enqueue a new one. - } - if (sub->second.checkPausedDuringSendAttempt(diff)) { - return; - } - const envoy::api::v2::DeltaDiscoveryRequest& request = - sub->second.populateRequestWithDiff(diff); - ENVOY_LOG(trace, "Sending DeltaDiscoveryRequest for {}: {}", diff.type_url_, - request.DebugString()); - // NOTE: at this point we are beyond the rate-limiting logic. sendMessage() unconditionally - // sends its argument over the gRPC stream. So, it is safe to unconditionally - // set_first_request_of_new_stream(false). - grpc_stream_.sendMessage(request); - sub->second.set_first_request_of_new_stream(false); - sub->second.buildCleanRequest(); - } - - void handleDiscoveryResponse(std::unique_ptr&& message) { + void + onDiscoveryResponse(std::unique_ptr&& message) override { ENVOY_LOG(debug, "Received DeltaDiscoveryResponse for {} at version {}", message->type_url(), message->system_version_info()); auto sub = subscriptions_.find(message->type_url()); if (sub == subscriptions_.end()) { ENVOY_LOG(warn, - "Dropping received DeltaDiscoveryResponse at version {} for non-existent " + "Dropping received DeltaDiscoveryResponse (with version {}) for non-existent " "subscription {}.", message->system_version_info(), message->type_url()); return; } - sub->second.handleResponse(message.get()); - // Queue up an ACK. - queueDiscoveryRequest( - ResourceNameDiff(message->type_url())); // no change to subscribed resources + kickOffDiscoveryRequestWithAck(message->type_url(), sub->second.handleResponse(message.get())); } - void handleStreamEstablished() override { - // Ensure that there's nothing leftover from previous streams in the request proto queue. - clearRequestQueue(); + void onStreamEstablished() override { for (auto& sub : subscriptions_) { sub.second.set_first_request_of_new_stream(true); - // Due to the first_request_of_new_stream logic, this "empty" diff will actually become a - // request populated with all the resource names passed to the DeltaSubscriptionState's - // constructor. - queueDiscoveryRequest(ResourceNameDiff(sub.first)); + kickOffDiscoveryRequest(sub.first); } } - void handleEstablishmentFailure() override { + void onEstablishmentFailure() override { for (auto& sub : subscriptions_) { sub.second.handleEstablishmentFailure(); } } - // Request queue management logic. - void queueDiscoveryRequest(const ResourceNameDiff& queue_item) { - request_queue_.push(queue_item); - drainRequests(); - } - void clearRequestQueue() { - grpc_stream_.maybeUpdateQueueSizeStat(0); - // TODO(fredlas) when we have C++17: request_queue_ = {}; - while (!request_queue_.empty()) { - request_queue_.pop(); - } + void onWriteable() override { trySendDiscoveryRequests(); } + + void kickOffDiscoveryRequest(const std::string& type_url) { + kickOffDiscoveryRequestWithAck(type_url, absl::nullopt); } - void drainRequests() override { - ENVOY_LOG(trace, "draining discovery requests {}", request_queue_.size()); - while (!request_queue_.empty() && grpc_stream_.checkRateLimitAllowsDrain()) { - // Process the request, if rate limiting is not enabled at all or if it is under rate limit. - sendDiscoveryRequest(request_queue_.front()); - request_queue_.pop(); - } - grpc_stream_.maybeUpdateQueueSizeStat(request_queue_.size()); + + void kickOffDiscoveryRequestWithAck(const std::string& type_url, absl::optional ack) { + request_queue_.push(PendingRequest(type_url, ack)); + trySendDiscoveryRequests(); } // TODO TODO remove, GrpcMux impersonation @@ -204,18 +153,73 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, Logger::Loggable& + grpcStreamForTest() { + return grpc_stream_; + } private: + void trySendDiscoveryRequests() { + while (!request_queue_.empty()) { + const PendingRequest& pending_request = request_queue_.front(); + auto sub = subscriptions_.find(pending_request.type_url_); + if (sub == subscriptions_.end()) { + ENVOY_LOG(warn, "Not sending queued request for non-existent subscription {}.", + pending_request.type_url_); + request_queue_.pop(); + continue; + } + if (shouldSendDiscoveryRequest(sub->first, &sub->second)) { + // TODO TODO hmm maybe should pass the ack into getNextRequest, to accomodate the + // templatization goal. + envoy::api::v2::DeltaDiscoveryRequest request = sub->second.getNextRequest(); + if (pending_request.ack_.has_value()) { + const UpdateAck& ack = pending_request.ack_.value(); + request.set_response_nonce(ack.nonce_); + request.mutable_error_detail()->CopyFrom(ack.error_detail_); + } + grpc_stream_.sendMessage(request); + request_queue_.pop(); + } else { + break; + } + } + grpc_stream_.maybeUpdateQueueSizeStat(request_queue_.size()); + } + + bool shouldSendDiscoveryRequest(absl::string_view type_url, DeltaSubscriptionState* sub) { + if (sub->paused()) { + ENVOY_LOG(trace, "API {} paused; discovery request on hold for now.", type_url); + return false; + } else if (!grpc_stream_.grpcStreamAvailable()) { + ENVOY_LOG(trace, "No stream available to send a DiscoveryRequest for {}.", type_url); + return false; + } else if (!grpc_stream_.checkRateLimitAllowsDrain()) { + ENVOY_LOG(trace, "{} DiscoveryRequest hit rate limit; will try later.", type_url); + return false; + } + return true; + } + const LocalInfo::LocalInfo& local_info_; Event::Dispatcher& dispatcher_; GrpcStream grpc_stream_; - // A queue to store requests while rate limited. Note that when requests cannot be sent due to the - // gRPC stream being down, this queue does not store them; rather, they are simply dropped. - std::queue request_queue_; + struct PendingRequest { + PendingRequest(absl::string_view type_url, absl::optional ack) + : type_url_(type_url), ack_(ack) {} + const std::string type_url_; + const absl::optional ack_; + }; + + // Discovery requests we're waiting to send, stored in the order that they should be sent in. All + // of our different API types are mixed together in this queue. Those that are ACKs have the ack_ + // field filled. + std::queue request_queue_; + // Map from type_url strings to a DeltaSubscriptionState for that type. std::unordered_map subscriptions_; }; diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index 1a871afe46abc..7c495d76f49db 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -14,11 +14,7 @@ GrpcMuxImpl::GrpcMuxImpl(const LocalInfo::LocalInfo& local_info, Grpc::AsyncClie Runtime::RandomGenerator& random, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings) : grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, - rate_limit_settings, - // callback for handling receipt of DiscoveryResponse protos. - [this](std::unique_ptr&& message) { - handleDiscoveryResponse(std::move(message)); - }), + rate_limit_settings), local_info_(local_info) { Config::Utility::checkLocalInfo("ads", local_info); @@ -32,8 +28,6 @@ GrpcMuxImpl::~GrpcMuxImpl() { } } -void GrpcMuxImpl::start() { grpc_stream_.establishNewStream(); } - void GrpcMuxImpl::sendDiscoveryRequest(const std::string& type_url) { if (!grpc_stream_.grpcStreamAvailable()) { ENVOY_LOG(debug, "No stream available to sendDiscoveryRequest for {}", type_url); @@ -70,7 +64,7 @@ void GrpcMuxImpl::sendDiscoveryRequest(const std::string& type_url) { } } -void GrpcMuxImpl::handleDiscoveryResponse( +void GrpcMuxImpl::onDiscoveryResponse( std::unique_ptr&& message) { const std::string& type_url = message->type_url(); ENVOY_LOG(debug, "Received gRPC message for {} at version {}", type_url, message->version_info()); @@ -174,6 +168,9 @@ GrpcMuxWatchPtr GrpcMuxImpl::subscribe(const std::string& type_url, // only send a single RDS/EDS update after the CDS/LDS update. queueDiscoveryRequest(type_url); + // This is idempotent, so it's fine to just always call when we have a new subscription. + grpc_stream_.establishStream(); + return watch; } @@ -198,13 +195,13 @@ void GrpcMuxImpl::resume(const std::string& type_url) { } } -void GrpcMuxImpl::handleStreamEstablished() { +void GrpcMuxImpl::onStreamEstablished() { for (const auto type_url : subscriptions_) { queueDiscoveryRequest(type_url); } } -void GrpcMuxImpl::handleEstablishmentFailure() { +void GrpcMuxImpl::onEstablishmentFailure() { for (const auto& api_state : api_state_) { for (auto watch : api_state.second.watches_) { watch->callbacks_.onConfigUpdateFailed(nullptr); @@ -212,6 +209,8 @@ void GrpcMuxImpl::handleEstablishmentFailure() { } } +void GrpcMuxImpl::onWriteable() { drainRequests(); } + void GrpcMuxImpl::queueDiscoveryRequest(const std::string& queue_item) { request_queue_.push(queue_item); drainRequests(); diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index a5019fd89e9f0..dfc92472f6bd9 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -22,6 +22,7 @@ namespace Config { */ class GrpcMuxImpl : public GrpcMux, public XdsGrpcContext, + public GrpcStreamCallbacks, public Logger::Loggable { public: GrpcMuxImpl(const LocalInfo::LocalInfo& local_info, Grpc::AsyncClientPtr async_client, @@ -30,18 +31,20 @@ class GrpcMuxImpl : public GrpcMux, const RateLimitSettings& rate_limit_settings); ~GrpcMuxImpl(); - void start() override; GrpcMuxWatchPtr subscribe(const std::string& type_url, const std::vector& resources, GrpcMuxCallbacks& callbacks) override; void sendDiscoveryRequest(const std::string& type_url); + // Config::GrpcStreamCallbacks + void onStreamEstablished() override; + void onEstablishmentFailure() override; + void onDiscoveryResponse(std::unique_ptr&& message) override; + void onWriteable() override; + // XdsGrpcContext - void handleStreamEstablished() override; - void handleEstablishmentFailure() override; void pause(const std::string& type_url) override; void resume(const std::string& type_url) override; - void drainRequests() override; void addSubscription(const std::vector&, const std::string&, SubscriptionCallbacks&, SubscriptionStats&, std::chrono::milliseconds) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; @@ -50,9 +53,15 @@ class GrpcMuxImpl : public GrpcMux, NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } void removeSubscription(const std::string&) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + void handleDiscoveryResponse(std::unique_ptr&& message); + + GrpcStream& + grpcStreamForTest() { + return grpc_stream_; + } private: - void handleDiscoveryResponse(std::unique_ptr&& message); + void drainRequests(); void setRetryTimer(); struct GrpcMuxWatchImpl : public GrpcMuxWatch { @@ -109,9 +118,9 @@ class GrpcMuxImpl : public GrpcMux, std::queue request_queue_; }; -class NullGrpcMuxImpl : public XdsGrpcContext { +class NullGrpcMuxImpl : public XdsGrpcContext, + GrpcStreamCallbacks { public: - void start() override {} GrpcMuxWatchPtr subscribe(const std::string&, const std::vector&, GrpcMuxCallbacks&) override { throw EnvoyException("ADS must be configured to support an ADS config source"); @@ -128,9 +137,10 @@ class NullGrpcMuxImpl : public XdsGrpcContext { } void removeSubscription(const std::string&) override {} - void drainRequests() override {} - void handleStreamEstablished() override {} - void handleEstablishmentFailure() override {} + void onWriteable() override {} + void onStreamEstablished() override {} + void onEstablishmentFailure() override {} + void onDiscoveryResponse(std::unique_ptr&&) override {} }; } // namespace Config diff --git a/source/common/config/grpc_stream.h b/source/common/config/grpc_stream.h index 2cc1dd8aa0630..b58334ae14c7c 100644 --- a/source/common/config/grpc_stream.h +++ b/source/common/config/grpc_stream.h @@ -2,7 +2,7 @@ #include -#include "envoy/config/xds_context.h" +#include "envoy/config/xds_grpc_context.h" #include "envoy/grpc/async_client.h" #include "common/common/backoff_strategy.h" @@ -19,44 +19,41 @@ template class GrpcStream : public Grpc::TypedAsyncStreamCallbacks, public Logger::Loggable { public: - GrpcStream(XdsGrpcContext* owning_context, Grpc::AsyncClientPtr async_client, + GrpcStream(GrpcStreamCallbacks* callbacks, Grpc::AsyncClientPtr async_client, const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, Event::Dispatcher& dispatcher, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, - std::function&&)> handle_response_cb) - : owning_context_(owning_context), async_client_(std::move(async_client)), + const RateLimitSettings& rate_limit_settings) + : callbacks_(callbacks), async_client_(std::move(async_client)), service_method_(service_method), control_plane_stats_(generateControlPlaneStats(scope)), random_(random), time_source_(dispatcher.timeSource()), - rate_limiting_enabled_(rate_limit_settings.enabled_), - handle_response_cb_(handle_response_cb) { - retry_timer_ = dispatcher.createTimer([this]() -> void { establishNewStream(); }); + rate_limiting_enabled_(rate_limit_settings.enabled_) { + retry_timer_ = dispatcher.createTimer([this]() -> void { establishStream(); }); if (rate_limiting_enabled_) { // Default Bucket contains 100 tokens maximum and refills at 10 tokens/sec. limit_request_ = std::make_unique( rate_limit_settings.max_tokens_, time_source_, rate_limit_settings.fill_rate_); - drain_request_timer_ = dispatcher.createTimer([this]() { owning_context_->drainRequests(); }); + drain_request_timer_ = dispatcher.createTimer([this]() { callbacks_->onWriteable(); }); } backoff_strategy_ = std::make_unique(RETRY_INITIAL_DELAY_MS, RETRY_MAX_DELAY_MS, random_); } - void establishNewStream() { - ENVOY_LOG(debug, "Establishing new gRPC bidi stream for {}", service_method_.DebugString()); - // TODO TODO TODO since this file also serves vanilla, need to DOUBLE CHECK whether this - // idempotency im adding here makes sense for vanilla too. + void establishStream() { if (stream_ != nullptr) { - ENVOY_LOG(warn, "gRPC bidi stream for {} already exists!", service_method_.DebugString()); - return; + ENVOY_LOG(debug, "gRPC bidi stream for {} already up and running", + service_method_.DebugString()); + return; // idempotent } + ENVOY_LOG(debug, "Establishing new gRPC bidi stream for {}", service_method_.DebugString()); stream_ = async_client_->start(service_method_, *this); if (stream_ == nullptr) { ENVOY_LOG(warn, "Unable to establish new stream"); - owning_context_->handleEstablishmentFailure(); + callbacks_->onEstablishmentFailure(); setRetryTimer(); return; } control_plane_stats_.connected_state_.set(1); - owning_context_->handleStreamEstablished(); + callbacks_->onStreamEstablished(); } bool grpcStreamAvailable() const { return stream_ != nullptr; } @@ -79,7 +76,7 @@ class GrpcStream : public Grpc::TypedAsyncStreamCallbacks, // have 0 until it is reconnected. Setting here ensures that it is consistent with the state of // management server connection. control_plane_stats_.connected_state_.set(1); - handle_response_cb_(std::move(message)); + callbacks_->onDiscoveryResponse(std::move(message)); } void onReceiveTrailingMetadata(Http::HeaderMapPtr&& metadata) override { @@ -90,7 +87,7 @@ class GrpcStream : public Grpc::TypedAsyncStreamCallbacks, ENVOY_LOG(warn, "gRPC config stream closed: {}, {}", status, message); stream_ = nullptr; control_plane_stats_.connected_state_.set(0); - owning_context_->handleEstablishmentFailure(); + callbacks_->onEstablishmentFailure(); setRetryTimer(); } @@ -129,7 +126,7 @@ class GrpcStream : public Grpc::TypedAsyncStreamCallbacks, POOL_GAUGE_PREFIX(scope, control_plane_prefix))}; } - XdsGrpcContext* const owning_context_; + GrpcStreamCallbacks* const callbacks_; // TODO(htuch): Make this configurable or some static. const uint32_t RETRY_INITIAL_DELAY_MS = 500; @@ -150,8 +147,6 @@ class GrpcStream : public Grpc::TypedAsyncStreamCallbacks, TokenBucketPtr limit_request_; const bool rate_limiting_enabled_; Event::TimerPtr drain_request_timer_; - - std::function&&)> handle_response_cb_; }; } // namespace Config diff --git a/source/common/config/grpc_subscription_impl.h b/source/common/config/grpc_subscription_impl.h index 066a826295cb3..1113620d9c3cf 100644 --- a/source/common/config/grpc_subscription_impl.h +++ b/source/common/config/grpc_subscription_impl.h @@ -30,7 +30,6 @@ class GrpcSubscriptionImpl : public Config::Subscription { Config::SubscriptionCallbacks& callbacks) override { // Subscribe first, so we get failure callbacks if grpc_mux_.start() fails. grpc_mux_subscription_.start(resources, callbacks); - grpc_mux_->start(); } void updateResources(const std::vector& resources) override { diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 734d255eca279..7c3adea571459 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -292,8 +292,6 @@ ClusterManagerImpl::ClusterManagerImpl( // clusters have already initialized. (E.g., if all static). init_helper_.onStaticLoadComplete(); - xds_grpc_context_->start(); - if (cm_config.has_load_stats_config()) { const auto& load_stats_config = cm_config.load_stats_config(); load_stats_reporter_ = diff --git a/test/common/config/BUILD b/test/common/config/BUILD index 7fb02b385b3ae..91e1120bdc544 100644 --- a/test/common/config/BUILD +++ b/test/common/config/BUILD @@ -11,10 +11,9 @@ load( envoy_package() envoy_cc_test( - name = "delta_subscription_impl_test", - srcs = ["delta_subscription_impl_test.cc"], + name = "delta_subscription_state_test", + srcs = ["delta_subscription_state_test.cc"], deps = [ - ":delta_subscription_test_harness", "//source/common/config:delta_subscription_lib", "//source/common/stats:isolated_store_lib", "//test/mocks:common_lib", diff --git a/test/common/config/delta_subscription_impl_test.cc b/test/common/config/delta_subscription_impl_test.cc deleted file mode 100644 index cbe980a526505..0000000000000 --- a/test/common/config/delta_subscription_impl_test.cc +++ /dev/null @@ -1,80 +0,0 @@ -#include "test/common/config/delta_subscription_test_harness.h" - -using testing::AnyNumber; -using testing::UnorderedElementsAre; - -namespace Envoy { -namespace Config { -namespace { - -class DeltaSubscriptionImplTest : public DeltaSubscriptionTestHarness, public testing::Test {}; - -TEST_F(DeltaSubscriptionImplTest, ResourceGoneLeadsToBlankInitialVersion) { - // Envoy is interested in three resources: name1, name2, and name3. - startSubscription({"name1", "name2", "name3"}); - - // Ignore these for now, although at the very end there is one we will care about. - EXPECT_CALL(async_stream_, sendMessage(_, _)).Times(AnyNumber()); - - // Semi-hack: we don't want the requests to actually get sent, since it would clear out the - // request_ that we want to inspect. pause() does the trick! - subscription_->pause(); - - // The xDS server's first update includes items for name1 and 2, but not 3. - Protobuf::RepeatedPtrField add1_2; - auto* resource = add1_2.Add(); - resource->set_name("name1"); - resource->set_version("version1A"); - resource = add1_2.Add(); - resource->set_name("name2"); - resource->set_version("version2A"); - subscription_->onConfigUpdate(add1_2, {}, "debugversion1"); - subscription_->handleStreamEstablished(); - envoy::api::v2::DeltaDiscoveryRequest cur_request = subscription_->internalRequestStateForTest(); - EXPECT_EQ("version1A", cur_request.initial_resource_versions().at("name1")); - EXPECT_EQ("version2A", cur_request.initial_resource_versions().at("name2")); - EXPECT_EQ(cur_request.initial_resource_versions().end(), - cur_request.initial_resource_versions().find("name3")); - - // The next update updates 1, removes 2, and adds 3. The map should then have 1 and 3. - Protobuf::RepeatedPtrField add1_3; - resource = add1_3.Add(); - resource->set_name("name1"); - resource->set_version("version1B"); - resource = add1_3.Add(); - resource->set_name("name3"); - resource->set_version("version3A"); - Protobuf::RepeatedPtrField remove2; - *remove2.Add() = "name2"; - subscription_->onConfigUpdate(add1_3, remove2, "debugversion2"); - subscription_->handleStreamEstablished(); - cur_request = subscription_->internalRequestStateForTest(); - EXPECT_EQ("version1B", cur_request.initial_resource_versions().at("name1")); - EXPECT_EQ(cur_request.initial_resource_versions().end(), - cur_request.initial_resource_versions().find("name2")); - EXPECT_EQ("version3A", cur_request.initial_resource_versions().at("name3")); - - // The next update removes 1 and 3. The map we send the server should be empty... - Protobuf::RepeatedPtrField remove1_3; - *remove1_3.Add() = "name1"; - *remove1_3.Add() = "name3"; - subscription_->onConfigUpdate({}, remove1_3, "debugversion3"); - subscription_->handleStreamEstablished(); - cur_request = subscription_->internalRequestStateForTest(); - EXPECT_TRUE(cur_request.initial_resource_versions().empty()); - - // ...but our own map should remember our interest. In particular, losing interest in all 3 should - // cause their names to appear in the resource_names_unsubscribe field of a DeltaDiscoveryRequest. - subscription_->resume(); // now we do want the request to actually get sendMessage()'d. - EXPECT_CALL(async_stream_, sendMessage(_, _)).WillOnce([](const Protobuf::Message& msg, bool) { - auto sent_request = static_cast(&msg); - EXPECT_THAT(sent_request->resource_names_subscribe(), UnorderedElementsAre("name4")); - EXPECT_THAT(sent_request->resource_names_unsubscribe(), - UnorderedElementsAre("name1", "name2", "name3")); - }); - subscription_->subscribe({"name4"}); // (implies "we no longer care about name1,2,3") -} - -} // namespace -} // namespace Config -} // namespace Envoy diff --git a/test/common/config/delta_subscription_state_test.cc b/test/common/config/delta_subscription_state_test.cc new file mode 100644 index 0000000000000..31dcdea72e080 --- /dev/null +++ b/test/common/config/delta_subscription_state_test.cc @@ -0,0 +1,116 @@ +#include "common/config/delta_subscription_state.h" +#include "common/config/utility.h" +#include "common/stats/isolated_store_impl.h" + +#include "test/mocks/config/mocks.h" +#include "test/mocks/event/mocks.h" +#include "test/mocks/local_info/mocks.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::NiceMock; +using testing::UnorderedElementsAre; + +namespace Envoy { +namespace Config { +namespace { + +const char TypeUrl[] = "type.googleapis.com/envoy.api.v2.Cluster"; + +void deliverDiscoveryResponse( + DeltaSubscriptionState& state, + const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& version_info) { + envoy::api::v2::DeltaDiscoveryResponse message; + *message.mutable_resources() = added_resources; + *message.mutable_removed_resources() = removed_resources; + message.set_system_version_info(version_info); + state.handleResponse(&message); +} + +TEST(DeltaSubscriptionImplTest, ResourceGoneLeadsToBlankInitialVersion) { + NiceMock> callbacks; + NiceMock local_info; + NiceMock dispatcher; + Stats::IsolatedStoreImpl store; + SubscriptionStats stats = Utility::generateStats(store); + // We start out interested in three resources: name1, name2, and name3. + DeltaSubscriptionState state(TypeUrl, {"name1", "name2", "name3"}, callbacks, local_info, + std::chrono::milliseconds(0U), dispatcher, stats); + { + envoy::api::v2::DeltaDiscoveryRequest cur_request = state.getNextRequest(); + EXPECT_THAT(cur_request.resource_names_subscribe(), + UnorderedElementsAre("name1", "name2", "name3")); + } + + { + // The xDS server's first update includes items for name1 and 2, but not 3. + Protobuf::RepeatedPtrField add1_2; + auto* resource = add1_2.Add(); + resource->set_name("name1"); + resource->set_version("version1A"); + resource = add1_2.Add(); + resource->set_name("name2"); + resource->set_version("version2A"); + deliverDiscoveryResponse(state, add1_2, {}, "debugversion1"); + state.set_first_request_of_new_stream(true); // simulate a stream reconnection + envoy::api::v2::DeltaDiscoveryRequest cur_request = state.getNextRequest(); + EXPECT_EQ("version1A", cur_request.initial_resource_versions().at("name1")); + EXPECT_EQ("version2A", cur_request.initial_resource_versions().at("name2")); + EXPECT_EQ(cur_request.initial_resource_versions().end(), + cur_request.initial_resource_versions().find("name3")); + } + + { + // The next update updates 1, removes 2, and adds 3. The map should then have 1 and 3. + Protobuf::RepeatedPtrField add1_3; + auto* resource = add1_3.Add(); + resource->set_name("name1"); + resource->set_version("version1B"); + resource = add1_3.Add(); + resource->set_name("name3"); + resource->set_version("version3A"); + Protobuf::RepeatedPtrField remove2; + *remove2.Add() = "name2"; + deliverDiscoveryResponse(state, add1_3, remove2, "debugversion2"); + state.set_first_request_of_new_stream(true); // simulate a stream reconnection + envoy::api::v2::DeltaDiscoveryRequest cur_request = state.getNextRequest(); + EXPECT_EQ("version1B", cur_request.initial_resource_versions().at("name1")); + EXPECT_EQ(cur_request.initial_resource_versions().end(), + cur_request.initial_resource_versions().find("name2")); + EXPECT_EQ("version3A", cur_request.initial_resource_versions().at("name3")); + } + + { + // The next update removes 1 and 3. The map we send the server should be empty... + Protobuf::RepeatedPtrField remove1_3; + *remove1_3.Add() = "name1"; + *remove1_3.Add() = "name3"; + deliverDiscoveryResponse(state, {}, remove1_3, "debugversion3"); + state.set_first_request_of_new_stream(true); // simulate a stream reconnection + envoy::api::v2::DeltaDiscoveryRequest cur_request = state.getNextRequest(); + EXPECT_TRUE(cur_request.initial_resource_versions().empty()); + } + + { + // ...but our own map should remember our interest. In particular, losing interest in all 3 + // should cause their names to appear in the resource_names_unsubscribe field of a + // DeltaDiscoveryRequest. + state.updateResourceInterest({"name4"}); // note the lack of 1, 2, and 3 + state.set_first_request_of_new_stream(true); // simulate a stream reconnection + envoy::api::v2::DeltaDiscoveryRequest cur_request = state.getNextRequest(); + EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4")); + EXPECT_THAT(cur_request.resource_names_unsubscribe(), + UnorderedElementsAre("name1", "name2", "name3")); + } +} + +// Checks that resources that were supposed to be present in the map last reconnect, but should not +// be present for this reconnect, are in fact not there. +TEST(DeltaSubscriptionImplTest, InitialVersionMapCleared) {} + +} // namespace +} // namespace Config +} // namespace Envoy diff --git a/test/common/config/delta_subscription_test_harness.h b/test/common/config/delta_subscription_test_harness.h index b7617b0716735..5764b619de08c 100644 --- a/test/common/config/delta_subscription_test_harness.h +++ b/test/common/config/delta_subscription_test_harness.h @@ -104,7 +104,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); expectSendMessage({}, {}, Grpc::Status::GrpcStatus::Internal, "bad config"); } - subscription_->onReceiveMessage(std::move(response)); + subscription_->onDiscoveryResponse(std::move(response)); Mock::VerifyAndClearExpectations(&async_stream_); } @@ -144,7 +144,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { Envoy::Config::RateLimitSettings rate_limit_settings_; Event::MockTimer* init_timeout_timer_; envoy::api::v2::core::Node node_; - NiceMock callbacks_; + NiceMock> callbacks_; }; } // namespace diff --git a/test/common/config/grpc_mux_impl_test.cc b/test/common/config/grpc_mux_impl_test.cc index f8e9f62cb8169..864cbcb9b07e7 100644 --- a/test/common/config/grpc_mux_impl_test.cc +++ b/test/common/config/grpc_mux_impl_test.cc @@ -107,7 +107,6 @@ TEST_F(GrpcMuxImplTest, MultipleTypeUrlStreams) { EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); expectSendMessage("foo", {"x", "y"}, ""); expectSendMessage("bar", {}, ""); - grpc_mux_->start(); EXPECT_EQ(1, stats_.gauge("control_plane.connected_state").value()); expectSendMessage("bar", {"z"}, ""); auto bar_z_sub = grpc_mux_->subscribe("bar", {"z"}, callbacks_); @@ -139,13 +138,12 @@ TEST_F(GrpcMuxImplTest, ResetStream) { expectSendMessage("foo", {"x", "y"}, ""); expectSendMessage("bar", {}, ""); expectSendMessage("baz", {"z"}, ""); - grpc_mux_->start(); EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)).Times(3); EXPECT_CALL(random_, random()); ASSERT_TRUE(timer != nullptr); // initialized from dispatcher mock. EXPECT_CALL(*timer, enableTimer(_)); - grpc_mux_->onRemoteClose(Grpc::Status::GrpcStatus::Canceled, ""); + grpc_mux_->grpcStreamForTest().onRemoteClose(Grpc::Status::GrpcStatus::Canceled, ""); EXPECT_EQ(0, stats_.gauge("control_plane.connected_state").value()); EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); expectSendMessage("foo", {"x", "y"}, ""); @@ -164,7 +162,6 @@ TEST_F(GrpcMuxImplTest, PauseResume) { auto foo_sub = grpc_mux_->subscribe("foo", {"x", "y"}, callbacks_); grpc_mux_->pause("foo"); EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); - grpc_mux_->start(); expectSendMessage("foo", {"x", "y"}, ""); grpc_mux_->resume("foo"); grpc_mux_->pause("bar"); @@ -189,13 +186,12 @@ TEST_F(GrpcMuxImplTest, TypeUrlMismatch) { EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); expectSendMessage("foo", {"x", "y"}, ""); - grpc_mux_->start(); { std::unique_ptr response( new envoy::api::v2::DiscoveryResponse()); response->set_type_url("bar"); - grpc_mux_->onReceiveMessage(std::move(response)); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); } { @@ -209,7 +205,7 @@ TEST_F(GrpcMuxImplTest, TypeUrlMismatch) { expectSendMessage("foo", {"x", "y"}, "", "", Grpc::Status::GrpcStatus::Internal, fmt::format("bar does not match foo type URL in DiscoveryResponse {}", invalid_response->DebugString())); - grpc_mux_->onReceiveMessage(std::move(invalid_response)); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(invalid_response)); } expectSendMessage("foo", {}, ""); } @@ -223,7 +219,6 @@ TEST_F(GrpcMuxImplTest, WildcardWatch) { auto foo_sub = grpc_mux_->subscribe(type_url, {}, callbacks_); EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); expectSendMessage(type_url, {}, ""); - grpc_mux_->start(); { std::unique_ptr response( @@ -243,7 +238,7 @@ TEST_F(GrpcMuxImplTest, WildcardWatch) { EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); })); expectSendMessage(type_url, {}, "1"); - grpc_mux_->onReceiveMessage(std::move(response)); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); } } @@ -259,7 +254,6 @@ TEST_F(GrpcMuxImplTest, WatchDemux) { EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); // Should dedupe the "x" resource. expectSendMessage(type_url, {"y", "z", "x"}, ""); - grpc_mux_->start(); { std::unique_ptr response( @@ -280,7 +274,7 @@ TEST_F(GrpcMuxImplTest, WatchDemux) { EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); })); expectSendMessage(type_url, {"y", "z", "x"}, "1"); - grpc_mux_->onReceiveMessage(std::move(response)); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); } { @@ -320,7 +314,7 @@ TEST_F(GrpcMuxImplTest, WatchDemux) { EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment_y)); })); expectSendMessage(type_url, {"y", "z", "x"}, "2"); - grpc_mux_->onReceiveMessage(std::move(response)); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); } expectSendMessage(type_url, {"x", "y"}, "2"); @@ -337,7 +331,6 @@ TEST_F(GrpcMuxImplTest, MultipleWatcherWithEmptyUpdates) { EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); expectSendMessage(type_url, {"x", "y"}, ""); - grpc_mux_->start(); std::unique_ptr response( new envoy::api::v2::DiscoveryResponse()); @@ -346,7 +339,7 @@ TEST_F(GrpcMuxImplTest, MultipleWatcherWithEmptyUpdates) { EXPECT_CALL(foo_callbacks, onConfigUpdate(_, "1")).Times(0); expectSendMessage(type_url, {"x", "y"}, "1"); - grpc_mux_->onReceiveMessage(std::move(response)); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); expectSendMessage(type_url, {}, "1"); } @@ -360,7 +353,6 @@ TEST_F(GrpcMuxImplTest, SingleWatcherWithEmptyUpdates) { EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); expectSendMessage(type_url, {}, ""); - grpc_mux_->start(); std::unique_ptr response( new envoy::api::v2::DiscoveryResponse()); @@ -371,7 +363,7 @@ TEST_F(GrpcMuxImplTest, SingleWatcherWithEmptyUpdates) { .WillOnce(Invoke([](const Protobuf::RepeatedPtrField& resources, const std::string&) { EXPECT_TRUE(resources.empty()); })); expectSendMessage(type_url, {}, "1"); - grpc_mux_->onReceiveMessage(std::move(response)); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); } // Exactly one test requires a mock time system to provoke behavior that cannot @@ -408,13 +400,12 @@ TEST_F(GrpcMuxImplTestWithMockTimeSystem, TooManyRequestsWithDefaultSettings) { response->set_version_info("baz"); response->set_nonce("bar"); response->set_type_url("foo"); - grpc_mux_->onReceiveMessage(std::move(response)); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); } }; auto foo_sub = grpc_mux_->subscribe("foo", {"x"}, callbacks_); expectSendMessage("foo", {"x"}, ""); - grpc_mux_->start(); // Exhausts the limit. onReceiveMessage(99); @@ -461,13 +452,12 @@ TEST_F(GrpcMuxImplTestWithMockTimeSystem, TooManyRequestsWithEmptyRateLimitSetti response->set_version_info("baz"); response->set_nonce("bar"); response->set_type_url("foo"); - grpc_mux_->onReceiveMessage(std::move(response)); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); } }; auto foo_sub = grpc_mux_->subscribe("foo", {"x"}, callbacks_); expectSendMessage("foo", {"x"}, ""); - grpc_mux_->start(); // Validate that drain_request_timer is enabled when there are no tokens. EXPECT_CALL(*drain_request_timer, enableTimer(std::chrono::milliseconds(100))); @@ -515,13 +505,12 @@ TEST_F(GrpcMuxImplTest, TooManyRequestsWithCustomRateLimitSettings) { response->set_version_info("baz"); response->set_nonce("bar"); response->set_type_url("foo"); - grpc_mux_->onReceiveMessage(std::move(response)); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); } }; auto foo_sub = grpc_mux_->subscribe("foo", {"x"}, callbacks_); expectSendMessage("foo", {"x"}, ""); - grpc_mux_->start(); // Validate that rate limit is not enforced for 100 requests. onReceiveMessage(100); @@ -549,7 +538,6 @@ TEST_F(GrpcMuxImplTest, UnwatchedTypeAcceptsEmptyResources) { const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; - grpc_mux_->start(); { // subscribe and unsubscribe to simulate a cluster added and removed expectSendMessage(type_url, {"y"}, ""); @@ -565,7 +553,7 @@ TEST_F(GrpcMuxImplTest, UnwatchedTypeAcceptsEmptyResources) { response->set_type_url(type_url); // This contains zero resources. No discovery request should be sent. - grpc_mux_->onReceiveMessage(std::move(response)); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); // when we add the new subscription version should be 1 and nonce should be bar expectSendMessage(type_url, {"x"}, "1", "bar"); @@ -583,7 +571,6 @@ TEST_F(GrpcMuxImplTest, UnwatchedTypeRejectsResources) { const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; - grpc_mux_->start(); // subscribe and unsubscribe (by not keeping the return watch) so that the type is known to envoy expectSendMessage(type_url, {"y"}, ""); expectSendMessage(type_url, {}, ""); @@ -604,7 +591,7 @@ TEST_F(GrpcMuxImplTest, UnwatchedTypeRejectsResources) { // The message should be rejected. expectSendMessage(type_url, {}, "", "bar"); EXPECT_LOG_CONTAINS("warning", "Ignoring unwatched type URL " + type_url, - grpc_mux_->onReceiveMessage(std::move(response))); + grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response))); } TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyClusterName) { diff --git a/test/common/config/grpc_stream_test.cc b/test/common/config/grpc_stream_test.cc new file mode 100644 index 0000000000000..b4a97675fe916 --- /dev/null +++ b/test/common/config/grpc_stream_test.cc @@ -0,0 +1,133 @@ +#include "envoy/api/v2/discovery.pb.h" + +#include "common/config/grpc_stream.h" +#include "common/protobuf/protobuf.h" + +#include "test/mocks/config/mocks.h" +#include "test/mocks/event/mocks.h" +#include "test/mocks/grpc/mocks.h" +#include "test/mocks/upstream/mocks.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::NiceMock; +using testing::Return; + +namespace Envoy { +namespace Config { +namespace { + +class GrpcStreamTest : public testing::Test { +protected: + GrpcStreamTest() + : async_client_owner_(std::make_unique()), + async_client_(async_client_owner_.get()), + grpc_stream_(&callbacks_, std::move(async_client_owner_), + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.api.v2.EndpointDiscoveryService.StreamEndpoints"), + random_, dispatcher_, stats_, rate_limit_settings_) {} + + NiceMock dispatcher_; + Grpc::MockAsyncStream async_stream_; + Stats::IsolatedStoreImpl stats_; + NiceMock random_; + Envoy::Config::RateLimitSettings rate_limit_settings_; + NiceMock callbacks_; + std::unique_ptr async_client_owner_; + Grpc::MockAsyncClient* async_client_; + + GrpcStream grpc_stream_; +}; + +// Tests that establishNewStream() establishes it, a second call does nothing, and a third call +// after the stream was disconnected re-establishes it. +TEST_F(GrpcStreamTest, EstablishNewStream) { + EXPECT_FALSE(grpc_stream_.grpcStreamAvailable()); + // Successful establishment + { + EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); + EXPECT_CALL(callbacks_, onStreamEstablished()); + grpc_stream_.establishNewStream(); + EXPECT_TRUE(grpc_stream_.grpcStreamAvailable()); + } + // Idempotency: do nothing (other than logging a warning) if already connected + { + EXPECT_CALL(*async_client_, start(_, _)).Times(0); + EXPECT_CALL(callbacks_, onStreamEstablished()).Times(0); + grpc_stream_.establishNewStream(); + EXPECT_TRUE(grpc_stream_.grpcStreamAvailable()); + } + grpc_stream_.onRemoteClose(Grpc::Status::GrpcStatus::Ok, ""); + EXPECT_FALSE(grpc_stream_.grpcStreamAvailable()); + // Successful re-establishment + { + EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); + EXPECT_CALL(callbacks_, onStreamEstablished()); + grpc_stream_.establishNewStream(); + EXPECT_TRUE(grpc_stream_.grpcStreamAvailable()); + } +} + +// A failure in the underlying gRPC machinery should result in grpcStreamAvailable() false. Calling +// sendMessage would segfault. +TEST_F(GrpcStreamTest, FailToEstablishNewStream) { + EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(nullptr)); + EXPECT_CALL(callbacks_, onEstablishmentFailure()); + grpc_stream_.establishNewStream(); + EXPECT_FALSE(grpc_stream_.grpcStreamAvailable()); +} + +// Checks that sendMessage correctly passes a DiscoveryRequest down to the underlying gRPC +// machinery. +TEST_F(GrpcStreamTest, SendMessage) { + EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); + grpc_stream_.establishNewStream(); + envoy::api::v2::DiscoveryRequest request; + request.set_response_nonce("grpc_stream_test_noncense"); + EXPECT_CALL(async_stream_, sendMessage(ProtoEq(request), false)); + grpc_stream_.sendMessage(request); +} + +// Tests that, upon a call of the GrpcStream::onReceiveMessage() callback, which is called by the +// underlying gRPC machinery, the received proto will make it up to the GrpcStreamCallbacks that the +// GrpcStream was given. +TEST_F(GrpcStreamTest, ReceiveMessage) { + envoy::api::v2::DiscoveryResponse response_copy; + response_copy.set_type_url("faketypeURL"); + auto response = std::make_unique(response_copy); + envoy::api::v2::DiscoveryResponse received_message; + EXPECT_CALL(callbacks_, onDiscoveryResponse(_)) + .WillOnce([&received_message](std::unique_ptr&& message) { + received_message = *message; + }); + grpc_stream_.onReceiveMessage(std::move(response)); + EXPECT_TRUE(TestUtility::protoEqual(response_copy, received_message)); +} + +// If the value has only ever been 0, the stat should remain unused, including after an attempt to +// write a 0 to it. +TEST_F(GrpcStreamTest, QueueSizeStat) { + grpc_stream_.maybeUpdateQueueSizeStat(0); + EXPECT_FALSE(stats_.gauge("control_plane.pending_requests").used()); + grpc_stream_.maybeUpdateQueueSizeStat(123); + EXPECT_EQ(123, stats_.gauge("control_plane.pending_requests").value()); + grpc_stream_.maybeUpdateQueueSizeStat(0); + EXPECT_EQ(0, stats_.gauge("control_plane.pending_requests").value()); +} + +// Just to add coverage to the no-op implementations of these callbacks (without exposing us to +// crashes from a badly behaved peer like NOT_IMPLEMENTED_GCOVR_EXCL_LINE would). +TEST_F(GrpcStreamTest, HeaderTrailerJustForCodeCoverage) { + Http::HeaderMapPtr response_headers{new Http::TestHeaderMapImpl{}}; + grpc_stream_.onReceiveInitialMetadata(std::move(response_headers)); + Http::TestHeaderMapImpl request_headers; + grpc_stream_.onCreateInitialMetadata(request_headers); + Http::HeaderMapPtr trailers{new Http::TestHeaderMapImpl{}}; + grpc_stream_.onReceiveTrailingMetadata(std::move(trailers)); +} + +} // namespace +} // namespace Config +} // namespace Envoy diff --git a/test/common/config/grpc_subscription_impl_test.cc b/test/common/config/grpc_subscription_impl_test.cc index 1293e48b203f2..762bad93821d0 100644 --- a/test/common/config/grpc_subscription_impl_test.cc +++ b/test/common/config/grpc_subscription_impl_test.cc @@ -37,12 +37,11 @@ TEST_F(GrpcSubscriptionImplTest, StreamCreationFailure) { TEST_F(GrpcSubscriptionImplTest, RemoteStreamClose) { startSubscription({"cluster0", "cluster1"}); verifyStats(1, 0, 0, 0, 0); - Http::HeaderMapPtr trailers{new Http::TestHeaderMapImpl{}}; - subscription_->grpcMux().onReceiveTrailingMetadata(std::move(trailers)); EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); EXPECT_CALL(*timer_, enableTimer(_)); EXPECT_CALL(random_, random()); - subscription_->grpcMux().onRemoteClose(Grpc::Status::GrpcStatus::Canceled, ""); + subscription_->grpcMux()->grpcStreamForTest()->onRemoteClose(Grpc::Status::GrpcStatus::Canceled, + ""); verifyStats(2, 0, 0, 1, 0); verifyControlPlaneStats(0); diff --git a/test/common/config/grpc_subscription_test_harness.h b/test/common/config/grpc_subscription_test_harness.h index 11b5a44208ae1..1faa8645ab0a9 100644 --- a/test/common/config/grpc_subscription_test_harness.h +++ b/test/common/config/grpc_subscription_test_harness.h @@ -80,12 +80,6 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { last_cluster_names_ = cluster_names; expectSendMessage(last_cluster_names_, ""); subscription_->start(cluster_names, callbacks_); - // These are just there to add coverage to the null implementations of these - // callbacks. - Http::HeaderMapPtr response_headers{new Http::TestHeaderMapImpl{}}; - subscription_->grpcMux().onReceiveInitialMetadata(std::move(response_headers)); - Http::TestHeaderMapImpl request_headers; - subscription_->grpcMux().onCreateInitialMetadata(request_headers); } void deliverConfigUpdate(const std::vector& cluster_names, @@ -115,7 +109,7 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { expectSendMessage(last_cluster_names_, version_, Grpc::Status::GrpcStatus::Internal, "bad config"); } - subscription_->grpcMux().onReceiveMessage(std::move(response)); + subscription_->grpcMux()->grpcStreamForTest().onDiscoveryResponse(std::move(response)); Mock::VerifyAndClearExpectations(&async_stream_); } diff --git a/test/mocks/config/BUILD b/test/mocks/config/BUILD index 212d7c9718e55..4ffbef91dbbfb 100644 --- a/test/mocks/config/BUILD +++ b/test/mocks/config/BUILD @@ -15,7 +15,7 @@ envoy_cc_mock( deps = [ "//include/envoy/config:grpc_mux_interface", "//include/envoy/config:subscription_interface", - "//include/envoy/config:xds_context_interface", + "//include/envoy/config:xds_grpc_context_interface", "//source/common/config:resources_lib", "//source/common/protobuf:utility_lib", "@envoy_api//envoy/api/v2:cds_cc", diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index 0f51d6857a353..18177e04136c7 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -3,7 +3,7 @@ #include "envoy/api/v2/eds.pb.h" #include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" -#include "envoy/config/xds_context.h" +#include "envoy/config/xds_grpc_context.h" #include "common/config/resources.h" #include "common/protobuf/utility.h" From bc7e515a8d198b94c8a3153c824b491cdc1def16 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 3 May 2019 11:59:13 -0400 Subject: [PATCH 05/56] CDS integration test working again Signed-off-by: Fred Douglas --- include/envoy/config/xds_grpc_context.h | 9 +++---- .../common/config/delta_subscription_impl.h | 6 ++--- .../common/config/delta_subscription_state.h | 15 ++++++++--- source/common/config/grpc_delta_xds_context.h | 25 ++++++++++--------- source/common/config/grpc_mux_impl.cc | 4 +-- source/common/config/grpc_mux_impl.h | 10 +++----- .../config/grpc_mux_subscription_impl.h | 2 +- test/mocks/config/mocks.cc | 5 +--- test/mocks/config/mocks.h | 8 +++--- 9 files changed, 42 insertions(+), 42 deletions(-) diff --git a/include/envoy/config/xds_grpc_context.h b/include/envoy/config/xds_grpc_context.h index 8bfafc2a756f2..8fe03356715a2 100644 --- a/include/envoy/config/xds_grpc_context.h +++ b/include/envoy/config/xds_grpc_context.h @@ -28,14 +28,13 @@ class XdsGrpcContext { * @param init_fetch_timeout how long the first fetch has to complete before onConfigUpdateFailed * will be called. */ - virtual void addSubscription(const std::vector& resources, - const std::string& type_url, SubscriptionCallbacks& callbacks, - SubscriptionStats& stats, + virtual void addSubscription(const std::set& resources, const std::string& type_url, + SubscriptionCallbacks& callbacks, SubscriptionStats& stats, std::chrono::milliseconds init_fetch_timeout) PURE; // Enqueues and attempts to send a discovery request, (un)subscribing to resources missing from / // added to the passed 'resources' argument, relative to resource_versions_. - virtual void updateResources(const std::vector& resources, + virtual void updateResources(const std::set& resources, const std::string& type_url) PURE; virtual void removeSubscription(const std::string& type_url) PURE; @@ -45,7 +44,7 @@ class XdsGrpcContext { // TODO TODO remove virtual GrpcMuxWatchPtr subscribe(const std::string& type_url, - const std::vector& resources, + const std::set& resources, GrpcMuxCallbacks& callbacks) PURE; }; diff --git a/source/common/config/delta_subscription_impl.h b/source/common/config/delta_subscription_impl.h index 5028a5d9cf221..e8a164050b4a6 100644 --- a/source/common/config/delta_subscription_impl.h +++ b/source/common/config/delta_subscription_impl.h @@ -42,12 +42,12 @@ class DeltaSubscriptionImpl : public Subscription { void resume() { context_->resume(type_url_); } // Config::DeltaSubscription - void start(const std::vector& resources, SubscriptionCallbacks& callbacks) override { + void start(const std::set& resources, SubscriptionCallbacks& callbacks) override { context_->addSubscription(resources, type_url_, callbacks, stats_, init_fetch_timeout_); } - void updateResources(const std::vector& resources) override { - context_->updateResources(resources, type_url_); + virtual void updateResources(const std::set& update_to_these_names) override { + context_->updateResources(update_to_these_names, type_url_); } private: diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index 6fc716b93c08c..9603ef4f1b7b7 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -99,7 +99,14 @@ class DeltaSubscriptionState : public Logger::Loggable { return !names_added_.empty() || !names_removed_.empty(); } - void set_first_request_of_new_stream(bool val) { first_request_of_new_stream_ = val; } + void markStreamFresh() { any_request_sent_yet_in_current_stream_ = false; } + + // Not having sent any requests yet counts as an "update pending" since you're supposed to resend + // the entirety of your interest at the start of a stream, even if nothing has changed. + bool subscriptionUpdatePending() const { + return !names_added_.empty() || !names_removed_.empty() || + !any_request_sent_yet_in_current_stream_; + } UpdateAck handleResponse(envoy::api::v2::DeltaDiscoveryResponse* message) { // We *always* copy the response's nonce into the next request, even if we're going to make that @@ -157,7 +164,8 @@ class DeltaSubscriptionState : public Logger::Loggable { envoy::api::v2::DeltaDiscoveryRequest getNextRequestAckless() { envoy::api::v2::DeltaDiscoveryRequest request; - if (first_request_of_new_stream_) { + if (!any_request_sent_yet_in_current_stream_) { + any_request_sent_yet_in_current_stream_ = true; // initial_resource_versions "must be populated for first request in a stream". // Also, since this might be a new server, we must explicitly state *all* of our subscription // interest. @@ -174,7 +182,6 @@ class DeltaSubscriptionState : public Logger::Loggable { names_added_.insert(resource.first); } names_removed_.clear(); - first_request_of_new_stream_ = false; } std::copy(names_added_.begin(), names_added_.end(), Protobuf::RepeatedFieldBackInserter(request.mutable_resource_names_subscribe())); @@ -257,7 +264,7 @@ class DeltaSubscriptionState : public Logger::Loggable { Event::TimerPtr init_fetch_timeout_timer_; bool paused_{}; - bool first_request_of_new_stream_{true}; + bool any_request_sent_yet_in_current_stream_{}; // Tracking of the delta in our subscription interest since the previous DeltaDiscoveryRequest was // sent. Can't use unordered_set due to ordering issues in gTest expectation matching. Feel free diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h index 0bc47c8b043c0..fcdfadb94a2a8 100644 --- a/source/common/config/grpc_delta_xds_context.h +++ b/source/common/config/grpc_delta_xds_context.h @@ -62,7 +62,7 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, rate_limit_settings) {} - void addSubscription(const std::vector& resources, const std::string& type_url, + void addSubscription(const std::set& resources, const std::string& type_url, SubscriptionCallbacks& callbacks, SubscriptionStats& stats, std::chrono::milliseconds init_fetch_timeout) override { std::set HACK_resources_as_set(resources.begin(), resources.end()); @@ -74,7 +74,7 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, // Enqueues and attempts to send a discovery request, (un)subscribing to resources missing from / // added to the passed 'resources' argument, relative to resource_versions_. - void updateResources(const std::vector& resources, + void updateResources(const std::set& resources, const std::string& type_url) override { auto sub = subscriptions_.find(type_url); if (sub == subscriptions_.end()) { @@ -139,12 +139,12 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, void onWriteable() override { trySendDiscoveryRequests(); } void kickOffAck(UpdateAck ack) { - request_queue_.push(ack); + ack_queue_.push(ack); trySendDiscoveryRequests(); } // TODO TODO remove, GrpcMux impersonation - virtual GrpcMuxWatchPtr subscribe(const std::string&, const std::vector&, + virtual GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, GrpcMuxCallbacks&) override { return nullptr; } @@ -158,22 +158,23 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, void trySendDiscoveryRequests() { while (true) { // Do any of our subscriptions (by type_url) even want to send a request? - absl::optional next_request_type = wantToSendDiscoveryRequest(); - if (!next_request_type.has_value()) { - break; - } - // Try again later if paused/rate limited/stream down. - if (!canSendDiscoveryRequest(next_request_type, &sub->second)) { + absl::optional maybe_request_type = wantToSendDiscoveryRequest(); + if (!maybe_request_type.has_value()) { break; } + std::string next_request_type_url = maybe_request_type.value(); // If this request's type_url is one we don't have, drop it. - auto sub = subscriptions_.find(pending_request.type_url_); + auto sub = subscriptions_.find(next_request_type_url); if (sub == subscriptions_.end()) { ENVOY_LOG(warn, "Not sending queued ACK for non-existent subscription {}.", - pending_request.type_url_); + next_request_type_url); ack_queue_.pop(); continue; } + // Try again later if paused/rate limited/stream down. + if (!canSendDiscoveryRequest(next_request_type_url, &sub->second)) { + break; + } // Get our subscription state to generate the appropriate DeltaDiscoveryRequest, and send. if (!ack_queue_.empty()) { grpc_stream_.sendMessage(sub->second.getNextRequestWithAck(ack_queue_.front())); diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index 3e760a05c4697..45d1dc201fce2 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -28,8 +28,6 @@ GrpcMuxImpl::~GrpcMuxImpl() { } } -void GrpcMuxImpl::start() { grpc_stream_.establishStream(); } - void GrpcMuxImpl::sendDiscoveryRequest(const std::string& type_url) { if (!grpc_stream_.grpcStreamAvailable()) { ENVOY_LOG(debug, "No stream available to sendDiscoveryRequest for {}", type_url); @@ -67,7 +65,7 @@ void GrpcMuxImpl::sendDiscoveryRequest(const std::string& type_url) { } GrpcMuxWatchPtr GrpcMuxImpl::subscribe(const std::string& type_url, - const std::vector& resources, + const std::set& resources, GrpcMuxCallbacks& callbacks) { auto watch = std::unique_ptr(new GrpcMuxWatchImpl(resources, callbacks, type_url, *this)); diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index 0ae4f97f8dc1b..3852b6b2584e0 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -31,18 +31,17 @@ class GrpcMuxImpl : public GrpcMux, const RateLimitSettings& rate_limit_settings); ~GrpcMuxImpl(); - void start() override; GrpcMuxWatchPtr subscribe(const std::string& type_url, const std::set& resources, GrpcMuxCallbacks& callbacks) override; // XdsGrpcContext void pause(const std::string& type_url) override; void resume(const std::string& type_url) override; - void addSubscription(const std::vector&, const std::string&, SubscriptionCallbacks&, + void addSubscription(const std::set&, const std::string&, SubscriptionCallbacks&, SubscriptionStats&, std::chrono::milliseconds) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } - void updateResources(const std::vector&, const std::string&) override { + void updateResources(const std::set&, const std::string&) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } void removeSubscription(const std::string&) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } @@ -122,7 +121,6 @@ class GrpcMuxImpl : public GrpcMux, class NullGrpcMuxImpl : public XdsGrpcContext, GrpcStreamCallbacks { public: - void start() override {} GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, GrpcMuxCallbacks&) override { throw EnvoyException("ADS must be configured to support an ADS config source"); @@ -130,11 +128,11 @@ class NullGrpcMuxImpl : public XdsGrpcContext, void pause(const std::string&) override {} void resume(const std::string&) override {} - void addSubscription(const std::vector&, const std::string&, SubscriptionCallbacks&, + void addSubscription(const std::set&, const std::string&, SubscriptionCallbacks&, SubscriptionStats&, std::chrono::milliseconds) override { throw EnvoyException("ADS must be configured to support an ADS config source"); } - void updateResources(const std::vector&, const std::string&) override { + void updateResources(const std::set&, const std::string&) override { throw EnvoyException("ADS must be configured to support an ADS config source"); } diff --git a/source/common/config/grpc_mux_subscription_impl.h b/source/common/config/grpc_mux_subscription_impl.h index 1528f35a81aba..258019403a984 100644 --- a/source/common/config/grpc_mux_subscription_impl.h +++ b/source/common/config/grpc_mux_subscription_impl.h @@ -50,7 +50,7 @@ class GrpcMuxSubscriptionImpl : public Subscription, // previously watched resources and the new ones (we may have lost interest in some of the // previously watched ones). watch_.reset(); - watch_ = grpc_mux_.subscribe(type_url_, update_to_these_names, *this); + watch_ = grpc_mux_->subscribe(type_url_, update_to_these_names, *this); stats_.update_attempt_.inc(); } diff --git a/test/mocks/config/mocks.cc b/test/mocks/config/mocks.cc index b85cbfe13b623..3dd496a2f6853 100644 --- a/test/mocks/config/mocks.cc +++ b/test/mocks/config/mocks.cc @@ -13,18 +13,15 @@ MockGrpcMuxWatch::~MockGrpcMuxWatch() { cancel(); } MockGrpcMux::MockGrpcMux() {} MockGrpcMux::~MockGrpcMux() {} -<<<<<<< HEAD MockXdsGrpcContext::MockXdsGrpcContext() {} MockXdsGrpcContext::~MockXdsGrpcContext() {} GrpcMuxWatchPtr MockXdsGrpcContext::subscribe(const std::string& type_url, - const std::vector& resources, + const std::set& resources, GrpcMuxCallbacks& callbacks) { return GrpcMuxWatchPtr(subscribe_(type_url, resources, callbacks)); } -======= MockGrpcStreamCallbacks::MockGrpcStreamCallbacks() {} MockGrpcStreamCallbacks::~MockGrpcStreamCallbacks() {} ->>>>>>> upstream/master GrpcMuxWatchPtr MockGrpcMux::subscribe(const std::string& type_url, const std::set& resources, diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index ee9665c4c8558..f2f0db8b21584 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -74,11 +74,11 @@ class MockXdsGrpcContext : public XdsGrpcContext { virtual ~MockXdsGrpcContext(); MOCK_METHOD5(addSubscription, - void(const std::vector& resources, const std::string& type_url, + void(const std::set& resources, const std::string& type_url, SubscriptionCallbacks& callbacks, SubscriptionStats& stats, std::chrono::milliseconds init_fetch_timeout)); MOCK_METHOD2(updateResources, - void(const std::vector& resources, const std::string& type_url)); + void(const std::set& resources, const std::string& type_url)); MOCK_METHOD1(removeSubscription, void(const std::string& type_url)); MOCK_METHOD1(pause, void(const std::string& type_url)); @@ -91,9 +91,9 @@ class MockXdsGrpcContext : public XdsGrpcContext { // GrpcMux TODO TODO remove MOCK_METHOD0(start, void()); MOCK_METHOD3(subscribe_, - GrpcMuxWatch*(const std::string& type_url, const std::vector& resources, + GrpcMuxWatch*(const std::string& type_url, const std::set& resources, GrpcMuxCallbacks& callbacks)); - GrpcMuxWatchPtr subscribe(const std::string& type_url, const std::vector& resources, + GrpcMuxWatchPtr subscribe(const std::string& type_url, const std::set& resources, GrpcMuxCallbacks& callbacks); }; From 0f7bcdb5b480457ca0e3de29b2ea3c0cec75fec2 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 3 May 2019 12:13:10 -0400 Subject: [PATCH 06/56] CDS integration passes with 6729 merged Signed-off-by: Fred Douglas --- source/common/config/BUILD | 12 +- .../common/config/delta_subscription_state.cc | 209 +++++++++++++++++ .../common/config/delta_subscription_state.h | 221 ++---------------- source/common/config/grpc_delta_xds_context.h | 2 +- 4 files changed, 242 insertions(+), 202 deletions(-) create mode 100644 source/common/config/delta_subscription_state.cc diff --git a/source/common/config/BUILD b/source/common/config/BUILD index 43930f81a009a..cba18b9ab8c3b 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -113,14 +113,16 @@ envoy_cc_library( ) envoy_cc_library( - name = "delta_subscription_state", + name = "delta_subscription_state_lib", + srcs = ["delta_subscription_state.cc"], hdrs = ["delta_subscription_state.h"], deps = [ - ":utility_lib", "//include/envoy/config:subscription_interface", "//include/envoy/event:dispatcher_interface", - "//include/envoy/local_info:local_info_interface", - "//source/common/common:hash_lib", + "//source/common/common:backoff_lib", + "//source/common/common:minimal_logger_lib", + "//source/common/common:token_bucket_impl_lib", + "//source/common/grpc:common_lib", "//source/common/protobuf", "@envoy_api//envoy/api/v2:discovery_cc", ], @@ -227,7 +229,7 @@ envoy_cc_library( name = "grpc_delta_xds_context_lib", hdrs = ["grpc_delta_xds_context.h"], deps = [ - ":delta_subscription_state", + ":delta_subscription_state_lib", "//include/envoy/event:dispatcher_interface", "//include/envoy/grpc:async_client_interface", ], diff --git a/source/common/config/delta_subscription_state.cc b/source/common/config/delta_subscription_state.cc new file mode 100644 index 0000000000000..8c43409d72872 --- /dev/null +++ b/source/common/config/delta_subscription_state.cc @@ -0,0 +1,209 @@ +#include "common/config/delta_subscription_state.h" + +namespace Envoy { +namespace Config { + +DeltaSubscriptionState::DeltaSubscriptionState(const std::string& type_url, + const std::set& resource_names, + SubscriptionCallbacks& callbacks, + const LocalInfo::LocalInfo& local_info, + std::chrono::milliseconds init_fetch_timeout, + Event::Dispatcher& dispatcher, + SubscriptionStats& stats) + : type_url_(type_url), callbacks_(callbacks), local_info_(local_info), + init_fetch_timeout_(init_fetch_timeout), stats_(stats) { + // In normal usage of updateResourceInterest(), the caller is supposed to cause a discovery + // request to be queued if it returns true. We don't need to do that because we know that the + // subscription gRPC stream is not yet established, and establishment causes a request. + updateResourceInterest(resource_names); + setInitFetchTimeout(dispatcher); +} + +void DeltaSubscriptionState::setInitFetchTimeout(Event::Dispatcher& dispatcher) { + if (init_fetch_timeout_.count() > 0 && !init_fetch_timeout_timer_) { + init_fetch_timeout_timer_ = dispatcher.createTimer([this]() -> void { + ENVOY_LOG(warn, "delta config: initial fetch timed out for {}", type_url_); + callbacks_.onConfigUpdateFailed(nullptr); + }); + init_fetch_timeout_timer_->enableTimer(init_fetch_timeout_); + } +} + +void DeltaSubscriptionState::pause() { + ENVOY_LOG(debug, "Pausing discovery requests for {}", type_url_); + ASSERT(!paused_); + paused_ = true; +} + +void DeltaSubscriptionState::resume() { + ENVOY_LOG(debug, "Resuming discovery requests for {}", type_url_); + ASSERT(paused_); + paused_ = false; +} + +// Returns true if there is any meaningful change in our subscription interest, worth reporting to +// the server. +void DeltaSubscriptionState::updateResourceInterest( + const std::set& update_to_these_names) { + std::vector cur_added; + std::vector cur_removed; + + std::set_difference(update_to_these_names.begin(), update_to_these_names.end(), + resource_names_.begin(), resource_names_.end(), + std::inserter(cur_added, cur_added.begin())); + std::set_difference(resource_names_.begin(), resource_names_.end(), update_to_these_names.begin(), + update_to_these_names.end(), std::inserter(cur_removed, cur_removed.begin())); + + for (const auto& a : cur_added) { + setResourceWaitingForServer(a); + // Removed->added requires us to keep track of it as a "new" addition, since our user may have + // forgotten its copy of the resource after instructing us to remove it, and so needs to be + // reminded of it. + names_removed_.erase(a); + names_added_.insert(a); + } + for (const auto& r : cur_removed) { + setLostInterestInResource(r); + // Ideally, when a resource is added-then-removed in between requests, we would avoid putting + // a superfluous "unsubscribe [resource that was never subscribed]" in the request. However, + // the removed-then-added case *does* need to go in the request, and due to how we accomplish + // that, it's difficult to distinguish remove-add-remove from add-remove (because "remove-add" + // has to be treated as equivalent to just "add"). + names_added_.erase(r); + names_removed_.insert(r); + } +} + +// Not having sent any requests yet counts as an "update pending" since you're supposed to resend +// the entirety of your interest at the start of a stream, even if nothing has changed. +bool DeltaSubscriptionState::subscriptionUpdatePending() const { + return !names_added_.empty() || !names_removed_.empty() || + !any_request_sent_yet_in_current_stream_; +} + +UpdateAck +DeltaSubscriptionState::handleResponse(const envoy::api::v2::DeltaDiscoveryResponse& message) { + // We *always* copy the response's nonce into the next request, even if we're going to make that + // request a NACK by setting error_detail. + UpdateAck ack(message.nonce(), type_url_); + stats_.update_attempt_.inc(); + try { + handleGoodResponse(message); + } catch (const EnvoyException& e) { + handleBadResponse(e, ack); + } + return ack; +} + +void DeltaSubscriptionState::handleGoodResponse( + const envoy::api::v2::DeltaDiscoveryResponse& message) { + disableInitFetchTimeoutTimer(); + callbacks_.onConfigUpdate(message.resources(), message.removed_resources(), + message.system_version_info()); + for (const auto& resource : message.resources()) { + setResourceVersion(resource.name(), resource.version()); + } + // If a resource is gone, there is no longer a meaningful version for it that makes sense to + // provide to the server upon stream reconnect: either it will continue to not exist, in which + // case saying nothing is fine, or the server will bring back something new, which we should + // receive regardless (which is the logic that not specifying a version will get you). + // + // So, leave the version map entry present but blank. It will be left out of + // initial_resource_versions messages, but will remind us to explicitly tell the server "I'm + // cancelling my subscription" when we lose interest. + for (const auto& resource_name : message.removed_resources()) { + if (resource_names_.find(resource_name) != resource_names_.end()) { + setResourceWaitingForServer(resource_name); + } + } + stats_.update_success_.inc(); + stats_.version_.set(HashUtil::xxHash64(message.system_version_info())); + ENVOY_LOG(debug, "Delta config for {} accepted with {} resources added, {} removed", type_url_, + message.resources().size(), message.removed_resources().size()); +} + +void DeltaSubscriptionState::handleBadResponse(const EnvoyException& e, UpdateAck& ack) { + // Note that error_detail being set is what indicates that a DeltaDiscoveryRequest is a NACK. + ack.error_detail_.set_code(Grpc::Status::GrpcStatus::Internal); + ack.error_detail_.set_message(e.what()); + disableInitFetchTimeoutTimer(); + stats_.update_rejected_.inc(); + ENVOY_LOG(warn, "delta config for {} rejected: {}", type_url_, e.what()); + callbacks_.onConfigUpdateFailed(&e); +} + +void DeltaSubscriptionState::handleEstablishmentFailure() { + disableInitFetchTimeoutTimer(); + stats_.update_failure_.inc(); + stats_.update_attempt_.inc(); + callbacks_.onConfigUpdateFailed(nullptr); +} + +envoy::api::v2::DeltaDiscoveryRequest DeltaSubscriptionState::getNextRequestAckless() { + envoy::api::v2::DeltaDiscoveryRequest request; + if (!any_request_sent_yet_in_current_stream_) { + any_request_sent_yet_in_current_stream_ = true; + // initial_resource_versions "must be populated for first request in a stream". + // Also, since this might be a new server, we must explicitly state *all* of our subscription + // interest. + for (auto const& resource : resource_versions_) { + // Populate initial_resource_versions with the resource versions we currently have. + // Resources we are interested in, but are still waiting to get any version of from the + // server, do not belong in initial_resource_versions. (But do belong in new subscriptions!) + if (!resource.second.waitingForServer()) { + (*request.mutable_initial_resource_versions())[resource.first] = resource.second.version(); + } + // As mentioned above, fill resource_names_subscribe with everything, including names we + // have yet to receive any resource for. + names_added_.insert(resource.first); + } + names_removed_.clear(); + } + std::copy(names_added_.begin(), names_added_.end(), + Protobuf::RepeatedFieldBackInserter(request.mutable_resource_names_subscribe())); + std::copy(names_removed_.begin(), names_removed_.end(), + Protobuf::RepeatedFieldBackInserter(request.mutable_resource_names_unsubscribe())); + names_added_.clear(); + names_removed_.clear(); + + request.set_type_url(type_url_); + request.mutable_node()->MergeFrom(local_info_.node()); + return request; +} + +envoy::api::v2::DeltaDiscoveryRequest +DeltaSubscriptionState::getNextRequestWithAck(const UpdateAck& ack) { + envoy::api::v2::DeltaDiscoveryRequest request = getNextRequestAckless(); + request.set_response_nonce(ack.nonce_); + if (ack.error_detail_.code() != Grpc::Status::GrpcStatus::Ok) { + // Don't needlessly make the field present-but-empty if status is ok. + request.mutable_error_detail()->CopyFrom(ack.error_detail_); + } + return request; +} + +void DeltaSubscriptionState::disableInitFetchTimeoutTimer() { + if (init_fetch_timeout_timer_) { + init_fetch_timeout_timer_->disableTimer(); + init_fetch_timeout_timer_.reset(); + } +} + +void DeltaSubscriptionState::setResourceVersion(const std::string& resource_name, + const std::string& resource_version) { + resource_versions_[resource_name] = ResourceVersion(resource_version); + resource_names_.insert(resource_name); +} + +void DeltaSubscriptionState::setResourceWaitingForServer(const std::string& resource_name) { + resource_versions_[resource_name] = ResourceVersion(); + resource_names_.insert(resource_name); +} + +void DeltaSubscriptionState::setLostInterestInResource(const std::string& resource_name) { + resource_versions_.erase(resource_name); + resource_names_.erase(resource_name); +} + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index 9603ef4f1b7b7..f430f8a7877af 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -19,199 +19,39 @@ struct UpdateAck { ::google::rpc::Status error_detail_; }; +// Tracks the xDS protocol state of an individual ongoing delta xDS session. class DeltaSubscriptionState : public Logger::Loggable { public: DeltaSubscriptionState(const std::string& type_url, const std::set& resource_names, SubscriptionCallbacks& callbacks, const LocalInfo::LocalInfo& local_info, std::chrono::milliseconds init_fetch_timeout, - Event::Dispatcher& dispatcher, SubscriptionStats& stats) - : type_url_(type_url), callbacks_(callbacks), local_info_(local_info), - init_fetch_timeout_(init_fetch_timeout), stats_(stats) { - // In normal usage of updateResourceInterest(), the caller is supposed to cause a discovery - // request to be queued if it returns true. We don't need to do that because we know that the - // subscription gRPC stream is not yet established, and establishment causes a request. - updateResourceInterest(resource_names); - setInitFetchTimeout(dispatcher); + Event::Dispatcher& dispatcher, SubscriptionStats& stats); - // The attempt stat here is maintained for the purposes of having consistency between ADS and - // individual DeltaSubscriptions. Since ADS is push based and muxed, the notion of an - // "attempt" for a given xDS API combined by ADS is not really that meaningful. - stats_.update_attempt_.inc(); - } + void setInitFetchTimeout(Event::Dispatcher& dispatcher); - void setInitFetchTimeout(Event::Dispatcher& dispatcher) { - if (init_fetch_timeout_.count() > 0 && !init_fetch_timeout_timer_) { - init_fetch_timeout_timer_ = dispatcher.createTimer([this]() -> void { - ENVOY_LOG(warn, "delta config: initial fetch timed out for {}", type_url_); - callbacks_.onConfigUpdateFailed(nullptr); - }); - init_fetch_timeout_timer_->enableTimer(init_fetch_timeout_); - } - } - - void pause() { - ENVOY_LOG(debug, "Pausing discovery requests for {}", type_url_); - ASSERT(!paused_); - paused_ = true; - } - - void resume() { - ENVOY_LOG(debug, "Resuming discovery requests for {}", type_url_); - ASSERT(paused_); - paused_ = false; - } - - bool paused() { return paused_; } + void pause(); + void resume(); + bool paused() const { return paused_; } - // Returns true if there is any meaningful change in our subscription interest, worth reporting to - // the server. - bool updateResourceInterest(const std::set& update_to_these_names) { - std::vector cur_added; - std::vector cur_removed; + // Update which resources we're interested in subscribing to. + void updateResourceInterest(const std::set& update_to_these_names); - std::set_difference(update_to_these_names.begin(), update_to_these_names.end(), - resource_names_.begin(), resource_names_.end(), - std::inserter(cur_added, cur_added.begin())); - std::set_difference(resource_names_.begin(), resource_names_.end(), - update_to_these_names.begin(), update_to_these_names.end(), - std::inserter(cur_removed, cur_removed.begin())); - - for (const auto& a : cur_added) { - setResourceWaitingForServer(a); - // Removed->added requires us to keep track of it as a "new" addition, since our user may have - // forgotten its copy of the resource after instructing us to remove it, and so needs to be - // reminded of it. - names_removed_.erase(a); - names_added_.insert(a); - } - for (const auto& r : cur_removed) { - lostInterestInResource(r); - // Ideally, when a resource is added-then-removed in between requests, we would avoid putting - // a superfluous "unsubscribe [resource that was never subscribed]" in the request. However, - // the removed-then-added case *does* need to go in the request, and due to how we accomplish - // that, it's difficult to distinguish remove-add-remove from add-remove (because "remove-add" - // has to be treated as equivalent to just "add"). - names_added_.erase(r); - names_removed_.insert(r); - } - stats_.update_attempt_.inc(); - // Tell the server about our new interests only if there are any. - return !names_added_.empty() || !names_removed_.empty(); - } + // Whether there was a change in our subscription interest we have yet to inform the server of. + bool subscriptionUpdatePending() const; void markStreamFresh() { any_request_sent_yet_in_current_stream_ = false; } - // Not having sent any requests yet counts as an "update pending" since you're supposed to resend - // the entirety of your interest at the start of a stream, even if nothing has changed. - bool subscriptionUpdatePending() const { - return !names_added_.empty() || !names_removed_.empty() || - !any_request_sent_yet_in_current_stream_; - } - - UpdateAck handleResponse(envoy::api::v2::DeltaDiscoveryResponse* message) { - // We *always* copy the response's nonce into the next request, even if we're going to make that - // request a NACK by setting error_detail. - UpdateAck ack(message->nonce(), type_url_); - std::cerr << "handleResponse" << std::endl; - try { - disableInitFetchTimeoutTimer(); - callbacks_.onConfigUpdate(message->resources(), message->removed_resources(), - message->system_version_info()); - for (const auto& resource : message->resources()) { - std::cerr << "setResourceVersion " << resource.name() << " " << resource.version() - << std::endl; - setResourceVersion(resource.name(), resource.version()); - } - std::cerr << "done setting versions" << std::endl; - // If a resource is gone, there is no longer a meaningful version for it that makes sense to - // provide to the server upon stream reconnect: either it will continue to not exist, in which - // case saying nothing is fine, or the server will bring back something new, which we should - // receive regardless (which is the logic that not specifying a version will get you). - // - // So, leave the version map entry present but blank. It will be left out of - // initial_resource_versions messages, but will remind us to explicitly tell the server "I'm - // cancelling my subscription" when we lose interest. - for (const auto& resource_name : message->removed_resources()) { - if (resource_names_.find(resource_name) != resource_names_.end()) { - std::cerr << resource_name << " waiting for server" << std::endl; - setResourceWaitingForServer(resource_name); - } - } - stats_.update_success_.inc(); - stats_.update_attempt_.inc(); - stats_.version_.set(HashUtil::xxHash64(message->system_version_info())); - ENVOY_LOG(debug, "Delta config for {} accepted with {} resources added, {} removed", - type_url_, message->resources().size(), message->removed_resources().size()); - } catch (const EnvoyException& e) { - std::cerr << "UH OH EXCEPTION" << std::endl; - // Note that error_detail being set is what indicates that a DeltaDiscoveryRequest is a NACK. - ack.error_detail_.set_code(Grpc::Status::GrpcStatus::Internal); - ack.error_detail_.set_message(e.what()); - disableInitFetchTimeoutTimer(); - stats_.update_rejected_.inc(); - ENVOY_LOG(warn, "delta config for {} rejected: {}", type_url_, e.what()); - stats_.update_attempt_.inc(); - callbacks_.onConfigUpdateFailed(&e); - } - return ack; - } - - void handleEstablishmentFailure() { - stats_.update_failure_.inc(); - stats_.update_attempt_.inc(); - callbacks_.onConfigUpdateFailed(nullptr); - } - - envoy::api::v2::DeltaDiscoveryRequest getNextRequestAckless() { - envoy::api::v2::DeltaDiscoveryRequest request; - if (!any_request_sent_yet_in_current_stream_) { - any_request_sent_yet_in_current_stream_ = true; - // initial_resource_versions "must be populated for first request in a stream". - // Also, since this might be a new server, we must explicitly state *all* of our subscription - // interest. - for (auto const& resource : resource_versions_) { - // Populate initial_resource_versions with the resource versions we currently have. - // Resources we are interested in, but are still waiting to get any version of from the - // server, do not belong in initial_resource_versions. (But do belong in new subscriptions!) - if (!resource.second.waitingForServer()) { - (*request.mutable_initial_resource_versions())[resource.first] = - resource.second.version(); - } - // As mentioned above, fill resource_names_subscribe with everything, including names we - // have yet to receive any resource for. - names_added_.insert(resource.first); - } - names_removed_.clear(); - } - std::copy(names_added_.begin(), names_added_.end(), - Protobuf::RepeatedFieldBackInserter(request.mutable_resource_names_subscribe())); - std::copy(names_removed_.begin(), names_removed_.end(), - Protobuf::RepeatedFieldBackInserter(request.mutable_resource_names_unsubscribe())); - names_added_.clear(); - names_removed_.clear(); + UpdateAck handleResponse(const envoy::api::v2::DeltaDiscoveryResponse& message); - request.set_type_url(type_url_); - request.mutable_node()->MergeFrom(local_info_.node()); - return request; - } + void handleEstablishmentFailure(); - envoy::api::v2::DeltaDiscoveryRequest getNextRequestWithAck(const UpdateAck& ack) { - envoy::api::v2::DeltaDiscoveryRequest request = getNextRequestAckless(); - request.set_response_nonce(ack.nonce_); - if (ack.error_detail_.code() != Grpc::Status::GrpcStatus::Ok) { - // Don't needlessly make the field present-but-empty if status is ok. - request.mutable_error_detail()->CopyFrom(ack.error_detail_); - } - return request; - } + envoy::api::v2::DeltaDiscoveryRequest getNextRequestAckless(); + envoy::api::v2::DeltaDiscoveryRequest getNextRequestWithAck(const UpdateAck& ack); private: - void disableInitFetchTimeoutTimer() { - if (init_fetch_timeout_timer_) { - init_fetch_timeout_timer_->disableTimer(); - init_fetch_timeout_timer_.reset(); - } - } + void handleGoodResponse(const envoy::api::v2::DeltaDiscoveryResponse& message); + void handleBadResponse(const EnvoyException& e, UpdateAck& ack); + void disableInitFetchTimeoutTimer(); class ResourceVersion { public: @@ -232,21 +72,10 @@ class DeltaSubscriptionState : public Logger::Loggable { absl::optional version_; }; - // Use these helpers to avoid forgetting to update both at once. - void setResourceVersion(const std::string& resource_name, const std::string& resource_version) { - resource_versions_[resource_name] = ResourceVersion(resource_version); - resource_names_.insert(resource_name); - } - - void setResourceWaitingForServer(const std::string& resource_name) { - resource_versions_[resource_name] = ResourceVersion(); - resource_names_.insert(resource_name); - } - - void lostInterestInResource(const std::string& resource_name) { - resource_versions_.erase(resource_name); - resource_names_.erase(resource_name); - } + // Use these helpers to ensure resource_versions_ and resource_names_ get updated together. + void setResourceVersion(const std::string& resource_name, const std::string& resource_version); + void setResourceWaitingForServer(const std::string& resource_name); + void setLostInterestInResource(const std::string& resource_name); // A map from resource name to per-resource version. The keys of this map are exactly the resource // names we are currently interested in. Those in the waitingForServer state currently don't have @@ -255,7 +84,7 @@ class DeltaSubscriptionState : public Logger::Loggable { std::unordered_map resource_versions_; // The keys of resource_versions_. Only tracked separately because std::map does not provide an // iterator into just its keys, e.g. for use in std::set_difference. - std::unordered_set resource_names_; + std::set resource_names_; const std::string type_url_; SubscriptionCallbacks& callbacks_; @@ -266,9 +95,9 @@ class DeltaSubscriptionState : public Logger::Loggable { bool paused_{}; bool any_request_sent_yet_in_current_stream_{}; - // Tracking of the delta in our subscription interest since the previous DeltaDiscoveryRequest was - // sent. Can't use unordered_set due to ordering issues in gTest expectation matching. Feel free - // to change if you can figure out how to make it work. + // Tracks changes in our subscription interest since the previous DeltaDiscoveryRequest we sent. + // Can't use unordered_set due to ordering issues in gTest expectation matching. + // Feel free to change to unordered if you can figure out how to make it work. std::set names_added_; std::set names_removed_; diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h index fcdfadb94a2a8..c636f18cfaf8f 100644 --- a/source/common/config/grpc_delta_xds_context.h +++ b/source/common/config/grpc_delta_xds_context.h @@ -120,7 +120,7 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, message->system_version_info(), message->type_url()); return; } - kickOffAck(sub->second.handleResponse(message.get())); + kickOffAck(sub->second.handleResponse(*message)); } void onStreamEstablished() override { From 62f42dcb669e3ed8a81753324a93b62d9839bf76 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 3 May 2019 12:29:52 -0400 Subject: [PATCH 07/56] lol polymorphism Signed-off-by: Fred Douglas --- source/common/config/grpc_delta_xds_context.h | 126 ++++++++++-------- 1 file changed, 67 insertions(+), 59 deletions(-) diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h index c636f18cfaf8f..a37d831187246 100644 --- a/source/common/config/grpc_delta_xds_context.h +++ b/source/common/config/grpc_delta_xds_context.h @@ -59,8 +59,11 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, const RateLimitSettings& rate_limit_settings, const LocalInfo::LocalInfo& local_info) : local_info_(local_info), dispatcher_(dispatcher), - grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, - rate_limit_settings) {} + grpc_stream_(TypedGrpcStreamUnion::makeDelta( + std::make_unique>( + this, std::move(async_client), service_method, random, dispatcher, scope, + rate_limit_settings))) {} void addSubscription(const std::set& resources, const std::string& type_url, SubscriptionCallbacks& callbacks, SubscriptionStats& stats, @@ -149,10 +152,10 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, return nullptr; } - GrpcStream& - grpcStreamForTest() { - return grpc_stream_; - } + // GrpcStream& + // grpcStreamForTest() { + // return grpc_stream_; + // } private: void trySendDiscoveryRequests() { @@ -223,79 +226,84 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, Event::Dispatcher& dispatcher_; /* NOTE NOTE TODO TODO: all of this is just to support the eventual sotw+delta merger. - * for current purposes, just sticking with this grpc_stream_ is fine:*/ + * for current purposes, just sticking with this grpc_stream_ is fine: GrpcStream - grpc_stream_; - - /* + grpc_stream_;*/ // A horrid little class that provides polymorphism for these two protobuf types. class TypedGrpcStreamUnion { + public: static TypedGrpcStreamUnion - makeDelta(std::unique_ptr>&& delta) { return TypedGrpcStreamUnion(std::move(delta)); - } + makeDelta(std::unique_ptr>&& delta) { + return TypedGrpcStreamUnion(std::move(delta)); + } static TypedGrpcStreamUnion - makeSotw(std::unique_ptr>&& sotw) { return TypedGrpcStreamUnion(std::move(sotw)); } + makeSotw(std::unique_ptr>&& sotw) { + return TypedGrpcStreamUnion(std::move(sotw)); + } - void establishStream() { - delta_stream_ ? delta_stream_->establishStream(); - : sotw_stream_->establishStream(); + void establishStream() { + delta_stream_ ? delta_stream_->establishStream() : sotw_stream_->establishStream(); } - bool grpcStreamAvailable() const { - return delta_stream_ ? delta_stream_->grpcStreamAvailable(); - : sotw_stream_->grpcStreamAvailable(); + bool grpcStreamAvailable() const { + return delta_stream_ ? delta_stream_->grpcStreamAvailable() + : sotw_stream_->grpcStreamAvailable(); } - void sendMessage(const plain_ol_untyped_Message& request) { - delta_stream_ - ? delta_stream_->sendMessage( - static_cast(request); - : sotw_stream_->sendMessage( - static_cast(request); - } + void sendMessage(const Protobuf::Message& request) { + delta_stream_ ? delta_stream_->sendMessage( + static_cast(request)) + : sotw_stream_->sendMessage( + static_cast(request)); + } - void onReceiveMessage(std::unique_ptr&& message) { - delta_stream_ - ? delta_stream_->onReceiveMessage( - static_cast&&>(message); - : sotw_stream_->onReceiveMessage( - static_cast&&>(message); - } + /* void onReceiveMessage(std::unique_ptr&& message) { + void* ew = reinterpret_cast(message.release()); + delta_stream_ + ? delta_stream_->onReceiveMessage( + std::unique_ptr( + reinterpret_cast(ew))) + : sotw_stream_->onReceiveMessage( + std::unique_ptr( + reinterpret_cast(ew))); + } - void onRemoteClose(Grpc::Status::GrpcStatus status, const std::string& message) { - delta_stream_ ? delta_stream_->onRemoteClose(status, message); - : sotw_stream_->onRemoteClose(status, message); - } + void onRemoteClose(Grpc::Status::GrpcStatus status, const std::string& message) { + delta_stream_ ? delta_stream_->onRemoteClose(status, message) + : sotw_stream_->onRemoteClose(status, message); + }*/ - void maybeUpdateQueueSizeStat(uint64_t size) { - delta_stream_ ? delta_stream_->maybeUpdateQueueSizeStat(size); - : sotw_stream_->maybeUpdateQueueSizeStat(size); + void maybeUpdateQueueSizeStat(uint64_t size) { + delta_stream_ ? delta_stream_->maybeUpdateQueueSizeStat(size) + : sotw_stream_->maybeUpdateQueueSizeStat(size); } - bool checkRateLimitAllowsDrain() { - return delta_stream_ ? delta_stream_->checkRateLimitAllowsDrain(); - : sotw_stream_->checkRateLimitAllowsDrain(); + bool checkRateLimitAllowsDrain() { + return delta_stream_ ? delta_stream_->checkRateLimitAllowsDrain() + : sotw_stream_->checkRateLimitAllowsDrain(); } private: - explicit TypedGrpcStreamUnion(std::unique_ptr>&& sotw) : delta_stream_(nullptr), - sotw_stream_(std::move(sotw)) {} - - explicit TypedGrpcStreamUnion(std::unique_ptr>&& delta) : delta_stream_(std::move(delta)), - sotw_stream_(nullptr) {} - - std::unique_ptr> delta_stream_; - std::unique_ptr> sotw_stream_; + explicit TypedGrpcStreamUnion( + std::unique_ptr< + GrpcStream>&& sotw) + : delta_stream_(nullptr), sotw_stream_(std::move(sotw)) {} + + explicit TypedGrpcStreamUnion( + std::unique_ptr>&& delta) + : delta_stream_(std::move(delta)), sotw_stream_(nullptr) {} + + std::unique_ptr< + GrpcStream> + delta_stream_; + std::unique_ptr> + sotw_stream_; }; - - */ + TypedGrpcStreamUnion grpc_stream_; // Resource (N)ACKs we're waiting to send, stored in the order that they should be sent in. All // of our different resource types' ACKs are mixed together in this queue. From 08068427f6906f26830aabedb99de90880084293 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 3 May 2019 16:10:00 -0400 Subject: [PATCH 08/56] fix unit tests Signed-off-by: Fred Douglas --- source/common/config/grpc_delta_xds_context.h | 14 +++++++----- .../config/delta_subscription_state_test.cc | 22 +++++++++---------- test/common/config/grpc_stream_test.cc | 16 +++++++------- .../config/grpc_subscription_impl_test.cc | 5 ----- .../config/grpc_subscription_test_harness.h | 6 +---- 5 files changed, 28 insertions(+), 35 deletions(-) diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h index a37d831187246..efb4dba2271d3 100644 --- a/source/common/config/grpc_delta_xds_context.h +++ b/source/common/config/grpc_delta_xds_context.h @@ -160,16 +160,17 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, private: void trySendDiscoveryRequests() { while (true) { - // Do any of our subscriptions (by type_url) even want to send a request? + // Do any of our subscriptions even want to send a request? absl::optional maybe_request_type = wantToSendDiscoveryRequest(); if (!maybe_request_type.has_value()) { break; } + // If so, which one (by type_url)? std::string next_request_type_url = maybe_request_type.value(); - // If this request's type_url is one we don't have, drop it. + // If we don't have a subscription object for this request's type_url, drop the request. auto sub = subscriptions_.find(next_request_type_url); if (sub == subscriptions_.end()) { - ENVOY_LOG(warn, "Not sending queued ACK for non-existent subscription {}.", + ENVOY_LOG(warn, "Not sending discovery request for non-existent subscription {}.", next_request_type_url); ack_queue_.pop(); continue; @@ -196,10 +197,10 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, ENVOY_LOG(trace, "API {} paused; discovery request on hold for now.", type_url); return false; } else if (!grpc_stream_.grpcStreamAvailable()) { - ENVOY_LOG(trace, "No stream available to send a DiscoveryRequest for {}.", type_url); + ENVOY_LOG(trace, "No stream available to send a discovery request for {}.", type_url); return false; } else if (!grpc_stream_.checkRateLimitAllowsDrain()) { - ENVOY_LOG(trace, "{} DiscoveryRequest hit rate limit; will try later.", type_url); + ENVOY_LOG(trace, "{} discovery request hit rate limit; will try later.", type_url); return false; } return true; @@ -207,7 +208,8 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, // Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or // a subscription update. (Does not check whether we *can* send that DeltaDiscoveryRequest). - // Returns the type_url of the resource type we should send for (if any). + // Returns the type_url of the resource type we should send the DeltaDiscoveryRequest for (if + // any). absl::optional wantToSendDiscoveryRequest() { // All ACKs are sent before plain updates. trySendDiscoveryRequests() relies on this. So, choose // type_url from ack_queue_ if possible, before looking at pending updates. diff --git a/test/common/config/delta_subscription_state_test.cc b/test/common/config/delta_subscription_state_test.cc index 31dcdea72e080..4076666675e08 100644 --- a/test/common/config/delta_subscription_state_test.cc +++ b/test/common/config/delta_subscription_state_test.cc @@ -27,7 +27,7 @@ void deliverDiscoveryResponse( *message.mutable_resources() = added_resources; *message.mutable_removed_resources() = removed_resources; message.set_system_version_info(version_info); - state.handleResponse(&message); + state.handleResponse(message); } TEST(DeltaSubscriptionImplTest, ResourceGoneLeadsToBlankInitialVersion) { @@ -40,7 +40,7 @@ TEST(DeltaSubscriptionImplTest, ResourceGoneLeadsToBlankInitialVersion) { DeltaSubscriptionState state(TypeUrl, {"name1", "name2", "name3"}, callbacks, local_info, std::chrono::milliseconds(0U), dispatcher, stats); { - envoy::api::v2::DeltaDiscoveryRequest cur_request = state.getNextRequest(); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state.getNextRequestAckless(); EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name1", "name2", "name3")); } @@ -55,8 +55,8 @@ TEST(DeltaSubscriptionImplTest, ResourceGoneLeadsToBlankInitialVersion) { resource->set_name("name2"); resource->set_version("version2A"); deliverDiscoveryResponse(state, add1_2, {}, "debugversion1"); - state.set_first_request_of_new_stream(true); // simulate a stream reconnection - envoy::api::v2::DeltaDiscoveryRequest cur_request = state.getNextRequest(); + state.markStreamFresh(); // simulate a stream reconnection + envoy::api::v2::DeltaDiscoveryRequest cur_request = state.getNextRequestAckless(); EXPECT_EQ("version1A", cur_request.initial_resource_versions().at("name1")); EXPECT_EQ("version2A", cur_request.initial_resource_versions().at("name2")); EXPECT_EQ(cur_request.initial_resource_versions().end(), @@ -75,8 +75,8 @@ TEST(DeltaSubscriptionImplTest, ResourceGoneLeadsToBlankInitialVersion) { Protobuf::RepeatedPtrField remove2; *remove2.Add() = "name2"; deliverDiscoveryResponse(state, add1_3, remove2, "debugversion2"); - state.set_first_request_of_new_stream(true); // simulate a stream reconnection - envoy::api::v2::DeltaDiscoveryRequest cur_request = state.getNextRequest(); + state.markStreamFresh(); // simulate a stream reconnection + envoy::api::v2::DeltaDiscoveryRequest cur_request = state.getNextRequestAckless(); EXPECT_EQ("version1B", cur_request.initial_resource_versions().at("name1")); EXPECT_EQ(cur_request.initial_resource_versions().end(), cur_request.initial_resource_versions().find("name2")); @@ -89,8 +89,8 @@ TEST(DeltaSubscriptionImplTest, ResourceGoneLeadsToBlankInitialVersion) { *remove1_3.Add() = "name1"; *remove1_3.Add() = "name3"; deliverDiscoveryResponse(state, {}, remove1_3, "debugversion3"); - state.set_first_request_of_new_stream(true); // simulate a stream reconnection - envoy::api::v2::DeltaDiscoveryRequest cur_request = state.getNextRequest(); + state.markStreamFresh(); // simulate a stream reconnection + envoy::api::v2::DeltaDiscoveryRequest cur_request = state.getNextRequestAckless(); EXPECT_TRUE(cur_request.initial_resource_versions().empty()); } @@ -98,9 +98,9 @@ TEST(DeltaSubscriptionImplTest, ResourceGoneLeadsToBlankInitialVersion) { // ...but our own map should remember our interest. In particular, losing interest in all 3 // should cause their names to appear in the resource_names_unsubscribe field of a // DeltaDiscoveryRequest. - state.updateResourceInterest({"name4"}); // note the lack of 1, 2, and 3 - state.set_first_request_of_new_stream(true); // simulate a stream reconnection - envoy::api::v2::DeltaDiscoveryRequest cur_request = state.getNextRequest(); + state.updateResourceInterest({"name4"}); // note the lack of 1, 2, and 3 + state.markStreamFresh(); // simulate a stream reconnection + envoy::api::v2::DeltaDiscoveryRequest cur_request = state.getNextRequestAckless(); EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4")); EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name1", "name2", "name3")); diff --git a/test/common/config/grpc_stream_test.cc b/test/common/config/grpc_stream_test.cc index b4a97675fe916..91e50c406cbb1 100644 --- a/test/common/config/grpc_stream_test.cc +++ b/test/common/config/grpc_stream_test.cc @@ -41,22 +41,22 @@ class GrpcStreamTest : public testing::Test { GrpcStream grpc_stream_; }; -// Tests that establishNewStream() establishes it, a second call does nothing, and a third call +// Tests that establishStream() establishes it, a second call does nothing, and a third call // after the stream was disconnected re-establishes it. -TEST_F(GrpcStreamTest, EstablishNewStream) { +TEST_F(GrpcStreamTest, EstablishStream) { EXPECT_FALSE(grpc_stream_.grpcStreamAvailable()); // Successful establishment { EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); EXPECT_CALL(callbacks_, onStreamEstablished()); - grpc_stream_.establishNewStream(); + grpc_stream_.establishStream(); EXPECT_TRUE(grpc_stream_.grpcStreamAvailable()); } - // Idempotency: do nothing (other than logging a warning) if already connected + // Idempotent { EXPECT_CALL(*async_client_, start(_, _)).Times(0); EXPECT_CALL(callbacks_, onStreamEstablished()).Times(0); - grpc_stream_.establishNewStream(); + grpc_stream_.establishStream(); EXPECT_TRUE(grpc_stream_.grpcStreamAvailable()); } grpc_stream_.onRemoteClose(Grpc::Status::GrpcStatus::Ok, ""); @@ -65,7 +65,7 @@ TEST_F(GrpcStreamTest, EstablishNewStream) { { EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); EXPECT_CALL(callbacks_, onStreamEstablished()); - grpc_stream_.establishNewStream(); + grpc_stream_.establishStream(); EXPECT_TRUE(grpc_stream_.grpcStreamAvailable()); } } @@ -75,7 +75,7 @@ TEST_F(GrpcStreamTest, EstablishNewStream) { TEST_F(GrpcStreamTest, FailToEstablishNewStream) { EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(nullptr)); EXPECT_CALL(callbacks_, onEstablishmentFailure()); - grpc_stream_.establishNewStream(); + grpc_stream_.establishStream(); EXPECT_FALSE(grpc_stream_.grpcStreamAvailable()); } @@ -83,7 +83,7 @@ TEST_F(GrpcStreamTest, FailToEstablishNewStream) { // machinery. TEST_F(GrpcStreamTest, SendMessage) { EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); - grpc_stream_.establishNewStream(); + grpc_stream_.establishStream(); envoy::api::v2::DiscoveryRequest request; request.set_response_nonce("grpc_stream_test_noncense"); EXPECT_CALL(async_stream_, sendMessage(ProtoEq(request), false)); diff --git a/test/common/config/grpc_subscription_impl_test.cc b/test/common/config/grpc_subscription_impl_test.cc index 1ca9f6ff733ca..490a74c228e39 100644 --- a/test/common/config/grpc_subscription_impl_test.cc +++ b/test/common/config/grpc_subscription_impl_test.cc @@ -40,13 +40,8 @@ TEST_F(GrpcSubscriptionImplTest, RemoteStreamClose) { EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); EXPECT_CALL(*timer_, enableTimer(_)); EXPECT_CALL(random_, random()); -<<<<<<< HEAD - subscription_->grpcMux()->grpcStreamForTest()->onRemoteClose(Grpc::Status::GrpcStatus::Canceled, - ""); -======= subscription_->grpcMux().grpcStreamForTest().onRemoteClose(Grpc::Status::GrpcStatus::Canceled, ""); ->>>>>>> upstream/master verifyStats(2, 0, 0, 1, 0); verifyControlPlaneStats(0); diff --git a/test/common/config/grpc_subscription_test_harness.h b/test/common/config/grpc_subscription_test_harness.h index a54a3f4831b80..8e656a5e9eaf3 100644 --- a/test/common/config/grpc_subscription_test_harness.h +++ b/test/common/config/grpc_subscription_test_harness.h @@ -109,11 +109,7 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { expectSendMessage(last_cluster_names_, version_, Grpc::Status::GrpcStatus::Internal, "bad config"); } -<<<<<<< HEAD - subscription_->grpcMux()->grpcStreamForTest().onDiscoveryResponse(std::move(response)); -======= - subscription_->grpcMux().onDiscoveryResponse(std::move(response)); ->>>>>>> upstream/master + subscription_->grpcMux()->onDiscoveryResponse(std::move(response)); Mock::VerifyAndClearExpectations(&async_stream_); } From fd8b1be54116ad737836c66f23c4c57193161b50 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 3 May 2019 16:40:29 -0400 Subject: [PATCH 09/56] small XdsGrpcContext comment change Signed-off-by: Fred Douglas --- include/envoy/config/xds_grpc_context.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/envoy/config/xds_grpc_context.h b/include/envoy/config/xds_grpc_context.h index 8fe03356715a2..d45c97653c198 100644 --- a/include/envoy/config/xds_grpc_context.h +++ b/include/envoy/config/xds_grpc_context.h @@ -33,7 +33,8 @@ class XdsGrpcContext { std::chrono::milliseconds init_fetch_timeout) PURE; // Enqueues and attempts to send a discovery request, (un)subscribing to resources missing from / - // added to the passed 'resources' argument, relative to resource_versions_. + // added to the passed 'resources' argument, relative to the resource subscription interest + // currently known for type_url. virtual void updateResources(const std::set& resources, const std::string& type_url) PURE; From d7d5b148e7c058116d08f2a67fd51219e0487ee8 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 10 May 2019 13:02:31 -0400 Subject: [PATCH 10/56] quick commit of beginnings of subclass stuff before merge Signed-off-by: Fred Douglas --- include/envoy/config/BUILD | 11 +- include/envoy/config/ads_mux.h | 54 ----- include/envoy/config/grpc_mux.h | 74 ++++-- include/envoy/config/subscription_state.h | 57 +++++ include/envoy/config/xds_grpc_context.h | 83 ------- include/envoy/upstream/BUILD | 1 - include/envoy/upstream/cluster_manager.h | 5 +- source/common/config/BUILD | 1 - .../common/config/delta_subscription_impl.h | 7 +- .../common/config/delta_subscription_state.cc | 2 + .../common/config/delta_subscription_state.h | 18 +- source/common/config/grpc_delta_xds_context.h | 221 ++++++++---------- source/common/config/grpc_mux_impl.h | 5 +- .../config/grpc_mux_subscription_impl.h | 4 +- source/common/config/grpc_stream.h | 2 +- source/common/config/grpc_subscription_impl.h | 2 +- source/common/config/subscription_factory.h | 3 +- source/common/upstream/cds_api_impl.cc | 8 +- .../common/upstream/cluster_manager_impl.cc | 8 +- source/common/upstream/cluster_manager_impl.h | 10 +- source/server/lds_api.cc | 4 +- source/server/server.cc | 4 +- .../config/delta_subscription_test_harness.h | 43 +++- test/common/upstream/cds_api_impl_test.cc | 32 +-- test/mocks/config/BUILD | 1 - test/mocks/config/mocks.cc | 10 +- test/mocks/config/mocks.h | 22 -- test/mocks/upstream/mocks.cc | 2 +- test/mocks/upstream/mocks.h | 4 +- 29 files changed, 317 insertions(+), 381 deletions(-) delete mode 100644 include/envoy/config/ads_mux.h create mode 100644 include/envoy/config/subscription_state.h delete mode 100644 include/envoy/config/xds_grpc_context.h diff --git a/include/envoy/config/BUILD b/include/envoy/config/BUILD index e3a04e4fa8e9c..c40bfe6b9814e 100644 --- a/include/envoy/config/BUILD +++ b/include/envoy/config/BUILD @@ -32,6 +32,7 @@ envoy_cc_library( name = "grpc_mux_interface", hdrs = ["grpc_mux.h"], deps = [ + ":subscription_interface", "//include/envoy/stats:stats_macros", "//source/common/protobuf", ], @@ -54,13 +55,3 @@ envoy_cc_library( "//source/common/protobuf", ], ) - -envoy_cc_library( - name = "xds_grpc_context_interface", - hdrs = ["xds_grpc_context.h"], - deps = [ - ":subscription_interface", - "//include/envoy/local_info:local_info_interface", - "//source/common/protobuf", - ], -) diff --git a/include/envoy/config/ads_mux.h b/include/envoy/config/ads_mux.h deleted file mode 100644 index 78eafc21f9677..0000000000000 --- a/include/envoy/config/ads_mux.h +++ /dev/null @@ -1,54 +0,0 @@ -#pragma once - -namespace Envoy { -namespace Config { - -/** - * Manage one or more gRPC subscriptions on a single stream to management server. This can be used - * for a single xDS API, e.g. EDS, or to combined multiple xDS APIs for ADS. - */ -class AdsMux : public XdsGrpcContext { -public: - virtual ~AdsMux() {} - - /** - * Initiate stream with management server. - */ - virtual void start() PURE; - - /** - * Start a configuration subscription asynchronously for some API type and resources. - * @param type_url type URL corresponding to xDS API, e.g. - * type.googleapis.com/envoy.api.v2.Cluster. - * @param resources vector of resource names to watch for. If this is empty, then all - * resources for type_url will result in callbacks. - * @param callbacks the callbacks to be notified of configuration updates. These must be valid - * until GrpcMuxWatch is destroyed. - * @return GrpcMuxWatchPtr a handle to cancel the subscription with. E.g. when a cluster goes - * away, its EDS updates should be cancelled by destroying the GrpcMuxWatchPtr. - */ - // TODO TODO no dont return that............ - virtual GrpcMuxWatchPtr subscribe(const std::string& type_url, - const std::vector& resources, - GrpcMuxCallbacks& callbacks) PURE; - - /** - * Pause discovery requests for a given API type. This is useful when we're processing an update - * for LDS or CDS and don't want a flood of updates for RDS or EDS respectively. Discovery - * requests may later be resumed with resume(). - * @param type_url type URL corresponding to xDS API, e.g. - * type.googleapis.com/envoy.api.v2.Cluster. - */ - virtual void pause(const std::string& type_url) PURE; - - /** - * Resume discovery requests for a given API type. This will send a discovery request if one would - * have been sent during the pause. - * @param type_url type URL corresponding to xDS API, - * e.g.type.googleapis.com/envoy.api.v2.Cluster. - */ - virtual void resume(const std::string& type_url) PURE; -}; - -} // namespace Config -} // namespace Envoy diff --git a/include/envoy/config/grpc_mux.h b/include/envoy/config/grpc_mux.h index 98a0ceaf05f6a..e1a6a3032ed55 100644 --- a/include/envoy/config/grpc_mux.h +++ b/include/envoy/config/grpc_mux.h @@ -2,6 +2,7 @@ #include "envoy/common/exception.h" #include "envoy/common/pure.h" +#include "envoy/config/subscription.h" #include "envoy/stats/stats_macros.h" #include "common/protobuf/protobuf.h" @@ -26,6 +27,7 @@ struct ControlPlaneStats { ALL_CONTROL_PLANE_STATS(GENERATE_COUNTER_STRUCT,GENERATE_GAUGE_STRUCT) }; +// TODO TODO remove. remove this whole file. class GrpcMuxCallbacks { public: virtual ~GrpcMuxCallbacks() {} @@ -65,28 +67,69 @@ class GrpcMuxWatch { typedef std::unique_ptr GrpcMuxWatchPtr; +/** + * A grouping of callbacks that a GrpcMux should provide to its GrpcStream. + */ +template class GrpcStreamCallbacks { +public: + virtual ~GrpcStreamCallbacks() {} + + /** + * For the GrpcStream to prompt the context to take appropriate action in response to the + * gRPC stream having been successfully established. + */ + virtual void onStreamEstablished() PURE; + + /** + * For the GrpcStream to prompt the context to take appropriate action in response to + * failure to establish the gRPC stream. + */ + virtual void onEstablishmentFailure() PURE; + + /** + * For the GrpcStream to pass received protos to the context. + */ + virtual void onDiscoveryResponse(std::unique_ptr&& message) PURE; + + /** + * For the GrpcStream to call when its rate limiting logic allows more requests to be sent. + */ + virtual void onWriteable() PURE; +}; + /** * Manage one or more gRPC subscriptions on a single stream to management server. This can be used - * for a single xDS API, e.g. EDS, or to combined multiple xDS APIs for ADS. + * for a single xDS API, e.g. EDS, or to combined multiple xDS APIs for ADS. Both delta and state-of-the-world implement this same interface - whether a GrpcSubscriptionImpl is delta or SotW is determined entirely by which type of GrpcMux it works with. */ class GrpcMux { public: virtual ~GrpcMux() {} - /** - * Start a configuration subscription asynchronously for some API type and resources. If the gRPC stream to the management server is not already up, starts it. + * Starts a configuration subscription asynchronously for some API type and resources. If the gRPC + * stream to the management server is not already up, starts it. + * @param resources vector of resource names to watch for. If this is empty, then all + * resources for type_url will result in callbacks. * @param type_url type URL corresponding to xDS API, e.g. * type.googleapis.com/envoy.api.v2.Cluster. - * @param resources set of resource names to watch for. If this is empty, then all - * resources for type_url will result in callbacks. * @param callbacks the callbacks to be notified of configuration updates. These must be valid - * until GrpcMuxWatch is destroyed. - * @return GrpcMuxWatchPtr a handle to cancel the subscription with. E.g. when a cluster goes - * away, its EDS updates should be cancelled by destroying the GrpcMuxWatchPtr. + * until this GrpcMux is destroyed. + * @param stats reference to a stats object, which should be owned by the SubscriptionImpl, for + * the GrpcMux to record stats specific to this one subscription. + * @param init_fetch_timeout how long the first fetch has to complete before onConfigUpdateFailed + * will be called. */ - virtual GrpcMuxWatchPtr subscribe(const std::string& type_url, - const std::set& resources, - GrpcMuxCallbacks& callbacks) PURE; + virtual void addSubscription(const std::set& resources, const std::string& type_url, + SubscriptionCallbacks& callbacks, SubscriptionStats& stats, + std::chrono::milliseconds init_fetch_timeout) PURE; + + // (Un)subscribes to resources missing from / added to the passed 'resources' argument, relative + // to the resource subscription interest currently known for type_url. + // Attempts to send a discovery request if there is any such change. + virtual void updateResources(const std::set& resources, + const std::string& type_url) PURE; + + // Ends the given subscription, and drops the relevant parts of the protocol state. + virtual void removeSubscription(const std::string& type_url) PURE; /** * Pause discovery requests for a given API type. This is useful when we're processing an update @@ -96,7 +139,7 @@ class GrpcMux { * type.googleapis.com/envoy.api.v2.Cluster. */ virtual void pause(const std::string& type_url) PURE; - + /** * Resume discovery requests for a given API type. This will send a discovery request if one would * have been sent during the pause. @@ -104,9 +147,12 @@ class GrpcMux { * e.g.type.googleapis.com/envoy.api.v2.Cluster. */ virtual void resume(const std::string& type_url) PURE; -}; -typedef std::unique_ptr GrpcMuxPtr; + // TODO(fredlas) remove, only here for compatibility with old-style GrpcMuxImpl. + virtual GrpcMuxWatchPtr subscribe(const std::string& type_url, + const std::set& resources, + GrpcMuxCallbacks& callbacks) PURE; +}; } // namespace Config } // namespace Envoy diff --git a/include/envoy/config/subscription_state.h b/include/envoy/config/subscription_state.h new file mode 100644 index 0000000000000..739db89849ff6 --- /dev/null +++ b/include/envoy/config/subscription_state.h @@ -0,0 +1,57 @@ +#pragma once + +#include +#include + +#include "envoy/common/pure.h" + +#include "common/protobuf/protobuf.h" + +namespace Envoy { +namespace Config { + +struct UpdateAck { + UpdateAck(absl::string_view nonce, absl::string_view type_url) + : nonce_(nonce), type_url_(type_url) {} + std::string nonce_; + std::string type_url_; + ::google::rpc::Status error_detail_; +}; + +class SubscriptionState { +public: + virtual ~SubscriptionState() {} + + void setInitFetchTimeout(Event::Dispatcher& dispatcher) PURE; + + void pause() PURE; + void resume() PURE; + bool paused() const PURE; + + // Update which resources we're interested in subscribing to. + void updateResourceInterest(const std::set& update_to_these_names) PURE; + + // Whether there was a change in our subscription interest we have yet to inform the server of. + bool subscriptionUpdatePending() const PURE; + + void markStreamFresh() PURE; + + // Argument should have been static_cast from GrpcStream's ResponseProto type. + UpdateAck handleResponse(const Protobuf::Message& message) PURE; + + void handleEstablishmentFailure() PURE; + + // Will have been static_cast from GrpcStream's RequestProto type, and should be static_cast back before handing to GrpcStream::sendMessage. + Protobuf::Message getNextRequestAckless() PURE; + // Will have been static_cast from GrpcStream's RequestProto type, and should be static_cast back before handing to GrpcStream::sendMessage. + Protobuf::Message getNextRequestWithAck(const UpdateAck& ack) PURE; +}; + +class SubscriptionStateFactory { +public: + SubscriptionState makeSubscriptionState(const std::string& type_url, const std::set& resource_names, + SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout, SubscriptionStats& stats) PURE; +}; + +} // namespace Config +} // namespace Envoy diff --git a/include/envoy/config/xds_grpc_context.h b/include/envoy/config/xds_grpc_context.h deleted file mode 100644 index d45c97653c198..0000000000000 --- a/include/envoy/config/xds_grpc_context.h +++ /dev/null @@ -1,83 +0,0 @@ -#pragma once - -#include "envoy/common/pure.h" -#include "envoy/config/grpc_mux.h" // TODO TODO remove -#include "envoy/config/subscription.h" -#include "envoy/event/dispatcher.h" -#include "envoy/local_info/local_info.h" - -#include "common/protobuf/protobuf.h" - -namespace Envoy { -namespace Config { - -class XdsGrpcContext { -public: - virtual ~XdsGrpcContext() {} - /** - * Starts a configuration subscription asynchronously for some API type and resources. If the gRPC - * stream to the management server is not already up, starts it. - * @param resources vector of resource names to watch for. If this is empty, then all - * resources for type_url will result in callbacks. - * @param type_url type URL corresponding to xDS API, e.g. - * type.googleapis.com/envoy.api.v2.Cluster. - * @param callbacks the callbacks to be notified of configuration updates. These must be valid - * until this XdsGrpcContext is destroyed. - * @param stats reference to a stats object, which should be owned by the SubscriptionImpl, for - * the XdsGrpcContext to record stats specific to this one subscription. - * @param init_fetch_timeout how long the first fetch has to complete before onConfigUpdateFailed - * will be called. - */ - virtual void addSubscription(const std::set& resources, const std::string& type_url, - SubscriptionCallbacks& callbacks, SubscriptionStats& stats, - std::chrono::milliseconds init_fetch_timeout) PURE; - - // Enqueues and attempts to send a discovery request, (un)subscribing to resources missing from / - // added to the passed 'resources' argument, relative to the resource subscription interest - // currently known for type_url. - virtual void updateResources(const std::set& resources, - const std::string& type_url) PURE; - - virtual void removeSubscription(const std::string& type_url) PURE; - - virtual void pause(const std::string& type_url) PURE; - virtual void resume(const std::string& type_url) PURE; - - // TODO TODO remove - virtual GrpcMuxWatchPtr subscribe(const std::string& type_url, - const std::set& resources, - GrpcMuxCallbacks& callbacks) PURE; -}; - -/** - * A grouping of callbacks that an XdsGrpcContext should provide to its GrpcStream. - */ -template class GrpcStreamCallbacks { -public: - virtual ~GrpcStreamCallbacks() {} - - /** - * For the GrpcStream to prompt the context to take appropriate action in response to the - * gRPC stream having been successfully established. - */ - virtual void onStreamEstablished() PURE; - - /** - * For the GrpcStream to prompt the context to take appropriate action in response to - * failure to establish the gRPC stream. - */ - virtual void onEstablishmentFailure() PURE; - - /** - * For the GrpcStream to pass received protos to the context. - */ - virtual void onDiscoveryResponse(std::unique_ptr&& message) PURE; - - /** - * For the GrpcStream to call when its rate limiting logic allows more requests to be sent. - */ - virtual void onWriteable() PURE; -}; - -} // namespace Config -} // namespace Envoy diff --git a/include/envoy/upstream/BUILD b/include/envoy/upstream/BUILD index 6cd764666e0dd..e20c85e1d24ec 100644 --- a/include/envoy/upstream/BUILD +++ b/include/envoy/upstream/BUILD @@ -18,7 +18,6 @@ envoy_cc_library( ":upstream_interface", "//include/envoy/access_log:access_log_interface", "//include/envoy/config:grpc_mux_interface", - "//include/envoy/config:xds_grpc_context_interface", "//include/envoy/grpc:async_client_manager_interface", "//include/envoy/http:async_client_interface", "//include/envoy/http:conn_pool_interface", diff --git a/include/envoy/upstream/cluster_manager.h b/include/envoy/upstream/cluster_manager.h index 6c00043aeef7f..b544e2a48cacb 100644 --- a/include/envoy/upstream/cluster_manager.h +++ b/include/envoy/upstream/cluster_manager.h @@ -11,7 +11,6 @@ #include "envoy/api/v2/cds.pb.h" #include "envoy/config/bootstrap/v2/bootstrap.pb.h" #include "envoy/config/grpc_mux.h" -#include "envoy/config/xds_grpc_context.h" #include "envoy/grpc/async_client_manager.h" #include "envoy/http/async_client.h" #include "envoy/http/conn_pool.h" @@ -205,9 +204,9 @@ class ClusterManager { * logically to the management of clusters but instead is required early in ClusterManager/server * initialization and in various sites that need ClusterManager for xDS API interfacing. * - * @return XdsGrpcContext& ADS API provider referencee. + * @return GrpcMux& ADS API provider referencee. */ - virtual std::shared_ptr xdsGrpcContext() PURE; + virtual std::shared_ptr adsMux() PURE; /** * @return Grpc::AsyncClientManager& the cluster manager's gRPC client manager. diff --git a/source/common/config/BUILD b/source/common/config/BUILD index cba18b9ab8c3b..f4469e0365bb8 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -135,7 +135,6 @@ envoy_cc_library( ":utility_lib", "//include/envoy/config:grpc_mux_interface", "//include/envoy/config:subscription_interface", - "//include/envoy/config:xds_grpc_context_interface", "//include/envoy/grpc:async_client_interface", "//include/envoy/upstream:cluster_manager_interface", "//source/common/common:backoff_lib", diff --git a/source/common/config/delta_subscription_impl.h b/source/common/config/delta_subscription_impl.h index e8a164050b4a6..75fd36010c9b6 100644 --- a/source/common/config/delta_subscription_impl.h +++ b/source/common/config/delta_subscription_impl.h @@ -1,9 +1,8 @@ #pragma once #include "envoy/config/subscription.h" -#include "envoy/config/xds_grpc_context.h" +#include "envoy/config/grpc_mux.h" -#include "common/config/grpc_delta_xds_context.h" #include "common/config/utility.h" #include "common/grpc/common.h" #include "common/protobuf/protobuf.h" @@ -22,7 +21,7 @@ namespace Config { // TODO(fredlas) someday this class will be named SubscriptionImpl (without any changes to its code) class DeltaSubscriptionImpl : public Subscription { public: - DeltaSubscriptionImpl(std::shared_ptr context, absl::string_view type_url, + DeltaSubscriptionImpl(std::shared_ptr context, absl::string_view type_url, SubscriptionStats stats, std::chrono::milliseconds init_fetch_timeout) : context_(context), type_url_(type_url), stats_(stats), init_fetch_timeout_(init_fetch_timeout) {} @@ -51,7 +50,7 @@ class DeltaSubscriptionImpl : public Subscription { } private: - std::shared_ptr context_; + std::shared_ptr context_; const std::string type_url_; SubscriptionStats stats_; const std::chrono::milliseconds init_fetch_timeout_; diff --git a/source/common/config/delta_subscription_state.cc b/source/common/config/delta_subscription_state.cc index 8c43409d72872..7332376cecd0a 100644 --- a/source/common/config/delta_subscription_state.cc +++ b/source/common/config/delta_subscription_state.cc @@ -1,5 +1,7 @@ #include "common/config/delta_subscription_state.h" +#include "common/common/hash.h" + namespace Envoy { namespace Config { diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index f430f8a7877af..d39960e4704ad 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -6,7 +6,8 @@ #include "envoy/grpc/status.h" #include "envoy/local_info/local_info.h" -#include "common/common/hash.h" +#include "common/common/assert.h" +#include "common/common/logger.h" namespace Envoy { namespace Config { @@ -104,5 +105,20 @@ class DeltaSubscriptionState : public Logger::Loggable { SubscriptionStats& stats_; }; +class DeltaSubscriptionStateFactory : public SubscriptionStateFactory { +public: + DeltaSubscriptionStateFactory(Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info) + : dispatcher_(dispatcher), local_info_(local_info) {} + + SubscriptionState makeSubscriptionState(const std::string& type_url, const std::set& resource_names, + SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout, SubscriptionStats& stats) override { + return DeltaSubscriptionState(type_url, resources, callbacks, local_info_, + init_fetch_timeout, dispatcher_, stats); + } +private: + Event::Dispatcher& dispatcher_; + const LocalInfo::LocalInfo& local_info_; +}; + } // namespace Config } // namespace Envoy diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h index efb4dba2271d3..248e44caad097 100644 --- a/source/common/config/grpc_delta_xds_context.h +++ b/source/common/config/grpc_delta_xds_context.h @@ -5,7 +5,6 @@ #include "envoy/api/v2/discovery.pb.h" #include "envoy/common/token_bucket.h" #include "envoy/config/subscription.h" -#include "envoy/config/xds_grpc_context.h" #include "common/common/assert.h" #include "common/common/backoff_strategy.h" @@ -31,7 +30,7 @@ namespace Config { // non-aggregated case is handled by running the aggregated logic, and just happening to only have 1 // xDS subscription type to "aggregate", i.e., GrpcDeltaXdsContext only has one // DeltaSubscriptionState entry in its map. The sole implementation difference: when the bootstrap -// specifies ADS, the method string set on the gRPC client that the XdsGrpcContext holds is +// specifies ADS, the method string set on the gRPC client that the GrpcMux holds is // {Delta,Stream}AggregatedResources, as opposed to e.g. {Delta,Stream}Clusters. This distinction is // necessary for the server to know what resources should be provided. // @@ -40,39 +39,27 @@ namespace Config { // DeltaSubscriptionState is equally applicable to non- and aggregated. // // Delta vs state-of-the-world is a question of wire format: the protos in question are named -// [Delta]Discovery{Request,Response}. That is what the XdsGrpcContext interface is useful for: its +// [Delta]Discovery{Request,Response}. That is what the GrpcMux interface is useful for: its // GrpcDeltaXdsContext implementation works with DeltaDiscovery{Request,Response} and has // delta-specific logic, and its GrpxMuxImpl implementation works with Discovery{Request,Response} // and has SotW-specific logic. A DeltaSubscriptionImpl (TODO rename to not delta-specific since -// it's ideally going to cover ALL 4 subscription "meta-types") has its shared_ptr. +// it's ideally going to cover ALL 4 subscription "meta-types") has its shared_ptr. // Both GrpcDeltaXdsContext (delta) or GrpcMuxImpl (SotW) will work just fine. The shared_ptr allows // for both non- and aggregated: if non-aggregated, you'll be the only holder of that shared_ptr. By // those two mechanisms, the single class (TODO rename) DeltaSubscriptionImpl handles all 4 // delta/SotW and non-/aggregated combinations. -class GrpcDeltaXdsContext : public XdsGrpcContext, +class GrpcXdsContext : public GrpcMux, public GrpcStreamCallbacks, Logger::Loggable { public: - GrpcDeltaXdsContext(Grpc::AsyncClientPtr async_client, Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, - Runtime::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, - const LocalInfo::LocalInfo& local_info) - : local_info_(local_info), dispatcher_(dispatcher), - grpc_stream_(TypedGrpcStreamUnion::makeDelta( - std::make_unique>( - this, std::move(async_client), service_method, random, dispatcher, scope, - rate_limit_settings))) {} - void addSubscription(const std::set& resources, const std::string& type_url, SubscriptionCallbacks& callbacks, SubscriptionStats& stats, std::chrono::milliseconds init_fetch_timeout) override { - std::set HACK_resources_as_set(resources.begin(), resources.end()); subscriptions_.emplace(std::make_pair( - type_url, DeltaSubscriptionState(type_url, HACK_resources_as_set, callbacks, local_info_, + type_url, subscription_factory_.makeSubscriptionState(type_url, resources, callbacks, local_info_, init_fetch_timeout, dispatcher_, stats))); - grpc_stream_.establishStream(); // (idempotent) + subscription_ordering_.emplace_back(type_url); + establishStream(); // (idempotent) } // Enqueues and attempts to send a discovery request, (un)subscribing to resources missing from / @@ -84,13 +71,23 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, ENVOY_LOG(warn, "Not updating non-existent subscription {}.", type_url); return; } - std::set HACK_resources_as_set(resources.begin(), resources.end()); - sub->second.updateResourceInterest(HACK_resources_as_set); + sub->second.updateResourceInterest(resources); // Tell the server about our new interests, if there are any. trySendDiscoveryRequests(); } - void removeSubscription(const std::string& type_url) override { subscriptions_.erase(type_url); } + void removeSubscription(const std::string& type_url) override { + subscriptions_.erase(type_url); + // And remove from the subscription_ordering_ list. + auto it = subscription_ordering_.begin(); + while (it != subscription_ordering_.end()) { + if (*it == type_url) { + it = subscription_ordering_.erase(it); + } else { + ++it; + } + } + } void pause(const std::string& type_url) override { auto sub = subscriptions_.find(type_url); @@ -146,22 +143,32 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, trySendDiscoveryRequests(); } - // TODO TODO remove, GrpcMux impersonation - virtual GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, - GrpcMuxCallbacks&) override { + // TODO TODO remove, GrpcMux impersonation. Basically combination of addSubscription and updateResources. + virtual GrpcMuxWatchPtr subscribe(const std::string&, + const std::set& , + GrpcMuxCallbacks& ) override { +// don't need any implementation here. only grpc_mux_subscription_impl ever calls it, and there would never be a GrpcDeltaXdsContext held by one of those. + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; return nullptr; } - - // GrpcStream& - // grpcStreamForTest() { - // return grpc_stream_; - // } + +protected: + GrpcXdsContext(Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info) + : dispatcher_(dispatcher), local_info_(local_info) {} + + // Everything related to GrpcStream must remain abstract. GrpcStream (and the gRPC-using classes that underlie it) are templated on protobufs. That means that a single implementation that supports different types of protobufs cannot use polymorphism to share code. The workaround: the GrpcStream will be owned by a derived class, and all code that would touch grpc_stream_ is seen here in the base class as abstract. + virtual void establishStream() PURE; + virtual void sendDiscoveryRequestWithAck(SubscriptionState* sub, UpdateAck ack) PURE; + virtual void sendDiscoveryRequestAckless(SubscriptionState* sub) PURE; + virtual void maybeUpdateQueueSizeStat(uint64_t size) PURE; + virtual bool grpcStreamAvailable() const PURE; + virtual bool checkRateLimitAllowsDrain() PURE; private: void trySendDiscoveryRequests() { while (true) { // Do any of our subscriptions even want to send a request? - absl::optional maybe_request_type = wantToSendDiscoveryRequest(); + absl::optional maybe_request_type = whoWantsToSendDiscoveryRequest(); if (!maybe_request_type.has_value()) { break; } @@ -172,6 +179,7 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, if (sub == subscriptions_.end()) { ENVOY_LOG(warn, "Not sending discovery request for non-existent subscription {}.", next_request_type_url); + // It's safe to assume the front of the ACK queue is of this type, because that's the only way whoWantsToSendDiscoveryRequest() could return something for a non-existent subscription. ack_queue_.pop(); continue; } @@ -181,13 +189,14 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, } // Get our subscription state to generate the appropriate DeltaDiscoveryRequest, and send. if (!ack_queue_.empty()) { - grpc_stream_.sendMessage(sub->second.getNextRequestWithAck(ack_queue_.front())); + // Because ACKs take precedence over plain requests, if there is anything in the queue, it's safe to assume it's what we want to send here. + sendDiscoveryRequestWithAck(&sub->second, ack_queue_.front()); ack_queue_.pop(); } else { - grpc_stream_.sendMessage(sub->second.getNextRequestAckless()); + sendDiscoveryRequestAckless(&sub->second); } } - grpc_stream_.maybeUpdateQueueSizeStat(ack_queue_.size()); + maybeUpdateQueueSizeStat(ack_queue_.size()); } // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check @@ -196,10 +205,10 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, if (sub->paused()) { ENVOY_LOG(trace, "API {} paused; discovery request on hold for now.", type_url); return false; - } else if (!grpc_stream_.grpcStreamAvailable()) { + } else if (!grpcStreamAvailable()) { ENVOY_LOG(trace, "No stream available to send a discovery request for {}.", type_url); return false; - } else if (!grpc_stream_.checkRateLimitAllowsDrain()) { + } else if (!checkRateLimitAllowsDrain()) { ENVOY_LOG(trace, "{} discovery request hit rate limit; will try later.", type_url); return false; } @@ -209,112 +218,74 @@ class GrpcDeltaXdsContext : public XdsGrpcContext, // Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or // a subscription update. (Does not check whether we *can* send that DeltaDiscoveryRequest). // Returns the type_url of the resource type we should send the DeltaDiscoveryRequest for (if - // any). - absl::optional wantToSendDiscoveryRequest() { + // any). Prioritizes ACKs over non-ACK subscription interest updates. + absl::optional whoWantsToSendDiscoveryRequest() { // All ACKs are sent before plain updates. trySendDiscoveryRequests() relies on this. So, choose // type_url from ack_queue_ if possible, before looking at pending updates. if (!ack_queue_.empty()) { return ack_queue_.front().type_url_; } - for (auto& sub : subscriptions_) { - if (sub.second.subscriptionUpdatePending()) { - return sub.first; + // If we're looking to send multiple non-ACK requests, send them in the order that their subscriptions were initiated. + for (const auto& sub_type : subscription_ordering_) { + auto sub = subscriptions_.find(sub_type); + if (sub == subscriptions_.end()) { + continue; + } + if (sub->second.subscriptionUpdatePending()) { + return sub->first; } } return absl::nullopt; } - const LocalInfo::LocalInfo& local_info_; Event::Dispatcher& dispatcher_; - - /* NOTE NOTE TODO TODO: all of this is just to support the eventual sotw+delta merger. - * for current purposes, just sticking with this grpc_stream_ is fine: - GrpcStream - grpc_stream_;*/ - // A horrid little class that provides polymorphism for these two protobuf types. - class TypedGrpcStreamUnion { - public: - static TypedGrpcStreamUnion - makeDelta(std::unique_ptr>&& delta) { - return TypedGrpcStreamUnion(std::move(delta)); - } - - static TypedGrpcStreamUnion - makeSotw(std::unique_ptr>&& sotw) { - return TypedGrpcStreamUnion(std::move(sotw)); - } - - void establishStream() { - delta_stream_ ? delta_stream_->establishStream() : sotw_stream_->establishStream(); - } - - bool grpcStreamAvailable() const { - return delta_stream_ ? delta_stream_->grpcStreamAvailable() - : sotw_stream_->grpcStreamAvailable(); - } - - void sendMessage(const Protobuf::Message& request) { - delta_stream_ ? delta_stream_->sendMessage( - static_cast(request)) - : sotw_stream_->sendMessage( - static_cast(request)); - } - - /* void onReceiveMessage(std::unique_ptr&& message) { - void* ew = reinterpret_cast(message.release()); - delta_stream_ - ? delta_stream_->onReceiveMessage( - std::unique_ptr( - reinterpret_cast(ew))) - : sotw_stream_->onReceiveMessage( - std::unique_ptr( - reinterpret_cast(ew))); - } - - void onRemoteClose(Grpc::Status::GrpcStatus status, const std::string& message) { - delta_stream_ ? delta_stream_->onRemoteClose(status, message) - : sotw_stream_->onRemoteClose(status, message); - }*/ - - void maybeUpdateQueueSizeStat(uint64_t size) { - delta_stream_ ? delta_stream_->maybeUpdateQueueSizeStat(size) - : sotw_stream_->maybeUpdateQueueSizeStat(size); - } - - bool checkRateLimitAllowsDrain() { - return delta_stream_ ? delta_stream_->checkRateLimitAllowsDrain() - : sotw_stream_->checkRateLimitAllowsDrain(); - } - - private: - explicit TypedGrpcStreamUnion( - std::unique_ptr< - GrpcStream>&& sotw) - : delta_stream_(nullptr), sotw_stream_(std::move(sotw)) {} - - explicit TypedGrpcStreamUnion( - std::unique_ptr>&& delta) - : delta_stream_(std::move(delta)), sotw_stream_(nullptr) {} - - std::unique_ptr< - GrpcStream> - delta_stream_; - std::unique_ptr> - sotw_stream_; - }; - TypedGrpcStreamUnion grpc_stream_; + const LocalInfo::LocalInfo& local_info_; // Resource (N)ACKs we're waiting to send, stored in the order that they should be sent in. All // of our different resource types' ACKs are mixed together in this queue. std::queue ack_queue_; // Map from type_url strings to a DeltaSubscriptionState for that type. - // TODO TODO to merge with SotW, State will become an interface, and this Context will need to - // hold a factory for building these States. std::unordered_map subscriptions_; + SubscriptionStateFactory subscription_factory_; + + // Determines the order of initial discovery requests. (Assumes that subscriptions are added in the order of Envoy's dependency ordering). + std::list subscription_ordering_; +}; + +class GrpcDeltaXdsContext : public GrpcXdsContext { +public: + GrpcDeltaXdsContext(Grpc::AsyncClientPtr async_client, Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, + Runtime::RandomGenerator& random, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info) + : GrpcXdsContext(dispatcher, local_info), + grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, + rate_limit_settings) {} + +protected: + void establishStream() override { + grpc_stream_.establishStream(); + } + void sendDiscoveryRequestWithAck(SubscriptionState* sub, UpdateAck ack) override { + grpc_stream_.sendMessage(static_cast(sub)->getNextRequestWithAck(ack)); + } + void sendDiscoveryRequestAckless(SubscriptionState* sub) override { + grpc_stream_.sendMessage(static_cast(sub)->getNextRequestAckless()); + } + void maybeUpdateQueueSizeStat(uint64_t size) override { + grpc_stream_.maybeUpdateQueueSizeStat(size); + } + bool grpcStreamAvailable() const override { + return grpc_stream_.grpcStreamAvailable(); + } + bool checkRateLimitAllowsDrain() override { + return grpc_stream_.checkRateLimitAllowsDrain(); + } +private: + GrpcStream + grpc_stream_; }; typedef std::shared_ptr GrpcDeltaXdsContextSharedPtr; diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index 3852b6b2584e0..646fd83ef1bd5 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -21,7 +21,6 @@ namespace Config { * ADS API implementation that fetches via gRPC. */ class GrpcMuxImpl : public GrpcMux, - public XdsGrpcContext, public GrpcStreamCallbacks, public Logger::Loggable { public: @@ -34,7 +33,7 @@ class GrpcMuxImpl : public GrpcMux, GrpcMuxWatchPtr subscribe(const std::string& type_url, const std::set& resources, GrpcMuxCallbacks& callbacks) override; - // XdsGrpcContext + // GrpcMux void pause(const std::string& type_url) override; void resume(const std::string& type_url) override; void addSubscription(const std::set&, const std::string&, SubscriptionCallbacks&, @@ -118,7 +117,7 @@ class GrpcMuxImpl : public GrpcMux, std::queue request_queue_; }; -class NullGrpcMuxImpl : public XdsGrpcContext, +class NullGrpcMuxImpl : public GrpcMux, GrpcStreamCallbacks { public: GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, diff --git a/source/common/config/grpc_mux_subscription_impl.h b/source/common/config/grpc_mux_subscription_impl.h index 258019403a984..1c11de2017668 100644 --- a/source/common/config/grpc_mux_subscription_impl.h +++ b/source/common/config/grpc_mux_subscription_impl.h @@ -20,7 +20,7 @@ class GrpcMuxSubscriptionImpl : public Subscription, GrpcMuxCallbacks, Logger::Loggable { public: - GrpcMuxSubscriptionImpl(std::shared_ptr grpc_mux, SubscriptionStats stats, + GrpcMuxSubscriptionImpl(std::shared_ptr grpc_mux, SubscriptionStats stats, absl::string_view type_url, Event::Dispatcher& dispatcher, std::chrono::milliseconds init_fetch_timeout) : grpc_mux_(grpc_mux), stats_(stats), type_url_(type_url), dispatcher_(dispatcher), @@ -96,7 +96,7 @@ class GrpcMuxSubscriptionImpl : public Subscription, } } - std::shared_ptr grpc_mux_; + std::shared_ptr grpc_mux_; SubscriptionStats stats_; const std::string type_url_; SubscriptionCallbacks* callbacks_{}; diff --git a/source/common/config/grpc_stream.h b/source/common/config/grpc_stream.h index b58334ae14c7c..2cfb73a5021a7 100644 --- a/source/common/config/grpc_stream.h +++ b/source/common/config/grpc_stream.h @@ -2,7 +2,7 @@ #include -#include "envoy/config/xds_grpc_context.h" +#include "envoy/config/grpc_mux.h" #include "envoy/grpc/async_client.h" #include "common/common/backoff_strategy.h" diff --git a/source/common/config/grpc_subscription_impl.h b/source/common/config/grpc_subscription_impl.h index 8ac06a6a055c0..a8d75d2942ceb 100644 --- a/source/common/config/grpc_subscription_impl.h +++ b/source/common/config/grpc_subscription_impl.h @@ -36,7 +36,7 @@ class GrpcSubscriptionImpl : public Config::Subscription { grpc_mux_subscription_.updateResources(update_to_these_names); } - std::shared_ptr grpcMux() { return grpc_mux_; } + std::shared_ptr grpcMux() { return grpc_mux_; } private: std::shared_ptr grpc_mux_; diff --git a/source/common/config/subscription_factory.h b/source/common/config/subscription_factory.h index ed65cd8ae54fe..9d55e79515ac8 100644 --- a/source/common/config/subscription_factory.h +++ b/source/common/config/subscription_factory.h @@ -10,6 +10,7 @@ #include "common/config/delta_subscription_impl.h" #include "common/config/filesystem_subscription_impl.h" +#include "common/config/grpc_delta_xds_context.h" #include "common/config/grpc_mux_subscription_impl.h" #include "common/config/grpc_subscription_impl.h" #include "common/config/http_subscription_impl.h" @@ -99,7 +100,7 @@ class SubscriptionFactory { } case envoy::api::v2::core::ConfigSource::kAds: { result = std::make_unique( - cm.xdsGrpcContext(), stats, type_url, dispatcher, + cm.adsMux(), stats, type_url, dispatcher, Utility::configSourceInitialFetchTimeout(config)); break; } diff --git a/source/common/upstream/cds_api_impl.cc b/source/common/upstream/cds_api_impl.cc index 2b478321e2bd1..24a5b6fd78f5f 100644 --- a/source/common/upstream/cds_api_impl.cc +++ b/source/common/upstream/cds_api_impl.cc @@ -66,9 +66,9 @@ void CdsApiImpl::onConfigUpdate( const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info) { - cm_.xdsGrpcContext()->pause(Config::TypeUrl::get().ClusterLoadAssignment); + cm_.adsMux()->pause(Config::TypeUrl::get().ClusterLoadAssignment); Cleanup eds_resume( - [this] { cm_.xdsGrpcContext()->resume(Config::TypeUrl::get().ClusterLoadAssignment); }); + [this] { cm_.adsMux()->resume(Config::TypeUrl::get().ClusterLoadAssignment); }); std::vector exception_msgs; std::unordered_set cluster_names; @@ -102,10 +102,10 @@ void CdsApiImpl::onConfigUpdate( // ever changes, we would need to correct the following logic. if (state == ClusterManager::ClusterWarmingState::Starting && cm_.warmingClusterCount() == 1) { - cm_.xdsGrpcContext()->pause(Config::TypeUrl::get().Cluster); + cm_.adsMux()->pause(Config::TypeUrl::get().Cluster); } else if (state == ClusterManager::ClusterWarmingState::Finished && cm_.warmingClusterCount() == 0) { - cm_.xdsGrpcContext()->resume(Config::TypeUrl::get().Cluster); + cm_.adsMux()->resume(Config::TypeUrl::get().Cluster); } })) { ENVOY_LOG(debug, "cds: add/update cluster '{}'", cluster.name()); diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 736fbbe28b31c..248108debc087 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -208,7 +208,7 @@ ClusterManagerImpl::ClusterManagerImpl( // Now setup ADS if needed, this might rely on a primary cluster. // This is the only point where distinction between delta ADS and state-of-the-world ADS is made. - // After here, we just have an XdsGrpcContext interface held in xds_grpc_context_, which hides + // After here, we just have a GrpcMux interface held in ads_mux_, which hides // whether the backing implementation is delta or SotW. if (bootstrap.dynamic_resources().has_ads_config()) { if (bootstrap.dynamic_resources().ads_config().api_type() == @@ -217,7 +217,7 @@ ClusterManagerImpl::ClusterManagerImpl( bootstrap.dynamic_resources().has_ads_config() ? bootstrap.dynamic_resources().ads_config() : bootstrap.dynamic_resources().cds_config().api_config_source(); - xds_grpc_context_ = std::make_shared( + ads_mux_ = std::make_shared( Config::Utility::factoryForGrpcApiConfigSource(*async_client_manager_, api_config_source, stats) ->create(), @@ -229,7 +229,7 @@ ClusterManagerImpl::ClusterManagerImpl( bootstrap.dynamic_resources().ads_config()), local_info); } else { - xds_grpc_context_ = std::make_shared( + ads_mux_ = std::make_shared( local_info, Config::Utility::factoryForGrpcApiConfigSource( *async_client_manager_, bootstrap.dynamic_resources().ads_config(), stats) @@ -242,7 +242,7 @@ ClusterManagerImpl::ClusterManagerImpl( bootstrap.dynamic_resources().ads_config())); } } else { - xds_grpc_context_ = std::make_unique(); + ads_mux_ = std::make_unique(); } // After ADS is initialized, load EDS static clusters as EDS config may potentially need ADS. diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index 4ac62af3d0e2b..1506762322568 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -205,8 +205,8 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable xdsGrpcContext() override { return xds_grpc_context_; } + std::shared_ptr adsMux() override { return ads_mux_; } Grpc::AsyncClientManager& grpcAsyncClientManager() override { return *async_client_manager_; } const std::string& localClusterName() const override { return local_cluster_name_; } @@ -456,9 +456,7 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable - xds_grpc_context_; // TODO actually probably should have "ADS" in the name after all, maybe - // ads_context_. + std::shared_ptr ads_mux_; LoadStatsReporterPtr load_stats_reporter_; // The name of the local cluster of this Envoy instance if defined, else the empty string. std::string local_cluster_name_; diff --git a/source/server/lds_api.cc b/source/server/lds_api.cc index cd339ed708642..444032d1218b1 100644 --- a/source/server/lds_api.cc +++ b/source/server/lds_api.cc @@ -33,9 +33,9 @@ LdsApiImpl::LdsApiImpl(const envoy::api::v2::core::ConfigSource& lds_config, void LdsApiImpl::onConfigUpdate(const Protobuf::RepeatedPtrField& resources, const std::string& version_info) { - cm_.xdsGrpcContext()->pause(Config::TypeUrl::get().RouteConfiguration); + cm_.adsMux()->pause(Config::TypeUrl::get().RouteConfiguration); Cleanup rds_resume( - [this] { cm_.xdsGrpcContext()->resume(Config::TypeUrl::get().RouteConfiguration); }); + [this] { cm_.adsMux()->resume(Config::TypeUrl::get().RouteConfiguration); }); std::vector listeners; for (const auto& listener_blob : resources) { diff --git a/source/server/server.cc b/source/server/server.cc index 021da0e575801..5184b50fcac90 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -455,14 +455,14 @@ RunHelper::RunHelper(Instance& instance, const Options& options, Event::Dispatch // Pause RDS to ensure that we don't send any requests until we've // subscribed to all the RDS resources. The subscriptions happen in the init callbacks, // so we pause RDS until we've completed all the callbacks. - cm.xdsGrpcContext()->pause(Config::TypeUrl::get().RouteConfiguration); + cm.adsMux()->pause(Config::TypeUrl::get().RouteConfiguration); ENVOY_LOG(info, "all clusters initialized. initializing init manager"); init_manager.initialize(init_watcher_); // Now that we're execute all the init callbacks we can resume RDS // as we've subscribed to all the statically defined RDS resources. - cm.xdsGrpcContext()->resume(Config::TypeUrl::get().RouteConfiguration); + cm.adsMux()->resume(Config::TypeUrl::get().RouteConfiguration); }); } diff --git a/test/common/config/delta_subscription_test_harness.h b/test/common/config/delta_subscription_test_harness.h index 64d4b966f1bcc..61d40bb2f83a7 100644 --- a/test/common/config/delta_subscription_test_harness.h +++ b/test/common/config/delta_subscription_test_harness.h @@ -37,10 +37,19 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { rate_limit_settings_, stats_, init_fetch_timeout); } + ~DeltaSubscriptionTestHarness() { + while (!nonce_acks_required_.empty()) { + EXPECT_FALSE(nonce_acks_sent_.empty()); + EXPECT_EQ(nonce_acks_required_.front(), nonce_acks_sent_.front()); + nonce_acks_required_.pop(); + nonce_acks_sent_.pop(); + } + EXPECT_TRUE(nonce_acks_sent_.empty()); + } + void startSubscription(const std::set& cluster_names) override { EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); last_cluster_names_ = cluster_names; - expectSendMessage({}, ""); expectSendMessage(last_cluster_names_, ""); subscription_->start(cluster_names, callbacks_); } @@ -48,12 +57,13 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { void expectSendMessage(const std::set& cluster_names, const std::string& version) override { UNREFERENCED_PARAMETER(version); - expectSendMessage(cluster_names, {}, Grpc::Status::GrpcStatus::Ok, ""); + expectSendMessage(cluster_names, {}, Grpc::Status::GrpcStatus::Ok, "", {}); } void expectSendMessage(const std::set& subscribe, const std::set& unsubscribe, const Protobuf::int32 error_code, - const std::string& error_message) { + const std::string& error_message, + std::map initial_resource_versions) { envoy::api::v2::DeltaDiscoveryRequest expected_request; expected_request.mutable_node()->CopyFrom(node_); std::copy( @@ -62,15 +72,30 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { std::copy( unsubscribe.begin(), unsubscribe.end(), Protobuf::RepeatedFieldBackInserter(expected_request.mutable_resource_names_unsubscribe())); - expected_request.set_response_nonce(last_response_nonce_); + if (!last_response_nonce_.empty()) { + nonce_acks_required_.push(last_response_nonce_); + last_response_nonce_ = ""; + } expected_request.set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); + for (auto const& resource : initial_resource_versions) { + (*expected_request.mutable_initial_resource_versions())[resource.first] = resource.second; + } + if (error_code != Grpc::Status::GrpcStatus::Ok) { ::google::rpc::Status* error_detail = expected_request.mutable_error_detail(); error_detail->set_code(error_code); error_detail->set_message(error_message); } - EXPECT_CALL(async_stream_, sendMessage(ProtoEq(expected_request), false)); + EXPECT_CALL(async_stream_, + sendMessage(ProtoEqIgnoringField(expected_request, "response_nonce"), false)) + .WillOnce([this](const Protobuf::Message& message, bool) { + const std::string nonce = + static_cast(message).response_nonce(); + if (!nonce.empty()) { + nonce_acks_sent_.push(nonce); + } + }); } void deliverConfigUpdate(const std::vector& cluster_names, @@ -100,7 +125,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { expectSendMessage({}, version); } else { EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); - expectSendMessage({}, {}, Grpc::Status::GrpcStatus::Internal, "bad config"); + expectSendMessage({}, {}, Grpc::Status::GrpcStatus::Internal, "bad config", {}); } subscription_->onDiscoveryResponse(std::move(response)); Mock::VerifyAndClearExpectations(&async_stream_); @@ -116,7 +141,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { cluster_names.begin(), cluster_names.end(), std::inserter(unsub, unsub.begin())); - expectSendMessage(sub, unsub, Grpc::Status::GrpcStatus::Ok, ""); + expectSendMessage(sub, unsub, Grpc::Status::GrpcStatus::Ok, "", {}); subscription_->updateResources(cluster_names); last_cluster_names_ = cluster_names; } @@ -139,7 +164,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { const Protobuf::MethodDescriptor* method_descriptor_; Grpc::MockAsyncClient* async_client_; Event::MockDispatcher dispatcher_; - Runtime::MockRandomGenerator random_; + NiceMock random_; NiceMock local_info_; Grpc::MockAsyncStream async_stream_; std::unique_ptr subscription_; @@ -149,6 +174,8 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { Event::MockTimer* init_timeout_timer_; envoy::api::v2::core::Node node_; NiceMock> callbacks_; + std::queue nonce_acks_required_; + std::queue nonce_acks_sent_; }; } // namespace diff --git a/test/common/upstream/cds_api_impl_test.cc b/test/common/upstream/cds_api_impl_test.cc index bb3ebfbc0fb8e..83fbd093d8330 100644 --- a/test/common/upstream/cds_api_impl_test.cc +++ b/test/common/upstream/cds_api_impl_test.cc @@ -396,16 +396,16 @@ version_info: '0' )EOF"; // Two clusters updated, both warmed up. - EXPECT_CALL(cm_.xds_grpc_context_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); cm_.expectAddWithWarming("cluster1", "0"); cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.xds_grpc_context_, pause(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); cm_.expectAddWithWarming("cluster2", "0"); cm_.expectWarmingClusterCount(); EXPECT_CALL(initialized_, ready()); cm_.expectWarmingClusterCount(2); - EXPECT_CALL(cm_.xds_grpc_context_, resume(Config::TypeUrl::get().Cluster)).Times(1); - EXPECT_CALL(cm_.xds_grpc_context_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); EXPECT_CALL(*interval_timer_, enableTimer(_)); cm_.clustersToWarmUp({"cluster1", "cluster2"}); callbacks_->onSuccess(parseResponseMessageFromYaml(response1_yaml)); @@ -431,15 +431,15 @@ version_info: '1' path: eds path )EOF"; - EXPECT_CALL(cm_.xds_grpc_context_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); cm_.expectAddWithWarming("cluster1", "1"); cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.xds_grpc_context_, pause(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); cm_.expectAddWithWarming("cluster3", "1"); cm_.expectWarmingClusterCount(); EXPECT_CALL(initialized_, ready()); cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.xds_grpc_context_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); EXPECT_CALL(*interval_timer_, enableTimer(_)); resetCdsInitializedCb(); cm_.clustersToWarmUp({"cluster1"}); @@ -460,13 +460,13 @@ version_info: '2' path: eds path )EOF"; - EXPECT_CALL(cm_.xds_grpc_context_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); cm_.expectAddWithWarming("cluster4", "2"); cm_.expectWarmingClusterCount(); EXPECT_CALL(initialized_, ready()); cm_.expectWarmingClusterCount(2); - EXPECT_CALL(cm_.xds_grpc_context_, resume(Config::TypeUrl::get().Cluster)).Times(1); - EXPECT_CALL(cm_.xds_grpc_context_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); EXPECT_CALL(*interval_timer_, enableTimer(_)); resetCdsInitializedCb(); cm_.clustersToWarmUp({"cluster4", "cluster3"}); @@ -493,19 +493,19 @@ version_info: '3' )EOF"; // Two clusters updated, first one warmed up before processing of the second one starts. - EXPECT_CALL(cm_.xds_grpc_context_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); cm_.expectAddWithWarming("cluster5", "3", true); cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.xds_grpc_context_, pause(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.xds_grpc_context_, resume(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); cm_.expectAddWithWarming("cluster6", "3"); cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.xds_grpc_context_, pause(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); EXPECT_CALL(initialized_, ready()); cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.xds_grpc_context_, resume(Config::TypeUrl::get().Cluster)).Times(1); - EXPECT_CALL(cm_.xds_grpc_context_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); EXPECT_CALL(*interval_timer_, enableTimer(_)); resetCdsInitializedCb(); cm_.clustersToWarmUp({"cluster6"}); diff --git a/test/mocks/config/BUILD b/test/mocks/config/BUILD index 4ffbef91dbbfb..f23e0337aa505 100644 --- a/test/mocks/config/BUILD +++ b/test/mocks/config/BUILD @@ -15,7 +15,6 @@ envoy_cc_mock( deps = [ "//include/envoy/config:grpc_mux_interface", "//include/envoy/config:subscription_interface", - "//include/envoy/config:xds_grpc_context_interface", "//source/common/config:resources_lib", "//source/common/protobuf:utility_lib", "@envoy_api//envoy/api/v2:cds_cc", diff --git a/test/mocks/config/mocks.cc b/test/mocks/config/mocks.cc index 3dd496a2f6853..8eb9290f306ec 100644 --- a/test/mocks/config/mocks.cc +++ b/test/mocks/config/mocks.cc @@ -13,9 +13,7 @@ MockGrpcMuxWatch::~MockGrpcMuxWatch() { cancel(); } MockGrpcMux::MockGrpcMux() {} MockGrpcMux::~MockGrpcMux() {} -MockXdsGrpcContext::MockXdsGrpcContext() {} -MockXdsGrpcContext::~MockXdsGrpcContext() {} -GrpcMuxWatchPtr MockXdsGrpcContext::subscribe(const std::string& type_url, +GrpcMuxWatchPtr MockGrpcMux::subscribe(const std::string& type_url, const std::set& resources, GrpcMuxCallbacks& callbacks) { return GrpcMuxWatchPtr(subscribe_(type_url, resources, callbacks)); @@ -23,12 +21,6 @@ GrpcMuxWatchPtr MockXdsGrpcContext::subscribe(const std::string& type_url, MockGrpcStreamCallbacks::MockGrpcStreamCallbacks() {} MockGrpcStreamCallbacks::~MockGrpcStreamCallbacks() {} -GrpcMuxWatchPtr MockGrpcMux::subscribe(const std::string& type_url, - const std::set& resources, - GrpcMuxCallbacks& callbacks) { - return GrpcMuxWatchPtr(subscribe_(type_url, resources, callbacks)); -} - MockGrpcMuxCallbacks::MockGrpcMuxCallbacks() { ON_CALL(*this, resourceName(testing::_)) .WillByDefault(testing::Invoke([](const ProtobufWkt::Any& resource) -> std::string { diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index f2f0db8b21584..f5cd3ddc81b19 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -3,7 +3,6 @@ #include "envoy/api/v2/eds.pb.h" #include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" -#include "envoy/config/xds_grpc_context.h" #include "common/config/resources.h" #include "common/protobuf/utility.h" @@ -66,13 +65,6 @@ class MockGrpcMux : public GrpcMux { GrpcMuxCallbacks& callbacks); MOCK_METHOD1(pause, void(const std::string& type_url)); MOCK_METHOD1(resume, void(const std::string& type_url)); -}; - -class MockXdsGrpcContext : public XdsGrpcContext { -public: - MockXdsGrpcContext(); - virtual ~MockXdsGrpcContext(); - MOCK_METHOD5(addSubscription, void(const std::set& resources, const std::string& type_url, SubscriptionCallbacks& callbacks, SubscriptionStats& stats, @@ -81,20 +73,6 @@ class MockXdsGrpcContext : public XdsGrpcContext { void(const std::set& resources, const std::string& type_url)); MOCK_METHOD1(removeSubscription, void(const std::string& type_url)); - MOCK_METHOD1(pause, void(const std::string& type_url)); - MOCK_METHOD1(resume, void(const std::string& type_url)); - - MOCK_METHOD0(drainRequests, void()); - MOCK_METHOD0(handleStreamEstablished, void()); - MOCK_METHOD0(handleEstablishmentFailure, void()); - - // GrpcMux TODO TODO remove - MOCK_METHOD0(start, void()); - MOCK_METHOD3(subscribe_, - GrpcMuxWatch*(const std::string& type_url, const std::set& resources, - GrpcMuxCallbacks& callbacks)); - GrpcMuxWatchPtr subscribe(const std::string& type_url, const std::set& resources, - GrpcMuxCallbacks& callbacks); }; class MockGrpcMuxCallbacks : public GrpcMuxCallbacks { diff --git a/test/mocks/upstream/mocks.cc b/test/mocks/upstream/mocks.cc index 380dcc31d7bea..3116cb87fe586 100644 --- a/test/mocks/upstream/mocks.cc +++ b/test/mocks/upstream/mocks.cc @@ -117,7 +117,7 @@ MockClusterManager::MockClusterManager() { ON_CALL(*this, httpAsyncClientForCluster(_)).WillByDefault(ReturnRef(async_client_)); ON_CALL(*this, httpAsyncClientForCluster(_)).WillByDefault((ReturnRef(async_client_))); ON_CALL(*this, bindConfig()).WillByDefault(ReturnRef(bind_config_)); - ON_CALL(*this, xdsGrpcContext()).WillByDefault(Return(xds_grpc_context_)); + ON_CALL(*this, adsMux()).WillByDefault(Return(ads_mux_)); ON_CALL(*this, grpcAsyncClientManager()).WillByDefault(ReturnRef(async_client_manager_)); ON_CALL(*this, localClusterName()).WillByDefault((ReturnRef(local_cluster_name_))); diff --git a/test/mocks/upstream/mocks.h b/test/mocks/upstream/mocks.h index b34bb822085e0..56f412120d461 100644 --- a/test/mocks/upstream/mocks.h +++ b/test/mocks/upstream/mocks.h @@ -304,7 +304,7 @@ class MockClusterManager : public ClusterManager { MOCK_METHOD1(removeCluster, bool(const std::string& cluster)); MOCK_METHOD0(shutdown, void()); MOCK_CONST_METHOD0(bindConfig, const envoy::api::v2::core::BindConfig&()); - MOCK_METHOD0(xdsGrpcContext, std::shared_ptr()); + MOCK_METHOD0(adsMux, std::shared_ptr()); MOCK_METHOD0(grpcAsyncClientManager, Grpc::AsyncClientManager&()); MOCK_CONST_METHOD0(versionInfo, const std::string()); MOCK_CONST_METHOD0(localClusterName, const std::string&()); @@ -317,7 +317,7 @@ class MockClusterManager : public ClusterManager { NiceMock tcp_conn_pool_; NiceMock thread_local_cluster_; envoy::api::v2::core::BindConfig bind_config_; - std::shared_ptr> xds_grpc_context_; + std::shared_ptr> ads_mux_; NiceMock async_client_manager_; std::string local_cluster_name_; NiceMock cluster_manager_factory_; From 67ccc26066ea49998a9bd9c27c064ca20b30e69d Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 10 May 2019 16:36:49 -0400 Subject: [PATCH 11/56] tests partially passing Signed-off-by: Fred Douglas --- include/envoy/config/subscription_state.h | 13 ++- .../common/config/delta_subscription_impl.cc | 32 +++--- .../common/config/delta_subscription_impl.h | 10 +- .../common/config/delta_subscription_state.cc | 2 +- .../common/config/delta_subscription_state.h | 15 --- source/common/config/grpc_delta_xds_context.h | 102 +++++++----------- source/common/config/grpc_mux_impl.h | 3 +- .../config/grpc_mux_subscription_impl.cc | 7 +- .../config/grpc_mux_subscription_impl.h | 4 +- source/common/config/grpc_subscription_impl.h | 2 +- source/server/lds_api.cc | 3 +- .../config/delta_subscription_impl_test.cc | 9 +- .../config/delta_subscription_state_test.cc | 4 +- .../config/delta_subscription_test_harness.h | 23 ++-- test/common/config/grpc_mux_impl_test.cc | 2 +- .../config/grpc_subscription_impl_test.cc | 4 +- test/mocks/config/mocks.cc | 4 +- 17 files changed, 105 insertions(+), 134 deletions(-) diff --git a/include/envoy/config/subscription_state.h b/include/envoy/config/subscription_state.h index 739db89849ff6..78352248f7903 100644 --- a/include/envoy/config/subscription_state.h +++ b/include/envoy/config/subscription_state.h @@ -41,16 +41,21 @@ class SubscriptionState { void handleEstablishmentFailure() PURE; - // Will have been static_cast from GrpcStream's RequestProto type, and should be static_cast back before handing to GrpcStream::sendMessage. + // Will have been static_cast from GrpcStream's RequestProto type, and should be static_cast back + // before handing to GrpcStream::sendMessage. Protobuf::Message getNextRequestAckless() PURE; - // Will have been static_cast from GrpcStream's RequestProto type, and should be static_cast back before handing to GrpcStream::sendMessage. + // Will have been static_cast from GrpcStream's RequestProto type, and should be static_cast back + // before handing to GrpcStream::sendMessage. Protobuf::Message getNextRequestWithAck(const UpdateAck& ack) PURE; }; class SubscriptionStateFactory { public: - SubscriptionState makeSubscriptionState(const std::string& type_url, const std::set& resource_names, - SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout, SubscriptionStats& stats) PURE; + SubscriptionState makeSubscriptionState(const std::string& type_url, + const std::set& resource_names, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout, + SubscriptionStats& stats) PURE; }; } // namespace Config diff --git a/source/common/config/delta_subscription_impl.cc b/source/common/config/delta_subscription_impl.cc index 44a9a536f0b20..190838797fc9b 100644 --- a/source/common/config/delta_subscription_impl.cc +++ b/source/common/config/delta_subscription_impl.cc @@ -3,27 +3,27 @@ namespace Envoy { namespace Config { -DeltaSubscriptionImpl::DeltaSubscriptionImpl(std::shared_ptr context, absl::string_view type_url, - SubscriptionStats stats, std::chrono::milliseconds init_fetch_timeout) - : context_(context), type_url_(type_url), stats_(stats), - init_fetch_timeout_(init_fetch_timeout) {} +DeltaSubscriptionImpl::DeltaSubscriptionImpl(std::shared_ptr context, + absl::string_view type_url, SubscriptionStats stats, + std::chrono::milliseconds init_fetch_timeout) + : context_(context), type_url_(type_url), stats_(stats), + init_fetch_timeout_(init_fetch_timeout) {} - DeltaSubscriptionImpl::~DeltaSubscriptionImpl() { - context_->removeSubscription(type_url_); - } +DeltaSubscriptionImpl::~DeltaSubscriptionImpl() { context_->removeSubscription(type_url_); } - void DeltaSubscriptionImpl::pause() { context_->pause(type_url_); } +void DeltaSubscriptionImpl::pause() { context_->pause(type_url_); } - void DeltaSubscriptionImpl::resume() { context_->resume(type_url_); } +void DeltaSubscriptionImpl::resume() { context_->resume(type_url_); } - // Config::DeltaSubscription - void DeltaSubscriptionImpl::start(const std::set& resources, SubscriptionCallbacks& callbacks) override { - context_->addSubscription(resources, type_url_, callbacks, stats_, init_fetch_timeout_); - } +// Config::DeltaSubscription +void DeltaSubscriptionImpl::start(const std::set& resources, + SubscriptionCallbacks& callbacks) { + context_->addSubscription(resources, type_url_, callbacks, stats_, init_fetch_timeout_); +} - void DeltaSubscriptionImpl::updateResources(const std::set& update_to_these_names) override { - context_->updateResources(update_to_these_names, type_url_); - } +void DeltaSubscriptionImpl::updateResources(const std::set& update_to_these_names) { + context_->updateResources(update_to_these_names, type_url_); +} } // namespace Config } // namespace Envoy diff --git a/source/common/config/delta_subscription_impl.h b/source/common/config/delta_subscription_impl.h index caddd9e332178..a4ba62f958694 100644 --- a/source/common/config/delta_subscription_impl.h +++ b/source/common/config/delta_subscription_impl.h @@ -1,8 +1,8 @@ #pragma once #include "envoy/config/subscription.h" -#include "envoy/config/grpc_mux.h" +#include "common/config/grpc_delta_xds_context.h" #include "common/config/utility.h" namespace Envoy { @@ -18,10 +18,10 @@ namespace Config { // TODO(fredlas) someday this class will be named SubscriptionImpl (without any changes to its code) class DeltaSubscriptionImpl : public Subscription { public: - DeltaSubscriptionImpl(std::shared_ptr context, absl::string_view type_url, + DeltaSubscriptionImpl(std::shared_ptr context, absl::string_view type_url, SubscriptionStats stats, std::chrono::milliseconds init_fetch_timeout); ~DeltaSubscriptionImpl(); - + void pause(); void resume(); @@ -30,8 +30,10 @@ class DeltaSubscriptionImpl : public Subscription { void start(const std::set& resources, SubscriptionCallbacks& callbacks) override; void updateResources(const std::set& update_to_these_names) override; + std::shared_ptr getContextForTest() { return context_; } + private: - std::shared_ptr context_; + std::shared_ptr context_; const std::string type_url_; SubscriptionStats stats_; const std::chrono::milliseconds init_fetch_timeout_; diff --git a/source/common/config/delta_subscription_state.cc b/source/common/config/delta_subscription_state.cc index e8699d5c27956..225a3ce0e81b2 100644 --- a/source/common/config/delta_subscription_state.cc +++ b/source/common/config/delta_subscription_state.cc @@ -1,7 +1,7 @@ #include "common/config/delta_subscription_state.h" -#include "common/common/hash.h" #include "common/common/assert.h" +#include "common/common/hash.h" namespace Envoy { namespace Config { diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index d39960e4704ad..3c252cd73610c 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -105,20 +105,5 @@ class DeltaSubscriptionState : public Logger::Loggable { SubscriptionStats& stats_; }; -class DeltaSubscriptionStateFactory : public SubscriptionStateFactory { -public: - DeltaSubscriptionStateFactory(Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info) - : dispatcher_(dispatcher), local_info_(local_info) {} - - SubscriptionState makeSubscriptionState(const std::string& type_url, const std::set& resource_names, - SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout, SubscriptionStats& stats) override { - return DeltaSubscriptionState(type_url, resources, callbacks, local_info_, - init_fetch_timeout, dispatcher_, stats); - } -private: - Event::Dispatcher& dispatcher_; - const LocalInfo::LocalInfo& local_info_; -}; - } // namespace Config } // namespace Envoy diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h index 248e44caad097..346abf272f10b 100644 --- a/source/common/config/grpc_delta_xds_context.h +++ b/source/common/config/grpc_delta_xds_context.h @@ -48,18 +48,27 @@ namespace Config { // for both non- and aggregated: if non-aggregated, you'll be the only holder of that shared_ptr. By // those two mechanisms, the single class (TODO rename) DeltaSubscriptionImpl handles all 4 // delta/SotW and non-/aggregated combinations. -class GrpcXdsContext : public GrpcMux, +class GrpcDeltaXdsContext : public GrpcMux, public GrpcStreamCallbacks, Logger::Loggable { public: + GrpcDeltaXdsContext(Grpc::AsyncClientPtr async_client, Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, + Runtime::RandomGenerator& random, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info) + : dispatcher_(dispatcher), local_info_(local_info), + grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, + rate_limit_settings) {} + void addSubscription(const std::set& resources, const std::string& type_url, SubscriptionCallbacks& callbacks, SubscriptionStats& stats, std::chrono::milliseconds init_fetch_timeout) override { - subscriptions_.emplace(std::make_pair( - type_url, subscription_factory_.makeSubscriptionState(type_url, resources, callbacks, local_info_, - init_fetch_timeout, dispatcher_, stats))); + subscriptions_.emplace( + std::make_pair(type_url, DeltaSubscriptionState(type_url, resources, callbacks, local_info_, + init_fetch_timeout, dispatcher_, stats))); subscription_ordering_.emplace_back(type_url); - establishStream(); // (idempotent) + grpc_stream_.establishStream(); // (idempotent) } // Enqueues and attempts to send a discovery request, (un)subscribing to resources missing from / @@ -77,7 +86,7 @@ class GrpcXdsContext : public GrpcMux, } void removeSubscription(const std::string& type_url) override { - subscriptions_.erase(type_url); + subscriptions_.erase(type_url); // And remove from the subscription_ordering_ list. auto it = subscription_ordering_.begin(); while (it != subscription_ordering_.end()) { @@ -143,26 +152,15 @@ class GrpcXdsContext : public GrpcMux, trySendDiscoveryRequests(); } - // TODO TODO remove, GrpcMux impersonation. Basically combination of addSubscription and updateResources. - virtual GrpcMuxWatchPtr subscribe(const std::string&, - const std::set& , - GrpcMuxCallbacks& ) override { -// don't need any implementation here. only grpc_mux_subscription_impl ever calls it, and there would never be a GrpcDeltaXdsContext held by one of those. + // TODO TODO remove, GrpcMux impersonation. Basically combination of addSubscription and + // updateResources. + virtual GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, + GrpcMuxCallbacks&) override { + // don't need any implementation here. only grpc_mux_subscription_impl ever calls it, and there + // would never be a GrpcDeltaXdsContext held by one of those. NOT_IMPLEMENTED_GCOVR_EXCL_LINE; return nullptr; } - -protected: - GrpcXdsContext(Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info) - : dispatcher_(dispatcher), local_info_(local_info) {} - - // Everything related to GrpcStream must remain abstract. GrpcStream (and the gRPC-using classes that underlie it) are templated on protobufs. That means that a single implementation that supports different types of protobufs cannot use polymorphism to share code. The workaround: the GrpcStream will be owned by a derived class, and all code that would touch grpc_stream_ is seen here in the base class as abstract. - virtual void establishStream() PURE; - virtual void sendDiscoveryRequestWithAck(SubscriptionState* sub, UpdateAck ack) PURE; - virtual void sendDiscoveryRequestAckless(SubscriptionState* sub) PURE; - virtual void maybeUpdateQueueSizeStat(uint64_t size) PURE; - virtual bool grpcStreamAvailable() const PURE; - virtual bool checkRateLimitAllowsDrain() PURE; private: void trySendDiscoveryRequests() { @@ -179,7 +177,9 @@ class GrpcXdsContext : public GrpcMux, if (sub == subscriptions_.end()) { ENVOY_LOG(warn, "Not sending discovery request for non-existent subscription {}.", next_request_type_url); - // It's safe to assume the front of the ACK queue is of this type, because that's the only way whoWantsToSendDiscoveryRequest() could return something for a non-existent subscription. + // It's safe to assume the front of the ACK queue is of this type, because that's the only + // way whoWantsToSendDiscoveryRequest() could return something for a non-existent + // subscription. ack_queue_.pop(); continue; } @@ -189,14 +189,15 @@ class GrpcXdsContext : public GrpcMux, } // Get our subscription state to generate the appropriate DeltaDiscoveryRequest, and send. if (!ack_queue_.empty()) { - // Because ACKs take precedence over plain requests, if there is anything in the queue, it's safe to assume it's what we want to send here. - sendDiscoveryRequestWithAck(&sub->second, ack_queue_.front()); + // Because ACKs take precedence over plain requests, if there is anything in the queue, it's + // safe to assume it's what we want to send here. + grpc_stream_.sendMessage(sub->second.getNextRequestWithAck(ack_queue_.front())); ack_queue_.pop(); } else { - sendDiscoveryRequestAckless(&sub->second); + grpc_stream_.sendMessage(sub->second.getNextRequestAckless()); } } - maybeUpdateQueueSizeStat(ack_queue_.size()); + grpc_stream_.maybeUpdateQueueSizeStat(ack_queue_.size()); } // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check @@ -205,10 +206,10 @@ class GrpcXdsContext : public GrpcMux, if (sub->paused()) { ENVOY_LOG(trace, "API {} paused; discovery request on hold for now.", type_url); return false; - } else if (!grpcStreamAvailable()) { + } else if (!grpc_stream_.grpcStreamAvailable()) { ENVOY_LOG(trace, "No stream available to send a discovery request for {}.", type_url); return false; - } else if (!checkRateLimitAllowsDrain()) { + } else if (!grpc_stream_.checkRateLimitAllowsDrain()) { ENVOY_LOG(trace, "{} discovery request hit rate limit; will try later.", type_url); return false; } @@ -225,7 +226,8 @@ class GrpcXdsContext : public GrpcMux, if (!ack_queue_.empty()) { return ack_queue_.front().type_url_; } - // If we're looking to send multiple non-ACK requests, send them in the order that their subscriptions were initiated. + // If we're looking to send multiple non-ACK requests, send them in the order that their + // subscriptions were initiated. for (const auto& sub_type : subscription_ordering_) { auto sub = subscriptions_.find(sub_type); if (sub == subscriptions_.end()) { @@ -247,43 +249,11 @@ class GrpcXdsContext : public GrpcMux, // Map from type_url strings to a DeltaSubscriptionState for that type. std::unordered_map subscriptions_; - SubscriptionStateFactory subscription_factory_; - - // Determines the order of initial discovery requests. (Assumes that subscriptions are added in the order of Envoy's dependency ordering). - std::list subscription_ordering_; -}; -class GrpcDeltaXdsContext : public GrpcXdsContext { -public: - GrpcDeltaXdsContext(Grpc::AsyncClientPtr async_client, Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, - Runtime::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, - const LocalInfo::LocalInfo& local_info) - : GrpcXdsContext(dispatcher, local_info), - grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, - rate_limit_settings) {} + // Determines the order of initial discovery requests. (Assumes that subscriptions are added in + // the order of Envoy's dependency ordering). + std::list subscription_ordering_; -protected: - void establishStream() override { - grpc_stream_.establishStream(); - } - void sendDiscoveryRequestWithAck(SubscriptionState* sub, UpdateAck ack) override { - grpc_stream_.sendMessage(static_cast(sub)->getNextRequestWithAck(ack)); - } - void sendDiscoveryRequestAckless(SubscriptionState* sub) override { - grpc_stream_.sendMessage(static_cast(sub)->getNextRequestAckless()); - } - void maybeUpdateQueueSizeStat(uint64_t size) override { - grpc_stream_.maybeUpdateQueueSizeStat(size); - } - bool grpcStreamAvailable() const override { - return grpc_stream_.grpcStreamAvailable(); - } - bool checkRateLimitAllowsDrain() override { - return grpc_stream_.checkRateLimitAllowsDrain(); - } -private: GrpcStream grpc_stream_; }; diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index 646fd83ef1bd5..54e8a7bf3df3e 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -117,8 +117,7 @@ class GrpcMuxImpl : public GrpcMux, std::queue request_queue_; }; -class NullGrpcMuxImpl : public GrpcMux, - GrpcStreamCallbacks { +class NullGrpcMuxImpl : public GrpcMux, GrpcStreamCallbacks { public: GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, GrpcMuxCallbacks&) override { diff --git a/source/common/config/grpc_mux_subscription_impl.cc b/source/common/config/grpc_mux_subscription_impl.cc index 5b7ffe2d084d0..4bfca2126ee82 100644 --- a/source/common/config/grpc_mux_subscription_impl.cc +++ b/source/common/config/grpc_mux_subscription_impl.cc @@ -9,7 +9,8 @@ namespace Envoy { namespace Config { -GrpcMuxSubscriptionImpl::GrpcMuxSubscriptionImpl(GrpcMux& grpc_mux, SubscriptionStats stats, +GrpcMuxSubscriptionImpl::GrpcMuxSubscriptionImpl(std::shared_ptr grpc_mux, + SubscriptionStats stats, absl::string_view type_url, Event::Dispatcher& dispatcher, std::chrono::milliseconds init_fetch_timeout) @@ -29,7 +30,7 @@ void GrpcMuxSubscriptionImpl::start(const std::set& resources, init_fetch_timeout_timer_->enableTimer(init_fetch_timeout_); } - watch_ = grpc_mux_.subscribe(type_url_, resources, *this); + watch_ = grpc_mux_->subscribe(type_url_, resources, *this); // The attempt stat here is maintained for the purposes of having consistency between ADS and // gRPC/filesystem/REST Subscriptions. Since ADS is push based and muxed, the notion of an // "attempt" for a given xDS API combined by ADS is not really that meaningful. @@ -41,7 +42,7 @@ void GrpcMuxSubscriptionImpl::updateResources(const std::set& updat // previously watched resources and the new ones (we may have lost interest in some of the // previously watched ones). watch_.reset(); - watch_ = grpc_mux_.subscribe(type_url_, update_to_these_names, *this); + watch_ = grpc_mux_->subscribe(type_url_, update_to_these_names, *this); stats_.update_attempt_.inc(); } diff --git a/source/common/config/grpc_mux_subscription_impl.h b/source/common/config/grpc_mux_subscription_impl.h index aa0098a6df25d..469a481c28676 100644 --- a/source/common/config/grpc_mux_subscription_impl.h +++ b/source/common/config/grpc_mux_subscription_impl.h @@ -17,8 +17,8 @@ class GrpcMuxSubscriptionImpl : public Subscription, GrpcMuxCallbacks, Logger::Loggable { public: - GrpcMuxSubscriptionImpl(GrpcMux& grpc_mux, SubscriptionStats stats, absl::string_view type_url, - Event::Dispatcher& dispatcher, + GrpcMuxSubscriptionImpl(std::shared_ptr grpc_mux, SubscriptionStats stats, + absl::string_view type_url, Event::Dispatcher& dispatcher, std::chrono::milliseconds init_fetch_timeout); // Config::Subscription diff --git a/source/common/config/grpc_subscription_impl.h b/source/common/config/grpc_subscription_impl.h index a8d75d2942ceb..0cec98487289d 100644 --- a/source/common/config/grpc_subscription_impl.h +++ b/source/common/config/grpc_subscription_impl.h @@ -36,7 +36,7 @@ class GrpcSubscriptionImpl : public Config::Subscription { grpc_mux_subscription_.updateResources(update_to_these_names); } - std::shared_ptr grpcMux() { return grpc_mux_; } + std::shared_ptr grpcMux() { return grpc_mux_; } private: std::shared_ptr grpc_mux_; diff --git a/source/server/lds_api.cc b/source/server/lds_api.cc index 444032d1218b1..2ecb7c9253aaf 100644 --- a/source/server/lds_api.cc +++ b/source/server/lds_api.cc @@ -34,8 +34,7 @@ LdsApiImpl::LdsApiImpl(const envoy::api::v2::core::ConfigSource& lds_config, void LdsApiImpl::onConfigUpdate(const Protobuf::RepeatedPtrField& resources, const std::string& version_info) { cm_.adsMux()->pause(Config::TypeUrl::get().RouteConfiguration); - Cleanup rds_resume( - [this] { cm_.adsMux()->resume(Config::TypeUrl::get().RouteConfiguration); }); + Cleanup rds_resume([this] { cm_.adsMux()->resume(Config::TypeUrl::get().RouteConfiguration); }); std::vector listeners; for (const auto& listener_blob : resources) { diff --git a/test/common/config/delta_subscription_impl_test.cc b/test/common/config/delta_subscription_impl_test.cc index 2f8577a393c19..54dcb03ebaa05 100644 --- a/test/common/config/delta_subscription_impl_test.cc +++ b/test/common/config/delta_subscription_impl_test.cc @@ -62,8 +62,9 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { resource->set_version("version1A"); const std::string nonce = std::to_string(HashUtil::xxHash64("version1A")); message->set_nonce(nonce); + message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); nonce_acks_required_.push(nonce); - subscription_->onDiscoveryResponse(std::move(message)); + subscription_->getContextForTest()->onDiscoveryResponse(std::move(message)); } // The server gives us our first version of resource name2. // subscription_ now wants to ACK name1 and then name2 (but can't due to pause). @@ -74,8 +75,9 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { resource->set_version("version2A"); const std::string nonce = std::to_string(HashUtil::xxHash64("version2A")); message->set_nonce(nonce); + message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); nonce_acks_required_.push(nonce); - subscription_->onDiscoveryResponse(std::move(message)); + subscription_->getContextForTest()->onDiscoveryResponse(std::move(message)); } // The server gives us an updated version of resource name1. // subscription_ now wants to ACK name1A, then name2, then name1B (but can't due to pause). @@ -86,8 +88,9 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { resource->set_version("version1B"); const std::string nonce = std::to_string(HashUtil::xxHash64("version1B")); message->set_nonce(nonce); + message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); nonce_acks_required_.push(nonce); - subscription_->onDiscoveryResponse(std::move(message)); + subscription_->getContextForTest()->onDiscoveryResponse(std::move(message)); } // All ACK sendMessage()s will happen upon calling resume(). EXPECT_CALL(async_stream_, sendMessage(_, _)) diff --git a/test/common/config/delta_subscription_state_test.cc b/test/common/config/delta_subscription_state_test.cc index 3ca88b8fb39fc..3b017789781c3 100644 --- a/test/common/config/delta_subscription_state_test.cc +++ b/test/common/config/delta_subscription_state_test.cc @@ -231,7 +231,7 @@ TEST_F(DeltaSubscriptionStateTest, ResourceGoneLeadsToBlankInitialVersion) { *remove2.Add() = "name2"; deliverDiscoveryResponse(add1_3, remove2, "debugversion2"); state_.markStreamFresh(); // simulate a stream reconnection - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAcklessAckless(); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_EQ("version1B", cur_request.initial_resource_versions().at("name1")); EXPECT_EQ(cur_request.initial_resource_versions().end(), cur_request.initial_resource_versions().find("name2")); @@ -245,7 +245,7 @@ TEST_F(DeltaSubscriptionStateTest, ResourceGoneLeadsToBlankInitialVersion) { *remove1_3.Add() = "name3"; deliverDiscoveryResponse({}, remove1_3, "debugversion3"); state_.markStreamFresh(); // simulate a stream reconnection - envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAcklessAckless(); + envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_TRUE(cur_request.initial_resource_versions().empty()); } diff --git a/test/common/config/delta_subscription_test_harness.h b/test/common/config/delta_subscription_test_harness.h index 61d40bb2f83a7..4e5473c907a54 100644 --- a/test/common/config/delta_subscription_test_harness.h +++ b/test/common/config/delta_subscription_test_harness.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "common/config/delta_subscription_impl.h" #include "test/common/config/subscription_test_harness.h" @@ -32,14 +34,20 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { EXPECT_CALL(local_info_, node()).WillRepeatedly(testing::ReturnRef(node_)); EXPECT_CALL(dispatcher_, createTimer_(_)); subscription_ = std::make_unique( - local_info_, std::unique_ptr(async_client_), dispatcher_, - *method_descriptor_, Config::TypeUrl::get().ClusterLoadAssignment, random_, stats_store_, - rate_limit_settings_, stats_, init_fetch_timeout); + std::make_shared(std::unique_ptr(async_client_), + dispatcher_, *method_descriptor_, random_, + stats_store_, rate_limit_settings_, local_info_), + Config::TypeUrl::get().ClusterLoadAssignment, stats_, init_fetch_timeout); } ~DeltaSubscriptionTestHarness() { while (!nonce_acks_required_.empty()) { - EXPECT_FALSE(nonce_acks_sent_.empty()); + if (nonce_acks_sent_.empty()) { + // It's not enough to EXPECT_FALSE(nonce_acks_sent_.empty()), we need to skip the following + // EXPECT_EQ, otherwise the undefined .front() can get pretty bad. + EXPECT_FALSE(nonce_acks_sent_.empty()); + break; + } EXPECT_EQ(nonce_acks_required_.front(), nonce_acks_sent_.front()); nonce_acks_required_.pop(); nonce_acks_sent_.pop(); @@ -100,12 +108,11 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { void deliverConfigUpdate(const std::vector& cluster_names, const std::string& version, bool accept) override { - std::unique_ptr response( - new envoy::api::v2::DeltaDiscoveryResponse()); - + auto response = std::make_unique(); last_response_nonce_ = std::to_string(HashUtil::xxHash64(version)); response->set_nonce(last_response_nonce_); response->set_system_version_info(version); + response->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); Protobuf::RepeatedPtrField typed_resources; for (const auto& cluster : cluster_names) { @@ -127,7 +134,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); expectSendMessage({}, {}, Grpc::Status::GrpcStatus::Internal, "bad config", {}); } - subscription_->onDiscoveryResponse(std::move(response)); + subscription_->getContextForTest()->onDiscoveryResponse(std::move(response)); Mock::VerifyAndClearExpectations(&async_stream_); } diff --git a/test/common/config/grpc_mux_impl_test.cc b/test/common/config/grpc_mux_impl_test.cc index 864cbcb9b07e7..0c234c20a6b1f 100644 --- a/test/common/config/grpc_mux_impl_test.cc +++ b/test/common/config/grpc_mux_impl_test.cc @@ -82,7 +82,7 @@ class GrpcMuxImplTestBase : public testing::Test { } NiceMock dispatcher_; - Runtime::MockRandomGenerator random_; + NiceMock random_; Grpc::MockAsyncClient* async_client_; Grpc::MockAsyncStream async_stream_; std::unique_ptr grpc_mux_; diff --git a/test/common/config/grpc_subscription_impl_test.cc b/test/common/config/grpc_subscription_impl_test.cc index 490a74c228e39..75f8d3cfa8023 100644 --- a/test/common/config/grpc_subscription_impl_test.cc +++ b/test/common/config/grpc_subscription_impl_test.cc @@ -40,8 +40,8 @@ TEST_F(GrpcSubscriptionImplTest, RemoteStreamClose) { EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); EXPECT_CALL(*timer_, enableTimer(_)); EXPECT_CALL(random_, random()); - subscription_->grpcMux().grpcStreamForTest().onRemoteClose(Grpc::Status::GrpcStatus::Canceled, - ""); + subscription_->grpcMux()->grpcStreamForTest().onRemoteClose(Grpc::Status::GrpcStatus::Canceled, + ""); verifyStats(2, 0, 0, 1, 0); verifyControlPlaneStats(0); diff --git a/test/mocks/config/mocks.cc b/test/mocks/config/mocks.cc index 40fe7ac6a8bf3..50e21f3ecf171 100644 --- a/test/mocks/config/mocks.cc +++ b/test/mocks/config/mocks.cc @@ -14,8 +14,8 @@ MockGrpcMux::MockGrpcMux() {} MockGrpcMux::~MockGrpcMux() {} GrpcMuxWatchPtr MockGrpcMux::subscribe(const std::string& type_url, - const std::set& resources, - GrpcMuxCallbacks& callbacks) { + const std::set& resources, + GrpcMuxCallbacks& callbacks) { return GrpcMuxWatchPtr(subscribe_(type_url, resources, callbacks)); } MockGrpcStreamCallbacks::MockGrpcStreamCallbacks() {} From d24744c3c4a362f17aade950d014c50a0dd57e32 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 10 May 2019 16:54:50 -0400 Subject: [PATCH 12/56] revert some test changes Signed-off-by: Fred Douglas --- include/envoy/config/grpc_mux.h | 1 + source/common/config/grpc_delta_xds_context.h | 9 +++++++-- source/common/config/grpc_mux_impl.cc | 2 ++ source/common/config/grpc_mux_impl.h | 1 + test/common/config/grpc_mux_impl_test.cc | 13 +++++++++++++ test/integration/cds_integration_test.cc | 8 ++++++++ 6 files changed, 32 insertions(+), 2 deletions(-) diff --git a/include/envoy/config/grpc_mux.h b/include/envoy/config/grpc_mux.h index e1a6a3032ed55..aa713b8d358c9 100644 --- a/include/envoy/config/grpc_mux.h +++ b/include/envoy/config/grpc_mux.h @@ -149,6 +149,7 @@ class GrpcMux { virtual void resume(const std::string& type_url) PURE; // TODO(fredlas) remove, only here for compatibility with old-style GrpcMuxImpl. + virtual void start() PURE; virtual GrpcMuxWatchPtr subscribe(const std::string& type_url, const std::set& resources, GrpcMuxCallbacks& callbacks) PURE; diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h index 346abf272f10b..978aad56db945 100644 --- a/source/common/config/grpc_delta_xds_context.h +++ b/source/common/config/grpc_delta_xds_context.h @@ -154,13 +154,18 @@ class GrpcDeltaXdsContext : public GrpcMux, // TODO TODO remove, GrpcMux impersonation. Basically combination of addSubscription and // updateResources. - virtual GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, - GrpcMuxCallbacks&) override { + GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, + GrpcMuxCallbacks&) override { // don't need any implementation here. only grpc_mux_subscription_impl ever calls it, and there // would never be a GrpcDeltaXdsContext held by one of those. NOT_IMPLEMENTED_GCOVR_EXCL_LINE; return nullptr; } + void start() override { + // don't need any implementation here. only grpc_mux_subscription_impl ever calls it, and there + // would never be a GrpcDeltaXdsContext held by one of those. + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } private: void trySendDiscoveryRequests() { diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index 45d1dc201fce2..094167e31b7c5 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -28,6 +28,8 @@ GrpcMuxImpl::~GrpcMuxImpl() { } } +void GrpcMuxImpl::start() { grpc_stream_.establishStream(); } + void GrpcMuxImpl::sendDiscoveryRequest(const std::string& type_url) { if (!grpc_stream_.grpcStreamAvailable()) { ENVOY_LOG(debug, "No stream available to sendDiscoveryRequest for {}", type_url); diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index 54e8a7bf3df3e..114b7d3b82877 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -30,6 +30,7 @@ class GrpcMuxImpl : public GrpcMux, const RateLimitSettings& rate_limit_settings); ~GrpcMuxImpl(); + void start() override; GrpcMuxWatchPtr subscribe(const std::string& type_url, const std::set& resources, GrpcMuxCallbacks& callbacks) override; diff --git a/test/common/config/grpc_mux_impl_test.cc b/test/common/config/grpc_mux_impl_test.cc index 0c234c20a6b1f..d33237e33b569 100644 --- a/test/common/config/grpc_mux_impl_test.cc +++ b/test/common/config/grpc_mux_impl_test.cc @@ -107,6 +107,7 @@ TEST_F(GrpcMuxImplTest, MultipleTypeUrlStreams) { EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); expectSendMessage("foo", {"x", "y"}, ""); expectSendMessage("bar", {}, ""); + grpc_mux_->start(); EXPECT_EQ(1, stats_.gauge("control_plane.connected_state").value()); expectSendMessage("bar", {"z"}, ""); auto bar_z_sub = grpc_mux_->subscribe("bar", {"z"}, callbacks_); @@ -138,6 +139,7 @@ TEST_F(GrpcMuxImplTest, ResetStream) { expectSendMessage("foo", {"x", "y"}, ""); expectSendMessage("bar", {}, ""); expectSendMessage("baz", {"z"}, ""); + grpc_mux_->start(); EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)).Times(3); EXPECT_CALL(random_, random()); @@ -146,6 +148,7 @@ TEST_F(GrpcMuxImplTest, ResetStream) { grpc_mux_->grpcStreamForTest().onRemoteClose(Grpc::Status::GrpcStatus::Canceled, ""); EXPECT_EQ(0, stats_.gauge("control_plane.connected_state").value()); EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); + grpc_mux_->start(); expectSendMessage("foo", {"x", "y"}, ""); expectSendMessage("bar", {}, ""); expectSendMessage("baz", {"z"}, ""); @@ -186,6 +189,7 @@ TEST_F(GrpcMuxImplTest, TypeUrlMismatch) { EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); expectSendMessage("foo", {"x", "y"}, ""); + grpc_mux_->start(); { std::unique_ptr response( @@ -219,6 +223,7 @@ TEST_F(GrpcMuxImplTest, WildcardWatch) { auto foo_sub = grpc_mux_->subscribe(type_url, {}, callbacks_); EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); expectSendMessage(type_url, {}, ""); + grpc_mux_->start(); { std::unique_ptr response( @@ -254,6 +259,7 @@ TEST_F(GrpcMuxImplTest, WatchDemux) { EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); // Should dedupe the "x" resource. expectSendMessage(type_url, {"y", "z", "x"}, ""); + grpc_mux_->start(); { std::unique_ptr response( @@ -331,6 +337,7 @@ TEST_F(GrpcMuxImplTest, MultipleWatcherWithEmptyUpdates) { EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); expectSendMessage(type_url, {"x", "y"}, ""); + grpc_mux_->start(); std::unique_ptr response( new envoy::api::v2::DiscoveryResponse()); @@ -353,6 +360,7 @@ TEST_F(GrpcMuxImplTest, SingleWatcherWithEmptyUpdates) { EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); expectSendMessage(type_url, {}, ""); + grpc_mux_->start(); std::unique_ptr response( new envoy::api::v2::DiscoveryResponse()); @@ -406,6 +414,7 @@ TEST_F(GrpcMuxImplTestWithMockTimeSystem, TooManyRequestsWithDefaultSettings) { auto foo_sub = grpc_mux_->subscribe("foo", {"x"}, callbacks_); expectSendMessage("foo", {"x"}, ""); + grpc_mux_->start(); // Exhausts the limit. onReceiveMessage(99); @@ -458,6 +467,7 @@ TEST_F(GrpcMuxImplTestWithMockTimeSystem, TooManyRequestsWithEmptyRateLimitSetti auto foo_sub = grpc_mux_->subscribe("foo", {"x"}, callbacks_); expectSendMessage("foo", {"x"}, ""); + grpc_mux_->start(); // Validate that drain_request_timer is enabled when there are no tokens. EXPECT_CALL(*drain_request_timer, enableTimer(std::chrono::milliseconds(100))); @@ -511,6 +521,7 @@ TEST_F(GrpcMuxImplTest, TooManyRequestsWithCustomRateLimitSettings) { auto foo_sub = grpc_mux_->subscribe("foo", {"x"}, callbacks_); expectSendMessage("foo", {"x"}, ""); + grpc_mux_->start(); // Validate that rate limit is not enforced for 100 requests. onReceiveMessage(100); @@ -538,6 +549,7 @@ TEST_F(GrpcMuxImplTest, UnwatchedTypeAcceptsEmptyResources) { const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; + grpc_mux_->start(); { // subscribe and unsubscribe to simulate a cluster added and removed expectSendMessage(type_url, {"y"}, ""); @@ -571,6 +583,7 @@ TEST_F(GrpcMuxImplTest, UnwatchedTypeRejectsResources) { const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; + grpc_mux_->start(); // subscribe and unsubscribe (by not keeping the return watch) so that the type is known to envoy expectSendMessage(type_url, {"y"}, ""); expectSendMessage(type_url, {}, ""); diff --git a/test/integration/cds_integration_test.cc b/test/integration/cds_integration_test.cc index caa76ccd4836d..9ebce9094c624 100644 --- a/test/integration/cds_integration_test.cc +++ b/test/integration/cds_integration_test.cc @@ -56,6 +56,7 @@ class CdsIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, public H // config that you use! setUpstreamCount(1); // the CDS cluster setUpstreamProtocol(FakeHttpConnection::Type::HTTP2); // CDS uses gRPC uses HTTP2. + // HttpIntegrationTest::initialize() does many things: // 1) It appends to fake_upstreams_ as many as you asked for via setUpstreamCount(). // 2) It updates your bootstrap config with the ports your fake upstreams are actually listening @@ -67,6 +68,7 @@ class CdsIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, public H // However, this test needs to defer all of that to later. defer_listener_finalization_ = true; HttpIntegrationTest::initialize(); + // Create the regular (i.e. not an xDS server) upstreams. We create them manually here after // initialize() because finalize() expects all fake_upstreams_ to correspond to a static // cluster in the bootstrap config - which we don't want since we're testing dynamic CDS! @@ -82,15 +84,19 @@ class CdsIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, public H cluster2_ = ConfigHelper::buildCluster( ClusterName2, fake_upstreams_[UpstreamIndex2]->localAddress()->ip()->port(), Network::Test::getLoopbackAddressString(ipVersion())); + // Let Envoy establish its connection to the CDS server. acceptXdsConnection(); + // Do the initial compareDiscoveryRequest / sendDiscoveryResponse for cluster_1. // Split out into its own function so that DeltaCdsIntegrationTest can override it. giveInitialCluster(); + // We can continue the test once we're sure that Envoy's ClusterManager has made use of // the DiscoveryResponse describing cluster_1 that we sent. // 2 because the statically specified CDS server itself counts as a cluster. test_server_->waitForGaugeGe("cluster_manager.active_clusters", 2); + // Wait for our statically specified listener to become ready, and register its port in the // test framework's downstream listener port map. test_server_->waitUntilListenersReady(); @@ -243,12 +249,14 @@ TEST_P(DeltaCdsIntegrationTest, CdsClusterUpDownUp) { // Calls CdsIntegrationTest::initialize(), which includes establishing a listener, route, and // cluster. testRouterHeaderOnlyRequestAndResponse(nullptr, UpstreamIndex1, "/cluster1"); + // Tell Envoy that cluster_1 is gone. EXPECT_TRUE(compareDeltaDiscoveryRequest(Config::TypeUrl::get().Cluster, {}, {})); sendDeltaDiscoveryResponse({}, {ClusterName1}, "42"); // We can continue the test once we're sure that Envoy's ClusterManager has made use of // the DiscoveryResponse that says cluster_1 is gone. test_server_->waitForCounterGe("cluster_manager.cluster_removed", 1); + // Now that cluster_1 is gone, the listener (with its routing to cluster_1) should 503. BufferingStreamDecoderPtr response = IntegrationUtil::makeSingleRequest( lookupPort("http"), "GET", "/cluster1", "", downstream_protocol_, version_, "foo.com"); From 804459c0e6dc8e7dc1d481a09fe72490402969f2 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Mon, 13 May 2019 11:46:16 -0400 Subject: [PATCH 13/56] revert establishNewStream idempotency Signed-off-by: Fred Douglas --- source/common/config/grpc_delta_xds_context.h | 2 +- source/common/config/grpc_mux_impl.cc | 7 ++++--- source/common/config/grpc_stream.h | 11 +++++------ test/common/config/grpc_mux_impl_test.cc | 3 +-- test/common/config/grpc_stream_test.cc | 12 ++++++------ 5 files changed, 17 insertions(+), 18 deletions(-) diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h index 978aad56db945..c8c8532c9490e 100644 --- a/source/common/config/grpc_delta_xds_context.h +++ b/source/common/config/grpc_delta_xds_context.h @@ -68,7 +68,7 @@ class GrpcDeltaXdsContext : public GrpcMux, std::make_pair(type_url, DeltaSubscriptionState(type_url, resources, callbacks, local_info_, init_fetch_timeout, dispatcher_, stats))); subscription_ordering_.emplace_back(type_url); - grpc_stream_.establishStream(); // (idempotent) + grpc_stream_.establishNewStream(); // (idempotent) TODO TODO does print warning though... } // Enqueues and attempts to send a discovery request, (un)subscribing to resources missing from / diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index 094167e31b7c5..3f82ecae00bcc 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -28,7 +28,7 @@ GrpcMuxImpl::~GrpcMuxImpl() { } } -void GrpcMuxImpl::start() { grpc_stream_.establishStream(); } +void GrpcMuxImpl::start() { grpc_stream_.establishNewStream(); } void GrpcMuxImpl::sendDiscoveryRequest(const std::string& type_url) { if (!grpc_stream_.grpcStreamAvailable()) { @@ -90,8 +90,9 @@ GrpcMuxWatchPtr GrpcMuxImpl::subscribe(const std::string& type_url, // only send a single RDS/EDS update after the CDS/LDS update. queueDiscoveryRequest(type_url); - // This is idempotent, so it's fine to just always call when we have a new subscription. - grpc_stream_.establishStream(); + // TODO TODO well, that takes fiddling with the tests... This is idempotent, so it's fine to just + // always call when we have a new subscription. + // TODO TODO well, that takes fiddling with the tests... grpc_stream_.establishNewStream(); return watch; } diff --git a/source/common/config/grpc_stream.h b/source/common/config/grpc_stream.h index 2cfb73a5021a7..9cb445a486a74 100644 --- a/source/common/config/grpc_stream.h +++ b/source/common/config/grpc_stream.h @@ -27,7 +27,7 @@ class GrpcStream : public Grpc::TypedAsyncStreamCallbacks, service_method_(service_method), control_plane_stats_(generateControlPlaneStats(scope)), random_(random), time_source_(dispatcher.timeSource()), rate_limiting_enabled_(rate_limit_settings.enabled_) { - retry_timer_ = dispatcher.createTimer([this]() -> void { establishStream(); }); + retry_timer_ = dispatcher.createTimer([this]() -> void { establishNewStream(); }); if (rate_limiting_enabled_) { // Default Bucket contains 100 tokens maximum and refills at 10 tokens/sec. limit_request_ = std::make_unique( @@ -38,13 +38,12 @@ class GrpcStream : public Grpc::TypedAsyncStreamCallbacks, RETRY_MAX_DELAY_MS, random_); } - void establishStream() { + void establishNewStream() { + ENVOY_LOG(debug, "Establishing new gRPC bidi stream for {}", service_method_.DebugString()); if (stream_ != nullptr) { - ENVOY_LOG(debug, "gRPC bidi stream for {} already up and running", - service_method_.DebugString()); - return; // idempotent + ENVOY_LOG(warn, "gRPC bidi stream for {} already exists!", service_method_.DebugString()); + return; } - ENVOY_LOG(debug, "Establishing new gRPC bidi stream for {}", service_method_.DebugString()); stream_ = async_client_->start(service_method_, *this); if (stream_ == nullptr) { ENVOY_LOG(warn, "Unable to establish new stream"); diff --git a/test/common/config/grpc_mux_impl_test.cc b/test/common/config/grpc_mux_impl_test.cc index d33237e33b569..3c4680504c498 100644 --- a/test/common/config/grpc_mux_impl_test.cc +++ b/test/common/config/grpc_mux_impl_test.cc @@ -140,7 +140,6 @@ TEST_F(GrpcMuxImplTest, ResetStream) { expectSendMessage("bar", {}, ""); expectSendMessage("baz", {"z"}, ""); grpc_mux_->start(); - EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)).Times(3); EXPECT_CALL(random_, random()); ASSERT_TRUE(timer != nullptr); // initialized from dispatcher mock. @@ -148,7 +147,6 @@ TEST_F(GrpcMuxImplTest, ResetStream) { grpc_mux_->grpcStreamForTest().onRemoteClose(Grpc::Status::GrpcStatus::Canceled, ""); EXPECT_EQ(0, stats_.gauge("control_plane.connected_state").value()); EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); - grpc_mux_->start(); expectSendMessage("foo", {"x", "y"}, ""); expectSendMessage("bar", {}, ""); expectSendMessage("baz", {"z"}, ""); @@ -165,6 +163,7 @@ TEST_F(GrpcMuxImplTest, PauseResume) { auto foo_sub = grpc_mux_->subscribe("foo", {"x", "y"}, callbacks_); grpc_mux_->pause("foo"); EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); + grpc_mux_->start(); expectSendMessage("foo", {"x", "y"}, ""); grpc_mux_->resume("foo"); grpc_mux_->pause("bar"); diff --git a/test/common/config/grpc_stream_test.cc b/test/common/config/grpc_stream_test.cc index 91e50c406cbb1..8dd536e54d638 100644 --- a/test/common/config/grpc_stream_test.cc +++ b/test/common/config/grpc_stream_test.cc @@ -41,7 +41,7 @@ class GrpcStreamTest : public testing::Test { GrpcStream grpc_stream_; }; -// Tests that establishStream() establishes it, a second call does nothing, and a third call +// Tests that establishNewStream() establishes it, a second call does nothing, and a third call // after the stream was disconnected re-establishes it. TEST_F(GrpcStreamTest, EstablishStream) { EXPECT_FALSE(grpc_stream_.grpcStreamAvailable()); @@ -49,14 +49,14 @@ TEST_F(GrpcStreamTest, EstablishStream) { { EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); EXPECT_CALL(callbacks_, onStreamEstablished()); - grpc_stream_.establishStream(); + grpc_stream_.establishNewStream(); EXPECT_TRUE(grpc_stream_.grpcStreamAvailable()); } // Idempotent { EXPECT_CALL(*async_client_, start(_, _)).Times(0); EXPECT_CALL(callbacks_, onStreamEstablished()).Times(0); - grpc_stream_.establishStream(); + grpc_stream_.establishNewStream(); EXPECT_TRUE(grpc_stream_.grpcStreamAvailable()); } grpc_stream_.onRemoteClose(Grpc::Status::GrpcStatus::Ok, ""); @@ -65,7 +65,7 @@ TEST_F(GrpcStreamTest, EstablishStream) { { EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); EXPECT_CALL(callbacks_, onStreamEstablished()); - grpc_stream_.establishStream(); + grpc_stream_.establishNewStream(); EXPECT_TRUE(grpc_stream_.grpcStreamAvailable()); } } @@ -75,7 +75,7 @@ TEST_F(GrpcStreamTest, EstablishStream) { TEST_F(GrpcStreamTest, FailToEstablishNewStream) { EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(nullptr)); EXPECT_CALL(callbacks_, onEstablishmentFailure()); - grpc_stream_.establishStream(); + grpc_stream_.establishNewStream(); EXPECT_FALSE(grpc_stream_.grpcStreamAvailable()); } @@ -83,7 +83,7 @@ TEST_F(GrpcStreamTest, FailToEstablishNewStream) { // machinery. TEST_F(GrpcStreamTest, SendMessage) { EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); - grpc_stream_.establishStream(); + grpc_stream_.establishNewStream(); envoy::api::v2::DiscoveryRequest request; request.set_response_nonce("grpc_stream_test_noncense"); EXPECT_CALL(async_stream_, sendMessage(ProtoEq(request), false)); From 83dfc89586f2b653c65f2600a54d535eba4b71ae Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Wed, 22 May 2019 14:51:59 -0400 Subject: [PATCH 14/56] DISALLOW_COPY_AND_ASSIGN saves lives Signed-off-by: Fred Douglas --- .../common/config/delta_subscription_state.h | 3 ++ source/common/config/grpc_delta_xds_context.h | 28 ++++++++-------- source/common/config/grpc_mux_impl.h | 1 + source/common/config/grpc_subscription_impl.h | 1 + .../common/upstream/cluster_manager_impl.cc | 2 ++ test/common/upstream/cds_api_impl_test.cc | 32 +++++++++---------- 6 files changed, 37 insertions(+), 30 deletions(-) diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index 3c252cd73610c..5d8786ea089da 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -103,6 +103,9 @@ class DeltaSubscriptionState : public Logger::Loggable { std::set names_removed_; SubscriptionStats& stats_; + + DeltaSubscriptionState(const DeltaSubscriptionState&) = delete; + DeltaSubscriptionState& operator=(const DeltaSubscriptionState&) = delete; }; } // namespace Config diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h index c8c8532c9490e..1421ba295a52f 100644 --- a/source/common/config/grpc_delta_xds_context.h +++ b/source/common/config/grpc_delta_xds_context.h @@ -64,9 +64,9 @@ class GrpcDeltaXdsContext : public GrpcMux, void addSubscription(const std::set& resources, const std::string& type_url, SubscriptionCallbacks& callbacks, SubscriptionStats& stats, std::chrono::milliseconds init_fetch_timeout) override { - subscriptions_.emplace( - std::make_pair(type_url, DeltaSubscriptionState(type_url, resources, callbacks, local_info_, - init_fetch_timeout, dispatcher_, stats))); + subscriptions_.emplace(type_url, std::make_unique( + type_url, resources, callbacks, local_info_, + init_fetch_timeout, dispatcher_, stats)); subscription_ordering_.emplace_back(type_url); grpc_stream_.establishNewStream(); // (idempotent) TODO TODO does print warning though... } @@ -80,7 +80,7 @@ class GrpcDeltaXdsContext : public GrpcMux, ENVOY_LOG(warn, "Not updating non-existent subscription {}.", type_url); return; } - sub->second.updateResourceInterest(resources); + sub->second->updateResourceInterest(resources); // Tell the server about our new interests, if there are any. trySendDiscoveryRequests(); } @@ -104,7 +104,7 @@ class GrpcDeltaXdsContext : public GrpcMux, ENVOY_LOG(warn, "Not pausing non-existent subscription {}.", type_url); return; } - sub->second.pause(); + sub->second->pause(); } void resume(const std::string& type_url) override { @@ -113,7 +113,7 @@ class GrpcDeltaXdsContext : public GrpcMux, ENVOY_LOG(warn, "Not resuming non-existent subscription {}.", type_url); return; } - sub->second.resume(); + sub->second->resume(); trySendDiscoveryRequests(); } @@ -129,19 +129,19 @@ class GrpcDeltaXdsContext : public GrpcMux, message->system_version_info(), message->type_url()); return; } - kickOffAck(sub->second.handleResponse(*message)); + kickOffAck(sub->second->handleResponse(*message)); } void onStreamEstablished() override { for (auto& sub : subscriptions_) { - sub.second.markStreamFresh(); + sub.second->markStreamFresh(); } trySendDiscoveryRequests(); } void onEstablishmentFailure() override { for (auto& sub : subscriptions_) { - sub.second.handleEstablishmentFailure(); + sub.second->handleEstablishmentFailure(); } } @@ -189,17 +189,17 @@ class GrpcDeltaXdsContext : public GrpcMux, continue; } // Try again later if paused/rate limited/stream down. - if (!canSendDiscoveryRequest(next_request_type_url, &sub->second)) { + if (!canSendDiscoveryRequest(next_request_type_url, sub->second.get())) { break; } // Get our subscription state to generate the appropriate DeltaDiscoveryRequest, and send. if (!ack_queue_.empty()) { // Because ACKs take precedence over plain requests, if there is anything in the queue, it's // safe to assume it's what we want to send here. - grpc_stream_.sendMessage(sub->second.getNextRequestWithAck(ack_queue_.front())); + grpc_stream_.sendMessage(sub->second->getNextRequestWithAck(ack_queue_.front())); ack_queue_.pop(); } else { - grpc_stream_.sendMessage(sub->second.getNextRequestAckless()); + grpc_stream_.sendMessage(sub->second->getNextRequestAckless()); } } grpc_stream_.maybeUpdateQueueSizeStat(ack_queue_.size()); @@ -238,7 +238,7 @@ class GrpcDeltaXdsContext : public GrpcMux, if (sub == subscriptions_.end()) { continue; } - if (sub->second.subscriptionUpdatePending()) { + if (sub->second->subscriptionUpdatePending()) { return sub->first; } } @@ -253,7 +253,7 @@ class GrpcDeltaXdsContext : public GrpcMux, std::queue ack_queue_; // Map from type_url strings to a DeltaSubscriptionState for that type. - std::unordered_map subscriptions_; + absl::flat_hash_map> subscriptions_; // Determines the order of initial discovery requests. (Assumes that subscriptions are added in // the order of Envoy's dependency ordering). diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index 114b7d3b82877..8014a7e5827bf 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -120,6 +120,7 @@ class GrpcMuxImpl : public GrpcMux, class NullGrpcMuxImpl : public GrpcMux, GrpcStreamCallbacks { public: + void start() override {} GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, GrpcMuxCallbacks&) override { throw EnvoyException("ADS must be configured to support an ADS config source"); diff --git a/source/common/config/grpc_subscription_impl.h b/source/common/config/grpc_subscription_impl.h index 0cec98487289d..2eba99c01446c 100644 --- a/source/common/config/grpc_subscription_impl.h +++ b/source/common/config/grpc_subscription_impl.h @@ -30,6 +30,7 @@ class GrpcSubscriptionImpl : public Config::Subscription { Config::SubscriptionCallbacks& callbacks) override { // Subscribe first, so we get failure callbacks if grpc_mux_.start() fails. grpc_mux_subscription_.start(resources, callbacks); + grpc_mux_->start(); } void updateResources(const std::set& update_to_these_names) override { diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index ed21eac998d05..2d9d64dbd6895 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -302,6 +302,8 @@ ClusterManagerImpl::ClusterManagerImpl( // clusters have already initialized. (E.g., if all static). init_helper_.onStaticLoadComplete(); + ads_mux_->start(); + if (cm_config.has_load_stats_config()) { const auto& load_stats_config = cm_config.load_stats_config(); load_stats_reporter_ = diff --git a/test/common/upstream/cds_api_impl_test.cc b/test/common/upstream/cds_api_impl_test.cc index 83fbd093d8330..ecc08858f6288 100644 --- a/test/common/upstream/cds_api_impl_test.cc +++ b/test/common/upstream/cds_api_impl_test.cc @@ -396,16 +396,16 @@ version_info: '0' )EOF"; // Two clusters updated, both warmed up. - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(*cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); cm_.expectAddWithWarming("cluster1", "0"); cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(*cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); cm_.expectAddWithWarming("cluster2", "0"); cm_.expectWarmingClusterCount(); EXPECT_CALL(initialized_, ready()); cm_.expectWarmingClusterCount(2); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(*cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(*cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); EXPECT_CALL(*interval_timer_, enableTimer(_)); cm_.clustersToWarmUp({"cluster1", "cluster2"}); callbacks_->onSuccess(parseResponseMessageFromYaml(response1_yaml)); @@ -431,15 +431,15 @@ version_info: '1' path: eds path )EOF"; - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(*cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); cm_.expectAddWithWarming("cluster1", "1"); cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(*cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); cm_.expectAddWithWarming("cluster3", "1"); cm_.expectWarmingClusterCount(); EXPECT_CALL(initialized_, ready()); cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(*cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); EXPECT_CALL(*interval_timer_, enableTimer(_)); resetCdsInitializedCb(); cm_.clustersToWarmUp({"cluster1"}); @@ -460,13 +460,13 @@ version_info: '2' path: eds path )EOF"; - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(*cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); cm_.expectAddWithWarming("cluster4", "2"); cm_.expectWarmingClusterCount(); EXPECT_CALL(initialized_, ready()); cm_.expectWarmingClusterCount(2); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(*cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(*cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); EXPECT_CALL(*interval_timer_, enableTimer(_)); resetCdsInitializedCb(); cm_.clustersToWarmUp({"cluster4", "cluster3"}); @@ -493,19 +493,19 @@ version_info: '3' )EOF"; // Two clusters updated, first one warmed up before processing of the second one starts. - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(*cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); cm_.expectAddWithWarming("cluster5", "3", true); cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(*cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(*cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); cm_.expectAddWithWarming("cluster6", "3"); cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(*cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); EXPECT_CALL(initialized_, ready()); cm_.expectWarmingClusterCount(); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); - EXPECT_CALL(cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(*cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(*cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); EXPECT_CALL(*interval_timer_, enableTimer(_)); resetCdsInitializedCb(); cm_.clustersToWarmUp({"cluster6"}); From 0b963208770d3780981716e4c28e9fe5a77e711e Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Wed, 22 May 2019 15:42:33 -0400 Subject: [PATCH 15/56] snapshot Signed-off-by: Fred Douglas --- api/envoy/api/v2/discovery.proto | 7 ------- include/envoy/config/grpc_mux.h | 4 +++- source/common/config/grpc_delta_xds_context.h | 3 +-- source/common/config/grpc_mux_impl.cc | 8 ++------ 4 files changed, 6 insertions(+), 16 deletions(-) diff --git a/api/envoy/api/v2/discovery.proto b/api/envoy/api/v2/discovery.proto index 96958f027996c..6ff24288ec6a9 100644 --- a/api/envoy/api/v2/discovery.proto +++ b/api/envoy/api/v2/discovery.proto @@ -187,13 +187,6 @@ message DeltaDiscoveryResponse { // The version of the response data (used for debugging). string system_version_info = 1; - // Type of the resources that are being provided, e.g. - // "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment". This is implicit - // in requests made via singleton xDS APIs such as CDS, LDS, etc. but is - // required for ADS. - // Must match the type_url in the DeltaDiscoveryRequest. - string type_url = 3; - // The response resources. The Resource proto's Any field named 'resource' should be // of type type_url. repeated Resource resources = 2 [(gogoproto.nullable) = false]; diff --git a/include/envoy/config/grpc_mux.h b/include/envoy/config/grpc_mux.h index aa713b8d358c9..83f99df3a5c23 100644 --- a/include/envoy/config/grpc_mux.h +++ b/include/envoy/config/grpc_mux.h @@ -99,7 +99,9 @@ template class GrpcStreamCallbacks { /** * Manage one or more gRPC subscriptions on a single stream to management server. This can be used - * for a single xDS API, e.g. EDS, or to combined multiple xDS APIs for ADS. Both delta and state-of-the-world implement this same interface - whether a GrpcSubscriptionImpl is delta or SotW is determined entirely by which type of GrpcMux it works with. + * for a single xDS API, e.g. EDS, or to combined multiple xDS APIs for ADS. Both delta and + * state-of-the-world implement this same interface - whether a GrpcSubscriptionImpl is delta or + * SotW is determined entirely by which type of GrpcMux it works with. */ class GrpcMux { public: diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h index 1421ba295a52f..5edb5c5fb8a70 100644 --- a/source/common/config/grpc_delta_xds_context.h +++ b/source/common/config/grpc_delta_xds_context.h @@ -152,8 +152,7 @@ class GrpcDeltaXdsContext : public GrpcMux, trySendDiscoveryRequests(); } - // TODO TODO remove, GrpcMux impersonation. Basically combination of addSubscription and - // updateResources. + // TODO(fredlas) remove, only here for compatibility with old-style GrpcMuxImpl. GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, GrpcMuxCallbacks&) override { // don't need any implementation here. only grpc_mux_subscription_impl ever calls it, and there diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index 3f82ecae00bcc..fc654d7e1c1d6 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -90,10 +90,6 @@ GrpcMuxWatchPtr GrpcMuxImpl::subscribe(const std::string& type_url, // only send a single RDS/EDS update after the CDS/LDS update. queueDiscoveryRequest(type_url); - // TODO TODO well, that takes fiddling with the tests... This is idempotent, so it's fine to just - // always call when we have a new subscription. - // TODO TODO well, that takes fiddling with the tests... grpc_stream_.establishNewStream(); - return watch; } @@ -198,6 +194,8 @@ void GrpcMuxImpl::onDiscoveryResponse( queueDiscoveryRequest(type_url); } +void GrpcMuxImpl::onWriteable() { drainRequests(); } + void GrpcMuxImpl::onStreamEstablished() { for (const auto type_url : subscriptions_) { queueDiscoveryRequest(type_url); @@ -212,8 +210,6 @@ void GrpcMuxImpl::onEstablishmentFailure() { } } -void GrpcMuxImpl::onWriteable() { drainRequests(); } - void GrpcMuxImpl::queueDiscoveryRequest(const std::string& queue_item) { request_queue_.push(queue_item); drainRequests(); From 0f8a8f1e4e4b21a875d4c056ba973e9b4bd37d84 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Wed, 22 May 2019 17:04:43 -0400 Subject: [PATCH 16/56] documentation Signed-off-by: Fred Douglas --- api/envoy/api/v2/discovery.proto | 57 +++++++++++-------- source/common/config/README.md | 35 ++++++++++++ .../common/config/delta_subscription_impl.h | 2 +- .../common/config/delta_subscription_state.h | 5 +- source/common/config/grpc_delta_xds_context.h | 39 +++---------- 5 files changed, 82 insertions(+), 56 deletions(-) create mode 100644 source/common/config/README.md diff --git a/api/envoy/api/v2/discovery.proto b/api/envoy/api/v2/discovery.proto index 6ff24288ec6a9..ac3f13643bd0e 100644 --- a/api/envoy/api/v2/discovery.proto +++ b/api/envoy/api/v2/discovery.proto @@ -108,7 +108,7 @@ message DiscoveryResponse { // With Delta xDS, the DeltaDiscoveryResponses do not need to include a full // snapshot of the tracked resources. Instead, DeltaDiscoveryResponses are a // diff to the state of a xDS client. -// In Delta XDS there are per resource versions, which allow tracking state at +// In Delta XDS there are per-resource versions, which allow tracking state at // the resource granularity. // An xDS Delta session is always in the context of a gRPC bidirectional // stream. This allows the xDS server to keep track of the state of xDS clients @@ -119,22 +119,28 @@ message DiscoveryResponse { // Optionally, a response message level system_version_info is present for // debugging purposes only. // -// DeltaDiscoveryRequest can be sent in 3 situations: -// 1. Initial message in a xDS bidirectional gRPC stream. -// 2. As a ACK or NACK response to a previous DeltaDiscoveryResponse. -// In this case the response_nonce is set to the nonce value in the Response. -// ACK or NACK is determined by the absence or presence of error_detail. -// 3. Spontaneous DeltaDiscoveryRequest from the client. -// This can be done to dynamically add or remove elements from the tracked -// resource_names set. In this case response_nonce must be omitted. +// DeltaDiscoveryRequest plays two independent roles. Any DeltaDiscoveryRequest +// can be either or both of: +// 1. informing the server of what resources the client has gained/lost interest in, +// (resource_names_subscribe and resource_names_unsubscribe) +// 2. (N)ACKing an earlier resource update from the server +// (response_nonce, with presence of error_detail making it a NACK). +// Additionally, the first message (for a given type_url) of a reconnected gRPC stream +// has a third role: informing the server of the resources (and their versions) +// that the client already possesses, using the initial_resource_versions field. +// +// As with state-of-the-world, when multiple resource types are multiplexed (ADS), +// all requests/acknowledgments/updates are logically walled off by type_url: +// a Cluster ACK exists in a completely separate world from a prior Route NACK. +// In particular, initial_resource_versions being sent at the "start" of every +// gRPC stream actually entails a message for each type_url, each with its own +// initial_resource_versions. message DeltaDiscoveryRequest { // The node making the request. core.Node node = 1; // Type of the resource that is being requested, e.g. - // "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment". This is implicit - // in requests made via singleton xDS APIs such as CDS, LDS, etc. but is - // required for ADS. + // "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment". string type_url = 2; // DeltaDiscoveryRequests allow the client to add or remove individual @@ -142,20 +148,21 @@ message DeltaDiscoveryRequest { // All resource names in the resource_names_subscribe list are added to the // set of tracked resources and all resource names in the resource_names_unsubscribe // list are removed from the set of tracked resources. - // Unlike in state-of-the-world xDS, an empty resource_names_subscribe or + // + // *Unlike* state-of-the-world xDS, an empty resource_names_subscribe or // resource_names_unsubscribe list simply means that no resources are to be // added or removed to the resource list. - // The xDS server must send updates for all tracked resources but can also - // send updates for resources the client has not subscribed to. This behavior - // is similar to state-of-the-world xDS. - // These two fields can be set for all types of DeltaDiscoveryRequests - // (initial, ACK/NACK or spontaneous). + // *Like* state-of-the-world xDS, the server must send updates for all tracked + // resources, but can also send updates for resources the client has not subscribed to. // // NOTE: the server must respond with all resources listed in resource_names_subscribe, // even if it believes the client has the most recent version of them. The reason: // the client may have dropped them, but then regained interest before it had a chance // to send the unsubscribe message. See DeltaSubscriptionStateTest.RemoveThenAdd. // + // These two fields can be set in any DeltaDiscoveryRequest, including ACKs + // and initial_resource_versions. + // // A list of Resource names to add to the list of tracked resources. repeated string resource_names_subscribe = 3; @@ -163,12 +170,14 @@ message DeltaDiscoveryRequest { repeated string resource_names_unsubscribe = 4; // Informs the server of the versions of the resources the xDS client knows of, to enable an xDS - // session to continue in a reconnected gRPC stream. This map will only be populated in the first - // message of a stream, as it is not applicable in later messages. It will not be populated in the - // very first stream of a session, since the client will not yet have any resources. In ADS, the - // first message *of each type_url* of a reconnected stream should populate this map. The map's - // keys are names of xDS resources known to the xDS client. The values are opaque resource - // versions. + // session to continue in a reconnected gRPC stream. + // It will not be populated: + // * in the very first stream of a session, since the client will not yet have any resources. + // * in any message after the first in a stream (for a given type_url), since the server + // will already be correctly tracking the client's state. + // (In ADS, the first message *of each type_url* of a reconnected stream populates this map.) + // The map's keys are names of xDS resources known to the xDS client. + // The map's values are opaque resource versions. map initial_resource_versions = 5; // When the DeltaDiscoveryRequest is a ACK or NACK message in response diff --git a/source/common/config/README.md b/source/common/config/README.md new file mode 100644 index 0000000000000..57ab37026d806 --- /dev/null +++ b/source/common/config/README.md @@ -0,0 +1,35 @@ +tldr: xDS can be filesystem, REST, or gRPC, and gRPC xDS comes in four flavors, +but Envoy code uses all of that via the same Subscription interface. If you are +an Envoy developer with your hands on a valid Subscription object, you can +forget the filesystem/REST/gRPC distinction, and you can especially forget about +the gRPC flavors. All of that is specified in the bootstrap config, which is +read and put into action by ClusterManagerImpl. + +If you are working on Envoy's gRPC xDS client logic itself, read on. + +When using gRPC, xDS has two pairs of options: aggregated/non-aggregated, and +delta/state-of-the-world updates. All four combinations of these are usable. + +"Aggregated" means that EDS, CDS, etc resources are all carried by the same gRPC stream. +For Envoy's implementation of xDS client logic, there is effectively no difference +between aggregated xDS and non-aggregated: they both use the same request/response protos. The +non-aggregated case is handled by running the aggregated logic, and just happening to only have 1 +xDS subscription type to "aggregate", i.e., GrpcDeltaXdsContext only has one +DeltaSubscriptionState entry in its map. + +However, to the config server, there is a huge difference: when using ADS (caused +by the user providing an ads_config in the bootstrap config), the gRPC client sets +its method string to {Delta,Stream}AggregatedResources, as opposed to {Delta,Stream}Clusters, +{Delta,Stream}Routes, etc. So, despite using the same request/response protos, +and having identical client code, they're actually different gRPC services. + +Delta vs state-of-the-world is a question of wire format: the protos in question are named +[Delta]Discovery{Request,Response}. That is what the GrpcMux (TODO rename) interface is useful for: its +GrpcDeltaXdsContext implementation works with DeltaDiscovery{Request,Response} and has +delta-specific logic, and its GrpxMuxImpl implementation (TODO will be merged into GrpcDeltaXdsContext) works with Discovery{Request,Response} +and has SotW-specific logic. A GrpcSubscriptionImpl has its shared_ptr. +Both GrpcDeltaXdsContext (delta) or GrpcMuxImpl (SotW) will work just fine. The shared_ptr allows +for both non- and aggregated: if non-aggregated, you'll be the only holder of that shared_ptr. By +those two mechanisms, the single class (TODO rename) DeltaSubscriptionImpl handles all 4 +delta/SotW and non-/aggregated combinations. + diff --git a/source/common/config/delta_subscription_impl.h b/source/common/config/delta_subscription_impl.h index a4ba62f958694..916fc2aee5cac 100644 --- a/source/common/config/delta_subscription_impl.h +++ b/source/common/config/delta_subscription_impl.h @@ -15,7 +15,7 @@ namespace Config { * the only difference is that ADS has multiple DeltaSubscriptions sharing a single * GrpcDeltaXdsContext. */ -// TODO(fredlas) someday this class will be named SubscriptionImpl (without any changes to its code) +// TODO(fredlas) someday this class will be named GrpcSubscriptionImpl class DeltaSubscriptionImpl : public Subscription { public: DeltaSubscriptionImpl(std::shared_ptr context, absl::string_view type_url, diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index 5d8786ea089da..f727592c2dd5f 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -20,7 +20,10 @@ struct UpdateAck { ::google::rpc::Status error_detail_; }; -// Tracks the xDS protocol state of an individual ongoing delta xDS session. +// Tracks the xDS protocol state of an individual ongoing delta xDS session, i.e. a single type_url. +// There can be multiple DeltaSubscriptionStates active. They will always all be +// blissfully unaware of each other's existence, even when their messages are +// being multiplexed together by ADS. class DeltaSubscriptionState : public Logger::Loggable { public: DeltaSubscriptionState(const std::string& type_url, const std::set& resource_names, diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h index 5edb5c5fb8a70..61ff1b7d6cfd2 100644 --- a/source/common/config/grpc_delta_xds_context.h +++ b/source/common/config/grpc_delta_xds_context.h @@ -20,34 +20,11 @@ namespace Envoy { namespace Config { -// (TODO this will become more of a README level thing) -// When using gRPC, xDS has two pairs of options: aggregated/non-aggregated, and -// delta/state-of-the-world updates. All four combinations of these are usable. -// -// "Aggregated" means that EDS, CDS, etc resources are all carried by the same gRPC stream (not even -// channel). For Envoy's implementation of xDS client logic, there is effectively no difference -// between aggregated xDS and non-aggregated: they both use the same request/response protos. The -// non-aggregated case is handled by running the aggregated logic, and just happening to only have 1 -// xDS subscription type to "aggregate", i.e., GrpcDeltaXdsContext only has one -// DeltaSubscriptionState entry in its map. The sole implementation difference: when the bootstrap -// specifies ADS, the method string set on the gRPC client that the GrpcMux holds is -// {Delta,Stream}AggregatedResources, as opposed to e.g. {Delta,Stream}Clusters. This distinction is -// necessary for the server to know what resources should be provided. -// -// DeltaSubscriptionState is what handles the conceptual/application-level protocol state of a given -// resource type subscription, which may or may not be multiplexed with others. So, -// DeltaSubscriptionState is equally applicable to non- and aggregated. -// -// Delta vs state-of-the-world is a question of wire format: the protos in question are named -// [Delta]Discovery{Request,Response}. That is what the GrpcMux interface is useful for: its -// GrpcDeltaXdsContext implementation works with DeltaDiscovery{Request,Response} and has -// delta-specific logic, and its GrpxMuxImpl implementation works with Discovery{Request,Response} -// and has SotW-specific logic. A DeltaSubscriptionImpl (TODO rename to not delta-specific since -// it's ideally going to cover ALL 4 subscription "meta-types") has its shared_ptr. -// Both GrpcDeltaXdsContext (delta) or GrpcMuxImpl (SotW) will work just fine. The shared_ptr allows -// for both non- and aggregated: if non-aggregated, you'll be the only holder of that shared_ptr. By -// those two mechanisms, the single class (TODO rename) DeltaSubscriptionImpl handles all 4 -// delta/SotW and non-/aggregated combinations. +// Manages subscriptions to one or more type of resource. The logical protocol +// state of those subscription(s) is handled by DeltaSubscriptionState. +// This class owns the GrpcStream used to talk to the server, maintains queuing +// logic to properly order the subscription(s)' various messages, and allows +// starting/stopping/pausing of the subscriptions. class GrpcDeltaXdsContext : public GrpcMux, public GrpcStreamCallbacks, Logger::Loggable { @@ -222,8 +199,10 @@ class GrpcDeltaXdsContext : public GrpcMux, // Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or // a subscription update. (Does not check whether we *can* send that DeltaDiscoveryRequest). - // Returns the type_url of the resource type we should send the DeltaDiscoveryRequest for (if - // any). Prioritizes ACKs over non-ACK subscription interest updates. + // Returns the type_url we should send the DeltaDiscoveryRequest for (if any). + // First, prioritizes ACKs over non-ACK subscription interest updates. + // Then, prioritizes non-ACK updates in the order the various types + // of subscriptions were activated. absl::optional whoWantsToSendDiscoveryRequest() { // All ACKs are sent before plain updates. trySendDiscoveryRequests() relies on this. So, choose // type_url from ack_queue_ if possible, before looking at pending updates. From 8c4e94bbe8f9a596f6033c8bf181b00096186592 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 24 May 2019 16:42:01 -0400 Subject: [PATCH 17/56] is_delta plumbing stuff to make delta better fit with ADS Signed-off-by: Fred Douglas --- .../router/route_config_provider_manager.h | 3 +- include/envoy/server/listener_manager.h | 6 +- include/envoy/upstream/cluster_manager.h | 4 +- .../common/config/delta_subscription_impl.cc | 2 +- .../common/config/delta_subscription_impl.h | 7 ++- source/common/config/subscription_factory.h | 16 ++++-- source/common/router/rds_impl.cc | 18 +++--- source/common/router/rds_impl.h | 10 ++-- source/common/router/vhds.cc | 2 +- source/common/router/vhds.h | 2 +- source/common/secret/sds_api.cc | 3 +- source/common/upstream/cds_api_impl.cc | 21 ++++--- source/common/upstream/cds_api_impl.h | 12 ++-- .../common/upstream/cluster_manager_impl.cc | 22 +++++-- source/common/upstream/cluster_manager_impl.h | 5 +- source/common/upstream/eds.cc | 13 +++-- source/common/upstream/eds.h | 2 +- .../network/http_connection_manager/config.cc | 5 +- source/server/lds_api.cc | 15 +++-- source/server/lds_api.h | 2 +- source/server/listener_manager_impl.h | 9 +-- source/server/server.cc | 8 ++- test/common/grpc/grpc_client_integration.h | 7 ++- test/config/utility.cc | 33 +++++++++++ test/config/utility.h | 1 + test/integration/ads_integration_test.cc | 57 +++++++------------ test/mocks/router/mocks.h | 4 +- test/mocks/server/mocks.h | 11 ++-- test/mocks/upstream/mocks.h | 5 +- test/server/BUILD | 1 + test/server/lds_api_test.cc | 7 ++- 31 files changed, 196 insertions(+), 117 deletions(-) diff --git a/include/envoy/router/route_config_provider_manager.h b/include/envoy/router/route_config_provider_manager.h index 9bf02066a6009..e1854fe95e9db 100644 --- a/include/envoy/router/route_config_provider_manager.h +++ b/include/envoy/router/route_config_provider_manager.h @@ -36,7 +36,8 @@ class RouteConfigProviderManager { */ virtual RouteConfigProviderPtr createRdsRouteConfigProvider( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, - Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix) PURE; + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + bool is_delta) PURE; /** * Get a RouteConfigSharedPtr for a statically defined route. Ownership is as described for diff --git a/include/envoy/server/listener_manager.h b/include/envoy/server/listener_manager.h index fd2808cd6da76..c729216153fa5 100644 --- a/include/envoy/server/listener_manager.h +++ b/include/envoy/server/listener_manager.h @@ -39,7 +39,8 @@ class ListenerComponentFactory { * @return an LDS API provider. * @param lds_config supplies the management server configuration. */ - virtual LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config) PURE; + virtual LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config, + bool is_delta) PURE; /** * Creates a socket. @@ -118,7 +119,8 @@ class ListenerManager { * pieces of the server existing. * @param lds_config supplies the management server configuration. */ - virtual void createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config) PURE; + virtual void createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config, + bool is_delta) PURE; /** * @return std::vector> a list of the currently diff --git a/include/envoy/upstream/cluster_manager.h b/include/envoy/upstream/cluster_manager.h index b544e2a48cacb..f69ff91a0230b 100644 --- a/include/envoy/upstream/cluster_manager.h +++ b/include/envoy/upstream/cluster_manager.h @@ -236,6 +236,8 @@ class ClusterManager { virtual ClusterManagerFactory& clusterManagerFactory() PURE; virtual std::size_t warmingClusterCount() const PURE; + + virtual bool xdsIsDelta() const PURE; }; typedef std::unique_ptr ClusterManagerPtr; @@ -309,7 +311,7 @@ class ClusterManagerFactory { /** * Create a CDS API provider from configuration proto. */ - virtual CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource& cds_config, + virtual CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource& cds_config, bool is_delta, ClusterManager& cm) PURE; /** diff --git a/source/common/config/delta_subscription_impl.cc b/source/common/config/delta_subscription_impl.cc index 190838797fc9b..bea462ef3c443 100644 --- a/source/common/config/delta_subscription_impl.cc +++ b/source/common/config/delta_subscription_impl.cc @@ -3,7 +3,7 @@ namespace Envoy { namespace Config { -DeltaSubscriptionImpl::DeltaSubscriptionImpl(std::shared_ptr context, +DeltaSubscriptionImpl::DeltaSubscriptionImpl(std::shared_ptr context, absl::string_view type_url, SubscriptionStats stats, std::chrono::milliseconds init_fetch_timeout) : context_(context), type_url_(type_url), stats_(stats), diff --git a/source/common/config/delta_subscription_impl.h b/source/common/config/delta_subscription_impl.h index 916fc2aee5cac..4cb64cd253f94 100644 --- a/source/common/config/delta_subscription_impl.h +++ b/source/common/config/delta_subscription_impl.h @@ -18,7 +18,7 @@ namespace Config { // TODO(fredlas) someday this class will be named GrpcSubscriptionImpl class DeltaSubscriptionImpl : public Subscription { public: - DeltaSubscriptionImpl(std::shared_ptr context, absl::string_view type_url, + DeltaSubscriptionImpl(std::shared_ptr context, absl::string_view type_url, SubscriptionStats stats, std::chrono::milliseconds init_fetch_timeout); ~DeltaSubscriptionImpl(); @@ -30,10 +30,11 @@ class DeltaSubscriptionImpl : public Subscription { void start(const std::set& resources, SubscriptionCallbacks& callbacks) override; void updateResources(const std::set& update_to_these_names) override; - std::shared_ptr getContextForTest() { return context_; } + std::shared_ptr getContextForTest() { return context_; } private: - std::shared_ptr context_; + std::shared_ptr + context_; // TODO TODO does it need to be GrpcDeltaXdsContext? hopefully not? const std::string type_url_; SubscriptionStats stats_; const std::chrono::milliseconds init_fetch_timeout_; diff --git a/source/common/config/subscription_factory.h b/source/common/config/subscription_factory.h index 9d55e79515ac8..72d3a00a6bb62 100644 --- a/source/common/config/subscription_factory.h +++ b/source/common/config/subscription_factory.h @@ -38,11 +38,12 @@ class SubscriptionFactory { * service description). * @param api reference to the Api object */ + // TODO(fredlas) remove is_delta once delta and SotW are more unified static std::unique_ptr subscriptionFromConfigSource( const envoy::api::v2::core::ConfigSource& config, const LocalInfo::LocalInfo& local_info, Event::Dispatcher& dispatcher, Upstream::ClusterManager& cm, Runtime::RandomGenerator& random, Stats::Scope& scope, const std::string& rest_method, const std::string& grpc_method, - absl::string_view type_url, Api::Api& api) { + absl::string_view type_url, Api::Api& api, bool is_delta) { std::unique_ptr result; SubscriptionStats stats = Utility::generateStats(scope); switch (config.config_source_specifier_case()) { @@ -99,9 +100,16 @@ class SubscriptionFactory { break; } case envoy::api::v2::core::ConfigSource::kAds: { - result = std::make_unique( - cm.adsMux(), stats, type_url, dispatcher, - Utility::configSourceInitialFetchTimeout(config)); + if (is_delta) { + std::cerr << "making a delta " << type_url << std::endl; + result = std::make_unique( + cm.adsMux(), type_url, stats, Utility::configSourceInitialFetchTimeout(config)); + } else { + std::cerr << "making a sotw" << type_url << std::endl; + result = std::make_unique( + cm.adsMux(), stats, type_url, dispatcher, + Utility::configSourceInitialFetchTimeout(config)); + } break; } default: diff --git a/source/common/router/rds_impl.cc b/source/common/router/rds_impl.cc index 5bba8f9f0a990..4aef95dd494d3 100644 --- a/source/common/router/rds_impl.cc +++ b/source/common/router/rds_impl.cc @@ -23,7 +23,7 @@ RouteConfigProviderPtr RouteConfigProviderUtil::create( const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& config, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, - RouteConfigProviderManager& route_config_provider_manager) { + RouteConfigProviderManager& route_config_provider_manager, bool is_delta) { switch (config.route_specifier_case()) { case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager:: kRouteConfig: @@ -31,7 +31,7 @@ RouteConfigProviderPtr RouteConfigProviderUtil::create( factory_context); case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager::kRds: return route_config_provider_manager.createRdsRouteConfigProvider(config.rds(), factory_context, - stat_prefix); + stat_prefix, is_delta); default: NOT_REACHED_GCOVR_EXCL_LINE; } @@ -56,7 +56,7 @@ RdsRouteConfigSubscription::RdsRouteConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, - Envoy::Router::RouteConfigProviderManagerImpl& route_config_provider_manager) + Envoy::Router::RouteConfigProviderManagerImpl& route_config_provider_manager, bool is_delta) : route_config_name_(rds.route_config_name()), factory_context_(factory_context), init_target_(fmt::format("RdsRouteConfigSubscription {}", route_config_name_), [this]() { subscription_->start({route_config_name_}, *this); }), @@ -66,13 +66,14 @@ RdsRouteConfigSubscription::RdsRouteConfigSubscription( manager_identifier_(manager_identifier) { Envoy::Config::Utility::checkLocalInfo("rds", factory_context.localInfo()); + const std::string grpc_method = is_delta ? "envoy.api.v2.RouteDiscoveryService.DeltaRoutes" + : "envoy.api.v2.RouteDiscoveryService.StreamRoutes"; subscription_ = Envoy::Config::SubscriptionFactory::subscriptionFromConfigSource( rds.config_source(), factory_context.localInfo(), factory_context.dispatcher(), factory_context.clusterManager(), factory_context.random(), *scope_, - "envoy.api.v2.RouteDiscoveryService.FetchRoutes", - "envoy.api.v2.RouteDiscoveryService.StreamRoutes", + "envoy.api.v2.RouteDiscoveryService.FetchRoutes", grpc_method, Grpc::Common::typeUrl(envoy::api::v2::RouteConfiguration().GetDescriptor()->full_name()), - factory_context.api()); + factory_context.api(), is_delta); config_update_info_ = std::make_unique(factory_context.timeSource()); @@ -211,7 +212,8 @@ RouteConfigProviderManagerImpl::RouteConfigProviderManagerImpl(Server::Admin& ad Router::RouteConfigProviderPtr RouteConfigProviderManagerImpl::createRdsRouteConfigProvider( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, - Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix) { + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + bool is_delta) { // RdsRouteConfigSubscriptions are unique based on their serialized RDS config. const uint64_t manager_identifier = MessageUtil::hash(rds); @@ -224,7 +226,7 @@ Router::RouteConfigProviderPtr RouteConfigProviderManagerImpl::createRdsRouteCon // around it. However, since this is not a performance critical path we err on the side // of simplicity. subscription.reset(new RdsRouteConfigSubscription(rds, manager_identifier, factory_context, - stat_prefix, *this)); + stat_prefix, *this, is_delta)); factory_context.initManager().add(subscription->init_target_); diff --git a/source/common/router/rds_impl.h b/source/common/router/rds_impl.h index 015dd7e89dda2..8421c2d3ab6da 100644 --- a/source/common/router/rds_impl.h +++ b/source/common/router/rds_impl.h @@ -45,7 +45,7 @@ class RouteConfigProviderUtil { create(const envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager& config, Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, - RouteConfigProviderManager& route_config_provider_manager); + RouteConfigProviderManager& route_config_provider_manager, bool is_delta); }; class RouteConfigProviderManagerImpl; @@ -123,8 +123,8 @@ class RdsRouteConfigSubscription : Envoy::Config::SubscriptionCallbacks, RdsRouteConfigSubscription( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, const uint64_t manager_identifier, Server::Configuration::FactoryContext& factory_context, - const std::string& stat_prefix, - RouteConfigProviderManagerImpl& route_config_provider_manager); + const std::string& stat_prefix, RouteConfigProviderManagerImpl& route_config_provider_manager, + bool is_delta); bool validateUpdateSize(int num_resources); @@ -192,8 +192,8 @@ class RouteConfigProviderManagerImpl : public RouteConfigProviderManager, // RouteConfigProviderManager RouteConfigProviderPtr createRdsRouteConfigProvider( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, - Server::Configuration::FactoryContext& factory_context, - const std::string& stat_prefix) override; + Server::Configuration::FactoryContext& factory_context, const std::string& stat_prefix, + bool is_delta) override; RouteConfigProviderPtr createStaticRouteConfigProvider(const envoy::api::v2::RouteConfiguration& route_config, diff --git a/source/common/router/vhds.cc b/source/common/router/vhds.cc index 07bc3c2cbdf27..b2a2b0aefeb69 100644 --- a/source/common/router/vhds.cc +++ b/source/common/router/vhds.cc @@ -45,7 +45,7 @@ VhdsSubscription::VhdsSubscription(RouteConfigUpdatePtr& config_update_info, factory_context.dispatcher(), factory_context.clusterManager(), factory_context.random(), *scope_, "none", "envoy.api.v2.VirtualHostDiscoveryService.DeltaVirtualHosts", Grpc::Common::typeUrl(envoy::api::v2::route::VirtualHost().GetDescriptor()->full_name()), - factory_context.api()); + factory_context.api(), /*is_delta=*/true); } void VhdsSubscription::onConfigUpdateFailed(const EnvoyException*) { diff --git a/source/common/router/vhds.h b/source/common/router/vhds.h index 704d021ef5b3b..fe7b1fda76675 100644 --- a/source/common/router/vhds.h +++ b/source/common/router/vhds.h @@ -41,7 +41,7 @@ struct VhdsStats { typedef std::unique_ptr (*SubscriptionFactoryFunction)( const envoy::api::v2::core::ConfigSource&, const LocalInfo::LocalInfo&, Event::Dispatcher&, Upstream::ClusterManager&, Envoy::Runtime::RandomGenerator&, Stats::Scope&, const std::string&, - const std::string&, absl::string_view, Api::Api&); + const std::string&, absl::string_view, Api::Api&, bool); class VhdsSubscription : Envoy::Config::SubscriptionCallbacks, Logger::Loggable { diff --git a/source/common/secret/sds_api.cc b/source/common/secret/sds_api.cc index d9160858eb471..c5f52f707a1e7 100644 --- a/source/common/secret/sds_api.cc +++ b/source/common/secret/sds_api.cc @@ -79,7 +79,8 @@ void SdsApi::initialize() { sds_config_, local_info_, dispatcher_, cluster_manager_, random_, stats_, "envoy.service.discovery.v2.SecretDiscoveryService.FetchSecrets", "envoy.service.discovery.v2.SecretDiscoveryService.StreamSecrets", - Grpc::Common::typeUrl(envoy::api::v2::auth::Secret().GetDescriptor()->full_name()), api_); + Grpc::Common::typeUrl(envoy::api::v2::auth::Secret().GetDescriptor()->full_name()), api_, + /*is_delta=*/false); subscription_->start({sds_config_name_}, *this); } diff --git a/source/common/upstream/cds_api_impl.cc b/source/common/upstream/cds_api_impl.cc index 37c064f872088..f7a3dccfa6d50 100644 --- a/source/common/upstream/cds_api_impl.cc +++ b/source/common/upstream/cds_api_impl.cc @@ -16,28 +16,33 @@ namespace Envoy { namespace Upstream { -CdsApiPtr CdsApiImpl::create(const envoy::api::v2::core::ConfigSource& cds_config, +CdsApiPtr CdsApiImpl::create(const envoy::api::v2::core::ConfigSource& cds_config, bool is_delta, ClusterManager& cm, Event::Dispatcher& dispatcher, Runtime::RandomGenerator& random, const LocalInfo::LocalInfo& local_info, Stats::Scope& scope, Api::Api& api) { - return CdsApiPtr{new CdsApiImpl(cds_config, cm, dispatcher, random, local_info, scope, api)}; + return CdsApiPtr{ + new CdsApiImpl(cds_config, is_delta, cm, dispatcher, random, local_info, scope, api)}; } -CdsApiImpl::CdsApiImpl(const envoy::api::v2::core::ConfigSource& cds_config, ClusterManager& cm, - Event::Dispatcher& dispatcher, Runtime::RandomGenerator& random, - const LocalInfo::LocalInfo& local_info, Stats::Scope& scope, Api::Api& api) +// TODO(fredlas) the is_delta argument can be removed upon delta+SotW ADS Envoy code unification. It +// is only actually needed to choose the grpc_method, which is irrelevant if ADS is used. +// TODO(fredlas) actually the whole "pass the grpc_method as an argument to SubscriptionFactory" +// seems like it could be streamlined/centralized, given that we're also passing type_url. I'm +// thinking, have a (type_url, is_delta) -> grpc_method map that SubscriptionFactory can look up in. +CdsApiImpl::CdsApiImpl(const envoy::api::v2::core::ConfigSource& cds_config, bool is_delta, + ClusterManager& cm, Event::Dispatcher& dispatcher, + Runtime::RandomGenerator& random, const LocalInfo::LocalInfo& local_info, + Stats::Scope& scope, Api::Api& api) : cm_(cm), scope_(scope.createScope("cluster_manager.cds.")) { Config::Utility::checkLocalInfo("cds", local_info); - const bool is_delta = (cds_config.api_config_source().api_type() == - envoy::api::v2::core::ApiConfigSource::DELTA_GRPC); const std::string grpc_method = is_delta ? "envoy.api.v2.ClusterDiscoveryService.DeltaClusters" : "envoy.api.v2.ClusterDiscoveryService.StreamClusters"; subscription_ = Config::SubscriptionFactory::subscriptionFromConfigSource( cds_config, local_info, dispatcher, cm, random, *scope_, "envoy.api.v2.ClusterDiscoveryService.FetchClusters", grpc_method, - Grpc::Common::typeUrl(envoy::api::v2::Cluster().GetDescriptor()->full_name()), api); + Grpc::Common::typeUrl(envoy::api::v2::Cluster().GetDescriptor()->full_name()), api, is_delta); } void CdsApiImpl::onConfigUpdate(const Protobuf::RepeatedPtrField& resources, diff --git a/source/common/upstream/cds_api_impl.h b/source/common/upstream/cds_api_impl.h index cdd2fe9255419..0aae180c7c800 100644 --- a/source/common/upstream/cds_api_impl.h +++ b/source/common/upstream/cds_api_impl.h @@ -22,10 +22,10 @@ class CdsApiImpl : public CdsApi, Config::SubscriptionCallbacks, Logger::Loggable { public: - static CdsApiPtr create(const envoy::api::v2::core::ConfigSource& cds_config, ClusterManager& cm, - Event::Dispatcher& dispatcher, Runtime::RandomGenerator& random, - const LocalInfo::LocalInfo& local_info, Stats::Scope& scope, - Api::Api& api); + static CdsApiPtr create(const envoy::api::v2::core::ConfigSource& cds_config, bool is_delta, + ClusterManager& cm, Event::Dispatcher& dispatcher, + Runtime::RandomGenerator& random, const LocalInfo::LocalInfo& local_info, + Stats::Scope& scope, Api::Api& api); // Upstream::CdsApi void initialize() override { subscription_->start({}, *this); } @@ -46,8 +46,8 @@ class CdsApiImpl : public CdsApi, } private: - CdsApiImpl(const envoy::api::v2::core::ConfigSource& cds_config, ClusterManager& cm, - Event::Dispatcher& dispatcher, Runtime::RandomGenerator& random, + CdsApiImpl(const envoy::api::v2::core::ConfigSource& cds_config, bool is_delta, + ClusterManager& cm, Event::Dispatcher& dispatcher, Runtime::RandomGenerator& random, const LocalInfo::LocalInfo& local_info, Stats::Scope& scope, Api::Api& api); void runInitializeCallbackIfAny(); diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 2d9d64dbd6895..afb9e7fc98be3 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -284,8 +284,15 @@ ClusterManagerImpl::ClusterManagerImpl( }); // We can now potentially create the CDS API once the backing cluster exists. - if (bootstrap.dynamic_resources().has_cds_config()) { - cds_api_ = factory_.createCds(bootstrap.dynamic_resources().cds_config(), *this); + const auto& dyn_resources = bootstrap.dynamic_resources(); + if (dyn_resources.has_cds_config()) { + const auto& cds_config = dyn_resources.cds_config(); + xds_is_delta_ = + cds_config.api_config_source().api_type() == + envoy::api::v2::core::ApiConfigSource::DELTA_GRPC || + (dyn_resources.has_ads_config() && dyn_resources.ads_config().api_type() == + envoy::api::v2::core::ApiConfigSource::DELTA_GRPC); + cds_api_ = factory_.createCds(cds_config, xds_is_delta_, *this); init_helper_.setCds(cds_api_.get()); } else { init_helper_.setCds(nullptr); @@ -302,7 +309,10 @@ ClusterManagerImpl::ClusterManagerImpl( // clusters have already initialized. (E.g., if all static). init_helper_.onStaticLoadComplete(); - ads_mux_->start(); + if (!xds_is_delta_) { + // TODO TODO i think just have the same approach for start() with both, i.e. remove the if. + ads_mux_->start(); + } if (cm_config.has_load_stats_config()) { const auto& load_stats_config = cm_config.load_stats_config(); @@ -1268,9 +1278,9 @@ ClusterSharedPtr ProdClusterManagerFactory::clusterFromProto( } CdsApiPtr ProdClusterManagerFactory::createCds(const envoy::api::v2::core::ConfigSource& cds_config, - ClusterManager& cm) { - return CdsApiImpl::create(cds_config, cm, main_thread_dispatcher_, random_, local_info_, stats_, - api_); + bool is_delta, ClusterManager& cm) { + return CdsApiImpl::create(cds_config, is_delta, cm, main_thread_dispatcher_, random_, local_info_, + stats_, api_); } } // namespace Upstream diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index d70ac1ac986de..a1adb8576457a 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -64,7 +64,7 @@ class ProdClusterManagerFactory : public ClusterManagerFactory { ClusterSharedPtr clusterFromProto(const envoy::api::v2::Cluster& cluster, ClusterManager& cm, Outlier::EventLoggerSharedPtr outlier_event_logger, bool added_via_api) override; - CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource& cds_config, + CdsApiPtr createCds(const envoy::api::v2::core::ConfigSource& cds_config, bool is_delta, ClusterManager& cm) override; Secret::SecretManager& secretManager() override { return secret_manager_; } @@ -226,6 +226,8 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable void { onAssignmentTimeout(); }); const auto& eds_config = cluster.eds_cluster_config().eds_config(); + const std::string grpc_method = is_delta + ? "envoy.api.v2.EndpointDiscoveryService.DeltaEndpoints" + : "envoy.api.v2.EndpointDiscoveryService.StreamEndpoints"; subscription_ = Config::SubscriptionFactory::subscriptionFromConfigSource( eds_config, local_info_, dispatcher, cm, random, info_->statsScope(), - "envoy.api.v2.EndpointDiscoveryService.FetchEndpoints", - "envoy.api.v2.EndpointDiscoveryService.StreamEndpoints", + "envoy.api.v2.EndpointDiscoveryService.FetchEndpoints", grpc_method, Grpc::Common::typeUrl(envoy::api::v2::ClusterLoadAssignment().GetDescriptor()->full_name()), - factory_context.api()); + factory_context.api(), is_delta); } void EdsClusterImpl::startPreInit() { subscription_->start({cluster_name_}, *this); } @@ -267,7 +269,8 @@ ClusterImplBaseSharedPtr EdsClusterFactory::createClusterImpl( } return std::make_unique(cluster, context.runtime(), socket_factory_context, - std::move(stats_scope), context.addedViaApi()); + std::move(stats_scope), context.addedViaApi(), + context.clusterManager().xdsIsDelta()); } /** diff --git a/source/common/upstream/eds.h b/source/common/upstream/eds.h index f1cae5638829c..d394ad826d9a8 100644 --- a/source/common/upstream/eds.h +++ b/source/common/upstream/eds.h @@ -24,7 +24,7 @@ class EdsClusterImpl : public BaseDynamicClusterImpl, Config::SubscriptionCallba public: EdsClusterImpl(const envoy::api::v2::Cluster& cluster, Runtime::Loader& runtime, Server::Configuration::TransportSocketFactoryContext& factory_context, - Stats::ScopePtr&& stats_scope, bool added_via_api); + Stats::ScopePtr&& stats_scope, bool added_via_api, bool is_delta); // Upstream::Cluster InitializePhase initializePhase() const override { return InitializePhase::Secondary; } diff --git a/source/extensions/filters/network/http_connection_manager/config.cc b/source/extensions/filters/network/http_connection_manager/config.cc index 92cce54bd3092..c68a4e9a22946 100644 --- a/source/extensions/filters/network/http_connection_manager/config.cc +++ b/source/extensions/filters/network/http_connection_manager/config.cc @@ -163,8 +163,9 @@ HttpConnectionManagerConfig::HttpConnectionManagerConfig( #endif ))) { - route_config_provider_ = Router::RouteConfigProviderUtil::create(config, context_, stats_prefix_, - route_config_provider_manager_); + route_config_provider_ = Router::RouteConfigProviderUtil::create( + config, context_, stats_prefix_, route_config_provider_manager_, + context_.clusterManager().xdsIsDelta()); switch (config.forward_client_cert_details()) { case envoy::config::filter::network::http_connection_manager::v2::HttpConnectionManager::SANITIZE: diff --git a/source/server/lds_api.cc b/source/server/lds_api.cc index e5b7a0a314a0a..b71474bad0df1 100644 --- a/source/server/lds_api.cc +++ b/source/server/lds_api.cc @@ -19,14 +19,17 @@ LdsApiImpl::LdsApiImpl(const envoy::api::v2::core::ConfigSource& lds_config, Upstream::ClusterManager& cm, Event::Dispatcher& dispatcher, Runtime::RandomGenerator& random, Init::Manager& init_manager, const LocalInfo::LocalInfo& local_info, Stats::Scope& scope, - ListenerManager& lm, Api::Api& api) + ListenerManager& lm, Api::Api& api, bool is_delta) : listener_manager_(lm), scope_(scope.createScope("listener_manager.lds.")), cm_(cm), init_target_("LDS", [this]() { subscription_->start({}, *this); }) { + const std::string grpc_method = is_delta + ? "envoy.api.v2.ListenerDiscoveryService.DeltaListeners" + : "envoy.api.v2.ListenerDiscoveryService.StreamListeners"; subscription_ = Envoy::Config::SubscriptionFactory::subscriptionFromConfigSource( lds_config, local_info, dispatcher, cm, random, *scope_, - "envoy.api.v2.ListenerDiscoveryService.FetchListeners", - "envoy.api.v2.ListenerDiscoveryService.StreamListeners", - Grpc::Common::typeUrl(envoy::api::v2::Listener().GetDescriptor()->full_name()), api); + "envoy.api.v2.ListenerDiscoveryService.FetchListeners", grpc_method, + Grpc::Common::typeUrl(envoy::api::v2::Listener().GetDescriptor()->full_name()), api, + is_delta); Config::Utility::checkLocalInfo("lds", local_info); init_manager.add(init_target_); } @@ -35,8 +38,8 @@ void LdsApiImpl::onConfigUpdate( const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info) { - cm_.adsMux().pause(Config::TypeUrl::get().RouteConfiguration); - Cleanup rds_resume([this] { cm_.adsMux().resume(Config::TypeUrl::get().RouteConfiguration); }); + cm_.adsMux()->pause(Config::TypeUrl::get().RouteConfiguration); + Cleanup rds_resume([this] { cm_.adsMux()->resume(Config::TypeUrl::get().RouteConfiguration); }); // We do all listener removals before adding the new listeners. This allows adding a new listener // with the same address as a listener that is to be removed. Do not change the order. diff --git a/source/server/lds_api.h b/source/server/lds_api.h index 75af148f18ae5..00aa3b2fd3505 100644 --- a/source/server/lds_api.h +++ b/source/server/lds_api.h @@ -25,7 +25,7 @@ class LdsApiImpl : public LdsApi, LdsApiImpl(const envoy::api::v2::core::ConfigSource& lds_config, Upstream::ClusterManager& cm, Event::Dispatcher& dispatcher, Runtime::RandomGenerator& random, Init::Manager& init_manager, const LocalInfo::LocalInfo& local_info, - Stats::Scope& scope, ListenerManager& lm, Api::Api& api); + Stats::Scope& scope, ListenerManager& lm, Api::Api& api, bool is_delta); // Server::LdsApi std::string versionInfo() const override { return system_version_info_; } diff --git a/source/server/listener_manager_impl.h b/source/server/listener_manager_impl.h index f90641e6ea10b..68330bb5c0f94 100644 --- a/source/server/listener_manager_impl.h +++ b/source/server/listener_manager_impl.h @@ -45,11 +45,12 @@ class ProdListenerComponentFactory : public ListenerComponentFactory, Configuration::ListenerFactoryContext& context); // Server::ListenerComponentFactory - LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config) override { + LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config, + bool is_delta) override { return std::make_unique(lds_config, server_.clusterManager(), server_.dispatcher(), server_.random(), server_.initManager(), server_.localInfo(), server_.stats(), - server_.listenerManager(), server_.api()); + server_.listenerManager(), server_.api(), is_delta); } std::vector createNetworkFilterFactoryList( const Protobuf::RepeatedPtrField& filters, @@ -111,9 +112,9 @@ class ListenerManagerImpl : public ListenerManager, Logger::Loggable> listeners() override; uint64_t numConnections() override; diff --git a/source/server/server.cc b/source/server/server.cc index 34457fcdc19a5..490b33d261127 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -384,7 +384,13 @@ void InstanceImpl::initialize(const Options& options, // Instruct the listener manager to create the LDS provider if needed. This must be done later // because various items do not yet exist when the listener manager is created. if (bootstrap_.dynamic_resources().has_lds_config()) { - listener_manager_->createLdsApi(bootstrap_.dynamic_resources().lds_config()); + const bool is_delta = + bootstrap_.dynamic_resources().lds_config().api_config_source().api_type() == + envoy::api::v2::core::ApiConfigSource::DELTA_GRPC || + (bootstrap_.dynamic_resources().has_ads_config() && + bootstrap_.dynamic_resources().ads_config().api_type() == + envoy::api::v2::core::ApiConfigSource::DELTA_GRPC); + listener_manager_->createLdsApi(bootstrap_.dynamic_resources().lds_config(), is_delta); } if (bootstrap_.has_hds_config()) { diff --git a/test/common/grpc/grpc_client_integration.h b/test/common/grpc/grpc_client_integration.h index 9a7178892055b..050cb86d533d1 100644 --- a/test/common/grpc/grpc_client_integration.h +++ b/test/common/grpc/grpc_client_integration.h @@ -55,7 +55,8 @@ class GrpcClientIntegrationParamTest }; class DeltaSotwIntegrationParamTest - : public testing::TestWithParam< + : public BaseGrpcClientIntegrationParamTest, + public testing::TestWithParam< std::tuple> { public: ~DeltaSotwIntegrationParamTest() override = default; @@ -67,8 +68,8 @@ class DeltaSotwIntegrationParamTest std::get<1>(p.param) == ClientType::GoogleGrpc ? "GoogleGrpc" : "EnvoyGrpc", std::get<2>(p.param) == SotwOrDelta::Delta ? "Delta" : "StateOfTheWorld"); } - Network::Address::IpVersion ipVersion() const { return std::get<0>(GetParam()); } - ClientType clientType() const { return std::get<1>(GetParam()); } + Network::Address::IpVersion ipVersion() const override { return std::get<0>(GetParam()); } + ClientType clientType() const override { return std::get<1>(GetParam()); } SotwOrDelta sotwOrDelta() const { return std::get<2>(GetParam()); } }; diff --git a/test/config/utility.cc b/test/config/utility.cc index 4c1c34986987f..5b0e924de55f3 100644 --- a/test/config/utility.cc +++ b/test/config/utility.cc @@ -181,6 +181,39 @@ std::string ConfigHelper::discoveredClustersBootstrap(const std::string& api_typ api_type); } +// TODO(#6327) cleaner approach to testing with static config. +std::string ConfigHelper::adsBootstrap(const std::string& api_type) { + return fmt::format( + R"EOF( +dynamic_resources: + lds_config: + ads: {{}} + cds_config: + ads: {{}} + ads_config: + api_type: {} +static_resources: + clusters: + name: dummy_cluster + connect_timeout: + seconds: 5 + type: STATIC + hosts: + socket_address: + address: 127.0.0.1 + port_value: 0 + lb_policy: ROUND_ROBIN + http2_protocol_options: {{}} +admin: + access_log_path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 0 +)EOF", + api_type); +} + envoy::api::v2::Cluster ConfigHelper::buildCluster(const std::string& name, int port, const std::string& ip_version) { return TestUtility::parseYaml(fmt::format(R"EOF( diff --git a/test/config/utility.h b/test/config/utility.h index bc0e2ba11e27b..ad69dc0d8acb6 100644 --- a/test/config/utility.h +++ b/test/config/utility.h @@ -89,6 +89,7 @@ class ConfigHelper { // Configuration for L7 proxying, with clusters cluster_1 and cluster_2 meant to be added via CDS. // api_type should be REST, GRPC, or DELTA_GRPC. static std::string discoveredClustersBootstrap(const std::string& api_type); + static std::string adsBootstrap(const std::string& api_type); // Builds a standard Cluster config fragment, with a single endpoint (at loopback:port). static envoy::api::v2::Cluster buildCluster(const std::string& name, int port, const std::string& ip_version); diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index 129947cf1bdf9..9d3b73be829bf 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -32,37 +32,16 @@ using testing::IsSubstring; namespace Envoy { namespace { -const std::string config = R"EOF( -dynamic_resources: - lds_config: {ads: {}} - cds_config: {ads: {}} - ads_config: - api_type: GRPC -static_resources: - clusters: - name: dummy_cluster - connect_timeout: { seconds: 5 } - type: STATIC - hosts: - socket_address: - address: 127.0.0.1 - port_value: 0 - lb_policy: ROUND_ROBIN - http2_protocol_options: {} -admin: - access_log_path: /dev/null - address: - socket_address: - address: 127.0.0.1 - port_value: 0 -)EOF"; - -class AdsIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, public HttpIntegrationTest { +class AdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public HttpIntegrationTest { public: - AdsIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), config) { + AdsIntegrationTest() + : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), + ConfigHelper::adsBootstrap( + sotwOrDelta() == Grpc::SotwOrDelta::Sotw ? "GRPC" : "DELTA_GRPC")) { use_lds_ = false; create_xds_upstream_ = true; tls_xds_upstream_ = true; + sotw_or_delta_ = sotwOrDelta(); } void TearDown() override { @@ -318,7 +297,7 @@ class AdsIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, public H Extensions::TransportSockets::Tls::ContextManagerImpl context_manager_{timeSystem()}; }; -INSTANTIATE_TEST_SUITE_P(IpVersionsClientType, AdsIntegrationTest, GRPC_CLIENT_INTEGRATION_PARAMS); +INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, AdsIntegrationTest, DELTA_INTEGRATION_PARAMS); // Validate basic config delivery and upgrade. TEST_P(AdsIntegrationTest, Basic) { @@ -813,13 +792,16 @@ TEST_P(AdsIntegrationTest, RdsAfterLdsInvalidated) { test_server_->waitForCounterGe("listener_manager.listener_create_success", 2); } -class AdsFailIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, +class AdsFailIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public HttpIntegrationTest { public: AdsFailIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), config) { + : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), + ConfigHelper::adsBootstrap( + sotwOrDelta() == Grpc::SotwOrDelta::Sotw ? "GRPC" : "DELTA_GRPC")) { create_xds_upstream_ = true; use_lds_ = false; + sotw_or_delta_ = sotwOrDelta(); } void TearDown() override { @@ -842,8 +824,8 @@ class AdsFailIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, } }; -INSTANTIATE_TEST_SUITE_P(IpVersionsClientType, AdsFailIntegrationTest, - GRPC_CLIENT_INTEGRATION_PARAMS); +INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, AdsFailIntegrationTest, + DELTA_INTEGRATION_PARAMS); // Validate that we don't crash on failed ADS stream. TEST_P(AdsFailIntegrationTest, ConnectDisconnect) { @@ -854,13 +836,16 @@ TEST_P(AdsFailIntegrationTest, ConnectDisconnect) { xds_stream_->finishGrpcStream(Grpc::Status::Internal); } -class AdsConfigIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, +class AdsConfigIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public HttpIntegrationTest { public: AdsConfigIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), config) { + : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), + ConfigHelper::adsBootstrap( + sotwOrDelta() == Grpc::SotwOrDelta::Sotw ? "GRPC" : "DELTA_GRPC")) { create_xds_upstream_ = true; use_lds_ = false; + sotw_or_delta_ = sotwOrDelta(); } void TearDown() override { @@ -891,8 +876,8 @@ class AdsConfigIntegrationTest : public Grpc::GrpcClientIntegrationParamTest, } }; -INSTANTIATE_TEST_SUITE_P(IpVersionsClientType, AdsConfigIntegrationTest, - GRPC_CLIENT_INTEGRATION_PARAMS); +INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, AdsConfigIntegrationTest, + DELTA_INTEGRATION_PARAMS); // This is s regression validating that we don't crash on EDS static Cluster that uses ADS. TEST_P(AdsConfigIntegrationTest, EdsClusterWithAdsConfigSource) { diff --git a/test/mocks/router/mocks.h b/test/mocks/router/mocks.h index 45979da33e5d1..79a19e1b6fec1 100644 --- a/test/mocks/router/mocks.h +++ b/test/mocks/router/mocks.h @@ -368,11 +368,11 @@ class MockRouteConfigProviderManager : public RouteConfigProviderManager { MockRouteConfigProviderManager(); ~MockRouteConfigProviderManager(); - MOCK_METHOD3(createRdsRouteConfigProvider, + MOCK_METHOD4(createRdsRouteConfigProvider, RouteConfigProviderPtr( const envoy::config::filter::network::http_connection_manager::v2::Rds& rds, Server::Configuration::FactoryContext& factory_context, - const std::string& stat_prefix)); + const std::string& stat_prefix, bool is_delta)); MOCK_METHOD2(createStaticRouteConfigProvider, RouteConfigProviderPtr(const envoy::api::v2::RouteConfiguration& route_config, Server::Configuration::FactoryContext& factory_context)); diff --git a/test/mocks/server/mocks.h b/test/mocks/server/mocks.h index 0a048f26c5a7a..826fb8ba5ecd4 100644 --- a/test/mocks/server/mocks.h +++ b/test/mocks/server/mocks.h @@ -221,11 +221,13 @@ class MockListenerComponentFactory : public ListenerComponentFactory { DrainManagerPtr createDrainManager(envoy::api::v2::Listener::DrainType drain_type) override { return DrainManagerPtr{createDrainManager_(drain_type)}; } - LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config) override { - return LdsApiPtr{createLdsApi_(lds_config)}; + LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config, + bool is_delta) override { + return LdsApiPtr{createLdsApi_(lds_config, is_delta)}; } - MOCK_METHOD1(createLdsApi_, LdsApi*(const envoy::api::v2::core::ConfigSource& lds_config)); + MOCK_METHOD2(createLdsApi_, + LdsApi*(const envoy::api::v2::core::ConfigSource& lds_config, bool is_delta)); MOCK_METHOD2(createNetworkFilterFactoryList, std::vector( const Protobuf::RepeatedPtrField& filters, @@ -252,7 +254,8 @@ class MockListenerManager : public ListenerManager { MOCK_METHOD3(addOrUpdateListener, bool(const envoy::api::v2::Listener& config, const std::string& version_info, bool modifiable)); - MOCK_METHOD1(createLdsApi, void(const envoy::api::v2::core::ConfigSource& lds_config)); + MOCK_METHOD2(createLdsApi, + void(const envoy::api::v2::core::ConfigSource& lds_config, bool is_delta)); MOCK_METHOD0(listeners, std::vector>()); MOCK_METHOD0(numConnections, uint64_t()); MOCK_METHOD1(removeListener, bool(const std::string& listener_name)); diff --git a/test/mocks/upstream/mocks.h b/test/mocks/upstream/mocks.h index 488776031132a..84738de405554 100644 --- a/test/mocks/upstream/mocks.h +++ b/test/mocks/upstream/mocks.h @@ -255,8 +255,8 @@ class MockClusterManagerFactory : public ClusterManagerFactory { Outlier::EventLoggerSharedPtr outlier_event_logger, bool added_via_api)); - MOCK_METHOD2(createCds, - CdsApiPtr(const envoy::api::v2::core::ConfigSource& cds_config, ClusterManager& cm)); + MOCK_METHOD3(createCds, CdsApiPtr(const envoy::api::v2::core::ConfigSource& cds_config, + bool is_delta, ClusterManager& cm)); private: NiceMock secret_manager_; @@ -318,6 +318,7 @@ class MockClusterManager : public ClusterManager { MOCK_METHOD1(addThreadLocalClusterUpdateCallbacks_, ClusterUpdateCallbacksHandle*(ClusterUpdateCallbacks& callbacks)); MOCK_CONST_METHOD0(warmingClusterCount, std::size_t()); + MOCK_CONST_METHOD0(xdsIsDelta, bool()); NiceMock conn_pool_; NiceMock async_client_; diff --git a/test/server/BUILD b/test/server/BUILD index f9fa7745c2ef0..0101fd216374d 100644 --- a/test/server/BUILD +++ b/test/server/BUILD @@ -147,6 +147,7 @@ envoy_cc_test( deps = [ "//source/common/protobuf:utility_lib", "//source/server:lds_api_lib", + "//test/mocks/config:config_mocks", "//test/mocks/server:server_mocks", "//test/test_common:environment_lib", "//test/test_common:utility_lib", diff --git a/test/server/lds_api_test.cc b/test/server/lds_api_test.cc index 731c0ec15cb55..c5f5e42c3b439 100644 --- a/test/server/lds_api_test.cc +++ b/test/server/lds_api_test.cc @@ -7,6 +7,7 @@ #include "server/lds_api.h" +#include "test/mocks/config/mocks.h" #include "test/mocks/server/mocks.h" #include "test/test_common/environment.h" #include "test/test_common/utility.h" @@ -26,7 +27,9 @@ namespace { class LdsApiTest : public testing::Test { public: - LdsApiTest() : request_(&cluster_manager_.async_client_), api_(Api::createApiForTest(store_)) { + LdsApiTest() + : grpc_mux_(std::make_shared>()), + request_(&cluster_manager_.async_client_), api_(Api::createApiForTest(store_)) { ON_CALL(init_manager_, add(_)).WillByDefault(Invoke([this](const Init::Target& target) { init_target_handle_ = target.createHandle("test"); })); @@ -49,6 +52,7 @@ class LdsApiTest : public testing::Test { Upstream::MockClusterMockPrioritySet cluster; cluster_map.emplace("foo_cluster", cluster); EXPECT_CALL(cluster_manager_, clusters()).WillOnce(Return(cluster_map)); + ON_CALL(cluster_manager_, adsMux()).WillByDefault(Return(grpc_mux_)); EXPECT_CALL(cluster, info()); EXPECT_CALL(*cluster.info_, addedViaApi()); EXPECT_CALL(cluster, info()); @@ -124,6 +128,7 @@ class LdsApiTest : public testing::Test { listeners.Add()->PackFrom(listener); } + std::shared_ptr> grpc_mux_; NiceMock cluster_manager_; Event::MockDispatcher dispatcher_; NiceMock random_; From 8d219e794eff049cab4941303a80ab4e1a2ab687 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Tue, 28 May 2019 17:42:04 -0400 Subject: [PATCH 18/56] add WatchMap initial untested implementation Signed-off-by: Fred Douglas --- include/envoy/config/grpc_mux.h | 80 ++------ .../common/config/delta_subscription_impl.cc | 1 + .../common/config/delta_subscription_state.cc | 34 +--- .../common/config/delta_subscription_state.h | 5 +- source/common/config/grpc_delta_xds_context.h | 109 +++++++---- source/common/config/watch_map.h | 185 ++++++++++++++++++ 6 files changed, 287 insertions(+), 127 deletions(-) create mode 100644 source/common/config/watch_map.h diff --git a/include/envoy/config/grpc_mux.h b/include/envoy/config/grpc_mux.h index 83f99df3a5c23..fb66a78abdcbb 100644 --- a/include/envoy/config/grpc_mux.h +++ b/include/envoy/config/grpc_mux.h @@ -2,7 +2,6 @@ #include "envoy/common/exception.h" #include "envoy/common/pure.h" -#include "envoy/config/subscription.h" #include "envoy/stats/stats_macros.h" #include "common/protobuf/protobuf.h" @@ -27,7 +26,6 @@ struct ControlPlaneStats { ALL_CONTROL_PLANE_STATS(GENERATE_COUNTER_STRUCT,GENERATE_GAUGE_STRUCT) }; -// TODO TODO remove. remove this whole file. class GrpcMuxCallbacks { public: virtual ~GrpcMuxCallbacks() {} @@ -68,70 +66,32 @@ class GrpcMuxWatch { typedef std::unique_ptr GrpcMuxWatchPtr; /** - * A grouping of callbacks that a GrpcMux should provide to its GrpcStream. + * Manage one or more gRPC subscriptions on a single stream to management server. This can be used + * for a single xDS API, e.g. EDS, or to combined multiple xDS APIs for ADS. */ -template class GrpcStreamCallbacks { +class GrpcMux { public: - virtual ~GrpcStreamCallbacks() {} - - /** - * For the GrpcStream to prompt the context to take appropriate action in response to the - * gRPC stream having been successfully established. - */ - virtual void onStreamEstablished() PURE; - - /** - * For the GrpcStream to prompt the context to take appropriate action in response to - * failure to establish the gRPC stream. - */ - virtual void onEstablishmentFailure() PURE; - - /** - * For the GrpcStream to pass received protos to the context. - */ - virtual void onDiscoveryResponse(std::unique_ptr&& message) PURE; + virtual ~GrpcMux() {} /** - * For the GrpcStream to call when its rate limiting logic allows more requests to be sent. + * Initiate stream with management server. */ - virtual void onWriteable() PURE; -}; + virtual void start() PURE; -/** - * Manage one or more gRPC subscriptions on a single stream to management server. This can be used - * for a single xDS API, e.g. EDS, or to combined multiple xDS APIs for ADS. Both delta and - * state-of-the-world implement this same interface - whether a GrpcSubscriptionImpl is delta or - * SotW is determined entirely by which type of GrpcMux it works with. - */ -class GrpcMux { -public: - virtual ~GrpcMux() {} /** - * Starts a configuration subscription asynchronously for some API type and resources. If the gRPC - * stream to the management server is not already up, starts it. - * @param resources vector of resource names to watch for. If this is empty, then all - * resources for type_url will result in callbacks. + * Start a configuration subscription asynchronously for some API type and resources. * @param type_url type URL corresponding to xDS API, e.g. * type.googleapis.com/envoy.api.v2.Cluster. + * @param resources set of resource names to watch for. If this is empty, then all + * resources for type_url will result in callbacks. * @param callbacks the callbacks to be notified of configuration updates. These must be valid - * until this GrpcMux is destroyed. - * @param stats reference to a stats object, which should be owned by the SubscriptionImpl, for - * the GrpcMux to record stats specific to this one subscription. - * @param init_fetch_timeout how long the first fetch has to complete before onConfigUpdateFailed - * will be called. + * until GrpcMuxWatch is destroyed. + * @return GrpcMuxWatchPtr a handle to cancel the subscription with. E.g. when a cluster goes + * away, its EDS updates should be cancelled by destroying the GrpcMuxWatchPtr. */ - virtual void addSubscription(const std::set& resources, const std::string& type_url, - SubscriptionCallbacks& callbacks, SubscriptionStats& stats, - std::chrono::milliseconds init_fetch_timeout) PURE; - - // (Un)subscribes to resources missing from / added to the passed 'resources' argument, relative - // to the resource subscription interest currently known for type_url. - // Attempts to send a discovery request if there is any such change. - virtual void updateResources(const std::set& resources, - const std::string& type_url) PURE; - - // Ends the given subscription, and drops the relevant parts of the protocol state. - virtual void removeSubscription(const std::string& type_url) PURE; + virtual GrpcMuxWatchPtr subscribe(const std::string& type_url, + const std::set& resources, + GrpcMuxCallbacks& callbacks) PURE; /** * Pause discovery requests for a given API type. This is useful when we're processing an update @@ -141,7 +101,7 @@ class GrpcMux { * type.googleapis.com/envoy.api.v2.Cluster. */ virtual void pause(const std::string& type_url) PURE; - + /** * Resume discovery requests for a given API type. This will send a discovery request if one would * have been sent during the pause. @@ -149,13 +109,9 @@ class GrpcMux { * e.g.type.googleapis.com/envoy.api.v2.Cluster. */ virtual void resume(const std::string& type_url) PURE; - - // TODO(fredlas) remove, only here for compatibility with old-style GrpcMuxImpl. - virtual void start() PURE; - virtual GrpcMuxWatchPtr subscribe(const std::string& type_url, - const std::set& resources, - GrpcMuxCallbacks& callbacks) PURE; }; +typedef std::unique_ptr GrpcMuxPtr; + } // namespace Config } // namespace Envoy diff --git a/source/common/config/delta_subscription_impl.cc b/source/common/config/delta_subscription_impl.cc index bea462ef3c443..4ca0fab99e4f7 100644 --- a/source/common/config/delta_subscription_impl.cc +++ b/source/common/config/delta_subscription_impl.cc @@ -22,6 +22,7 @@ void DeltaSubscriptionImpl::start(const std::set& resources, } void DeltaSubscriptionImpl::updateResources(const std::set& update_to_these_names) { + // TODO TODO this should really be like "updateWatch". context_->updateResources(update_to_these_names, type_url_); } diff --git a/source/common/config/delta_subscription_state.cc b/source/common/config/delta_subscription_state.cc index 73f4cdb03417d..cda8af9a9a308 100644 --- a/source/common/config/delta_subscription_state.cc +++ b/source/common/config/delta_subscription_state.cc @@ -7,7 +7,6 @@ namespace Envoy { namespace Config { DeltaSubscriptionState::DeltaSubscriptionState(const std::string& type_url, - const std::set& resource_names, SubscriptionCallbacks& callbacks, const LocalInfo::LocalInfo& local_info, std::chrono::milliseconds init_fetch_timeout, @@ -15,10 +14,6 @@ DeltaSubscriptionState::DeltaSubscriptionState(const std::string& type_url, SubscriptionStats& stats) : type_url_(type_url), callbacks_(callbacks), local_info_(local_info), init_fetch_timeout_(init_fetch_timeout), stats_(stats) { - // In normal usage of updateResourceInterest(), the caller is supposed to cause a discovery - // request to be queued if it returns true. We don't need to do that because we know that the - // subscription gRPC stream is not yet established, and establishment causes a request. - updateResourceInterest(resource_names); setInitFetchTimeout(dispatcher); } @@ -46,32 +41,23 @@ void DeltaSubscriptionState::resume() { // Returns true if there is any meaningful change in our subscription interest, worth reporting to // the server. -void DeltaSubscriptionState::updateResourceInterest( - const std::set& update_to_these_names) { - std::vector cur_added; - std::vector cur_removed; - - std::set_difference(update_to_these_names.begin(), update_to_these_names.end(), - resource_names_.begin(), resource_names_.end(), - std::inserter(cur_added, cur_added.begin())); - std::set_difference(resource_names_.begin(), resource_names_.end(), update_to_these_names.begin(), - update_to_these_names.end(), std::inserter(cur_removed, cur_removed.begin())); - +void DeltaSubscriptionState::updateResourceInterest(std::vector cur_added, + std::vector cur_removed) { for (const auto& a : cur_added) { setResourceWaitingForServer(a); - // Removed->added requires us to keep track of it as a "new" addition, since our user may have - // forgotten its copy of the resource after instructing us to remove it, and so needs to be - // reminded of it. + // If interest in a resource is removed-then-added (all before a discovery request + // can be sent), we must treat it as a "new" addition: our user may have forgotten its + // copy of the resource after instructing us to remove it, and need to be reminded of it. names_removed_.erase(a); names_added_.insert(a); } for (const auto& r : cur_removed) { setLostInterestInResource(r); - // Ideally, when a resource is added-then-removed in between requests, we would avoid putting - // a superfluous "unsubscribe [resource that was never subscribed]" in the request. However, - // the removed-then-added case *does* need to go in the request, and due to how we accomplish - // that, it's difficult to distinguish remove-add-remove from add-remove (because "remove-add" - // has to be treated as equivalent to just "add"). + // Ideally, when interest in a resource is added-then-removed in between requests, + // we would avoid putting a superfluous "unsubscribe [resource that was never subscribed]" + // in the request. However, the removed-then-added case *does* need to go in the request, + // and due to how we accomplish that, it's difficult to distinguish remove-add-remove from + // add-remove (because "remove-add" has to be treated as equivalent to just "add"). names_added_.erase(r); names_removed_.insert(r); } diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index f727592c2dd5f..62c39c333ef77 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -26,8 +26,8 @@ struct UpdateAck { // being multiplexed together by ADS. class DeltaSubscriptionState : public Logger::Loggable { public: - DeltaSubscriptionState(const std::string& type_url, const std::set& resource_names, - SubscriptionCallbacks& callbacks, const LocalInfo::LocalInfo& local_info, + DeltaSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, + const LocalInfo::LocalInfo& local_info, std::chrono::milliseconds init_fetch_timeout, Event::Dispatcher& dispatcher, SubscriptionStats& stats); @@ -91,6 +91,7 @@ class DeltaSubscriptionState : public Logger::Loggable { std::set resource_names_; const std::string type_url_; + // callbacks_ is expected to be a WatchMap. SubscriptionCallbacks& callbacks_; const LocalInfo::LocalInfo& local_info_; std::chrono::milliseconds init_fetch_timeout_; diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h index 61ff1b7d6cfd2..02efa67cb67d1 100644 --- a/source/common/config/grpc_delta_xds_context.h +++ b/source/common/config/grpc_delta_xds_context.h @@ -20,6 +20,18 @@ namespace Envoy { namespace Config { +// TODO TODO not actually needed; just an adapter to the old-style subscribe(). +class TokenizedGrpcMuxWatch : public GrpcMuxWatch { +public: + TokenizedGrpcMuxWatch(GrpcDeltaXdsContext& context, WatchToken token) + : context_(context), token_(token) {} + ~TokenizedGrpcMuxWatch() { context_.removeWatch(token_); } + +private: + WatchToken token_; + GrpcDeltaXdsContext& context_; +}; + // Manages subscriptions to one or more type of resource. The logical protocol // state of those subscription(s) is handled by DeltaSubscriptionState. // This class owns the GrpcStream used to talk to the server, maintains queuing @@ -38,43 +50,43 @@ class GrpcDeltaXdsContext : public GrpcMux, grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, rate_limit_settings) {} - void addSubscription(const std::set& resources, const std::string& type_url, - SubscriptionCallbacks& callbacks, SubscriptionStats& stats, - std::chrono::milliseconds init_fetch_timeout) override { - subscriptions_.emplace(type_url, std::make_unique( - type_url, resources, callbacks, local_info_, - init_fetch_timeout, dispatcher_, stats)); - subscription_ordering_.emplace_back(type_url); - grpc_stream_.establishNewStream(); // (idempotent) TODO TODO does print warning though... + WatchToken addWatch(const std::string& type_url, const std::set& resources, + SubscriptionCallbacks& callbacks) override { + if + type_url not in subscriptons_ / + watch_maps_ yet addSubscription() + + WatchToken watch_token = watch_maps_[type_url].addWatch(callbacks); + // updateWatch() queues a discovery request if any of 'resources' are not yet subscribed + updateWatch(type_url, watch_token, resources); + return watch_token; } - // Enqueues and attempts to send a discovery request, (un)subscribing to resources missing from / - // added to the passed 'resources' argument, relative to resource_versions_. - void updateResources(const std::set& resources, - const std::string& type_url) override { + void removeWatch(const std::string& type_url, WatchToken watch_token) { + // updateWatch() queues a discovery request if any resources were cared about only by this + // watch. + updateWatch(type_url, watch_token, {}); + if (watch_maps_[type_url].removeWatch(watch_token)) { + removeSubscription(type_url); + } + } + + // Updates the list of resource names watched by the given watch. If an added name is new across + // the whole subscription, or if a removed name has no other watch interested in it, then the + // subscription will enqueue and attempt to send an appropriate discovery request. + void updateWatch(const std::string& type_url, WatchToken watch_token, + const std::set& resources) override { + auto added_removed = watch_maps_[type_url].updateWatchInterest(watch_token, resources); auto sub = subscriptions_.find(type_url); if (sub == subscriptions_.end()) { - ENVOY_LOG(warn, "Not updating non-existent subscription {}.", type_url); + ENVOY_LOG(error, "Watch of {} has no subscription to update.", type_url); return; } - sub->second->updateResourceInterest(resources); + sub->second->updateResourceInterest(added_removed.first, added_removed.second); // Tell the server about our new interests, if there are any. trySendDiscoveryRequests(); } - void removeSubscription(const std::string& type_url) override { - subscriptions_.erase(type_url); - // And remove from the subscription_ordering_ list. - auto it = subscription_ordering_.begin(); - while (it != subscription_ordering_.end()) { - if (*it == type_url) { - it = subscription_ordering_.erase(it); - } else { - ++it; - } - } - } - void pause(const std::string& type_url) override { auto sub = subscriptions_.find(type_url); if (sub == subscriptions_.end()) { @@ -129,21 +141,39 @@ class GrpcDeltaXdsContext : public GrpcMux, trySendDiscoveryRequests(); } - // TODO(fredlas) remove, only here for compatibility with old-style GrpcMuxImpl. - GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, - GrpcMuxCallbacks&) override { - // don't need any implementation here. only grpc_mux_subscription_impl ever calls it, and there - // would never be a GrpcDeltaXdsContext held by one of those. - NOT_IMPLEMENTED_GCOVR_EXCL_LINE; - return nullptr; - } - void start() override { - // don't need any implementation here. only grpc_mux_subscription_impl ever calls it, and there - // would never be a GrpcDeltaXdsContext held by one of those. - NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + // TODO TODO but yeah this should just be gone!!!!!!!!! + GrpcMuxWatchPtr subscribe(const std::string& type_url, const std::set& resources, + GrpcMuxCallbacks& callbacks) override { + return std::make_unique(*this, addWatch(type_url, resources, callbacks)); } + void start() override { grpc_stream_.establishNewStream(); } private: + void addSubscription(const std::string& type_url, SubscriptionStats& stats, + std::chrono::milliseconds init_fetch_timeout) override { + watch_maps_.emplace(type_url, std::make_unique()); + subscriptions_.emplace(type_url, std::make_unique( + type_url, *watch_maps_[type_url], local_info_, + init_fetch_timeout, dispatcher_, stats)); + subscription_ordering_.emplace_back(type_url); + } + + // TODO do we need this? or should we just let subscriptions hang out even when nobody is + // interested in them? + void removeSubscription(const std::string& type_url) override { + subscriptions_.erase(type_url); + watch_maps_.erase(type_url); + // And remove from the subscription_ordering_ list. + auto it = subscription_ordering_.begin(); + while (it != subscription_ordering_.end()) { + if (*it == type_url) { + it = subscription_ordering_.erase(it); + } else { + ++it; + } + } + } + void trySendDiscoveryRequests() { while (true) { // Do any of our subscriptions even want to send a request? @@ -232,6 +262,7 @@ class GrpcDeltaXdsContext : public GrpcMux, // Map from type_url strings to a DeltaSubscriptionState for that type. absl::flat_hash_map> subscriptions_; + absl::flat_hash_map> watch_maps_; // Determines the order of initial discovery requests. (Assumes that subscriptions are added in // the order of Envoy's dependency ordering). diff --git a/source/common/config/watch_map.h b/source/common/config/watch_map.h new file mode 100644 index 0000000000000..d5cf9b49d8b2f --- /dev/null +++ b/source/common/config/watch_map.h @@ -0,0 +1,185 @@ +#pragma once + +namespace Envoy { +namespace Config { + +// An opaque token given out to users of WatchMap, to identify a given watch. +using WatchToken = uint64_t; + +// Manages "watches" of xDS resources. Several xDS callers might ask for a subscription to the same +// resource name "X". The xDS machinery must return to each their very own subscription to X. +// The xDS machinery's "watch" concept accomplishes that, while avoiding parallel reduntant xDS +// requests for X. Each of those subscriptions is viewed as a "watch" on X, while behind the scenes +// there is just a single real subscription to that resource name. +// This class maintains the mapping between those two: it +// 1) delivers updates to all interested watches, and +// 2) adds/removes resource names to/from the subscription when first/last watch is added/removed. +// +// #1 is accomplished by WatchMap's implementation of the SubscriptionCallbacks interface. +// This interface allows the xDS client to just throw each xDS update message it receives directly +// into WatchMap::onConfigUpdate, rather than having to track the various watches' callbacks. +// +// A WatchMap is assumed to be dedicated to a single type_url type of resource (EDS, CDS, etc). +class WatchMap : public SubscriptionCallbacks { +public: + WatchToken addWatch(SubscriptionCallbacks& callbacks) override { + WatchToken next_watch = next_watch_++; + watches_[next_watch] = Watch(callbacks); + } + + // Returns true if this was the very last watch in the map. + // Expects that the watch to be removed has already had all of its resource names removed via + // updateWatchInterest(). + bool removeWatch(WatchToken watch_token) { + watches_.erase(watch_token); + return watches_.empty(); + } + + // Returns resource names added and removed across all watches. That is: + // 1) if 'resources' contains X and no other watch cares about X, X will be in pair.first. + // 2) if 'resources' does not contain Y, and this watch was the only one that cared about Y, + // Y will be in pair.second. + std::pair, std::set> + updateWatchInterest(WatchToken watch_token, const std::set& update_to_these_names) { + std::vector newly_added_to_watch; + std::vector newly_removed_from_watch; + std::set_difference(update_to_these_names.begin(), update_to_these_names.end(), + watches_[watch_token].resource_names_.begin(), + watches_[watch_token].resource_names_.end(), + std::inserter(newly_added_to_watch, newly_added_to_watch.begin())); + std::set_difference(watches_[watch_token].resource_names_.begin(), + watches_[watch_token].resource_names_.end(), update_to_these_names.begin(), + update_to_these_names.end(), + std::inserter(newly_removed_from_watch, newly_removed_from_watch.begin())); + + watches_[watch_token].resource_names_ = update_to_these_names; + + return std::make_pair(findAdditions(newly_added_to_watch, watch_token), + findRemovals(newly_removed_from_watch, watch_token)); + } + + virtual void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) override { + if (watches_.empty()) { + ENVOY_LOG(warn, "WatchMap::onConfigUpdate: there are no watches!"); + return; + } + absl::flat_hash_map> per_watch_updates; + GrpcMuxCallbacks& name_getter = watches_.front()->callbacks_; + for (const auto& r : resources) { + const absl::flat_hash_set& interested_in_r = + watch_interest_.find(name_getter.resourceName(r)); + for (interested_token : interested_in_r) + per_watch_added[interested_token].Add()->CopyFrom(r); + } + + // We just bundled up the updates into nice per-watch packages. Now, deliver them. + for (const auto& updated : per_watch_updates) { + watches_[updated.first].callbacks_.onConfigUpdate(updated.second, version_info); + } + } + + virtual void + onConfigUpdate(const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) override { + absl::flat_hash_map> + per_watch_added; + absl::flat_hash_map> per_watch_removed; + for (const auto& r : added_resources) { + const absl::flat_hash_set& interested_in_r = watch_interest_.find(r.name()); + for (interested_token : interested_in_r) + per_watch_added[interested_token].Add()->CopyFrom(r); + } + for (const auto& r : removed_resources) { + const absl::flat_hash_set& interested_in_r = watch_interest_.find(r); + for (interested_token : interested_in_r) + *per_watch_removed[interested_token].Add() = r; + } + + // We just bundled up the updates into nice per-watch packages. Now, deliver them. + for (const auto& added : per_watch_added) { + auto removed = per_watch_removed.find(added.first); + if (removed == per_watch_removed.end()) { // additions only, no removals + watches_[added.first].callbacks_.onConfigUpdate(per_watch_added.second, {}, + system_version_info); + } else { // both additions and removals + watches_[added.first].callbacks_.onConfigUpdate(per_watch_added.second, removed.second, + system_version_info); + // Drop it now, so the final removals-only pass won't accidentally use it. + per_watch_removed.erase(removed); + } + } + // Any removals-only updates will not have been picked up in the per_watch_added loop. + for (const auto& removed : per_watch_removed) { + watches_[removed.first].callbacks_.onConfigUpdate({}, removed.second, system_version_info); + } + } + + virtual void onConfigUpdateFailed(const EnvoyException* e) override { + for (auto& watch : watches_) { + watch.second.onConfigUpdateFailed(e); + } + } + + virtual std::string resourceName(const ProtobufWkt::Any& resource) override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } + +private: + struct Watch { + std::set resource_names_; // must be sorted set, for set_difference. + GrpcMuxCallbacks& callbacks_; + }; + + // Given a list of names that are new to an individual watch, returns those names that are in fact + // new to the entire subscription. + std::set findAdditions(const std::vector& newly_added_to_watch, + WatchToken watch_token) { + std::set newly_added_to_subscription; + for (const auto& name : newly_added_to_watch) { + auto entry = watch_interest_.find(name); + if (entry == watch_interest_.end()) { + newly_added_to_subscription.insert(name); + watch_interest_.emplace(name, {watch_token}) + } else { + entry.second.insert(watch_token); + } + } + return newly_added_to_subscription; + } + + // Given a list of names that an individual watch no longer cares about, returns those names that + // in fact the entire subscription no longer cares about. + std::set findRemovals(const std::vector& newly_removed_from_watch, + WatchToken watch_token) { + std::set newly_removed_from_subscription; + for (const auto& name : newly_removed_from_watch) { + auto entry = watch_interest_.find(name); + if (entry == watch_interest_.end()) { + ENVOY_LOG(warn, "WatchMap: tried to remove a watch from untracked resource {}", name); + continue; + } + entry.second.erase(watch_token); + if (entry.second.empty()) { + watch_interest_.erase(entry); + } + newly_removed_from_subscription.insert(name); + } + return newly_removed_from_subscription; + } + + absl::flat_hash_map watches_; + // Maps a resource name to the set of watches interested in that resource. Has two purposes: + // 1) Acts as a reference count; no watches care anymore ==> the resource can be removed. + // 2) Enables efficient lookup of all interested watches when a resource has been updated. + absl::flat_hash_map> watch_interest_; + + WatchToken next_watch_{0}; + + WatchMap(const WatchMap&) = delete; + WatchMap& operator=(const WatchMap&) = delete; +}; + +} // namespace Config +} // namespace Envoy From b94c2eefb8d88771c412385c88f11fdc0c6f5fe9 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 30 May 2019 14:16:40 -0400 Subject: [PATCH 19/56] merge cleaned up WatchMap Signed-off-by: Fred Douglas --- source/common/config/BUILD | 12 ++ source/common/config/watch_map.cc | 186 ++++++++++++++++++++++++++++++ source/common/config/watch_map.h | 183 ++++++++++------------------- 3 files changed, 256 insertions(+), 125 deletions(-) create mode 100644 source/common/config/watch_map.cc diff --git a/source/common/config/BUILD b/source/common/config/BUILD index 81b560a9a0f16..949d6df7ce981 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -395,3 +395,15 @@ envoy_cc_library( "//source/common/protobuf", ], ) + +envoy_cc_library( + name = "watch_map_lib", + srcs = ["watch_map.cc"], + hdrs = ["watch_map.h"], + deps = [ + "//include/envoy/config:subscription_interface", + "//source/common/common:assert_lib", + "//source/common/common:minimal_logger_lib", + "//source/common/protobuf", + ], +) diff --git a/source/common/config/watch_map.cc b/source/common/config/watch_map.cc new file mode 100644 index 0000000000000..0aad8a31c934e --- /dev/null +++ b/source/common/config/watch_map.cc @@ -0,0 +1,186 @@ +#include "common/config/watch_map.h" + +namespace Envoy { +namespace Config { + +WatchMap::Token WatchMap::addWatch(SubscriptionCallbacks& callbacks) { + WatchMap::Token next_watch = next_watch_++; + watches_.emplace(next_watch, WatchMap::Watch(callbacks)); + return next_watch; +} + +bool WatchMap::removeWatch(WatchMap::Token token) { + watches_.erase(token); + return watches_.empty(); +} + +std::pair, std::set> +WatchMap::updateWatchInterest(WatchMap::Token token, + const std::set& update_to_these_names) { + auto watches_entry = watches_.find(token); + if (watches_entry == watches_.end()) { + ENVOY_LOG(error, "updateWatchInterest() called on nonexistent token!"); + return std::make_pair(std::set(), std::set()); + } + auto& watch = watches_entry->second; + + std::vector newly_added_to_watch; + std::set_difference(update_to_these_names.begin(), update_to_these_names.end(), + watch.resource_names_.begin(), watch.resource_names_.end(), + std::inserter(newly_added_to_watch, newly_added_to_watch.begin())); + + std::vector newly_removed_from_watch; + std::set_difference(watch.resource_names_.begin(), watch.resource_names_.end(), + update_to_these_names.begin(), update_to_these_names.end(), + std::inserter(newly_removed_from_watch, newly_removed_from_watch.begin())); + + watch.resource_names_ = update_to_these_names; + + return std::make_pair(findAdditions(newly_added_to_watch, token), + findRemovals(newly_removed_from_watch, token)); +} + +const absl::flat_hash_set& +WatchMap::tokensInterestedIn(const std::string& resource_name) { + auto entry = watch_interest_.find(resource_name); + if (entry == watch_interest_.end()) { + return empty_token_set_; + } + return entry->second; +} + +void WatchMap::onConfigUpdate(const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) { + if (watches_.empty()) { + ENVOY_LOG(warn, "WatchMap::onConfigUpdate: there are no watches!"); + return; + } + SubscriptionCallbacks& name_getter = watches_.begin()->second.callbacks_; + + // Build a map from watches, to the set of updated resources that each watch cares about. Each + // entry in the map is then a nice little bundle that can be fed directly into the individual + // onConfigUpdate()s. + absl::flat_hash_map> + per_watch_updates; + for (const auto& r : resources) { + const absl::flat_hash_set& interested_in_r = + tokensInterestedIn(name_getter.resourceName(r)); + for (const auto& interested_token : interested_in_r) { + per_watch_updates[interested_token].Add()->CopyFrom(r); + } + } + + // We just bundled up the updates into nice per-watch packages. Now, deliver them. + for (const auto& updated : per_watch_updates) { + auto entry = watches_.find(updated.first); + if (entry == watches_.end()) { + ENVOY_LOG(error, "A token referred to by watch_interest_ is not present in watches_!"); + continue; + } + entry->second.callbacks_.onConfigUpdate(updated.second, version_info); + } +} + +void WatchMap::tryDeliverConfigUpdate( + WatchMap::Token token, + const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) { + auto entry = watches_.find(token); + if (entry == watches_.end()) { + ENVOY_LOG(error, "A token referred to by watch_interest_ is not present in watches_!"); + return; + } + entry->second.callbacks_.onConfigUpdate(added_resources, removed_resources, system_version_info); +} + +void WatchMap::onConfigUpdate( + const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) { + if (watches_.empty()) { + ENVOY_LOG(warn, "WatchMap::onConfigUpdate: there are no watches!"); + return; + } + // Build a pair of maps: from watches, to the set of resources {added,removed} that each watch + // cares about. Each entry in the map-pair is then a nice little bundle that can be fed directly + // into the individual onConfigUpdate()s. + absl::flat_hash_map> + per_watch_added; + absl::flat_hash_map> per_watch_removed; + for (const auto& r : added_resources) { + const absl::flat_hash_set& interested_in_r = tokensInterestedIn(r.name()); + for (const auto& interested_token : interested_in_r) { + per_watch_added[interested_token].Add()->CopyFrom(r); + } + } + for (const auto& r : removed_resources) { + const absl::flat_hash_set& interested_in_r = tokensInterestedIn(r); + for (const auto& interested_token : interested_in_r) { + *per_watch_removed[interested_token].Add() = r; + } + } + + // We just bundled up the updates into nice per-watch packages. Now, deliver them. + for (const auto& added : per_watch_added) { + const WatchMap::Token& cur_token = added.first; + auto removed = per_watch_removed.find(cur_token); + if (removed == per_watch_removed.end()) { + // additions only, no removals + tryDeliverConfigUpdate(cur_token, added.second, {}, system_version_info); + } else { + // both additions and removals + tryDeliverConfigUpdate(cur_token, added.second, removed->second, system_version_info); + // Drop the removals now, so the final removals-only pass won't use them. + per_watch_removed.erase(removed); + } + } + // Any removals-only updates will not have been picked up in the per_watch_added loop. + for (const auto& removed : per_watch_removed) { + tryDeliverConfigUpdate(removed.first, {}, removed.second, system_version_info); + } +} + +void WatchMap::onConfigUpdateFailed(const EnvoyException* e) { + for (auto& watch : watches_) { + watch.second.callbacks_.onConfigUpdateFailed(e); + } +} + +std::set WatchMap::findAdditions(const std::vector& newly_added_to_watch, + WatchMap::Token token) { + std::set newly_added_to_subscription; + for (const auto& name : newly_added_to_watch) { + auto entry = watch_interest_.find(name); + if (entry == watch_interest_.end()) { + newly_added_to_subscription.insert(name); + watch_interest_[name] = {token}; + } else { + entry->second.insert(token); + } + } + return newly_added_to_subscription; +} + +std::set +WatchMap::findRemovals(const std::vector& newly_removed_from_watch, + WatchMap::Token token) { + std::set newly_removed_from_subscription; + for (const auto& name : newly_removed_from_watch) { + auto entry = watch_interest_.find(name); + if (entry == watch_interest_.end()) { + ENVOY_LOG(warn, "WatchMap: tried to remove a watch from untracked resource {}", name); + continue; + } + + entry->second.erase(token); + if (entry->second.empty()) { + watch_interest_.erase(entry); + newly_removed_from_subscription.insert(name); + } + } + return newly_removed_from_subscription; +} + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/watch_map.h b/source/common/config/watch_map.h index d5cf9b49d8b2f..d8bb112b23ee3 100644 --- a/source/common/config/watch_map.h +++ b/source/common/config/watch_map.h @@ -1,181 +1,114 @@ #pragma once +#include +#include +#include + +#include "envoy/config/subscription.h" + +#include "common/common/assert.h" +#include "common/common/logger.h" + +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" + namespace Envoy { namespace Config { -// An opaque token given out to users of WatchMap, to identify a given watch. -using WatchToken = uint64_t; - // Manages "watches" of xDS resources. Several xDS callers might ask for a subscription to the same // resource name "X". The xDS machinery must return to each their very own subscription to X. -// The xDS machinery's "watch" concept accomplishes that, while avoiding parallel reduntant xDS +// The xDS machinery's "watch" concept accomplishes that, while avoiding parallel redundant xDS // requests for X. Each of those subscriptions is viewed as a "watch" on X, while behind the scenes // there is just a single real subscription to that resource name. -// This class maintains the mapping between those two: it +// +// This class maintains the watches<-->subscription mapping: it // 1) delivers updates to all interested watches, and -// 2) adds/removes resource names to/from the subscription when first/last watch is added/removed. +// 2) tracks which resource names should be {added to,removed from} the subscription when the +// {first,last} watch on a resource name is {added,removed}. // // #1 is accomplished by WatchMap's implementation of the SubscriptionCallbacks interface. // This interface allows the xDS client to just throw each xDS update message it receives directly // into WatchMap::onConfigUpdate, rather than having to track the various watches' callbacks. // +// The information for #2 is returned by updateWatchInterest(); the caller should use it to +// update the subscription accordingly. +// // A WatchMap is assumed to be dedicated to a single type_url type of resource (EDS, CDS, etc). -class WatchMap : public SubscriptionCallbacks { +class WatchMap : public SubscriptionCallbacks, public Logger::Loggable { public: - WatchToken addWatch(SubscriptionCallbacks& callbacks) override { - WatchToken next_watch = next_watch_++; - watches_[next_watch] = Watch(callbacks); - } + WatchMap() {} + + // An opaque token given out to users of WatchMap, to identify a given watch. + using Token = uint64_t; + + // Adds 'callbacks' to the WatchMap, with no resource names being watched. + // (Use updateWatchInterest() to add some names). + // Returns a new token identifying the newly added watch. + Token addWatch(SubscriptionCallbacks& callbacks); // Returns true if this was the very last watch in the map. // Expects that the watch to be removed has already had all of its resource names removed via // updateWatchInterest(). - bool removeWatch(WatchToken watch_token) { - watches_.erase(watch_token); - return watches_.empty(); - } + bool removeWatch(Token token); - // Returns resource names added and removed across all watches. That is: + // Updates the set of resource names that the given watch should watch. + // Returns any resource name additions/removals that are unique across all watches. That is: // 1) if 'resources' contains X and no other watch cares about X, X will be in pair.first. // 2) if 'resources' does not contain Y, and this watch was the only one that cared about Y, // Y will be in pair.second. std::pair, std::set> - updateWatchInterest(WatchToken watch_token, const std::set& update_to_these_names) { - std::vector newly_added_to_watch; - std::vector newly_removed_from_watch; - std::set_difference(update_to_these_names.begin(), update_to_these_names.end(), - watches_[watch_token].resource_names_.begin(), - watches_[watch_token].resource_names_.end(), - std::inserter(newly_added_to_watch, newly_added_to_watch.begin())); - std::set_difference(watches_[watch_token].resource_names_.begin(), - watches_[watch_token].resource_names_.end(), update_to_these_names.begin(), - update_to_these_names.end(), - std::inserter(newly_removed_from_watch, newly_removed_from_watch.begin())); - - watches_[watch_token].resource_names_ = update_to_these_names; - - return std::make_pair(findAdditions(newly_added_to_watch, watch_token), - findRemovals(newly_removed_from_watch, watch_token)); - } + updateWatchInterest(Token token, const std::set& update_to_these_names); + // SubscriptionCallbacks virtual void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, - const std::string& version_info) override { - if (watches_.empty()) { - ENVOY_LOG(warn, "WatchMap::onConfigUpdate: there are no watches!"); - return; - } - absl::flat_hash_map> per_watch_updates; - GrpcMuxCallbacks& name_getter = watches_.front()->callbacks_; - for (const auto& r : resources) { - const absl::flat_hash_set& interested_in_r = - watch_interest_.find(name_getter.resourceName(r)); - for (interested_token : interested_in_r) - per_watch_added[interested_token].Add()->CopyFrom(r); - } - - // We just bundled up the updates into nice per-watch packages. Now, deliver them. - for (const auto& updated : per_watch_updates) { - watches_[updated.first].callbacks_.onConfigUpdate(updated.second, version_info); - } - } - + const std::string& version_info) override; virtual void onConfigUpdate(const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, - const std::string& system_version_info) override { - absl::flat_hash_map> - per_watch_added; - absl::flat_hash_map> per_watch_removed; - for (const auto& r : added_resources) { - const absl::flat_hash_set& interested_in_r = watch_interest_.find(r.name()); - for (interested_token : interested_in_r) - per_watch_added[interested_token].Add()->CopyFrom(r); - } - for (const auto& r : removed_resources) { - const absl::flat_hash_set& interested_in_r = watch_interest_.find(r); - for (interested_token : interested_in_r) - *per_watch_removed[interested_token].Add() = r; - } - - // We just bundled up the updates into nice per-watch packages. Now, deliver them. - for (const auto& added : per_watch_added) { - auto removed = per_watch_removed.find(added.first); - if (removed == per_watch_removed.end()) { // additions only, no removals - watches_[added.first].callbacks_.onConfigUpdate(per_watch_added.second, {}, - system_version_info); - } else { // both additions and removals - watches_[added.first].callbacks_.onConfigUpdate(per_watch_added.second, removed.second, - system_version_info); - // Drop it now, so the final removals-only pass won't accidentally use it. - per_watch_removed.erase(removed); - } - } - // Any removals-only updates will not have been picked up in the per_watch_added loop. - for (const auto& removed : per_watch_removed) { - watches_[removed.first].callbacks_.onConfigUpdate({}, removed.second, system_version_info); - } - } + const std::string& system_version_info) override; - virtual void onConfigUpdateFailed(const EnvoyException* e) override { - for (auto& watch : watches_) { - watch.second.onConfigUpdateFailed(e); - } - } + virtual void onConfigUpdateFailed(const EnvoyException* e) override; - virtual std::string resourceName(const ProtobufWkt::Any& resource) override { + virtual std::string resourceName(const ProtobufWkt::Any&) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } private: struct Watch { + Watch(SubscriptionCallbacks& callbacks) : callbacks_(callbacks) {} std::set resource_names_; // must be sorted set, for set_difference. - GrpcMuxCallbacks& callbacks_; + SubscriptionCallbacks& callbacks_; }; // Given a list of names that are new to an individual watch, returns those names that are in fact // new to the entire subscription. std::set findAdditions(const std::vector& newly_added_to_watch, - WatchToken watch_token) { - std::set newly_added_to_subscription; - for (const auto& name : newly_added_to_watch) { - auto entry = watch_interest_.find(name); - if (entry == watch_interest_.end()) { - newly_added_to_subscription.insert(name); - watch_interest_.emplace(name, {watch_token}) - } else { - entry.second.insert(watch_token); - } - } - return newly_added_to_subscription; - } + Token token); // Given a list of names that an individual watch no longer cares about, returns those names that // in fact the entire subscription no longer cares about. std::set findRemovals(const std::vector& newly_removed_from_watch, - WatchToken watch_token) { - std::set newly_removed_from_subscription; - for (const auto& name : newly_removed_from_watch) { - auto entry = watch_interest_.find(name); - if (entry == watch_interest_.end()) { - ENVOY_LOG(warn, "WatchMap: tried to remove a watch from untracked resource {}", name); - continue; - } - entry.second.erase(watch_token); - if (entry.second.empty()) { - watch_interest_.erase(entry); - } - newly_removed_from_subscription.insert(name); - } - return newly_removed_from_subscription; - } + Token token); + + // Calls watches_[token].callbacks_.onConfigUpdate(), or logs an error if token isn't in watches_. + void tryDeliverConfigUpdate( + Token token, const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info); + + // Does a lookup in watch_interest_, returning empty set if not found. + const absl::flat_hash_set& tokensInterestedIn(const std::string& resource_name); + // A little hack to allow tokensInterestedIn() to return a ref, rather than a copy. + const absl::flat_hash_set empty_token_set_{}; + + absl::flat_hash_map watches_; - absl::flat_hash_map watches_; // Maps a resource name to the set of watches interested in that resource. Has two purposes: // 1) Acts as a reference count; no watches care anymore ==> the resource can be removed. // 2) Enables efficient lookup of all interested watches when a resource has been updated. - absl::flat_hash_map> watch_interest_; + absl::flat_hash_map> watch_interest_; - WatchToken next_watch_{0}; + Token next_watch_{0}; WatchMap(const WatchMap&) = delete; WatchMap& operator=(const WatchMap&) = delete; From 4e174c1e98d80f6b62b92ede398dd0748e5a2b86 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 31 May 2019 13:25:03 -0400 Subject: [PATCH 20/56] config: move SubscriptionCallbacks to ctor, pointers become refs Signed-off-by: Fred Douglas --- include/envoy/config/subscription.h | 4 +-- .../common/config/delta_subscription_impl.cc | 17 +++++------ .../common/config/delta_subscription_impl.h | 6 ++-- .../config/filesystem_subscription_impl.cc | 21 +++++-------- .../config/filesystem_subscription_impl.h | 13 ++++---- .../config/grpc_mux_subscription_impl.cc | 21 +++++++------ .../config/grpc_mux_subscription_impl.h | 7 +++-- source/common/config/grpc_subscription_impl.h | 17 ++++++----- .../common/config/http_subscription_impl.cc | 23 +++++++------- source/common/config/http_subscription_impl.h | 8 ++--- source/common/config/subscription_factory.cc | 19 ++++++------ source/common/config/subscription_factory.h | 4 ++- source/common/router/rds_impl.cc | 4 +-- source/common/router/vhds.cc | 4 +-- source/common/router/vhds.h | 2 +- source/common/secret/sds_api.cc | 5 ++-- source/common/upstream/cds_api_impl.cc | 2 +- source/common/upstream/cds_api_impl.h | 2 +- source/common/upstream/eds.cc | 4 +-- source/server/lds_api.cc | 4 +-- .../config/delta_subscription_impl_test.cc | 2 +- .../config/delta_subscription_test_harness.h | 4 +-- .../filesystem_subscription_impl_test.cc | 6 ++-- .../filesystem_subscription_test_harness.h | 4 +-- .../config/grpc_subscription_impl_test.cc | 2 +- .../config/grpc_subscription_test_harness.h | 6 ++-- .../config/http_subscription_test_harness.h | 5 ++-- .../config/subscription_factory_test.cc | 30 +++++++++---------- test/common/router/vhds_test.cc | 14 ++++----- test/mocks/config/mocks.h | 5 ++-- 30 files changed, 132 insertions(+), 133 deletions(-) diff --git a/include/envoy/config/subscription.h b/include/envoy/config/subscription.h index 798168e7182ba..ab6b3a2d64166 100644 --- a/include/envoy/config/subscription.h +++ b/include/envoy/config/subscription.h @@ -68,10 +68,8 @@ class Subscription { * Start a configuration subscription asynchronously. This should be called once and will continue * to fetch throughout the lifetime of the Subscription object. * @param resources set of resource names to fetch. - * @param callbacks the callbacks to be notified of configuration updates. The callback must not - * result in the deletion of the Subscription object. */ - virtual void start(const std::set& resources, SubscriptionCallbacks& callbacks) PURE; + virtual void start(const std::set& resource_names) PURE; /** * Update the resources to fetch. diff --git a/source/common/config/delta_subscription_impl.cc b/source/common/config/delta_subscription_impl.cc index e3d285ec80106..d2ee8cc76f1f1 100644 --- a/source/common/config/delta_subscription_impl.cc +++ b/source/common/config/delta_subscription_impl.cc @@ -14,12 +14,12 @@ DeltaSubscriptionImpl::DeltaSubscriptionImpl( const LocalInfo::LocalInfo& local_info, Grpc::AsyncClientPtr async_client, Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, absl::string_view type_url, Runtime::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, SubscriptionStats stats, - std::chrono::milliseconds init_fetch_timeout) + const RateLimitSettings& rate_limit_settings, SubscriptionCallbacks& callbacks, + SubscriptionStats stats, std::chrono::milliseconds init_fetch_timeout) : grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, rate_limit_settings), - type_url_(type_url), local_info_(local_info), stats_(stats), dispatcher_(dispatcher), - init_fetch_timeout_(init_fetch_timeout) {} + type_url_(type_url), local_info_(local_info), callbacks_(callbacks), stats_(stats), + dispatcher_(dispatcher), init_fetch_timeout_(init_fetch_timeout) {} void DeltaSubscriptionImpl::pause() { state_->pause(); } void DeltaSubscriptionImpl::resume() { @@ -28,12 +28,11 @@ void DeltaSubscriptionImpl::resume() { } // Config::Subscription -void DeltaSubscriptionImpl::start(const std::set& resources, - SubscriptionCallbacks& callbacks) { - state_ = std::make_unique(type_url_, resources, callbacks, local_info_, - init_fetch_timeout_, dispatcher_, stats_); +void DeltaSubscriptionImpl::start(const std::set& resource_names) { + state_ = std::make_unique( + type_url_, resource_names, callbacks_, local_info_, init_fetch_timeout_, dispatcher_, stats_); grpc_stream_.establishNewStream(); - updateResources(resources); + updateResources(resource_names); } void DeltaSubscriptionImpl::updateResources(const std::set& update_to_these_names) { diff --git a/source/common/config/delta_subscription_impl.h b/source/common/config/delta_subscription_impl.h index 846de51d9eefb..7389947b0aa16 100644 --- a/source/common/config/delta_subscription_impl.h +++ b/source/common/config/delta_subscription_impl.h @@ -30,13 +30,14 @@ class DeltaSubscriptionImpl : public Subscription, const Protobuf::MethodDescriptor& service_method, absl::string_view type_url, Runtime::RandomGenerator& random, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, - SubscriptionStats stats, std::chrono::milliseconds init_fetch_timeout); + SubscriptionCallbacks& callbacks, SubscriptionStats stats, + std::chrono::milliseconds init_fetch_timeout); void pause(); void resume(); // Config::Subscription - void start(const std::set& resources, SubscriptionCallbacks& callbacks) override; + void start(const std::set& resource_names) override; void updateResources(const std::set& update_to_these_names) override; // Config::GrpcStreamCallbacks @@ -80,6 +81,7 @@ class DeltaSubscriptionImpl : public Subscription, std::queue ack_queue_; const LocalInfo::LocalInfo& local_info_; + SubscriptionCallbacks& callbacks_; SubscriptionStats stats_; Event::Dispatcher& dispatcher_; std::chrono::milliseconds init_fetch_timeout_; diff --git a/source/common/config/filesystem_subscription_impl.cc b/source/common/config/filesystem_subscription_impl.cc index aa7f1f7036a03..19b98ea1b75f7 100644 --- a/source/common/config/filesystem_subscription_impl.cc +++ b/source/common/config/filesystem_subscription_impl.cc @@ -10,10 +10,11 @@ namespace Config { FilesystemSubscriptionImpl::FilesystemSubscriptionImpl(Event::Dispatcher& dispatcher, absl::string_view path, + SubscriptionCallbacks& callbacks, SubscriptionStats stats, Api::Api& api) - : path_(path), watcher_(dispatcher.createFilesystemWatcher()), stats_(stats), api_(api) { - watcher_->addWatch(path_, Filesystem::Watcher::Events::MovedTo, [this](uint32_t events) { - UNREFERENCED_PARAMETER(events); + : path_(path), watcher_(dispatcher.createFilesystemWatcher()), callbacks_(callbacks), + stats_(stats), api_(api) { + watcher_->addWatch(path_, Filesystem::Watcher::Events::MovedTo, [this](uint32_t) { if (started_) { refresh(); } @@ -21,19 +22,13 @@ FilesystemSubscriptionImpl::FilesystemSubscriptionImpl(Event::Dispatcher& dispat } // Config::Subscription -void FilesystemSubscriptionImpl::start(const std::set& resources, - Config::SubscriptionCallbacks& callbacks) { - // We report all discovered resources in the watched file. - UNREFERENCED_PARAMETER(resources); - callbacks_ = &callbacks; +void FilesystemSubscriptionImpl::start(const std::set&) { started_ = true; // Attempt to read in case there is a file there already. refresh(); } -void FilesystemSubscriptionImpl::updateResources(const std::set& resources) { - // We report all discovered resources in the watched file. - UNREFERENCED_PARAMETER(resources); +void FilesystemSubscriptionImpl::updateResources(const std::set&) { // Bump stats for consistence behavior with other xDS. stats_.update_attempt_.inc(); } @@ -46,7 +41,7 @@ void FilesystemSubscriptionImpl::refresh() { try { MessageUtil::loadFromFile(path_, message, api_); config_update_available = true; - callbacks_->onConfigUpdate(message.resources(), message.version_info()); + callbacks_.onConfigUpdate(message.resources(), message.version_info()); stats_.version_.set(HashUtil::xxHash64(message.version_info())); stats_.update_success_.inc(); ENVOY_LOG(debug, "Filesystem config update accepted for {}: {}", path_, message.DebugString()); @@ -59,7 +54,7 @@ void FilesystemSubscriptionImpl::refresh() { ENVOY_LOG(warn, "Filesystem config update failure: {}", e.what()); stats_.update_failure_.inc(); } - callbacks_->onConfigUpdateFailed(&e); + callbacks_.onConfigUpdateFailed(&e); } } diff --git a/source/common/config/filesystem_subscription_impl.h b/source/common/config/filesystem_subscription_impl.h index 87249fc273aad..b13537c1ec22a 100644 --- a/source/common/config/filesystem_subscription_impl.h +++ b/source/common/config/filesystem_subscription_impl.h @@ -20,13 +20,14 @@ class FilesystemSubscriptionImpl : public Config::Subscription, Logger::Loggable { public: FilesystemSubscriptionImpl(Event::Dispatcher& dispatcher, absl::string_view path, - SubscriptionStats stats, Api::Api& api); + SubscriptionCallbacks& callbacks, SubscriptionStats stats, + Api::Api& api); // Config::Subscription - void start(const std::set& resources, - Config::SubscriptionCallbacks& callbacks) override; - - void updateResources(const std::set& resources) override; + // We report all discovered resources in the watched file, so the resource names arguments are + // unused, and updateResources is a no-op (other than updating a stat). + void start(const std::set&) override; + void updateResources(const std::set&) override; private: void refresh(); @@ -34,7 +35,7 @@ class FilesystemSubscriptionImpl : public Config::Subscription, bool started_{}; const std::string path_; std::unique_ptr watcher_; - SubscriptionCallbacks* callbacks_{}; + SubscriptionCallbacks& callbacks_; SubscriptionStats stats_; Api::Api& api_; }; diff --git a/source/common/config/grpc_mux_subscription_impl.cc b/source/common/config/grpc_mux_subscription_impl.cc index 5b7ffe2d084d0..8383036d4e522 100644 --- a/source/common/config/grpc_mux_subscription_impl.cc +++ b/source/common/config/grpc_mux_subscription_impl.cc @@ -9,22 +9,21 @@ namespace Envoy { namespace Config { -GrpcMuxSubscriptionImpl::GrpcMuxSubscriptionImpl(GrpcMux& grpc_mux, SubscriptionStats stats, +GrpcMuxSubscriptionImpl::GrpcMuxSubscriptionImpl(GrpcMux& grpc_mux, + SubscriptionCallbacks& callbacks, + SubscriptionStats stats, absl::string_view type_url, Event::Dispatcher& dispatcher, std::chrono::milliseconds init_fetch_timeout) - : grpc_mux_(grpc_mux), stats_(stats), type_url_(type_url), dispatcher_(dispatcher), - init_fetch_timeout_(init_fetch_timeout) {} + : grpc_mux_(grpc_mux), callbacks_(callbacks), stats_(stats), type_url_(type_url), + dispatcher_(dispatcher), init_fetch_timeout_(init_fetch_timeout) {} // Config::Subscription -void GrpcMuxSubscriptionImpl::start(const std::set& resources, - SubscriptionCallbacks& callbacks) { - callbacks_ = &callbacks; - +void GrpcMuxSubscriptionImpl::start(const std::set& resources) { if (init_fetch_timeout_.count() > 0) { init_fetch_timeout_timer_ = dispatcher_.createTimer([this]() -> void { ENVOY_LOG(warn, "gRPC config: initial fetch timed out for {}", type_url_); - callbacks_->onConfigUpdateFailed(nullptr); + callbacks_.onConfigUpdateFailed(nullptr); }); init_fetch_timeout_timer_->enableTimer(init_fetch_timeout_); } @@ -54,7 +53,7 @@ void GrpcMuxSubscriptionImpl::onConfigUpdate( // supply those versions to onConfigUpdate() along with the xDS response ("system") // version_info. This way, both types of versions can be tracked and exposed for debugging by // the configuration update targets. - callbacks_->onConfigUpdate(resources, version_info); + callbacks_.onConfigUpdate(resources, version_info); stats_.update_success_.inc(); stats_.update_attempt_.inc(); stats_.version_.set(HashUtil::xxHash64(version_info)); @@ -73,11 +72,11 @@ void GrpcMuxSubscriptionImpl::onConfigUpdateFailed(const EnvoyException* e) { ENVOY_LOG(warn, "gRPC config for {} rejected: {}", type_url_, e->what()); } stats_.update_attempt_.inc(); - callbacks_->onConfigUpdateFailed(e); + callbacks_.onConfigUpdateFailed(e); } std::string GrpcMuxSubscriptionImpl::resourceName(const ProtobufWkt::Any& resource) { - return callbacks_->resourceName(resource); + return callbacks_.resourceName(resource); } void GrpcMuxSubscriptionImpl::disableInitFetchTimeoutTimer() { diff --git a/source/common/config/grpc_mux_subscription_impl.h b/source/common/config/grpc_mux_subscription_impl.h index 933ddb8cff1dc..10d08ff49f3c6 100644 --- a/source/common/config/grpc_mux_subscription_impl.h +++ b/source/common/config/grpc_mux_subscription_impl.h @@ -17,12 +17,13 @@ class GrpcMuxSubscriptionImpl : public Subscription, GrpcMuxCallbacks, Logger::Loggable { public: - GrpcMuxSubscriptionImpl(GrpcMux& grpc_mux, SubscriptionStats stats, absl::string_view type_url, + GrpcMuxSubscriptionImpl(GrpcMux& grpc_mux, SubscriptionCallbacks& callbacks, + SubscriptionStats stats, absl::string_view type_url, Event::Dispatcher& dispatcher, std::chrono::milliseconds init_fetch_timeout); // Config::Subscription - void start(const std::set& resources, SubscriptionCallbacks& callbacks) override; + void start(const std::set& resource_names) override; void updateResources(const std::set& update_to_these_names) override; // Config::GrpcMuxCallbacks @@ -35,9 +36,9 @@ class GrpcMuxSubscriptionImpl : public Subscription, void disableInitFetchTimeoutTimer(); GrpcMux& grpc_mux_; + SubscriptionCallbacks& callbacks_; SubscriptionStats stats_; const std::string type_url_; - SubscriptionCallbacks* callbacks_{}; GrpcMuxWatchPtr watch_{}; Event::Dispatcher& dispatcher_; std::chrono::milliseconds init_fetch_timeout_; diff --git a/source/common/config/grpc_subscription_impl.h b/source/common/config/grpc_subscription_impl.h index 0c90b8a47f065..caace8dcf2cc0 100644 --- a/source/common/config/grpc_subscription_impl.h +++ b/source/common/config/grpc_subscription_impl.h @@ -17,18 +17,18 @@ class GrpcSubscriptionImpl : public Config::Subscription { GrpcSubscriptionImpl(const LocalInfo::LocalInfo& local_info, Grpc::AsyncClientPtr async_client, Event::Dispatcher& dispatcher, Runtime::RandomGenerator& random, const Protobuf::MethodDescriptor& service_method, absl::string_view type_url, - SubscriptionStats stats, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, + SubscriptionCallbacks& callbacks, SubscriptionStats stats, + Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, std::chrono::milliseconds init_fetch_timeout) - : grpc_mux_(local_info, std::move(async_client), dispatcher, service_method, random, scope, - rate_limit_settings), - grpc_mux_subscription_(grpc_mux_, stats, type_url, dispatcher, init_fetch_timeout) {} + : callbacks_(callbacks), grpc_mux_(local_info, std::move(async_client), dispatcher, + service_method, random, scope, rate_limit_settings), + grpc_mux_subscription_(grpc_mux_, callbacks_, stats, type_url, dispatcher, + init_fetch_timeout) {} // Config::Subscription - void start(const std::set& resources, - Config::SubscriptionCallbacks& callbacks) override { + void start(const std::set& resource_names) override { // Subscribe first, so we get failure callbacks if grpc_mux_.start() fails. - grpc_mux_subscription_.start(resources, callbacks); + grpc_mux_subscription_.start(resource_names); grpc_mux_.start(); } @@ -39,6 +39,7 @@ class GrpcSubscriptionImpl : public Config::Subscription { GrpcMuxImpl& grpcMux() { return grpc_mux_; } private: + Config::SubscriptionCallbacks& callbacks_; GrpcMuxImpl grpc_mux_; GrpcMuxSubscriptionImpl grpc_mux_subscription_; }; diff --git a/source/common/config/http_subscription_impl.cc b/source/common/config/http_subscription_impl.cc index 971d342e8c4e4..c4cfad33f1806 100644 --- a/source/common/config/http_subscription_impl.cc +++ b/source/common/config/http_subscription_impl.cc @@ -20,10 +20,12 @@ HttpSubscriptionImpl::HttpSubscriptionImpl( const std::string& remote_cluster_name, Event::Dispatcher& dispatcher, Runtime::RandomGenerator& random, std::chrono::milliseconds refresh_interval, std::chrono::milliseconds request_timeout, const Protobuf::MethodDescriptor& service_method, - SubscriptionStats stats, std::chrono::milliseconds init_fetch_timeout) + SubscriptionCallbacks& callbacks, SubscriptionStats stats, + std::chrono::milliseconds init_fetch_timeout) : Http::RestApiFetcher(cm, remote_cluster_name, dispatcher, random, refresh_interval, request_timeout), - stats_(stats), dispatcher_(dispatcher), init_fetch_timeout_(init_fetch_timeout) { + callbacks_(callbacks), stats_(stats), dispatcher_(dispatcher), + init_fetch_timeout_(init_fetch_timeout) { request_.mutable_node()->CopyFrom(local_info.node()); ASSERT(service_method.options().HasExtension(google::api::http)); const auto& http_rule = service_method.options().GetExtension(google::api::http); @@ -32,21 +34,18 @@ HttpSubscriptionImpl::HttpSubscriptionImpl( } // Config::Subscription -void HttpSubscriptionImpl::start(const std::set& resources, - Config::SubscriptionCallbacks& callbacks) { - ASSERT(callbacks_ == nullptr); - +void HttpSubscriptionImpl::start(const std::set& resource_names) { if (init_fetch_timeout_.count() > 0) { init_fetch_timeout_timer_ = dispatcher_.createTimer([this]() -> void { ENVOY_LOG(warn, "REST config: initial fetch timed out for", path_); - callbacks_->onConfigUpdateFailed(nullptr); + callbacks_.onConfigUpdateFailed(nullptr); }); init_fetch_timeout_timer_->enableTimer(init_fetch_timeout_); } - Protobuf::RepeatedPtrField resources_vector(resources.begin(), resources.end()); + Protobuf::RepeatedPtrField resources_vector(resource_names.begin(), + resource_names.end()); request_.mutable_resource_names()->Swap(&resources_vector); - callbacks_ = &callbacks; initialize(); } @@ -80,14 +79,14 @@ void HttpSubscriptionImpl::parseResponse(const Http::Message& response) { return; } try { - callbacks_->onConfigUpdate(message.resources(), message.version_info()); + callbacks_.onConfigUpdate(message.resources(), message.version_info()); request_.set_version_info(message.version_info()); stats_.version_.set(HashUtil::xxHash64(request_.version_info())); stats_.update_success_.inc(); } catch (const EnvoyException& e) { ENVOY_LOG(warn, "REST config update rejected: {}", e.what()); stats_.update_rejected_.inc(); - callbacks_->onConfigUpdateFailed(&e); + callbacks_.onConfigUpdateFailed(&e); } } @@ -101,7 +100,7 @@ void HttpSubscriptionImpl::onFetchFailure(const EnvoyException* e) { void HttpSubscriptionImpl::handleFailure(const EnvoyException* e) { stats_.update_failure_.inc(); - callbacks_->onConfigUpdateFailed(e); + callbacks_.onConfigUpdateFailed(e); } void HttpSubscriptionImpl::disableInitFetchTimeoutTimer() { diff --git a/source/common/config/http_subscription_impl.h b/source/common/config/http_subscription_impl.h index 97612b3897153..3faf5d6045bb6 100644 --- a/source/common/config/http_subscription_impl.h +++ b/source/common/config/http_subscription_impl.h @@ -24,12 +24,12 @@ class HttpSubscriptionImpl : public Http::RestApiFetcher, const std::string& remote_cluster_name, Event::Dispatcher& dispatcher, Runtime::RandomGenerator& random, std::chrono::milliseconds refresh_interval, std::chrono::milliseconds request_timeout, - const Protobuf::MethodDescriptor& service_method, SubscriptionStats stats, + const Protobuf::MethodDescriptor& service_method, + SubscriptionCallbacks& callbacks, SubscriptionStats stats, std::chrono::milliseconds init_fetch_timeout); // Config::Subscription - void start(const std::set& resources, - Config::SubscriptionCallbacks& callbacks) override; + void start(const std::set& resource_names) override; void updateResources(const std::set& update_to_these_names) override; // Http::RestApiFetcher @@ -44,8 +44,8 @@ class HttpSubscriptionImpl : public Http::RestApiFetcher, std::string path_; Protobuf::RepeatedPtrField resources_; - Config::SubscriptionCallbacks* callbacks_{}; envoy::api::v2::DiscoveryRequest request_; + Config::SubscriptionCallbacks& callbacks_; SubscriptionStats stats_; Event::Dispatcher& dispatcher_; std::chrono::milliseconds init_fetch_timeout_; diff --git a/source/common/config/subscription_factory.cc b/source/common/config/subscription_factory.cc index 5cd8db23cec8a..7da7f1cab08f7 100644 --- a/source/common/config/subscription_factory.cc +++ b/source/common/config/subscription_factory.cc @@ -15,14 +15,14 @@ std::unique_ptr SubscriptionFactory::subscriptionFromConfigSource( const envoy::api::v2::core::ConfigSource& config, const LocalInfo::LocalInfo& local_info, Event::Dispatcher& dispatcher, Upstream::ClusterManager& cm, Runtime::RandomGenerator& random, Stats::Scope& scope, const std::string& rest_method, const std::string& grpc_method, - absl::string_view type_url, Api::Api& api) { + absl::string_view type_url, Api::Api& api, SubscriptionCallbacks& callbacks) { std::unique_ptr result; SubscriptionStats stats = Utility::generateStats(scope); switch (config.config_source_specifier_case()) { case envoy::api::v2::core::ConfigSource::kPath: { Utility::checkFilesystemSubscriptionBackingPath(config.path(), api); - result = - std::make_unique(dispatcher, config.path(), stats, api); + result = std::make_unique(dispatcher, config.path(), + callbacks, stats, api); break; } case envoy::api::v2::core::ConfigSource::kApiConfigSource: { @@ -39,8 +39,8 @@ std::unique_ptr SubscriptionFactory::subscriptionFromConfigSource( local_info, cm, api_config_source.cluster_names()[0], dispatcher, random, Utility::apiConfigSourceRefreshDelay(api_config_source), Utility::apiConfigSourceRequestTimeout(api_config_source), - *Protobuf::DescriptorPool::generated_pool()->FindMethodByName(rest_method), stats, - Utility::configSourceInitialFetchTimeout(config)); + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName(rest_method), callbacks, + stats, Utility::configSourceInitialFetchTimeout(config)); break; case envoy::api::v2::core::ApiConfigSource::GRPC: result = std::make_unique( @@ -50,7 +50,7 @@ std::unique_ptr SubscriptionFactory::subscriptionFromConfigSource( ->create(), dispatcher, random, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName(grpc_method), type_url, - stats, scope, Utility::parseRateLimitSettings(api_config_source), + callbacks, stats, scope, Utility::parseRateLimitSettings(api_config_source), Utility::configSourceInitialFetchTimeout(config)); break; case envoy::api::v2::core::ApiConfigSource::DELTA_GRPC: { @@ -61,8 +61,8 @@ std::unique_ptr SubscriptionFactory::subscriptionFromConfigSource( api_config_source, scope) ->create(), dispatcher, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName(grpc_method), - type_url, random, scope, Utility::parseRateLimitSettings(api_config_source), stats, - Utility::configSourceInitialFetchTimeout(config)); + type_url, random, scope, Utility::parseRateLimitSettings(api_config_source), callbacks, + stats, Utility::configSourceInitialFetchTimeout(config)); break; } default: @@ -72,7 +72,8 @@ std::unique_ptr SubscriptionFactory::subscriptionFromConfigSource( } case envoy::api::v2::core::ConfigSource::kAds: { result = std::make_unique( - cm.adsMux(), stats, type_url, dispatcher, Utility::configSourceInitialFetchTimeout(config)); + cm.adsMux(), callbacks, stats, type_url, dispatcher, + Utility::configSourceInitialFetchTimeout(config)); break; } default: diff --git a/source/common/config/subscription_factory.h b/source/common/config/subscription_factory.h index 3a5ae196c503c..f9ab04678c2f4 100644 --- a/source/common/config/subscription_factory.h +++ b/source/common/config/subscription_factory.h @@ -26,12 +26,14 @@ class SubscriptionFactory { * @param grpc_method fully qualified name of v2 gRPC API bidi streaming method (as per protobuf * service description). * @param api reference to the Api object + * @param callbacks the callbacks needed by all Subscription objects, to deliver config updates. + * The callbacks must not result in the deletion of the Subscription object. */ static std::unique_ptr subscriptionFromConfigSource( const envoy::api::v2::core::ConfigSource& config, const LocalInfo::LocalInfo& local_info, Event::Dispatcher& dispatcher, Upstream::ClusterManager& cm, Runtime::RandomGenerator& random, Stats::Scope& scope, const std::string& rest_method, const std::string& grpc_method, - absl::string_view type_url, Api::Api& api); + absl::string_view type_url, Api::Api& api, SubscriptionCallbacks& callbacks); }; } // namespace Config diff --git a/source/common/router/rds_impl.cc b/source/common/router/rds_impl.cc index 5bba8f9f0a990..73d8568191752 100644 --- a/source/common/router/rds_impl.cc +++ b/source/common/router/rds_impl.cc @@ -59,7 +59,7 @@ RdsRouteConfigSubscription::RdsRouteConfigSubscription( Envoy::Router::RouteConfigProviderManagerImpl& route_config_provider_manager) : route_config_name_(rds.route_config_name()), factory_context_(factory_context), init_target_(fmt::format("RdsRouteConfigSubscription {}", route_config_name_), - [this]() { subscription_->start({route_config_name_}, *this); }), + [this]() { subscription_->start({route_config_name_}); }), scope_(factory_context.scope().createScope(stat_prefix + "rds." + route_config_name_ + ".")), stat_prefix_(stat_prefix), stats_({ALL_RDS_STATS(POOL_COUNTER(*scope_))}), route_config_provider_manager_(route_config_provider_manager), @@ -72,7 +72,7 @@ RdsRouteConfigSubscription::RdsRouteConfigSubscription( "envoy.api.v2.RouteDiscoveryService.FetchRoutes", "envoy.api.v2.RouteDiscoveryService.StreamRoutes", Grpc::Common::typeUrl(envoy::api::v2::RouteConfiguration().GetDescriptor()->full_name()), - factory_context.api()); + factory_context.api(), *this); config_update_info_ = std::make_unique(factory_context.timeSource()); diff --git a/source/common/router/vhds.cc b/source/common/router/vhds.cc index 07bc3c2cbdf27..ab3610597d73f 100644 --- a/source/common/router/vhds.cc +++ b/source/common/router/vhds.cc @@ -25,7 +25,7 @@ VhdsSubscription::VhdsSubscription(RouteConfigUpdatePtr& config_update_info, SubscriptionFactoryFunction factory_function) : config_update_info_(config_update_info), init_target_(fmt::format("VhdsConfigSubscription {}", config_update_info_->routeConfigName()), - [this]() { subscription_->start({}, *this); }), + [this]() { subscription_->start({}); }), scope_(factory_context.scope().createScope(stat_prefix + "vhds." + config_update_info_->routeConfigName() + ".")), stats_({ALL_VHDS_STATS(POOL_COUNTER(*scope_))}), @@ -45,7 +45,7 @@ VhdsSubscription::VhdsSubscription(RouteConfigUpdatePtr& config_update_info, factory_context.dispatcher(), factory_context.clusterManager(), factory_context.random(), *scope_, "none", "envoy.api.v2.VirtualHostDiscoveryService.DeltaVirtualHosts", Grpc::Common::typeUrl(envoy::api::v2::route::VirtualHost().GetDescriptor()->full_name()), - factory_context.api()); + factory_context.api(), *this); } void VhdsSubscription::onConfigUpdateFailed(const EnvoyException*) { diff --git a/source/common/router/vhds.h b/source/common/router/vhds.h index 704d021ef5b3b..04571a39c54fe 100644 --- a/source/common/router/vhds.h +++ b/source/common/router/vhds.h @@ -41,7 +41,7 @@ struct VhdsStats { typedef std::unique_ptr (*SubscriptionFactoryFunction)( const envoy::api::v2::core::ConfigSource&, const LocalInfo::LocalInfo&, Event::Dispatcher&, Upstream::ClusterManager&, Envoy::Runtime::RandomGenerator&, Stats::Scope&, const std::string&, - const std::string&, absl::string_view, Api::Api&); + const std::string&, absl::string_view, Api::Api&, Envoy::Config::SubscriptionCallbacks&); class VhdsSubscription : Envoy::Config::SubscriptionCallbacks, Logger::Loggable { diff --git a/source/common/secret/sds_api.cc b/source/common/secret/sds_api.cc index edf06432ebd22..6e0519ea80021 100644 --- a/source/common/secret/sds_api.cc +++ b/source/common/secret/sds_api.cc @@ -80,8 +80,9 @@ void SdsApi::initialize() { sds_config_, local_info_, dispatcher_, cluster_manager_, random_, stats_, "envoy.service.discovery.v2.SecretDiscoveryService.FetchSecrets", "envoy.service.discovery.v2.SecretDiscoveryService.StreamSecrets", - Grpc::Common::typeUrl(envoy::api::v2::auth::Secret().GetDescriptor()->full_name()), api_); - subscription_->start({sds_config_name_}, *this); + Grpc::Common::typeUrl(envoy::api::v2::auth::Secret().GetDescriptor()->full_name()), api_, + *this); + subscription_->start({sds_config_name_}); } } // namespace Secret diff --git a/source/common/upstream/cds_api_impl.cc b/source/common/upstream/cds_api_impl.cc index c0ded041a333b..76339ad74323e 100644 --- a/source/common/upstream/cds_api_impl.cc +++ b/source/common/upstream/cds_api_impl.cc @@ -37,7 +37,7 @@ CdsApiImpl::CdsApiImpl(const envoy::api::v2::core::ConfigSource& cds_config, Clu subscription_ = Config::SubscriptionFactory::subscriptionFromConfigSource( cds_config, local_info, dispatcher, cm, random, *scope_, "envoy.api.v2.ClusterDiscoveryService.FetchClusters", grpc_method, - Grpc::Common::typeUrl(envoy::api::v2::Cluster().GetDescriptor()->full_name()), api); + Grpc::Common::typeUrl(envoy::api::v2::Cluster().GetDescriptor()->full_name()), api, *this); } void CdsApiImpl::onConfigUpdate(const Protobuf::RepeatedPtrField& resources, diff --git a/source/common/upstream/cds_api_impl.h b/source/common/upstream/cds_api_impl.h index cdd2fe9255419..01e4d54006214 100644 --- a/source/common/upstream/cds_api_impl.h +++ b/source/common/upstream/cds_api_impl.h @@ -28,7 +28,7 @@ class CdsApiImpl : public CdsApi, Api::Api& api); // Upstream::CdsApi - void initialize() override { subscription_->start({}, *this); } + void initialize() override { subscription_->start({}); } void setInitializedCb(std::function callback) override { initialize_callback_ = callback; } diff --git a/source/common/upstream/eds.cc b/source/common/upstream/eds.cc index 57d77c055a620..e5c450a19a829 100644 --- a/source/common/upstream/eds.cc +++ b/source/common/upstream/eds.cc @@ -29,10 +29,10 @@ EdsClusterImpl::EdsClusterImpl( "envoy.api.v2.EndpointDiscoveryService.FetchEndpoints", "envoy.api.v2.EndpointDiscoveryService.StreamEndpoints", Grpc::Common::typeUrl(envoy::api::v2::ClusterLoadAssignment().GetDescriptor()->full_name()), - factory_context.api()); + factory_context.api(), *this); } -void EdsClusterImpl::startPreInit() { subscription_->start({cluster_name_}, *this); } +void EdsClusterImpl::startPreInit() { subscription_->start({cluster_name_}); } void EdsClusterImpl::BatchUpdateHelper::batchUpdate(PrioritySet::HostUpdateCb& host_update_cb) { std::unordered_map updated_hosts; diff --git a/source/server/lds_api.cc b/source/server/lds_api.cc index 04ff3b0bad8d9..c1113aa43c626 100644 --- a/source/server/lds_api.cc +++ b/source/server/lds_api.cc @@ -21,12 +21,12 @@ LdsApiImpl::LdsApiImpl(const envoy::api::v2::core::ConfigSource& lds_config, const LocalInfo::LocalInfo& local_info, Stats::Scope& scope, ListenerManager& lm, Api::Api& api) : listener_manager_(lm), scope_(scope.createScope("listener_manager.lds.")), cm_(cm), - init_target_("LDS", [this]() { subscription_->start({}, *this); }) { + init_target_("LDS", [this]() { subscription_->start({}); }) { subscription_ = Envoy::Config::SubscriptionFactory::subscriptionFromConfigSource( lds_config, local_info, dispatcher, cm, random, *scope_, "envoy.api.v2.ListenerDiscoveryService.FetchListeners", "envoy.api.v2.ListenerDiscoveryService.StreamListeners", - Grpc::Common::typeUrl(envoy::api::v2::Listener().GetDescriptor()->full_name()), api); + Grpc::Common::typeUrl(envoy::api::v2::Listener().GetDescriptor()->full_name()), api, *this); Config::Utility::checkLocalInfo("lds", local_info); init_manager.add(init_target_); } diff --git a/test/common/config/delta_subscription_impl_test.cc b/test/common/config/delta_subscription_impl_test.cc index 2f8577a393c19..7ed82165a09bd 100644 --- a/test/common/config/delta_subscription_impl_test.cc +++ b/test/common/config/delta_subscription_impl_test.cc @@ -108,7 +108,7 @@ TEST_F(DeltaSubscriptionImplTest, NoGrpcStream) { // start() also tries to start the GrpcStream. So, have that attempt return nullptr. EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(nullptr)); EXPECT_CALL(async_stream_, sendMessage(_, _)).Times(0); - subscription_->start({"name1"}, callbacks_); + subscription_->start({"name1"}); subscription_->updateResources({"name1", "name2"}); } diff --git a/test/common/config/delta_subscription_test_harness.h b/test/common/config/delta_subscription_test_harness.h index 61d40bb2f83a7..12b6d5d8e4e63 100644 --- a/test/common/config/delta_subscription_test_harness.h +++ b/test/common/config/delta_subscription_test_harness.h @@ -34,7 +34,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { subscription_ = std::make_unique( local_info_, std::unique_ptr(async_client_), dispatcher_, *method_descriptor_, Config::TypeUrl::get().ClusterLoadAssignment, random_, stats_store_, - rate_limit_settings_, stats_, init_fetch_timeout); + rate_limit_settings_, callbacks_, stats_, init_fetch_timeout); } ~DeltaSubscriptionTestHarness() { @@ -51,7 +51,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); last_cluster_names_ = cluster_names; expectSendMessage(last_cluster_names_, ""); - subscription_->start(cluster_names, callbacks_); + subscription_->start(cluster_names); } void expectSendMessage(const std::set& cluster_names, diff --git a/test/common/config/filesystem_subscription_impl_test.cc b/test/common/config/filesystem_subscription_impl_test.cc index 2feabde6921fd..80250f2e690b8 100644 --- a/test/common/config/filesystem_subscription_impl_test.cc +++ b/test/common/config/filesystem_subscription_impl_test.cc @@ -42,8 +42,10 @@ TEST(MiscFilesystemSubscriptionImplTest, BadWatch) { auto* watcher = new Filesystem::MockWatcher(); EXPECT_CALL(dispatcher, createFilesystemWatcher_()).WillOnce(Return(watcher)); EXPECT_CALL(*watcher, addWatch(_, _, _)).WillOnce(Throw(EnvoyException("bad path"))); - EXPECT_THROW_WITH_MESSAGE(FilesystemSubscriptionImpl(dispatcher, "##!@/dev/null", stats, *api), - EnvoyException, "bad path"); + NiceMock> callbacks; + EXPECT_THROW_WITH_MESSAGE( + FilesystemSubscriptionImpl(dispatcher, "##!@/dev/null", callbacks, stats, *api), + EnvoyException, "bad path"); } } // namespace diff --git a/test/common/config/filesystem_subscription_test_harness.h b/test/common/config/filesystem_subscription_test_harness.h index 912d21739419d..d0f0c726dbd42 100644 --- a/test/common/config/filesystem_subscription_test_harness.h +++ b/test/common/config/filesystem_subscription_test_harness.h @@ -29,7 +29,7 @@ class FilesystemSubscriptionTestHarness : public SubscriptionTestHarness { FilesystemSubscriptionTestHarness() : path_(TestEnvironment::temporaryPath("eds.json")), api_(Api::createApiForTest(stats_store_)), dispatcher_(api_->allocateDispatcher()), - subscription_(*dispatcher_, path_, stats_, *api_) {} + subscription_(*dispatcher_, path_, callbacks_, stats_, *api_) {} ~FilesystemSubscriptionTestHarness() { if (::access(path_.c_str(), F_OK) != -1) { @@ -40,7 +40,7 @@ class FilesystemSubscriptionTestHarness : public SubscriptionTestHarness { void startSubscription(const std::set& cluster_names) override { std::ifstream config_file(path_); file_at_start_ = config_file.good(); - subscription_.start(cluster_names, callbacks_); + subscription_.start(cluster_names); } void updateResources(const std::set& cluster_names) override { diff --git a/test/common/config/grpc_subscription_impl_test.cc b/test/common/config/grpc_subscription_impl_test.cc index 490a74c228e39..7b5302ff46618 100644 --- a/test/common/config/grpc_subscription_impl_test.cc +++ b/test/common/config/grpc_subscription_impl_test.cc @@ -18,7 +18,7 @@ TEST_F(GrpcSubscriptionImplTest, StreamCreationFailure) { EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); EXPECT_CALL(random_, random()); EXPECT_CALL(*timer_, enableTimer(_)); - subscription_->start({"cluster0", "cluster1"}, callbacks_); + subscription_->start({"cluster0", "cluster1"}); verifyStats(2, 0, 0, 1, 0); // Ensure this doesn't cause an issue by sending a request, since we don't // have a gRPC stream. diff --git a/test/common/config/grpc_subscription_test_harness.h b/test/common/config/grpc_subscription_test_harness.h index 5c23300dab288..097329b70d002 100644 --- a/test/common/config/grpc_subscription_test_harness.h +++ b/test/common/config/grpc_subscription_test_harness.h @@ -44,8 +44,8 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { })); subscription_ = std::make_unique( local_info_, std::unique_ptr(async_client_), dispatcher_, random_, - *method_descriptor_, Config::TypeUrl::get().ClusterLoadAssignment, stats_, stats_store_, - rate_limit_settings_, init_fetch_timeout); + *method_descriptor_, Config::TypeUrl::get().ClusterLoadAssignment, callbacks_, stats_, + stats_store_, rate_limit_settings_, init_fetch_timeout); } ~GrpcSubscriptionTestHarness() override { EXPECT_CALL(async_stream_, sendMessage(_, false)); } @@ -79,7 +79,7 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); last_cluster_names_ = cluster_names; expectSendMessage(last_cluster_names_, ""); - subscription_->start(cluster_names, callbacks_); + subscription_->start(cluster_names); } void deliverConfigUpdate(const std::vector& cluster_names, diff --git a/test/common/config/http_subscription_test_harness.h b/test/common/config/http_subscription_test_harness.h index 7e6cfc828092d..df9be283f6015 100644 --- a/test/common/config/http_subscription_test_harness.h +++ b/test/common/config/http_subscription_test_harness.h @@ -46,7 +46,8 @@ class HttpSubscriptionTestHarness : public SubscriptionTestHarness { })); subscription_ = std::make_unique( local_info_, cm_, "eds_cluster", dispatcher_, random_gen_, std::chrono::milliseconds(1), - std::chrono::milliseconds(1000), *method_descriptor_, stats_, init_fetch_timeout); + std::chrono::milliseconds(1000), *method_descriptor_, callbacks_, stats_, + init_fetch_timeout); } ~HttpSubscriptionTestHarness() { @@ -100,7 +101,7 @@ class HttpSubscriptionTestHarness : public SubscriptionTestHarness { version_ = ""; cluster_names_ = cluster_names; expectSendMessage(cluster_names, ""); - subscription_->start(cluster_names, callbacks_); + subscription_->start(cluster_names); } void updateResources(const std::set& cluster_names) override { diff --git a/test/common/config/subscription_factory_test.cc b/test/common/config/subscription_factory_test.cc index 3497f16a1826e..606601d9b15b7 100644 --- a/test/common/config/subscription_factory_test.cc +++ b/test/common/config/subscription_factory_test.cc @@ -38,7 +38,7 @@ class SubscriptionFactoryTest : public testing::Test { config, local_info_, dispatcher_, cm_, random_, stats_store_, "envoy.api.v2.EndpointDiscoveryService.FetchEndpoints", "envoy.api.v2.EndpointDiscoveryService.StreamEndpoints", - Config::TypeUrl::get().ClusterLoadAssignment, *api_); + Config::TypeUrl::get().ClusterLoadAssignment, *api_, callbacks_); } Upstream::MockClusterManager cm_; @@ -186,14 +186,13 @@ TEST_F(SubscriptionFactoryTest, FilesystemSubscription) { EXPECT_CALL(dispatcher_, createFilesystemWatcher_()).WillOnce(Return(watcher)); EXPECT_CALL(*watcher, addWatch(test_path, _, _)); EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); - subscriptionFromConfigSource(config)->start({"foo"}, callbacks_); + subscriptionFromConfigSource(config)->start({"foo"}); } TEST_F(SubscriptionFactoryTest, FilesystemSubscriptionNonExistentFile) { envoy::api::v2::core::ConfigSource config; config.set_path("/blahblah"); - EXPECT_THROW_WITH_MESSAGE(subscriptionFromConfigSource(config)->start({"foo"}, callbacks_), - EnvoyException, + EXPECT_THROW_WITH_MESSAGE(subscriptionFromConfigSource(config)->start({"foo"}), EnvoyException, "envoy::api::v2::Path must refer to an existing path in the system: " "'/blahblah' does not exist") } @@ -209,9 +208,8 @@ TEST_F(SubscriptionFactoryTest, LegacySubscription) { EXPECT_CALL(cm_, clusters()).WillOnce(Return(cluster_map)); EXPECT_CALL(cluster, info()).Times(2); EXPECT_CALL(*cluster.info_, addedViaApi()); - EXPECT_THROW_WITH_REGEX( - subscriptionFromConfigSource(config)->start({"static_cluster"}, callbacks_), EnvoyException, - "REST_LEGACY no longer a supported ApiConfigSource.*"); + EXPECT_THROW_WITH_REGEX(subscriptionFromConfigSource(config)->start({"static_cluster"}), + EnvoyException, "REST_LEGACY no longer a supported ApiConfigSource.*"); } TEST_F(SubscriptionFactoryTest, HttpSubscriptionCustomRequestTimeout) { @@ -232,7 +230,7 @@ TEST_F(SubscriptionFactoryTest, HttpSubscriptionCustomRequestTimeout) { EXPECT_CALL( cm_.async_client_, send_(_, _, Http::AsyncClient::RequestOptions().setTimeout(std::chrono::milliseconds(5000)))); - subscriptionFromConfigSource(config)->start({"static_cluster"}, callbacks_); + subscriptionFromConfigSource(config)->start({"static_cluster"}); } TEST_F(SubscriptionFactoryTest, HttpSubscription) { @@ -260,7 +258,7 @@ TEST_F(SubscriptionFactoryTest, HttpSubscription) { return &http_request_; })); EXPECT_CALL(http_request_, cancel()); - subscriptionFromConfigSource(config)->start({"static_cluster"}, callbacks_); + subscriptionFromConfigSource(config)->start({"static_cluster"}); } // Confirm error when no refresh delay is set (not checked by schema). @@ -275,9 +273,9 @@ TEST_F(SubscriptionFactoryTest, HttpSubscriptionNoRefreshDelay) { EXPECT_CALL(cm_, clusters()).WillOnce(Return(cluster_map)); EXPECT_CALL(cluster, info()).Times(2); EXPECT_CALL(*cluster.info_, addedViaApi()); - EXPECT_THROW_WITH_MESSAGE( - subscriptionFromConfigSource(config)->start({"static_cluster"}, callbacks_), EnvoyException, - "refresh_delay is required for REST API configuration sources"); + EXPECT_THROW_WITH_MESSAGE(subscriptionFromConfigSource(config)->start({"static_cluster"}), + EnvoyException, + "refresh_delay is required for REST API configuration sources"); } TEST_F(SubscriptionFactoryTest, GrpcSubscription) { @@ -304,7 +302,7 @@ TEST_F(SubscriptionFactoryTest, GrpcSubscription) { EXPECT_CALL(random_, random()); EXPECT_CALL(dispatcher_, createTimer_(_)); EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); - subscriptionFromConfigSource(config)->start({"static_cluster"}, callbacks_); + subscriptionFromConfigSource(config)->start({"static_cluster"}); } INSTANTIATE_TEST_SUITE_P(SubscriptionFactoryTestApiConfigSource, @@ -325,7 +323,7 @@ TEST_P(SubscriptionFactoryTestApiConfigSource, NonExistentCluster) { Upstream::ClusterManager::ClusterInfoMap cluster_map; EXPECT_CALL(cm_, clusters()).WillOnce(Return(cluster_map)); EXPECT_THROW_WITH_MESSAGE( - subscriptionFromConfigSource(config)->start({"static_cluster"}, callbacks_), EnvoyException, + subscriptionFromConfigSource(config)->start({"static_cluster"}), EnvoyException, "envoy::api::v2::core::ConfigSource must have a statically defined " "non-EDS cluster: 'static_cluster' does not exist, was added via api, or is an EDS cluster"); } @@ -347,7 +345,7 @@ TEST_P(SubscriptionFactoryTestApiConfigSource, DynamicCluster) { EXPECT_CALL(cluster, info()); EXPECT_CALL(*cluster.info_, addedViaApi()).WillOnce(Return(true)); EXPECT_THROW_WITH_MESSAGE( - subscriptionFromConfigSource(config)->start({"static_cluster"}, callbacks_), EnvoyException, + subscriptionFromConfigSource(config)->start({"static_cluster"}), EnvoyException, "envoy::api::v2::core::ConfigSource must have a statically defined " "non-EDS cluster: 'static_cluster' does not exist, was added via api, or is an EDS cluster"); } @@ -370,7 +368,7 @@ TEST_P(SubscriptionFactoryTestApiConfigSource, EDSClusterBackingEDSCluster) { EXPECT_CALL(*cluster.info_, addedViaApi()); EXPECT_CALL(*cluster.info_, type()).WillOnce(Return(envoy::api::v2::Cluster::EDS)); EXPECT_THROW_WITH_MESSAGE( - subscriptionFromConfigSource(config)->start({"static_cluster"}, callbacks_), EnvoyException, + subscriptionFromConfigSource(config)->start({"static_cluster"}), EnvoyException, "envoy::api::v2::core::ConfigSource must have a statically defined " "non-EDS cluster: 'static_cluster' does not exist, was added via api, or is an EDS cluster"); } diff --git a/test/common/router/vhds_test.cc b/test/common/router/vhds_test.cc index d96475f1095c5..5fa772d7e8ef5 100644 --- a/test/common/router/vhds_test.cc +++ b/test/common/router/vhds_test.cc @@ -41,13 +41,13 @@ namespace { class VhdsTest : public testing::Test { public: void SetUp() override { - factory_function_ = {[](const envoy::api::v2::core::ConfigSource&, const LocalInfo::LocalInfo&, - Event::Dispatcher&, Upstream::ClusterManager&, - Envoy::Runtime::RandomGenerator&, Stats::Scope&, const std::string&, - const std::string&, absl::string_view, - Api::Api&) -> std::unique_ptr { - return std::unique_ptr(); - }}; + factory_function_ = { + [](const envoy::api::v2::core::ConfigSource&, const LocalInfo::LocalInfo&, + Event::Dispatcher&, Upstream::ClusterManager&, Envoy::Runtime::RandomGenerator&, + Stats::Scope&, const std::string&, const std::string&, absl::string_view, Api::Api&, + Envoy::Config::SubscriptionCallbacks&) -> std::unique_ptr { + return std::unique_ptr(); + }}; default_vhds_config_ = R"EOF( name: my_route diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index 0d0c0df9df344..e901ee66d5ee1 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -42,9 +42,8 @@ template class MockSubscriptionCallbacks : public Subscript class MockSubscription : public Subscription { public: - MOCK_METHOD2_T(start, - void(const std::set& resources, SubscriptionCallbacks& callbacks)); - MOCK_METHOD1_T(updateResources, void(const std::set& update_to_these_names)); + MOCK_METHOD1(start, void(const std::set& resources)); + MOCK_METHOD1(updateResources, void(const std::set& update_to_these_names)); }; class MockGrpcMuxWatch : public GrpcMuxWatch { From 1bcbdf6ff7ed6a0e012b5dc77af92fc15e515823 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 31 May 2019 14:56:59 -0400 Subject: [PATCH 21/56] starting to move stats to DeltaSubscriptionImpl Signed-off-by: Fred Douglas --- include/envoy/config/grpc_mux.h | 38 +++++++++++++ source/common/config/BUILD | 1 + .../common/config/delta_subscription_impl.cc | 56 +++++++++++++++---- .../common/config/delta_subscription_impl.h | 25 +++++++-- .../common/config/delta_subscription_state.cc | 15 ++--- .../common/config/delta_subscription_state.h | 7 +-- source/common/config/grpc_delta_xds_context.h | 40 ++++++++----- source/common/config/grpc_mux_impl.h | 24 +++++--- source/common/config/subscription_factory.h | 2 +- source/common/config/watch_map.h | 2 +- source/common/upstream/eds.cc | 8 +-- 11 files changed, 159 insertions(+), 59 deletions(-) diff --git a/include/envoy/config/grpc_mux.h b/include/envoy/config/grpc_mux.h index fb66a78abdcbb..4fee7dedbc99d 100644 --- a/include/envoy/config/grpc_mux.h +++ b/include/envoy/config/grpc_mux.h @@ -2,6 +2,7 @@ #include "envoy/common/exception.h" #include "envoy/common/pure.h" +#include "envoy/config/subscription.h" #include "envoy/stats/stats_macros.h" #include "common/protobuf/protobuf.h" @@ -109,9 +110,46 @@ class GrpcMux { * e.g.type.googleapis.com/envoy.api.v2.Cluster. */ virtual void resume(const std::string& type_url) PURE; + + // For delta + virtual /*WatchMap::Token*/uint64_t addWatch(const std::string& type_url, const std::set& resources, + SubscriptionCallbacks& callbacks) PURE; + virtual void removeWatch(const std::string& type_url, /*WatchMap::Token*/uint64_t watch_token) PURE; + virtual void updateWatch(const std::string& type_url, /*WatchMap::Token*/uint64_t watch_token, + const std::set& resources) PURE; }; typedef std::unique_ptr GrpcMuxPtr; +/** + * A grouping of callbacks that a GrpcMux should provide to its GrpcStream. + */ +template class GrpcStreamCallbacks { +public: + virtual ~GrpcStreamCallbacks() {} + + /** + * For the GrpcStream to prompt the context to take appropriate action in response to the + * gRPC stream having been successfully established. + */ + virtual void onStreamEstablished() PURE; + + /** + * For the GrpcStream to prompt the context to take appropriate action in response to + * failure to establish the gRPC stream. + */ + virtual void onEstablishmentFailure() PURE; + + /** + * For the GrpcStream to pass received protos to the context. + */ + virtual void onDiscoveryResponse(std::unique_ptr&& message) PURE; + + /** + * For the GrpcStream to call when its rate limiting logic allows more requests to be sent. + */ + virtual void onWriteable() PURE; +}; + } // namespace Config } // namespace Envoy diff --git a/source/common/config/BUILD b/source/common/config/BUILD index de2a1cf2d4724..77d803177b5df 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -216,6 +216,7 @@ envoy_cc_library( hdrs = ["grpc_delta_xds_context.h"], deps = [ ":delta_subscription_state_lib", + ":watch_map_lib", "//include/envoy/event:dispatcher_interface", "//include/envoy/grpc:async_client_interface", ], diff --git a/source/common/config/delta_subscription_impl.cc b/source/common/config/delta_subscription_impl.cc index 8032c0f09d931..acf1a121b6b5b 100644 --- a/source/common/config/delta_subscription_impl.cc +++ b/source/common/config/delta_subscription_impl.cc @@ -4,30 +4,64 @@ namespace Envoy { namespace Config { DeltaSubscriptionImpl::DeltaSubscriptionImpl(std::shared_ptr context, - absl::string_view type_url, SubscriptionStats stats, + absl::string_view type_url, + SubscriptionCallbacks& callbacks, + SubscriptionStats stats, std::chrono::milliseconds init_fetch_timeout) - : context_(context), type_url_(type_url), stats_(stats) { - context_->setInitFetchTimeout(type_url_, init_fetch_timeout); + : context_(context), type_url_(type_url), callbacks_(callbacks), stats_(stats), + init_fetch_timeout_(init_fetch_timeout) { + // TODO TODO TRIM context_->setInitFetchTimeout(type_url_, init_fetch_timeout); } -DeltaSubscriptionImpl::~DeltaSubscriptionImpl() { - if (watch_token_ != WatchMap::InvalidToken) { - context_->removeWatch(type_url_, watch_token_); - } -} +DeltaSubscriptionImpl::~DeltaSubscriptionImpl() { context_->removeWatch(type_url_, watch_token_); } void DeltaSubscriptionImpl::pause() { context_->pause(type_url_); } void DeltaSubscriptionImpl::resume() { context_->resume(type_url_); } // Config::DeltaSubscription -void DeltaSubscriptionImpl::start(const std::set& resources, - SubscriptionCallbacks& callbacks) { - watch_token_ = context_->addWatch(type_url_, resources, callbacks, stats_); +void DeltaSubscriptionImpl::start(const std::set& resources, SubscriptionCallbacks&) { + watch_token_ = context_->addWatch(type_url_, resources, *this, init_fetch_timeout_); + stats_.update_attempt_.inc(); } void DeltaSubscriptionImpl::updateResources(const std::set& update_to_these_names) { context_->updateWatch(type_url_, watch_token_, update_to_these_names); + stats_.update_attempt_.inc(); +} + +// Config::SubscriptionCallbacks +void DeltaSubscriptionImpl::onConfigUpdate( + const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) { + callbacks_.onConfigUpdate(resources, version_info); + stats_.update_success_.inc(); + stats_.update_attempt_.inc(); + stats_.version_.set(HashUtil::xxHash64(version_info)); +} +void DeltaSubscriptionImpl::onConfigUpdate( + const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) { + callbacks_.onConfigUpdate(added_resources, removed_resources, system_version_info); + stats_.update_success_.inc(); + stats_.update_attempt_.inc(); + stats_.version_.set(HashUtil::xxHash64(system_version_info)); +} +void DeltaSubscriptionImpl::onConfigUpdateFailed(const EnvoyException* e) { + stats_.update_attempt_.inc(); + // TODO(htuch): Less fragile signal that this is failure vs. reject. + if (e == nullptr) { + stats_.update_failure_.inc(); + ENVOY_LOG(debug, "gRPC update for {} failed", type_url_); + } else { + stats_.update_rejected_.inc(); + ENVOY_LOG(warn, "gRPC config for {} rejected: {}", type_url_, e->what()); + } + callbacks_.onConfigUpdateFailed(e); +} +std::string DeltaSubscriptionImpl::resourceName(const ProtobufWkt::Any& resource) { + return callbacks_.resourceName(resource); } } // namespace Config diff --git a/source/common/config/delta_subscription_impl.h b/source/common/config/delta_subscription_impl.h index acb5cb16e6316..87bd5c9becd46 100644 --- a/source/common/config/delta_subscription_impl.h +++ b/source/common/config/delta_subscription_impl.h @@ -16,26 +16,43 @@ namespace Config { * GrpcDeltaXdsContext. */ // TODO(fredlas) someday this class will be named GrpcSubscriptionImpl -class DeltaSubscriptionImpl : public Subscription { +// TODO TODO better comment: DeltaSubscriptionImpl implements the SubscriptionCallbacks interface so +// that it can write to SubscriptionStats upon a config update. The idea is, DeltaSubscriptionImpl +// presents itself as the SubscriptionCallbacks, but actually the *real* SubscriptionCallbacks is +// held by DeltaSubscriptionImpl. DeltaSubscriptionImpl passes through all SubscriptionCallbacks +// calls to the held SubscriptionCallbacks. +class DeltaSubscriptionImpl : public Subscription, public SubscriptionCallbacks { public: DeltaSubscriptionImpl(std::shared_ptr context, absl::string_view type_url, - SubscriptionStats stats, std::chrono::milliseconds init_fetch_timeout); + SubscriptionCallbacks& callbacks, SubscriptionStats stats, + std::chrono::milliseconds init_fetch_timeout); ~DeltaSubscriptionImpl(); void pause(); void resume(); - // Config::DeltaSubscription - void start(const std::set& resources, SubscriptionCallbacks& callbacks) override; + // Config::Subscription + void start(const std::set& resources, SubscriptionCallbacks&) override; void updateResources(const std::set& update_to_these_names) override; + // Config::SubscriptionCallbacks (all pass through to callbacks_!) + void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) override; + void onConfigUpdate(const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) override; + void onConfigUpdateFailed(const EnvoyException* e) override; + std::string resourceName(const ProtobufWkt::Any& resource) override; + std::shared_ptr getContextForTest() { return context_; } private: std::shared_ptr context_; const std::string type_url_; + SubscriptionCallbacks& callbacks_; SubscriptionStats stats_; + std::chrono::milliseconds init_fetch_timeout_; WatchMap::Token watch_token_{WatchMap::InvalidToken}; }; diff --git a/source/common/config/delta_subscription_state.cc b/source/common/config/delta_subscription_state.cc index cda8af9a9a308..de7c8a581408b 100644 --- a/source/common/config/delta_subscription_state.cc +++ b/source/common/config/delta_subscription_state.cc @@ -10,10 +10,9 @@ DeltaSubscriptionState::DeltaSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, const LocalInfo::LocalInfo& local_info, std::chrono::milliseconds init_fetch_timeout, - Event::Dispatcher& dispatcher, - SubscriptionStats& stats) + Event::Dispatcher& dispatcher) : type_url_(type_url), callbacks_(callbacks), local_info_(local_info), - init_fetch_timeout_(init_fetch_timeout), stats_(stats) { + init_fetch_timeout_(init_fetch_timeout) { setInitFetchTimeout(dispatcher); } @@ -41,8 +40,8 @@ void DeltaSubscriptionState::resume() { // Returns true if there is any meaningful change in our subscription interest, worth reporting to // the server. -void DeltaSubscriptionState::updateResourceInterest(std::vector cur_added, - std::vector cur_removed) { +void DeltaSubscriptionState::updateSubscriptionInterest(const std::set& cur_added, + const std::set& cur_removed) { for (const auto& a : cur_added) { setResourceWaitingForServer(a); // If interest in a resource is removed-then-added (all before a discovery request @@ -75,7 +74,6 @@ DeltaSubscriptionState::handleResponse(const envoy::api::v2::DeltaDiscoveryRespo // We *always* copy the response's nonce into the next request, even if we're going to make that // request a NACK by setting error_detail. UpdateAck ack(message.nonce(), type_url_); - stats_.update_attempt_.inc(); try { handleGoodResponse(message); } catch (const EnvoyException& e) { @@ -119,8 +117,6 @@ void DeltaSubscriptionState::handleGoodResponse( setResourceWaitingForServer(resource_name); } } - stats_.update_success_.inc(); - stats_.version_.set(HashUtil::xxHash64(message.system_version_info())); ENVOY_LOG(debug, "Delta config for {} accepted with {} resources added, {} removed", type_url_, message.resources().size(), message.removed_resources().size()); } @@ -130,15 +126,12 @@ void DeltaSubscriptionState::handleBadResponse(const EnvoyException& e, UpdateAc ack.error_detail_.set_code(Grpc::Status::GrpcStatus::Internal); ack.error_detail_.set_message(e.what()); disableInitFetchTimeoutTimer(); - stats_.update_rejected_.inc(); ENVOY_LOG(warn, "delta config for {} rejected: {}", type_url_, e.what()); callbacks_.onConfigUpdateFailed(&e); } void DeltaSubscriptionState::handleEstablishmentFailure() { disableInitFetchTimeoutTimer(); - stats_.update_failure_.inc(); - stats_.update_attempt_.inc(); callbacks_.onConfigUpdateFailed(nullptr); } diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index 62c39c333ef77..9d00ce76c78c1 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -29,7 +29,7 @@ class DeltaSubscriptionState : public Logger::Loggable { DeltaSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, const LocalInfo::LocalInfo& local_info, std::chrono::milliseconds init_fetch_timeout, - Event::Dispatcher& dispatcher, SubscriptionStats& stats); + Event::Dispatcher& dispatcher); void setInitFetchTimeout(Event::Dispatcher& dispatcher); @@ -38,7 +38,8 @@ class DeltaSubscriptionState : public Logger::Loggable { bool paused() const { return paused_; } // Update which resources we're interested in subscribing to. - void updateResourceInterest(const std::set& update_to_these_names); + void updateSubscriptionInterest(const std::set& cur_added, + const std::set& cur_removed); // Whether there was a change in our subscription interest we have yet to inform the server of. bool subscriptionUpdatePending() const; @@ -106,8 +107,6 @@ class DeltaSubscriptionState : public Logger::Loggable { std::set names_added_; std::set names_removed_; - SubscriptionStats& stats_; - DeltaSubscriptionState(const DeltaSubscriptionState&) = delete; DeltaSubscriptionState& operator=(const DeltaSubscriptionState&) = delete; }; diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h index 042d946b82683..89d4f10473ddb 100644 --- a/source/common/config/grpc_delta_xds_context.h +++ b/source/common/config/grpc_delta_xds_context.h @@ -13,6 +13,7 @@ #include "common/config/delta_subscription_state.h" #include "common/config/grpc_stream.h" #include "common/config/utility.h" +#include "common/config/watch_map.h" #include "common/grpc/common.h" #include "common/protobuf/protobuf.h" #include "common/protobuf/utility.h" @@ -23,13 +24,14 @@ namespace Config { // TODO TODO not actually needed; just an adapter to the old-style subscribe(). class TokenizedGrpcMuxWatch : public GrpcMuxWatch { public: - TokenizedGrpcMuxWatch(GrpcDeltaXdsContext& context, WatchMap::Token token) - : context_(context), token_(token) {} - ~TokenizedGrpcMuxWatch() { context_.removeWatch(token_); } + TokenizedGrpcMuxWatch(GrpcMux& context, const std::string& type_url, WatchMap::Token token) + : context_(context), type_url_(type_url), token_(token) {} + ~TokenizedGrpcMuxWatch() { context_.removeWatch(type_url_, token_); } private: + GrpcMux& context_; + std::string type_url_; WatchMap::Token token_; - GrpcDeltaXdsContext& context_; }; // Manages subscriptions to one or more type of resource. The logical protocol @@ -58,16 +60,19 @@ class GrpcDeltaXdsContext : public GrpcMux, addSubscription(type_url); } - WatchMap::Token watch_token = watch_maps_[type_url].addWatch(callbacks); + WatchMap::Token watch_token = watch_maps_[type_url]->addWatch(callbacks); // updateWatch() queues a discovery request if any of 'resources' are not yet subscribed. updateWatch(type_url, watch_token, resources); return watch_token; } - void removeWatch(const std::string& type_url, WatchMap::Token watch_token) { + void removeWatch(const std::string& type_url, WatchMap::Token watch_token) override { + if (watch_token == WatchMap::InvalidToken) { + return; + } // updateWatch() queues a discovery request if any resources were watched only by this watch. updateWatch(type_url, watch_token, {}); - if (watch_maps_[type_url].removeWatch(watch_token)) { + if (watch_maps_[type_url]->removeWatch(watch_token)) { // removeWatch() told us that watch_token was the last watch on the entire subscription. removeSubscription(type_url); } @@ -78,13 +83,13 @@ class GrpcDeltaXdsContext : public GrpcMux, // subscription will enqueue and attempt to send an appropriate discovery request. void updateWatch(const std::string& type_url, WatchMap::Token watch_token, const std::set& resources) override { - auto added_removed = watch_maps_[type_url].updateWatchInterest(watch_token, resources); + auto added_removed = watch_maps_[type_url]->updateWatchInterest(watch_token, resources); auto sub = subscriptions_.find(type_url); if (sub == subscriptions_.end()) { ENVOY_LOG(error, "Watch of {} has no subscription to update.", type_url); return; } - sub->second->updateResourceInterest(added_removed.first, added_removed.second); + sub->second->updateSubscriptionInterest(added_removed.first, added_removed.second); // Tell the server about our new interests, if there are any. trySendDiscoveryRequests(); } @@ -144,9 +149,12 @@ class GrpcDeltaXdsContext : public GrpcMux, } // TODO TODO but yeah this should just be gone!!!!!!!!! - GrpcMuxWatchPtr subscribe(const std::string& type_url, const std::set& resources, - GrpcMuxCallbacks& callbacks) override { - return std::make_unique(*this, addWatch(type_url, resources, callbacks)); + GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, + GrpcMuxCallbacks&) override { + // not sure what GrpcMuxCallbacks is for, but we need a SubscriptionCallbacks here, so...... + // return std::make_unique(*this, addWatch(type_url, resources, + // callbacks)); + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } void start() override { grpc_stream_.establishNewStream(); } @@ -157,17 +165,17 @@ class GrpcDeltaXdsContext : public GrpcMux, } private: - void addSubscription(const std::string& type_url, SubscriptionStats& stats) override { + void addSubscription(const std::string& type_url) { watch_maps_.emplace(type_url, std::make_unique()); subscriptions_.emplace(type_url, std::make_unique( type_url, *watch_maps_[type_url], local_info_, - init_fetch_timeouts_[type_url], dispatcher_, stats)); + init_fetch_timeouts_[type_url], dispatcher_)); subscription_ordering_.emplace_back(type_url); } // TODO do we need this? or should we just let subscriptions hang out even when nobody is // interested in them? - void removeSubscription(const std::string& type_url) override { + void removeSubscription(const std::string& type_url) { subscriptions_.erase(type_url); watch_maps_.erase(type_url); // And remove from the subscription_ordering_ list. @@ -267,9 +275,11 @@ class GrpcDeltaXdsContext : public GrpcMux, // of our different resource types' ACKs are mixed together in this queue. std::queue ack_queue_; + // TODO TODO probably consolidate // Map from type_url strings to a DeltaSubscriptionState for that type. absl::flat_hash_map> subscriptions_; absl::flat_hash_map> watch_maps_; + absl::flat_hash_map init_fetch_timeouts_; // Determines the order of initial discovery requests. (Assumes that subscriptions are added in // the order of Envoy's dependency ordering). diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index c60f7b6b59b32..de45e786442af 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -38,14 +38,19 @@ class GrpcMuxImpl : public GrpcMux, // GrpcMux void pause(const std::string& type_url) override; void resume(const std::string& type_url) override; - void addSubscription(const std::set&, const std::string&, SubscriptionCallbacks&, - SubscriptionStats&, std::chrono::milliseconds) override { + + /*WatchMap::Token*/ uint64_t addWatch(const std::string&, const std::set&, + SubscriptionCallbacks&) override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } + virtual void removeWatch(const std::string&, /*WatchMap::Token*/ uint64_t) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } - void updateResources(const std::set&, const std::string&) override { + virtual void updateWatch(const std::string&, /*WatchMap::Token*/ uint64_t, + const std::set&) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } - void removeSubscription(const std::string&) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + void handleDiscoveryResponse(std::unique_ptr&& message); void sendDiscoveryRequest(const std::string& type_url); @@ -134,15 +139,18 @@ class NullGrpcMuxImpl : public GrpcMux, GrpcStreamCallbacks&, const std::string&, SubscriptionCallbacks&, - SubscriptionStats&, std::chrono::milliseconds) override { + /*WatchMap::Token*/ uint64_t addWatch(const std::string&, const std::set&, + SubscriptionCallbacks&) override { + throw EnvoyException("ADS must be configured to support an ADS config source"); + } + virtual void removeWatch(const std::string&, /*WatchMap::Token*/ uint64_t) override { throw EnvoyException("ADS must be configured to support an ADS config source"); } - void updateResources(const std::set&, const std::string&) override { + virtual void updateWatch(const std::string&, /*WatchMap::Token*/ uint64_t, + const std::set&) override { throw EnvoyException("ADS must be configured to support an ADS config source"); } - void removeSubscription(const std::string&) override {} void onWriteable() override {} void onStreamEstablished() override {} void onEstablishmentFailure() override {} diff --git a/source/common/config/subscription_factory.h b/source/common/config/subscription_factory.h index ebd9a914e9f4a..eaa3a621c54b8 100644 --- a/source/common/config/subscription_factory.h +++ b/source/common/config/subscription_factory.h @@ -32,7 +32,7 @@ class SubscriptionFactory { const envoy::api::v2::core::ConfigSource& config, const LocalInfo::LocalInfo& local_info, Event::Dispatcher& dispatcher, Upstream::ClusterManager& cm, Runtime::RandomGenerator& random, Stats::Scope& scope, const std::string& rest_method, const std::string& grpc_method, - absl::string_view type_url, Api::Api& api); + absl::string_view type_url, Api::Api& api, bool is_delta); }; } // namespace Config diff --git a/source/common/config/watch_map.h b/source/common/config/watch_map.h index 0f09d891c8817..77653a46af0a4 100644 --- a/source/common/config/watch_map.h +++ b/source/common/config/watch_map.h @@ -40,7 +40,7 @@ class WatchMap : public SubscriptionCallbacks, public Logger::Loggable::max(); + static constexpr Token InvalidToken = std::numeric_limits::max(); // Adds 'callbacks' to the WatchMap, with no resource names being watched. // (Use updateWatchInterest() to add some names). diff --git a/source/common/upstream/eds.cc b/source/common/upstream/eds.cc index 2f9fba2c740a3..f7f21ea12574e 100644 --- a/source/common/upstream/eds.cc +++ b/source/common/upstream/eds.cc @@ -270,10 +270,10 @@ EdsClusterFactory::createClusterImpl( } return std::make_pair( - return std::make_unique(cluster, context.runtime(), socket_factory_context, - std::move(stats_scope), context.addedViaApi(), - context.clusterManager().xdsIsDelta()), - nullptr); + std::make_unique(cluster, context.runtime(), socket_factory_context, + std::move(stats_scope), context.addedViaApi(), + context.clusterManager().xdsIsDelta()), + nullptr); } /** From de02e59e82c28f66ad9681ecf0488fd44b9a2d54 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 31 May 2019 15:42:14 -0400 Subject: [PATCH 22/56] whoops missed scoped RDS Signed-off-by: Fred Douglas --- source/common/router/scoped_rds.cc | 2 +- source/common/router/scoped_rds.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index e2208766b37cb..0d6dd03ffe7e5 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -53,7 +53,7 @@ ScopedRdsConfigSubscription::ScopedRdsConfigSubscription( "envoy.api.v2.ScopedRoutesDiscoveryService.StreamScopedRoutes", Grpc::Common::typeUrl( envoy::api::v2::ScopedRouteConfiguration().GetDescriptor()->full_name()), - factory_context.api()); + factory_context.api(), *this); } void ScopedRdsConfigSubscription::onConfigUpdate( diff --git a/source/common/router/scoped_rds.h b/source/common/router/scoped_rds.h index c32ec1012ce5e..68453ae475947 100644 --- a/source/common/router/scoped_rds.h +++ b/source/common/router/scoped_rds.h @@ -82,7 +82,7 @@ class ScopedRdsConfigSubscription : public Envoy::Config::DeltaConfigSubscriptio const std::string& name() const { return name_; } // Envoy::Config::ConfigSubscriptionCommonBase - void start() override { subscription_->start({}, *this); } + void start() override { subscription_->start({}); } // Envoy::Config::SubscriptionCallbacks void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, From ccd83110c542e0dd04c34cca05295282a173e6cd Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Mon, 3 Jun 2019 10:17:55 -0400 Subject: [PATCH 23/56] fix addWatch logic with cute recursion Signed-off-by: Fred Douglas --- source/common/config/delta_subscription_state.cc | 2 -- source/common/config/grpc_delta_xds_context.h | 1 + 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/source/common/config/delta_subscription_state.cc b/source/common/config/delta_subscription_state.cc index b614035db9108..d0ef188de98b2 100644 --- a/source/common/config/delta_subscription_state.cc +++ b/source/common/config/delta_subscription_state.cc @@ -38,8 +38,6 @@ void DeltaSubscriptionState::resume() { paused_ = false; } -// Returns true if there is any meaningful change in our subscription interest, worth reporting to -// the server. void DeltaSubscriptionState::updateSubscriptionInterest(const std::set& cur_added, const std::set& cur_removed) { for (const auto& a : cur_added) { diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h index e4875379b530a..06dde4a277658 100644 --- a/source/common/config/grpc_delta_xds_context.h +++ b/source/common/config/grpc_delta_xds_context.h @@ -59,6 +59,7 @@ class GrpcDeltaXdsContext : public GrpcMux, if (entry == subscriptions_.end()) { // We don't yet have a subscription for type_url! Make one! addSubscription(type_url, init_fetch_timeout); + return addWatch(type_url, resources, callbacks, init_fetch_timeout); } WatchMap::Token watch_token = entry->second->watch_map_.addWatch(callbacks); From 5e551a5bb8cd386d1193dd374138a04ccdabdc62 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Tue, 4 Jun 2019 17:14:40 -0400 Subject: [PATCH 24/56] align GrpcDeltaXdsContext::start usage with GrpcMuxImpl, most tests pass Signed-off-by: Fred Douglas --- api/envoy/api/v2/discovery.proto | 2 +- source/common/config/grpc_delta_xds_context.h | 14 +++++++++++- .../common/upstream/cluster_manager_impl.cc | 5 +---- .../config/delta_subscription_impl_test.cc | 14 +++++++++--- .../config/delta_subscription_state_test.cc | 2 +- .../config/delta_subscription_test_harness.h | 22 ++++++++++++++----- test/common/config/subscription_impl_test.cc | 2 ++ .../common/config/subscription_test_harness.h | 2 ++ 8 files changed, 48 insertions(+), 15 deletions(-) diff --git a/api/envoy/api/v2/discovery.proto b/api/envoy/api/v2/discovery.proto index ac3f13643bd0e..355d9ff7509a6 100644 --- a/api/envoy/api/v2/discovery.proto +++ b/api/envoy/api/v2/discovery.proto @@ -196,7 +196,7 @@ message DeltaDiscoveryResponse { // The version of the response data (used for debugging). string system_version_info = 1; - // The response resources. The Resource proto's Any field named 'resource' should be + // The response resources. The Resource proto's Any field ('resource') should be // of type type_url. repeated Resource resources = 2 [(gogoproto.nullable) = false]; diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h index 06dde4a277658..90fda5732a593 100644 --- a/source/common/config/grpc_delta_xds_context.h +++ b/source/common/config/grpc_delta_xds_context.h @@ -74,7 +74,13 @@ class GrpcDeltaXdsContext : public GrpcMux, } // updateWatch() queues a discovery request if any resources were watched only by this watch. updateWatch(type_url, watch_token, {}); - if (subscriptions_[type_url]->watch_map_.removeWatch(watch_token)) { + + auto entry = subscriptions_.find(type_url); + if (entry == subscriptions_.end()) { + ENVOY_LOG(error, "Asked to remove watch of {}, but there is no subscription...", type_url); + return; + } + if (entry->second->watch_map_.removeWatch(watch_token)) { // removeWatch() told us that watch_token was the last watch on the entire subscription. removeSubscription(type_url); } @@ -183,10 +189,12 @@ class GrpcDeltaXdsContext : public GrpcMux, } void trySendDiscoveryRequests() { + std::cerr << "====================================\ntrySendDiscoveryRequests....." << std::endl; while (true) { // Do any of our subscriptions even want to send a request? absl::optional maybe_request_type = whoWantsToSendDiscoveryRequest(); if (!maybe_request_type.has_value()) { + std::cerr << "nobody wants to send.\n====================================" << std::endl; break; } // If so, which one (by type_url)? @@ -194,6 +202,7 @@ class GrpcDeltaXdsContext : public GrpcMux, // If we don't have a subscription object for this request's type_url, drop the request. auto sub = subscriptions_.find(next_request_type_url); if (sub == subscriptions_.end()) { + std::cerr << "NONEXISTENT!!!!!!!!!!! SKIPPING!!!!!!!!!!!!" << std::endl; ENVOY_LOG(warn, "Not sending discovery request for non-existent subscription {}.", next_request_type_url); // It's safe to assume the front of the ACK queue is of this type, because that's the only @@ -204,15 +213,18 @@ class GrpcDeltaXdsContext : public GrpcMux, } // Try again later if paused/rate limited/stream down. if (!canSendDiscoveryRequest(next_request_type_url, sub->second->sub_state_)) { + std::cerr << "not available to send\n====================================" << std::endl; break; } // Get our subscription state to generate the appropriate DeltaDiscoveryRequest, and send. if (!ack_queue_.empty()) { // Because ACKs take precedence over plain requests, if there is anything in the queue, it's // safe to assume it's what we want to send here. + std::cerr << "sending " << next_request_type_url << " with ACK" << std::endl; grpc_stream_.sendMessage(sub->second->sub_state_.getNextRequestWithAck(ack_queue_.front())); ack_queue_.pop(); } else { + std::cerr << "sending " << next_request_type_url << " without ACK" << std::endl; grpc_stream_.sendMessage(sub->second->sub_state_.getNextRequestAckless()); } } diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 89a6d0a11591f..2c1f2f8103fa8 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -309,10 +309,7 @@ ClusterManagerImpl::ClusterManagerImpl( // clusters have already initialized. (E.g., if all static). init_helper_.onStaticLoadComplete(); - if (!xds_is_delta_) { - // TODO TODO i think just have the same approach for start() with both, i.e. remove the if. - ads_mux_->start(); - } + ads_mux_->start(); if (cm_config.has_load_stats_config()) { const auto& load_stats_config = cm_config.load_stats_config(); diff --git a/test/common/config/delta_subscription_impl_test.cc b/test/common/config/delta_subscription_impl_test.cc index 5d2e9c0dd65cf..5c327468e5429 100644 --- a/test/common/config/delta_subscription_impl_test.cc +++ b/test/common/config/delta_subscription_impl_test.cc @@ -7,6 +7,17 @@ namespace { class DeltaSubscriptionImplTest : public DeltaSubscriptionTestHarness, public testing::Test { protected: DeltaSubscriptionImplTest() : DeltaSubscriptionTestHarness() {} + + // We need to destroy the subscription before the test's destruction, because the subscription's + // destructor removes its watch from the GrpcDeltaXdsContext, and that removal process involves + // some things held by the test fixture. + void TearDown() override { + // if (subscription_started_) { + // EXPECT_CALL(async_stream_, sendMessage(_,_)); + // subscription_.reset(); + // } + doSubscriptionTearDown(); + } }; TEST_F(DeltaSubscriptionImplTest, UpdateResourcesCausesRequest) { @@ -110,9 +121,6 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { } TEST_F(DeltaSubscriptionImplTest, NoGrpcStream) { - // Have to call start() to get state_ populated (which this test needs to not segfault), but - // start() also tries to start the GrpcStream. So, have that attempt return nullptr. - EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(nullptr)); EXPECT_CALL(async_stream_, sendMessage(_, _)).Times(0); subscription_->start({"name1"}); subscription_->updateResources({"name1", "name2"}); diff --git a/test/common/config/delta_subscription_state_test.cc b/test/common/config/delta_subscription_state_test.cc index 21793a34ddba1..8c73c07d98423 100644 --- a/test/common/config/delta_subscription_state_test.cc +++ b/test/common/config/delta_subscription_state_test.cc @@ -79,7 +79,7 @@ populateRepeatedResource(std::vector> items) // Basic gaining/losing interest in resources should lead to (un)subscriptions. TEST_F(DeltaSubscriptionStateTest, SubscribeAndUnsubscribe) { { - state_.updateSubscriptionInterest({"name1"}, {"name4"}); + state_.updateSubscriptionInterest({"name4"}, {"name1"}); envoy::api::v2::DeltaDiscoveryRequest cur_request = state_.getNextRequestAckless(); EXPECT_THAT(cur_request.resource_names_subscribe(), UnorderedElementsAre("name4")); EXPECT_THAT(cur_request.resource_names_unsubscribe(), UnorderedElementsAre("name1")); diff --git a/test/common/config/delta_subscription_test_harness.h b/test/common/config/delta_subscription_test_harness.h index 69df17c936db6..27e169e0c7c45 100644 --- a/test/common/config/delta_subscription_test_harness.h +++ b/test/common/config/delta_subscription_test_harness.h @@ -33,11 +33,19 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { node_.set_id("fo0"); EXPECT_CALL(local_info_, node()).WillRepeatedly(testing::ReturnRef(node_)); EXPECT_CALL(dispatcher_, createTimer_(_)); + xds_context_ = std::make_shared( + std::unique_ptr(async_client_), dispatcher_, *method_descriptor_, + random_, stats_store_, rate_limit_settings_, local_info_); subscription_ = std::make_unique( - std::make_shared(std::unique_ptr(async_client_), - dispatcher_, *method_descriptor_, random_, - stats_store_, rate_limit_settings_, local_info_), - Config::TypeUrl::get().ClusterLoadAssignment, callbacks_, stats_, init_fetch_timeout); + xds_context_, Config::TypeUrl::get().ClusterLoadAssignment, callbacks_, stats_, + init_fetch_timeout); + } + + void doSubscriptionTearDown() override { + if (subscription_started_) { + EXPECT_CALL(async_stream_, sendMessage(_, _)); + subscription_.reset(); + } } ~DeltaSubscriptionTestHarness() { @@ -56,10 +64,12 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { } void startSubscription(const std::set& cluster_names) override { - EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); + subscription_started_ = true; last_cluster_names_ = cluster_names; expectSendMessage(last_cluster_names_, ""); + EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); subscription_->start(cluster_names); + xds_context_->start(); } void expectSendMessage(const std::set& cluster_names, @@ -175,6 +185,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { NiceMock random_; NiceMock local_info_; Grpc::MockAsyncStream async_stream_; + std::shared_ptr xds_context_; std::unique_ptr subscription_; std::string last_response_nonce_; std::set last_cluster_names_; @@ -184,6 +195,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { NiceMock> callbacks_; std::queue nonce_acks_required_; std::queue nonce_acks_sent_; + bool subscription_started_{}; }; } // namespace diff --git a/test/common/config/subscription_impl_test.cc b/test/common/config/subscription_impl_test.cc index 67a9566619c2a..412214496aac2 100644 --- a/test/common/config/subscription_impl_test.cc +++ b/test/common/config/subscription_impl_test.cc @@ -39,6 +39,8 @@ class SubscriptionImplTest : public testing::TestWithParam { } } + void TearDown() override { test_harness_->doSubscriptionTearDown(); } + void startSubscription(const std::set& cluster_names) { test_harness_->startSubscription(cluster_names); } diff --git a/test/common/config/subscription_test_harness.h b/test/common/config/subscription_test_harness.h index acb1c2caacfc4..75423403c0f9c 100644 --- a/test/common/config/subscription_test_harness.h +++ b/test/common/config/subscription_test_harness.h @@ -75,6 +75,8 @@ class SubscriptionTestHarness { virtual void callInitFetchTimeoutCb() PURE; + virtual void doSubscriptionTearDown() {} + Stats::IsolatedStoreImpl stats_store_; SubscriptionStats stats_; }; From 02dff3ed25dde488cce97bd5b0681d856fc724b4 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 6 Jun 2019 11:25:38 -0400 Subject: [PATCH 25/56] fix subscription_impl tests, non-delta ones mocked out the code that increments update_failure Signed-off-by: Fred Douglas --- test/common/config/delta_subscription_test_harness.h | 2 +- test/common/config/filesystem_subscription_test_harness.h | 3 ++- test/common/config/grpc_subscription_test_harness.h | 3 ++- test/common/config/http_subscription_test_harness.h | 3 ++- test/common/config/subscription_impl_test.cc | 2 +- 5 files changed, 8 insertions(+), 5 deletions(-) diff --git a/test/common/config/delta_subscription_test_harness.h b/test/common/config/delta_subscription_test_harness.h index 27e169e0c7c45..4a045a2fb85dc 100644 --- a/test/common/config/delta_subscription_test_harness.h +++ b/test/common/config/delta_subscription_test_harness.h @@ -66,8 +66,8 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { void startSubscription(const std::set& cluster_names) override { subscription_started_ = true; last_cluster_names_ = cluster_names; - expectSendMessage(last_cluster_names_, ""); EXPECT_CALL(*async_client_, start(_, _)).WillOnce(Return(&async_stream_)); + expectSendMessage(last_cluster_names_, ""); subscription_->start(cluster_names); xds_context_->start(); } diff --git a/test/common/config/filesystem_subscription_test_harness.h b/test/common/config/filesystem_subscription_test_harness.h index 9a14fc73dd5cf..aee1270ae4109 100644 --- a/test/common/config/filesystem_subscription_test_harness.h +++ b/test/common/config/filesystem_subscription_test_harness.h @@ -94,7 +94,8 @@ class FilesystemSubscriptionTestHarness : public SubscriptionTestHarness { } void expectConfigUpdateFailed() override { - // initial_fetch_timeout not implemented + // initial_fetch_timeout not implemented. Match the stat behavior of the others. + stats_.update_failure_.inc(); } void expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds timeout) override { diff --git a/test/common/config/grpc_subscription_test_harness.h b/test/common/config/grpc_subscription_test_harness.h index e0da24a3b7745..d831005ce403c 100644 --- a/test/common/config/grpc_subscription_test_harness.h +++ b/test/common/config/grpc_subscription_test_harness.h @@ -132,7 +132,8 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { } void expectConfigUpdateFailed() override { - EXPECT_CALL(callbacks_, onConfigUpdateFailed(nullptr)); + EXPECT_CALL(callbacks_, onConfigUpdateFailed(nullptr)) + .WillOnce(Invoke([this](const EnvoyException*) { stats_.update_failure_.inc(); })); } void expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds timeout) override { diff --git a/test/common/config/http_subscription_test_harness.h b/test/common/config/http_subscription_test_harness.h index c6d0a0572cbfa..feeef1185de0c 100644 --- a/test/common/config/http_subscription_test_harness.h +++ b/test/common/config/http_subscription_test_harness.h @@ -143,7 +143,8 @@ class HttpSubscriptionTestHarness : public SubscriptionTestHarness { } void expectConfigUpdateFailed() override { - EXPECT_CALL(callbacks_, onConfigUpdateFailed(nullptr)); + EXPECT_CALL(callbacks_, onConfigUpdateFailed(nullptr)) + .WillOnce(Invoke([this](const EnvoyException*) { stats_.update_failure_.inc(); })); } void expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds timeout) override { diff --git a/test/common/config/subscription_impl_test.cc b/test/common/config/subscription_impl_test.cc index 412214496aac2..2297b820fd7ca 100644 --- a/test/common/config/subscription_impl_test.cc +++ b/test/common/config/subscription_impl_test.cc @@ -151,7 +151,7 @@ TEST_P(SubscriptionImplInitFetchTimeoutTest, InitialFetchTimeout) { verifyStats(1, 0, 0, 0, 0); expectConfigUpdateFailed(); callInitFetchTimeoutCb(); - verifyStats(1, 0, 0, 0, 0); + verifyStats(1, 0, 0, 1, 0); } // Validate that initial fetch timer is disabled on config update From c0c3d987ca2a2139adeb1159ec77d7faf940aa98 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 6 Jun 2019 17:12:12 -0400 Subject: [PATCH 26/56] allow pausing nonexistent subs, also pick up WatchMap wildcard Signed-off-by: Fred Douglas --- .../common/config/delta_subscription_impl.cc | 5 +++ .../common/config/delta_subscription_state.cc | 14 +------- .../common/config/delta_subscription_state.h | 5 --- source/common/config/grpc_delta_xds_context.h | 36 ++++++++++--------- source/common/config/watch_map.cc | 22 +++++++++--- source/common/config/watch_map.h | 9 ++--- .../common/upstream/cluster_manager_impl.cc | 3 ++ test/common/upstream/BUILD | 1 + test/common/upstream/cds_api_impl_test.cc | 36 ++++++++++--------- test/mocks/config/mocks.h | 6 ++++ 10 files changed, 78 insertions(+), 59 deletions(-) diff --git a/source/common/config/delta_subscription_impl.cc b/source/common/config/delta_subscription_impl.cc index ea1c5134638af..c506020223cb1 100644 --- a/source/common/config/delta_subscription_impl.cc +++ b/source/common/config/delta_subscription_impl.cc @@ -19,6 +19,11 @@ void DeltaSubscriptionImpl::resume() { context_->resume(type_url_); } // Config::DeltaSubscription void DeltaSubscriptionImpl::start(const std::set& resources) { + // TODO TODO needs to be idempotent. GrpcSubscription is special-purposed to non-ADS, and so knows + // that it needs to call start. My DeltaSubscriptionImpl can be either ADS or not, and so needs to + // (safely) call start regardless. + context_->start(); + watch_token_ = context_->addWatch(type_url_, resources, *this, init_fetch_timeout_); stats_.update_attempt_.inc(); } diff --git a/source/common/config/delta_subscription_state.cc b/source/common/config/delta_subscription_state.cc index d0ef188de98b2..d33056477d6c1 100644 --- a/source/common/config/delta_subscription_state.cc +++ b/source/common/config/delta_subscription_state.cc @@ -26,18 +26,6 @@ void DeltaSubscriptionState::setInitFetchTimeout(Event::Dispatcher& dispatcher) } } -void DeltaSubscriptionState::pause() { - ENVOY_LOG(debug, "Pausing discovery requests for {}", type_url_); - ASSERT(!paused_); - paused_ = true; -} - -void DeltaSubscriptionState::resume() { - ENVOY_LOG(debug, "Resuming discovery requests for {}", type_url_); - ASSERT(paused_); - paused_ = false; -} - void DeltaSubscriptionState::updateSubscriptionInterest(const std::set& cur_added, const std::set& cur_removed) { for (const auto& a : cur_added) { @@ -96,7 +84,7 @@ void DeltaSubscriptionState::handleGoodResponse( fmt::format("duplicate name {} found in the union of added+removed resources", name)); } } - + std::cerr << "handleGoodResponse now callbacks_.onConfigUpdate()" << std::endl; callbacks_.onConfigUpdate(message.resources(), message.removed_resources(), message.system_version_info()); for (const auto& resource : message.resources()) { diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index 9d00ce76c78c1..f251150f75d63 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -33,10 +33,6 @@ class DeltaSubscriptionState : public Logger::Loggable { void setInitFetchTimeout(Event::Dispatcher& dispatcher); - void pause(); - void resume(); - bool paused() const { return paused_; } - // Update which resources we're interested in subscribing to. void updateSubscriptionInterest(const std::set& cur_added, const std::set& cur_removed); @@ -98,7 +94,6 @@ class DeltaSubscriptionState : public Logger::Loggable { std::chrono::milliseconds init_fetch_timeout_; Event::TimerPtr init_fetch_timeout_timer_; - bool paused_{}; bool any_request_sent_yet_in_current_stream_{}; // Tracks changes in our subscription interest since the previous DeltaDiscoveryRequest we sent. diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h index 90fda5732a593..de44c0ae92fa8 100644 --- a/source/common/config/grpc_delta_xds_context.h +++ b/source/common/config/grpc_delta_xds_context.h @@ -50,7 +50,9 @@ class GrpcDeltaXdsContext : public GrpcMux, const LocalInfo::LocalInfo& local_info) : dispatcher_(dispatcher), local_info_(local_info), grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, - rate_limit_settings) {} + rate_limit_settings) { + std::cerr << "making a GrpcDeltaXdsContext" << std::endl; + } WatchMap::Token addWatch(const std::string& type_url, const std::set& resources, SubscriptionCallbacks& callbacks, @@ -103,21 +105,16 @@ class GrpcDeltaXdsContext : public GrpcMux, } void pause(const std::string& type_url) override { - auto sub = subscriptions_.find(type_url); - if (sub == subscriptions_.end()) { - ENVOY_LOG(warn, "Not pausing non-existent subscription {}.", type_url); - return; - } - sub->second->sub_state_.pause(); + // It's ok to pause a subscription that doesn't exist yet. + auto& pause_entry = paused_[type_url]; + ASSERT(!pause_entry); + pause_entry = true; } void resume(const std::string& type_url) override { - auto sub = subscriptions_.find(type_url); - if (sub == subscriptions_.end()) { - ENVOY_LOG(warn, "Not resuming non-existent subscription {}.", type_url); - return; - } - sub->second->sub_state_.resume(); + auto& pause_entry = paused_[type_url]; + ASSERT(pause_entry); + pause_entry = false; trySendDiscoveryRequests(); } @@ -164,6 +161,9 @@ class GrpcDeltaXdsContext : public GrpcMux, // callbacks)); NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + // TODO TODO needs to be idempotent. GrpcSubscription is special-purposed to non-ADS, and so knows + // that it needs to call start. My DeltaSubscriptionImpl can be either ADS or not, and so needs to + // (safely) call start regardless. void start() override { grpc_stream_.establishNewStream(); } private: @@ -212,7 +212,7 @@ class GrpcDeltaXdsContext : public GrpcMux, continue; } // Try again later if paused/rate limited/stream down. - if (!canSendDiscoveryRequest(next_request_type_url, sub->second->sub_state_)) { + if (!canSendDiscoveryRequest(next_request_type_url)) { std::cerr << "not available to send\n====================================" << std::endl; break; } @@ -233,8 +233,8 @@ class GrpcDeltaXdsContext : public GrpcMux, // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check // whether we *want* to send a DeltaDiscoveryRequest). - bool canSendDiscoveryRequest(absl::string_view type_url, DeltaSubscriptionState& sub) { - if (sub.paused()) { + bool canSendDiscoveryRequest(absl::string_view type_url) { + if (paused_[type_url]) { ENVOY_LOG(trace, "API {} paused; discovery request on hold for now.", type_url); return false; } else if (!grpc_stream_.grpcStreamAvailable()) { @@ -297,6 +297,10 @@ class GrpcDeltaXdsContext : public GrpcMux, // Map key is type_url. absl::flat_hash_map> subscriptions_; + // It's ok for non-existent subs to be paused/resumed. The cleanest way to support that is to give + // the pause state its own map. (Map key is type_url.) + absl::flat_hash_map paused_; + // Determines the order of initial discovery requests. (Assumes that subscriptions are added in // the order of Envoy's dependency ordering). std::list subscription_ordering_; diff --git a/source/common/config/watch_map.cc b/source/common/config/watch_map.cc index 7d93f685fef51..76ec5fb0f3b1c 100644 --- a/source/common/config/watch_map.cc +++ b/source/common/config/watch_map.cc @@ -6,11 +6,13 @@ namespace Config { WatchMap::Token WatchMap::addWatch(SubscriptionCallbacks& callbacks) { WatchMap::Token next_watch = next_watch_++; watches_.emplace(next_watch, WatchMap::Watch(callbacks)); + wildcard_watches_.insert(next_watch); return next_watch; } bool WatchMap::removeWatch(WatchMap::Token token) { watches_.erase(token); + wildcard_watches_.erase(token); // may or may not be in there, but we want it gone. return watches_.empty(); } @@ -22,6 +24,12 @@ WatchMap::updateWatchInterest(WatchMap::Token token, ENVOY_LOG(error, "updateWatchInterest() called on nonexistent token!"); return std::make_pair(std::set(), std::set()); } + if (update_to_these_names.empty()) { + wildcard_watches_.insert(token); + } else { + wildcard_watches_.erase(token); + } + auto& watch = watches_entry->second; std::vector newly_added_to_watch; @@ -40,13 +48,17 @@ WatchMap::updateWatchInterest(WatchMap::Token token, findRemovals(newly_removed_from_watch, token)); } -const absl::flat_hash_set& +absl::flat_hash_set WatchMap::tokensInterestedIn(const std::string& resource_name) { - auto entry = watch_interest_.find(resource_name); - if (entry == watch_interest_.end()) { - return empty_token_set_; + // Note that std::set_union needs sorted sets. Better to do it ourselves with insert(). + absl::flat_hash_set ret = wildcard_watches_; + auto watches_interested = watch_interest_.find(resource_name); + if (watches_interested != watch_interest_.end()) { + for (const auto& watch : watches_interested->second) { + ret.insert(watch); + } } - return entry->second; + return ret; } void WatchMap::onConfigUpdate(const Protobuf::RepeatedPtrField& resources, diff --git a/source/common/config/watch_map.h b/source/common/config/watch_map.h index 77653a46af0a4..04b2444cdbe56 100644 --- a/source/common/config/watch_map.h +++ b/source/common/config/watch_map.h @@ -97,13 +97,14 @@ class WatchMap : public SubscriptionCallbacks, public Logger::Loggable& removed_resources, const std::string& system_version_info); - // Does a lookup in watch_interest_, returning empty set if not found. - const absl::flat_hash_set& tokensInterestedIn(const std::string& resource_name); - // A little hack to allow tokensInterestedIn() to return a ref, rather than a copy. - const absl::flat_hash_set empty_token_set_{}; + // Returns the union of watch_interest_[resource_name] and wildcard_watches_. + absl::flat_hash_set tokensInterestedIn(const std::string& resource_name); absl::flat_hash_map watches_; + // Watches whose interest set is currently empty, which is interpreted as "everything". + absl::flat_hash_set wildcard_watches_; + // Maps a resource name to the set of watches interested in that resource. Has two purposes: // 1) Acts as a reference count; no watches care anymore ==> the resource can be removed. // 2) Enables efficient lookup of all interested watches when a resource has been updated. diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 2c1f2f8103fa8..8fce7e0d34a96 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -227,6 +227,7 @@ ClusterManagerImpl::ClusterManagerImpl( bootstrap.dynamic_resources().has_ads_config() ? bootstrap.dynamic_resources().ads_config() : bootstrap.dynamic_resources().cds_config().api_config_source(); + std::cerr << "MAKING DELTA ADS" << std::endl; ads_mux_ = std::make_shared( Config::Utility::factoryForGrpcApiConfigSource(*async_client_manager_, api_config_source, stats) @@ -239,6 +240,7 @@ ClusterManagerImpl::ClusterManagerImpl( bootstrap.dynamic_resources().ads_config()), local_info); } else { + std::cerr << "MAKING SOTW ADS" << std::endl; ads_mux_ = std::make_shared( local_info, Config::Utility::factoryForGrpcApiConfigSource( @@ -252,6 +254,7 @@ ClusterManagerImpl::ClusterManagerImpl( bootstrap.dynamic_resources().ads_config())); } } else { + std::cerr << "MAKING NULLLLLLLLLL ADS" << std::endl; ads_mux_ = std::make_unique(); } diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD index 13985c09deb44..b606e0d7551f5 100644 --- a/test/common/upstream/BUILD +++ b/test/common/upstream/BUILD @@ -21,6 +21,7 @@ envoy_cc_test( "//source/common/json:json_loader_lib", "//source/common/protobuf:utility_lib", "//source/common/upstream:cds_api_lib", + "//test/mocks/config:config_mocks", "//test/mocks/local_info:local_info_mocks", "//test/mocks/protobuf:protobuf_mocks", "//test/mocks/upstream:upstream_mocks", diff --git a/test/common/upstream/cds_api_impl_test.cc b/test/common/upstream/cds_api_impl_test.cc index ebadfafa34095..d44e269b6753a 100644 --- a/test/common/upstream/cds_api_impl_test.cc +++ b/test/common/upstream/cds_api_impl_test.cc @@ -12,6 +12,7 @@ #include "common/upstream/cds_api_impl.h" #include "test/common/upstream/utility.h" +#include "test/mocks/config/mocks.h" #include "test/mocks/local_info/mocks.h" #include "test/mocks/protobuf/mocks.h" #include "test/mocks/upstream/mocks.h" @@ -55,6 +56,8 @@ class CdsApiImplTest : public testing::Test { envoy::api::v2::core::ConfigSource cds_config; TestUtility::loadFromYamlAndValidate(config_yaml, cds_config); cluster_map_.emplace("foo_cluster", mock_cluster_); + ads_mux_ = std::make_shared>(); + ON_CALL(cm_, adsMux()).WillByDefault(Return(ads_mux_)); EXPECT_CALL(cm_, clusters()).WillRepeatedly(Return(cluster_map_)); EXPECT_CALL(mock_cluster_, info()).Times(AnyNumber()); EXPECT_CALL(*mock_cluster_.info_, addedViaApi()); @@ -171,6 +174,7 @@ class CdsApiImplTest : public testing::Test { std::vector clusters_to_warm_up_; }; + std::shared_ptr> ads_mux_; NiceMock cm_; Upstream::ClusterManager::ClusterInfoMap cluster_map_; Upstream::MockClusterMockPrioritySet mock_cluster_; @@ -502,16 +506,16 @@ version_info: '0' )EOF"; // Two clusters updated, both warmed up. - EXPECT_CALL(*cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(*ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); cm_.expectAddWithWarming("cluster1", "0"); cm_.expectWarmingClusterCount(); - EXPECT_CALL(*cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(*ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); cm_.expectAddWithWarming("cluster2", "0"); cm_.expectWarmingClusterCount(); EXPECT_CALL(initialized_, ready()); cm_.expectWarmingClusterCount(2); - EXPECT_CALL(*cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); - EXPECT_CALL(*cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(*ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(*ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); EXPECT_CALL(*interval_timer_, enableTimer(_)); cm_.clustersToWarmUp({"cluster1", "cluster2"}); callbacks_->onSuccess(parseResponseMessageFromYaml(response1_yaml)); @@ -537,15 +541,15 @@ version_info: '1' path: eds path )EOF"; - EXPECT_CALL(*cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(*ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); cm_.expectAddWithWarming("cluster1", "1"); cm_.expectWarmingClusterCount(); - EXPECT_CALL(*cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(*ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); cm_.expectAddWithWarming("cluster3", "1"); cm_.expectWarmingClusterCount(); EXPECT_CALL(initialized_, ready()); cm_.expectWarmingClusterCount(); - EXPECT_CALL(*cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(*ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); EXPECT_CALL(*interval_timer_, enableTimer(_)); resetCdsInitializedCb(); cm_.clustersToWarmUp({"cluster1"}); @@ -566,13 +570,13 @@ version_info: '2' path: eds path )EOF"; - EXPECT_CALL(*cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(*ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); cm_.expectAddWithWarming("cluster4", "2"); cm_.expectWarmingClusterCount(); EXPECT_CALL(initialized_, ready()); cm_.expectWarmingClusterCount(2); - EXPECT_CALL(*cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); - EXPECT_CALL(*cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(*ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(*ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); EXPECT_CALL(*interval_timer_, enableTimer(_)); resetCdsInitializedCb(); cm_.clustersToWarmUp({"cluster4", "cluster3"}); @@ -599,19 +603,19 @@ version_info: '3' )EOF"; // Two clusters updated, first one warmed up before processing of the second one starts. - EXPECT_CALL(*cm_.ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(*ads_mux_, pause(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); cm_.expectAddWithWarming("cluster5", "3", true); cm_.expectWarmingClusterCount(); - EXPECT_CALL(*cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(*ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); cm_.expectWarmingClusterCount(); - EXPECT_CALL(*cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(*ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); cm_.expectAddWithWarming("cluster6", "3"); cm_.expectWarmingClusterCount(); - EXPECT_CALL(*cm_.ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(*ads_mux_, pause(Config::TypeUrl::get().Cluster)).Times(1); EXPECT_CALL(initialized_, ready()); cm_.expectWarmingClusterCount(); - EXPECT_CALL(*cm_.ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); - EXPECT_CALL(*cm_.ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); + EXPECT_CALL(*ads_mux_, resume(Config::TypeUrl::get().Cluster)).Times(1); + EXPECT_CALL(*ads_mux_, resume(Config::TypeUrl::get().ClusterLoadAssignment)).Times(1); EXPECT_CALL(*interval_timer_, enableTimer(_)); resetCdsInitializedCb(); cm_.clustersToWarmUp({"cluster6"}); diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index 4f22d2aebd1dd..caf48c40eaf6f 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -74,6 +74,12 @@ class MockGrpcMux : public GrpcMux { MOCK_METHOD2(updateResources, void(const std::set& resources, const std::string& type_url)); + MOCK_METHOD4(addWatch, uint64_t(const std::string&, const std::set&, + SubscriptionCallbacks&, std::chrono::milliseconds)); + MOCK_METHOD2(removeWatch, void(const std::string&, /*WatchMap::Token*/ uint64_t)); + MOCK_METHOD3(updateWatch, void(const std::string&, /*WatchMap::Token*/ uint64_t, + const std::set&)); + MOCK_METHOD1(removeSubscription, void(const std::string& type_url)); }; From 2a6beda1823941389caeabdf1f68456fc27743a1 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 7 Jun 2019 16:55:20 -0400 Subject: [PATCH 27/56] merge the switch of WatchMap from token to pointer Signed-off-by: Fred Douglas --- .../common/config/delta_subscription_impl.cc | 15 ++- .../common/config/delta_subscription_impl.h | 2 +- source/common/config/grpc_delta_xds_context.h | 6 - source/common/config/watch_map.cc | 127 +++++++----------- source/common/config/watch_map.h | 69 +++++----- 5 files changed, 96 insertions(+), 123 deletions(-) diff --git a/source/common/config/delta_subscription_impl.cc b/source/common/config/delta_subscription_impl.cc index c506020223cb1..9656d4a0aa251 100644 --- a/source/common/config/delta_subscription_impl.cc +++ b/source/common/config/delta_subscription_impl.cc @@ -11,7 +11,16 @@ DeltaSubscriptionImpl::DeltaSubscriptionImpl(std::shared_ptr context, : context_(context), type_url_(type_url), callbacks_(callbacks), stats_(stats), init_fetch_timeout_(init_fetch_timeout) {} -DeltaSubscriptionImpl::~DeltaSubscriptionImpl() { context_->removeWatch(type_url_, watch_token_); } +DeltaSubscriptionImpl::~DeltaSubscriptionImpl() { + // The Watch destruction process assumes all resource interest has already been removed. Doing + // that cleanly needs the context's involvement. So, do that now, just before destroying the + // watch. + context_->updateWatch(type_url_, watch_.get(), {}); + // A Watch must not outlive its owning WatchMap. Since DeltaSubscriptionImpl holds a shared_ptr to + // the context, which owns the WatchMap, the watch must be destroyed first in case this + // subscription holds the last shared_ptr. + watch_.reset(); +} void DeltaSubscriptionImpl::pause() { context_->pause(type_url_); } @@ -24,12 +33,12 @@ void DeltaSubscriptionImpl::start(const std::set& resources) { // (safely) call start regardless. context_->start(); - watch_token_ = context_->addWatch(type_url_, resources, *this, init_fetch_timeout_); + watch_ = context_->addWatch(type_url_, resources, *this, init_fetch_timeout_); stats_.update_attempt_.inc(); } void DeltaSubscriptionImpl::updateResources(const std::set& update_to_these_names) { - context_->updateWatch(type_url_, watch_token_, update_to_these_names); + context_->updateWatch(type_url_, watch_.get(), update_to_these_names); stats_.update_attempt_.inc(); } diff --git a/source/common/config/delta_subscription_impl.h b/source/common/config/delta_subscription_impl.h index c13fb41507021..775f2cad7d0ff 100644 --- a/source/common/config/delta_subscription_impl.h +++ b/source/common/config/delta_subscription_impl.h @@ -55,7 +55,7 @@ class DeltaSubscriptionImpl : public Subscription, public SubscriptionCallbacks // NOTE: if another subscription of the same type_url has already been started, this value will be // ignored in favor of the other subscription's. std::chrono::milliseconds init_fetch_timeout_; - WatchMap::Token watch_token_{WatchMap::InvalidToken}; + WatchPr watch_; }; } // namespace Config diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h index de44c0ae92fa8..976e2ceeb37af 100644 --- a/source/common/config/grpc_delta_xds_context.h +++ b/source/common/config/grpc_delta_xds_context.h @@ -189,12 +189,10 @@ class GrpcDeltaXdsContext : public GrpcMux, } void trySendDiscoveryRequests() { - std::cerr << "====================================\ntrySendDiscoveryRequests....." << std::endl; while (true) { // Do any of our subscriptions even want to send a request? absl::optional maybe_request_type = whoWantsToSendDiscoveryRequest(); if (!maybe_request_type.has_value()) { - std::cerr << "nobody wants to send.\n====================================" << std::endl; break; } // If so, which one (by type_url)? @@ -202,7 +200,6 @@ class GrpcDeltaXdsContext : public GrpcMux, // If we don't have a subscription object for this request's type_url, drop the request. auto sub = subscriptions_.find(next_request_type_url); if (sub == subscriptions_.end()) { - std::cerr << "NONEXISTENT!!!!!!!!!!! SKIPPING!!!!!!!!!!!!" << std::endl; ENVOY_LOG(warn, "Not sending discovery request for non-existent subscription {}.", next_request_type_url); // It's safe to assume the front of the ACK queue is of this type, because that's the only @@ -213,18 +210,15 @@ class GrpcDeltaXdsContext : public GrpcMux, } // Try again later if paused/rate limited/stream down. if (!canSendDiscoveryRequest(next_request_type_url)) { - std::cerr << "not available to send\n====================================" << std::endl; break; } // Get our subscription state to generate the appropriate DeltaDiscoveryRequest, and send. if (!ack_queue_.empty()) { // Because ACKs take precedence over plain requests, if there is anything in the queue, it's // safe to assume it's what we want to send here. - std::cerr << "sending " << next_request_type_url << " with ACK" << std::endl; grpc_stream_.sendMessage(sub->second->sub_state_.getNextRequestWithAck(ack_queue_.front())); ack_queue_.pop(); } else { - std::cerr << "sending " << next_request_type_url << " without ACK" << std::endl; grpc_stream_.sendMessage(sub->second->sub_state_.getNextRequestAckless()); } } diff --git a/source/common/config/watch_map.cc b/source/common/config/watch_map.cc index 76ec5fb0f3b1c..c8c0e11e2bb0c 100644 --- a/source/common/config/watch_map.cc +++ b/source/common/config/watch_map.cc @@ -3,55 +3,45 @@ namespace Envoy { namespace Config { -WatchMap::Token WatchMap::addWatch(SubscriptionCallbacks& callbacks) { - WatchMap::Token next_watch = next_watch_++; - watches_.emplace(next_watch, WatchMap::Watch(callbacks)); - wildcard_watches_.insert(next_watch); - return next_watch; +WatchPtr WatchMap::addWatch(SubscriptionCallbacks& callbacks) { + auto watch = std::make_unique(*this, callbacks); + wildcard_watches_.insert(watch.get()); + watches_.insert(watch.get()); + return watch; } -bool WatchMap::removeWatch(WatchMap::Token token) { - watches_.erase(token); - wildcard_watches_.erase(token); // may or may not be in there, but we want it gone. - return watches_.empty(); +void WatchMap::removeWatch(Watch* watch) { + wildcard_watches_.erase(watch); // may or may not be in there, but we want it gone. + watches_.erase(watch); } -std::pair, std::set> -WatchMap::updateWatchInterest(WatchMap::Token token, - const std::set& update_to_these_names) { - auto watches_entry = watches_.find(token); - if (watches_entry == watches_.end()) { - ENVOY_LOG(error, "updateWatchInterest() called on nonexistent token!"); - return std::make_pair(std::set(), std::set()); - } +AddedRemoved WatchMap::updateWatchInterest(Watch* watch, + const std::set& update_to_these_names) { if (update_to_these_names.empty()) { - wildcard_watches_.insert(token); + wildcard_watches_.insert(watch); } else { - wildcard_watches_.erase(token); + wildcard_watches_.erase(watch); } - auto& watch = watches_entry->second; - std::vector newly_added_to_watch; std::set_difference(update_to_these_names.begin(), update_to_these_names.end(), - watch.resource_names_.begin(), watch.resource_names_.end(), + watch->resource_names_.begin(), watch->resource_names_.end(), std::inserter(newly_added_to_watch, newly_added_to_watch.begin())); std::vector newly_removed_from_watch; - std::set_difference(watch.resource_names_.begin(), watch.resource_names_.end(), + std::set_difference(watch->resource_names_.begin(), watch->resource_names_.end(), update_to_these_names.begin(), update_to_these_names.end(), std::inserter(newly_removed_from_watch, newly_removed_from_watch.begin())); - watch.resource_names_ = update_to_these_names; + watch->resource_names_ = update_to_these_names; - return std::make_pair(findAdditions(newly_added_to_watch, token), - findRemovals(newly_removed_from_watch, token)); + return AddedRemoved(findAdditions(newly_added_to_watch, watch), + findRemovals(newly_removed_from_watch, watch)); } -absl::flat_hash_set -WatchMap::tokensInterestedIn(const std::string& resource_name) { +absl::flat_hash_set WatchMap::watchesInterestedIn(const std::string& resource_name) { // Note that std::set_union needs sorted sets. Better to do it ourselves with insert(). - absl::flat_hash_set ret = wildcard_watches_; + absl::flat_hash_set ret = wildcard_watches_; auto watches_interested = watch_interest_.find(resource_name); if (watches_interested != watch_interest_.end()) { for (const auto& watch : watches_interested->second) { @@ -67,120 +57,99 @@ void WatchMap::onConfigUpdate(const Protobuf::RepeatedPtrField ENVOY_LOG(warn, "WatchMap::onConfigUpdate: there are no watches!"); return; } - SubscriptionCallbacks& name_getter = watches_.begin()->second.callbacks_; + SubscriptionCallbacks& name_getter = (*watches_.begin())->callbacks_; // Build a map from watches, to the set of updated resources that each watch cares about. Each // entry in the map is then a nice little bundle that can be fed directly into the individual // onConfigUpdate()s. - absl::flat_hash_map> - per_watch_updates; + absl::flat_hash_map> per_watch_updates; for (const auto& r : resources) { - const absl::flat_hash_set& interested_in_r = - tokensInterestedIn(name_getter.resourceName(r)); - for (const auto& interested_token : interested_in_r) { - per_watch_updates[interested_token].Add()->CopyFrom(r); + const absl::flat_hash_set& interested_in_r = + watchesInterestedIn(name_getter.resourceName(r)); + for (const auto& interested_watch : interested_in_r) { + per_watch_updates[interested_watch].Add()->CopyFrom(r); } } // We just bundled up the updates into nice per-watch packages. Now, deliver them. for (auto& watch : watches_) { - auto this_watch_updates = per_watch_updates.find(watch.first); + auto this_watch_updates = per_watch_updates.find(watch); if (this_watch_updates == per_watch_updates.end()) { // This update included no resources this watch cares about - so we do an empty // onConfigUpdate(), to notify the watch that its resources - if they existed before this - // were dropped. - watch.second.callbacks_.onConfigUpdate({}, version_info); + watch->callbacks_.onConfigUpdate({}, version_info); } else { - watch.second.callbacks_.onConfigUpdate(this_watch_updates->second, version_info); + watch->callbacks_.onConfigUpdate(this_watch_updates->second, version_info); } } } -void WatchMap::tryDeliverConfigUpdate( - WatchMap::Token token, - const Protobuf::RepeatedPtrField& added_resources, - const Protobuf::RepeatedPtrField& removed_resources, - const std::string& system_version_info) { - auto entry = watches_.find(token); - if (entry == watches_.end()) { - ENVOY_LOG(error, "A token referred to by watch_interest_ is not present in watches_!"); - return; - } - entry->second.callbacks_.onConfigUpdate(added_resources, removed_resources, system_version_info); -} - void WatchMap::onConfigUpdate( const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info) { - if (watches_.empty()) { - ENVOY_LOG(warn, "WatchMap::onConfigUpdate: there are no watches!"); - return; - } - // Build a pair of maps: from watches, to the set of resources {added,removed} that each watch // cares about. Each entry in the map-pair is then a nice little bundle that can be fed directly // into the individual onConfigUpdate()s. - absl::flat_hash_map> - per_watch_added; + absl::flat_hash_map> per_watch_added; for (const auto& r : added_resources) { - const absl::flat_hash_set& interested_in_r = tokensInterestedIn(r.name()); - for (const auto& interested_token : interested_in_r) { - per_watch_added[interested_token].Add()->CopyFrom(r); + const absl::flat_hash_set& interested_in_r = watchesInterestedIn(r.name()); + for (const auto& interested_watch : interested_in_r) { + per_watch_added[interested_watch].Add()->CopyFrom(r); } } - absl::flat_hash_map> per_watch_removed; + absl::flat_hash_map> per_watch_removed; for (const auto& r : removed_resources) { - const absl::flat_hash_set& interested_in_r = tokensInterestedIn(r); - for (const auto& interested_token : interested_in_r) { - *per_watch_removed[interested_token].Add() = r; + const absl::flat_hash_set& interested_in_r = watchesInterestedIn(r); + for (const auto& interested_watch : interested_in_r) { + *per_watch_removed[interested_watch].Add() = r; } } // We just bundled up the updates into nice per-watch packages. Now, deliver them. for (const auto& added : per_watch_added) { - const WatchMap::Token& cur_token = added.first; - auto removed = per_watch_removed.find(cur_token); + const Watch* cur_watch = added.first; + auto removed = per_watch_removed.find(cur_watch); if (removed == per_watch_removed.end()) { // additions only, no removals - tryDeliverConfigUpdate(cur_token, added.second, {}, system_version_info); + cur_watch->callbacks_.onConfigUpdate(added.second, {}, system_version_info); } else { // both additions and removals - tryDeliverConfigUpdate(cur_token, added.second, removed->second, system_version_info); + cur_watch->callbacks_.onConfigUpdate(added.second, removed->second, system_version_info); // Drop the removals now, so the final removals-only pass won't use them. per_watch_removed.erase(removed); } } // Any removals-only updates will not have been picked up in the per_watch_added loop. - for (const auto& removed : per_watch_removed) { - tryDeliverConfigUpdate(removed.first, {}, removed.second, system_version_info); + for (auto& removed : per_watch_removed) { + removed.first->callbacks_.onConfigUpdate({}, removed.second, system_version_info); } } void WatchMap::onConfigUpdateFailed(const EnvoyException* e) { for (auto& watch : watches_) { - watch.second.callbacks_.onConfigUpdateFailed(e); + watch->callbacks_.onConfigUpdateFailed(e); } } std::set WatchMap::findAdditions(const std::vector& newly_added_to_watch, - WatchMap::Token token) { + Watch* watch) { std::set newly_added_to_subscription; for (const auto& name : newly_added_to_watch) { auto entry = watch_interest_.find(name); if (entry == watch_interest_.end()) { newly_added_to_subscription.insert(name); - watch_interest_[name] = {token}; + watch_interest_[name] = {watch}; } else { - entry->second.insert(token); + entry->second.insert(watch); } } return newly_added_to_subscription; } std::set -WatchMap::findRemovals(const std::vector& newly_removed_from_watch, - WatchMap::Token token) { +WatchMap::findRemovals(const std::vector& newly_removed_from_watch, Watch* watch) { std::set newly_removed_from_subscription; for (const auto& name : newly_removed_from_watch) { auto entry = watch_interest_.find(name); @@ -189,7 +158,7 @@ WatchMap::findRemovals(const std::vector& newly_removed_from_watch, continue; } - entry->second.erase(token); + entry->second.erase(watch); if (entry->second.empty()) { watch_interest_.erase(entry); newly_removed_from_subscription.insert(name); diff --git a/source/common/config/watch_map.h b/source/common/config/watch_map.h index 04b2444cdbe56..2c0107a2b0e24 100644 --- a/source/common/config/watch_map.h +++ b/source/common/config/watch_map.h @@ -15,6 +15,16 @@ namespace Envoy { namespace Config { +struct Watch; +using WatchPtr = std::unique_ptr; + +struct AddedRemoved { + AddedRemoved(std::set added, std::set removed) + : added_(added), removed_(removed) {} + std::set added_; + std::set removed_; +}; + // Manages "watches" of xDS resources. Several xDS callers might ask for a subscription to the same // resource name "X". The xDS machinery must return to each their very own subscription to X. // The xDS machinery's "watch" concept accomplishes that, while avoiding parallel redundant xDS @@ -38,27 +48,18 @@ class WatchMap : public SubscriptionCallbacks, public Logger::Loggable::max(); - // Adds 'callbacks' to the WatchMap, with no resource names being watched. // (Use updateWatchInterest() to add some names). - // Returns a new token identifying the newly added watch. - Token addWatch(SubscriptionCallbacks& callbacks); - - // Returns true if this was the very last watch in the map. - // Expects that the watch to be removed has already had all of its resource names removed via - // updateWatchInterest(). - bool removeWatch(Token token); + // Returns the newly added watch, to be used for updateWatchInterest. Destroy to remove from map. + WatchPtr addWatch(SubscriptionCallbacks& callbacks); // Updates the set of resource names that the given watch should watch. // Returns any resource name additions/removals that are unique across all watches. That is: - // 1) if 'resources' contains X and no other watch cares about X, X will be in pair.first. + // 1) if 'resources' contains X and no other watch cares about X, X will be in added_. // 2) if 'resources' does not contain Y, and this watch was the only one that cared about Y, - // Y will be in pair.second. - std::pair, std::set> - updateWatchInterest(Token token, const std::set& update_to_these_names); + // Y will be in removed_. + AddedRemoved updateWatchInterest(Watch* watch, + const std::set& update_to_these_names); // SubscriptionCallbacks virtual void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, @@ -75,46 +76,46 @@ class WatchMap : public SubscriptionCallbacks, public Logger::Loggable resource_names_; // must be sorted set, for set_difference. - SubscriptionCallbacks& callbacks_; - }; + friend struct Watch; + // Expects that the watch to be removed has already had all of its resource names removed via + // updateWatchInterest(). + void removeWatch(Watch* watch); // Given a list of names that are new to an individual watch, returns those names that are in fact // new to the entire subscription. std::set findAdditions(const std::vector& newly_added_to_watch, - Token token); + Watch* watch); // Given a list of names that an individual watch no longer cares about, returns those names that // in fact the entire subscription no longer cares about. std::set findRemovals(const std::vector& newly_removed_from_watch, - Token token); - - // Calls watches_[token].callbacks_.onConfigUpdate(), or logs an error if token isn't in watches_. - void tryDeliverConfigUpdate( - Token token, const Protobuf::RepeatedPtrField& added_resources, - const Protobuf::RepeatedPtrField& removed_resources, - const std::string& system_version_info); + Watch* watch); // Returns the union of watch_interest_[resource_name] and wildcard_watches_. - absl::flat_hash_set tokensInterestedIn(const std::string& resource_name); + absl::flat_hash_set watchesInterestedIn(const std::string& resource_name); - absl::flat_hash_map watches_; + absl::flat_hash_set watches_; // Watches whose interest set is currently empty, which is interpreted as "everything". - absl::flat_hash_set wildcard_watches_; + absl::flat_hash_set wildcard_watches_; // Maps a resource name to the set of watches interested in that resource. Has two purposes: // 1) Acts as a reference count; no watches care anymore ==> the resource can be removed. // 2) Enables efficient lookup of all interested watches when a resource has been updated. - absl::flat_hash_map> watch_interest_; - - Token next_watch_{0}; + absl::flat_hash_map> watch_interest_; WatchMap(const WatchMap&) = delete; WatchMap& operator=(const WatchMap&) = delete; }; +struct Watch { + Watch(WatchMap& owning_map, SubscriptionCallbacks& callbacks) + : owning_map_(owning_map), callbacks_(callbacks) {} + ~Watch() { owning_map_.removeWatch(this); } + WatchMap& owning_map_; + SubscriptionCallbacks& callbacks_; + std::set resource_names_; // must be sorted set, for set_difference. +}; + } // namespace Config } // namespace Envoy From 6f5b0526aefea10fb39cd46c56bb2c1950b4d6d3 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Sat, 15 Jun 2019 15:04:10 -0400 Subject: [PATCH 28/56] yay delta ADS works and all tests pass Signed-off-by: Fred Douglas --- include/envoy/config/BUILD | 10 + include/envoy/config/grpc_mux.h | 10 +- include/envoy/config/subscription.h | 2 +- include/envoy/config/subscription_state.h | 39 ++-- source/common/config/BUILD | 12 ++ source/common/config/README.md | 40 ++-- .../common/config/delta_subscription_impl.cc | 45 ++-- .../common/config/delta_subscription_impl.h | 42 ++-- .../common/config/delta_subscription_state.cc | 10 +- .../common/config/delta_subscription_state.h | 11 +- .../config/filesystem_subscription_impl.cc | 2 +- .../config/filesystem_subscription_impl.h | 4 +- source/common/config/grpc_delta_xds_context.h | 204 ++++++++---------- source/common/config/grpc_mux_impl.h | 14 +- .../config/grpc_mux_subscription_impl.cc | 3 +- .../config/grpc_mux_subscription_impl.h | 2 +- source/common/config/grpc_subscription_impl.h | 4 +- .../common/config/http_subscription_impl.cc | 3 +- source/common/config/http_subscription_impl.h | 2 +- source/common/config/pausable_ack_queue.cc | 53 +++++ source/common/config/pausable_ack_queue.h | 35 +++ source/common/config/subscription_factory.cc | 10 +- .../config/xDS_code_diagram_june2019.png | Bin 0 -> 92572 bytes source/common/router/vhds.cc | 4 +- source/common/router/vhds.h | 4 +- .../common/upstream/cluster_manager_impl.cc | 61 +++--- source/common/upstream/cluster_manager_impl.h | 5 +- .../config_validation/cluster_manager.cc | 4 +- .../config_validation/cluster_manager.h | 2 +- source/server/config_validation/server.h | 5 +- source/server/server.cc | 8 +- .../config/delta_subscription_impl_test.cc | 24 +-- .../config/delta_subscription_test_harness.h | 7 +- .../filesystem_subscription_test_harness.h | 4 +- .../config/grpc_subscription_impl_test.cc | 6 +- .../config/grpc_subscription_test_harness.h | 4 +- .../config/http_subscription_test_harness.h | 4 +- test/common/config/subscription_impl_test.cc | 6 +- .../common/config/subscription_test_harness.h | 2 +- test/integration/ads_integration_test.cc | 56 ++--- test/integration/integration.cc | 94 ++++---- test/integration/integration.h | 6 +- test/mocks/config/mocks.h | 14 +- test/server/lds_api_test.cc | 6 +- test/server/listener_manager_impl_test.cc | 4 +- 45 files changed, 510 insertions(+), 377 deletions(-) create mode 100644 source/common/config/pausable_ack_queue.cc create mode 100644 source/common/config/pausable_ack_queue.h create mode 100644 source/common/config/xDS_code_diagram_june2019.png diff --git a/include/envoy/config/BUILD b/include/envoy/config/BUILD index 5d08f816626b8..ba9d438af4d51 100644 --- a/include/envoy/config/BUILD +++ b/include/envoy/config/BUILD @@ -50,6 +50,16 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "subscription_state_interface", + hdrs = ["subscription_state.h"], + deps = [ + ":subscription_interface", + "//source/common/protobuf", + "@envoy_api//envoy/api/v2:discovery_cc", + ], +) + envoy_cc_library( name = "typed_metadata_interface", hdrs = ["typed_metadata.h"], diff --git a/include/envoy/config/grpc_mux.h b/include/envoy/config/grpc_mux.h index 9f404a19547b5..e79852b2e820d 100644 --- a/include/envoy/config/grpc_mux.h +++ b/include/envoy/config/grpc_mux.h @@ -111,12 +111,10 @@ class GrpcMux { virtual void resume(const std::string& type_url) PURE; // For delta - virtual WatchPtr addWatch(const std::string& type_url, const std::set& resources, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) PURE; - // TODO TODO TRIM? virtual void removeWatch(const std::string& type_url, Watch* watch) PURE; - virtual void updateWatch(const std::string& type_url, Watch* watch, - const std::set& resources) PURE; + virtual void addOrUpdateWatch(const std::string& type_url, WatchPtr& watch, + const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) PURE; }; typedef std::unique_ptr GrpcMuxPtr; diff --git a/include/envoy/config/subscription.h b/include/envoy/config/subscription.h index 707fab98ea3bd..ae94940196017 100644 --- a/include/envoy/config/subscription.h +++ b/include/envoy/config/subscription.h @@ -76,7 +76,7 @@ class Subscription { * @param resources vector of resource names to fetch. It's a (not unordered_)set so that it can * be passed to std::set_difference, which must be given sorted collections. */ - virtual void updateResources(const std::set& update_to_these_names) PURE; + virtual void updateResourceInterest(const std::set& update_to_these_names) PURE; }; /** diff --git a/include/envoy/config/subscription_state.h b/include/envoy/config/subscription_state.h index 78352248f7903..b844899772fd4 100644 --- a/include/envoy/config/subscription_state.h +++ b/include/envoy/config/subscription_state.h @@ -3,10 +3,14 @@ #include #include +#include "envoy/api/v2/discovery.pb.h" #include "envoy/common/pure.h" +#include "envoy/config/subscription.h" #include "common/protobuf/protobuf.h" +#include "absl/strings/string_view.h" + namespace Envoy { namespace Config { @@ -20,42 +24,41 @@ struct UpdateAck { class SubscriptionState { public: - virtual ~SubscriptionState() {} - - void setInitFetchTimeout(Event::Dispatcher& dispatcher) PURE; + virtual ~SubscriptionState() = default; - void pause() PURE; - void resume() PURE; - bool paused() const PURE; + virtual void pause() PURE; + virtual void resume() PURE; + virtual bool paused() const PURE; // Update which resources we're interested in subscribing to. - void updateResourceInterest(const std::set& update_to_these_names) PURE; + virtual void updateResourceInterest(const std::set& update_to_these_names) PURE; // Whether there was a change in our subscription interest we have yet to inform the server of. - bool subscriptionUpdatePending() const PURE; + virtual bool subscriptionUpdatePending() const PURE; - void markStreamFresh() PURE; + virtual void markStreamFresh() PURE; // Argument should have been static_cast from GrpcStream's ResponseProto type. - UpdateAck handleResponse(const Protobuf::Message& message) PURE; + virtual UpdateAck handleResponse(const Protobuf::Message& message) PURE; - void handleEstablishmentFailure() PURE; + virtual void handleEstablishmentFailure() PURE; // Will have been static_cast from GrpcStream's RequestProto type, and should be static_cast back // before handing to GrpcStream::sendMessage. - Protobuf::Message getNextRequestAckless() PURE; + virtual Protobuf::Message getNextRequestAckless() PURE; // Will have been static_cast from GrpcStream's RequestProto type, and should be static_cast back // before handing to GrpcStream::sendMessage. - Protobuf::Message getNextRequestWithAck(const UpdateAck& ack) PURE; + virtual Protobuf::Message getNextRequestWithAck(const UpdateAck& ack) PURE; }; class SubscriptionStateFactory { public: - SubscriptionState makeSubscriptionState(const std::string& type_url, - const std::set& resource_names, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout, - SubscriptionStats& stats) PURE; + virtual ~SubscriptionStateFactory() = default; + virtual SubscriptionState makeSubscriptionState(const std::string& type_url, + const std::set& resource_names, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout, + SubscriptionStats& stats) PURE; }; } // namespace Config diff --git a/source/common/config/BUILD b/source/common/config/BUILD index 61911c7696c1e..8ed9a4ba519c9 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -103,6 +103,7 @@ envoy_cc_library( hdrs = ["delta_subscription_state.h"], deps = [ "//include/envoy/config:subscription_interface", + "//include/envoy/config:subscription_state_interface", "//include/envoy/event:dispatcher_interface", "//source/common/common:assert_lib", "//source/common/common:backoff_lib", @@ -217,6 +218,7 @@ envoy_cc_library( hdrs = ["grpc_delta_xds_context.h"], deps = [ ":delta_subscription_state_lib", + ":pausable_ack_queue_lib", ":watch_map_lib", "//include/envoy/event:dispatcher_interface", "//include/envoy/grpc:async_client_interface", @@ -267,6 +269,16 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "pausable_ack_queue_lib", + srcs = ["pausable_ack_queue.cc"], + hdrs = ["pausable_ack_queue.h"], + deps = [ + "//include/envoy/config:subscription_state_interface", + "//source/common/common:assert_lib", + ], +) + envoy_cc_library( name = "protobuf_link_hacks", hdrs = ["protobuf_link_hacks.h"], diff --git a/source/common/config/README.md b/source/common/config/README.md index 57ab37026d806..7c0b860aaea5f 100644 --- a/source/common/config/README.md +++ b/source/common/config/README.md @@ -1,9 +1,20 @@ -tldr: xDS can be filesystem, REST, or gRPC, and gRPC xDS comes in four flavors, -but Envoy code uses all of that via the same Subscription interface. If you are -an Envoy developer with your hands on a valid Subscription object, you can -forget the filesystem/REST/gRPC distinction, and you can especially forget about -the gRPC flavors. All of that is specified in the bootstrap config, which is -read and put into action by ClusterManagerImpl. +xDS + +xDS stands for [fill in the blank] Discovery Service. It provides dynamic config discovery/updates. + +tldr: xDS can be filesystem, REST, or gRPC. gRPC xDS comes in four flavors. +However, Envoy code uses all of that via the same Subscription interface. +If you are an Envoy developer with your hands on a valid Subscription object, +you can mostly forget the filesystem/REST/gRPC distinction, and you can +especially forget about the gRPC flavors. All of that is specified in the +bootstrap config, which is read and put into action by ClusterManagerImpl. + +Note that there can be multiple active gRPC subscriptions for a single resource +type. This concept is called "resource watches". If one EDS subscription +subscribes to X and Y, and another subscribes to Y and Z, the underlying +subscription logic will maintain a subscription to the union: X Y and Z. Updates +to X will be delivered to the first object, Y to both, Z to the second. This +logic is implemented by WatchMap. If you are working on Envoy's gRPC xDS client logic itself, read on. @@ -24,12 +35,13 @@ its method string to {Delta,Stream}AggregatedResources, as opposed to {Delta,Str and having identical client code, they're actually different gRPC services. Delta vs state-of-the-world is a question of wire format: the protos in question are named -[Delta]Discovery{Request,Response}. That is what the GrpcMux (TODO rename) interface is useful for: its -GrpcDeltaXdsContext implementation works with DeltaDiscovery{Request,Response} and has -delta-specific logic, and its GrpxMuxImpl implementation (TODO will be merged into GrpcDeltaXdsContext) works with Discovery{Request,Response} -and has SotW-specific logic. A GrpcSubscriptionImpl has its shared_ptr. -Both GrpcDeltaXdsContext (delta) or GrpcMuxImpl (SotW) will work just fine. The shared_ptr allows -for both non- and aggregated: if non-aggregated, you'll be the only holder of that shared_ptr. By -those two mechanisms, the single class (TODO rename) DeltaSubscriptionImpl handles all 4 -delta/SotW and non-/aggregated combinations. +[Delta]Discovery{Request,Response}. That is what the GrpcMux interface is useful for: its +GrpcDeltaXdsContext (TODO may be renamed) implementation works with DeltaDiscovery{Request,Response} and has +delta-specific logic; its GrpxMuxImpl implementation (TODO will be merged into GrpcDeltaXdsContext) works with Discovery{Request,Response} +and has SotW-specific logic. Both the delta and SotW Subscription implementations (TODO will be merged) hold a shared_ptr. +The shared_ptr allows for both non- and aggregated: if non-aggregated, you'll be the only holder of that shared_ptr. + +![xDS_code_diagram_june2019](xDS_code_diagram_june2019.png) +Note that the orange flow does not necessarily have to happen in response to the blue flow; there can be spontaneous updates. ACKs are not shown in this diagram; they are also carred by the [Delta]DiscoveryRequest protos. +What does GrpcXdsContext even do in this diagram? Just own things and pass through function calls? Answer: it sequences the requests and ACKs that the various type_urls send. diff --git a/source/common/config/delta_subscription_impl.cc b/source/common/config/delta_subscription_impl.cc index 9656d4a0aa251..9222f655dc918 100644 --- a/source/common/config/delta_subscription_impl.cc +++ b/source/common/config/delta_subscription_impl.cc @@ -3,23 +3,23 @@ namespace Envoy { namespace Config { -DeltaSubscriptionImpl::DeltaSubscriptionImpl(std::shared_ptr context, - absl::string_view type_url, - SubscriptionCallbacks& callbacks, - SubscriptionStats stats, - std::chrono::milliseconds init_fetch_timeout) +DeltaSubscriptionImpl::DeltaSubscriptionImpl( + std::shared_ptr context, absl::string_view type_url, SubscriptionCallbacks& callbacks, + SubscriptionStats stats, std::chrono::milliseconds init_fetch_timeout, bool is_aggregated) : context_(context), type_url_(type_url), callbacks_(callbacks), stats_(stats), - init_fetch_timeout_(init_fetch_timeout) {} + init_fetch_timeout_(init_fetch_timeout), is_aggregated_(is_aggregated) {} DeltaSubscriptionImpl::~DeltaSubscriptionImpl() { - // The Watch destruction process assumes all resource interest has already been removed. Doing - // that cleanly needs the context's involvement. So, do that now, just before destroying the - // watch. - context_->updateWatch(type_url_, watch_.get(), {}); - // A Watch must not outlive its owning WatchMap. Since DeltaSubscriptionImpl holds a shared_ptr to - // the context, which owns the WatchMap, the watch must be destroyed first in case this - // subscription holds the last shared_ptr. - watch_.reset(); + if (watch_) { + // The Watch destruction process assumes all resource interest has already been removed. Doing + // that cleanly needs the context's involvement. So, do that now, just before destroying the + // watch. + context_->addOrUpdateWatch(type_url_, watch_, {}, *this, init_fetch_timeout_); + // A Watch must not outlive its owning WatchMap. Since DeltaSubscriptionImpl holds a shared_ptr + // to the context, which owns the WatchMap, the watch must be destroyed first in case this + // subscription holds the last shared_ptr. + watch_.reset(); + } } void DeltaSubscriptionImpl::pause() { context_->pause(type_url_); } @@ -28,17 +28,18 @@ void DeltaSubscriptionImpl::resume() { context_->resume(type_url_); } // Config::DeltaSubscription void DeltaSubscriptionImpl::start(const std::set& resources) { - // TODO TODO needs to be idempotent. GrpcSubscription is special-purposed to non-ADS, and so knows - // that it needs to call start. My DeltaSubscriptionImpl can be either ADS or not, and so needs to - // (safely) call start regardless. - context_->start(); - - watch_ = context_->addWatch(type_url_, resources, *this, init_fetch_timeout_); + // ADS initial request batching relies on the users of the GrpcMux *not* calling start on it, + // whereas non-ADS xDS users must call it themselves. + if (!is_aggregated_) { + context_->start(); + } + context_->addOrUpdateWatch(type_url_, watch_, resources, *this, init_fetch_timeout_); stats_.update_attempt_.inc(); } -void DeltaSubscriptionImpl::updateResources(const std::set& update_to_these_names) { - context_->updateWatch(type_url_, watch_.get(), update_to_these_names); +void DeltaSubscriptionImpl::updateResourceInterest( + const std::set& update_to_these_names) { + context_->addOrUpdateWatch(type_url_, watch_, update_to_these_names, *this, init_fetch_timeout_); stats_.update_attempt_.inc(); } diff --git a/source/common/config/delta_subscription_impl.h b/source/common/config/delta_subscription_impl.h index fadb93f986b50..a2d16c04b2eef 100644 --- a/source/common/config/delta_subscription_impl.h +++ b/source/common/config/delta_subscription_impl.h @@ -8,24 +8,33 @@ namespace Envoy { namespace Config { -/** - * DeltaSubscription manages the logic of a delta xDS subscription for a particular resource - * type. It uses a GrpcDeltaXdsContext to handle the actual gRPC communication with the xDS server. - * DeltaSubscription and GrpcDeltaXdsContext are both used for both ADS and non-aggregated xDS; - * the only difference is that ADS has multiple DeltaSubscriptions sharing a single - * GrpcDeltaXdsContext. - */ -// TODO(fredlas) someday this class will be named GrpcSubscriptionImpl -// TODO TODO better comment: DeltaSubscriptionImpl implements the SubscriptionCallbacks interface so -// that it can write to SubscriptionStats upon a config update. The idea is, DeltaSubscriptionImpl -// presents itself as the SubscriptionCallbacks, but actually the *real* SubscriptionCallbacks is -// held by DeltaSubscriptionImpl. DeltaSubscriptionImpl passes through all SubscriptionCallbacks -// calls to the held SubscriptionCallbacks. +// DeltaSubscriptionImpl provides a top-level interface to the Envoy's communication with an xDS +// server, for use by the various xDS users within Envoy. It is built around a (shared) +// GrpcDeltaXdsContext, and the further machinery underlying that. An xDS user indicates interest in +// various resources via start() and updateResourceInterest(). It receives updates to those +// resources via the SubscriptionCallbacks it provides. Multiple users can each have their own +// Subscription object for the same type_url; GrpcDeltaXdsContext maintains a subscription to the +// union of interested resources, and delivers to the users just the resource updates that they are +// "watching" for. +// +// DeltaSubscriptionImpl and GrpcDeltaXdsContext are both built to provide both regular xDS and ADS, +// distinguished by whether multiple DeltaSubscriptionImpls are sharing a single +// GrpcDeltaXdsContext. (And by the gRPC method string, but that's taken care of over in +// SubscriptionFactory). +// +// Why does DeltaSubscriptionImpl itself implement the SubscriptionCallbacks interface? So that it +// can write to SubscriptionStats (which needs to live out here in the DeltaSubscriptionImpl) upon a +// config update. The idea is, DeltaSubscriptionImpl presents itself to WatchMap as the +// SubscriptionCallbacks, and then passes (after incrementing stats) all callbacks through to +// callbacks_, which are the real SubscriptionCallbacks. class DeltaSubscriptionImpl : public Subscription, public SubscriptionCallbacks { public: + // is_aggregated: whether the underlying mux/context is providing ADS to us and others, or whether + // it's all ours. The practical difference is that we ourselves must call start() on it only in + // the latter case. DeltaSubscriptionImpl(std::shared_ptr context, absl::string_view type_url, SubscriptionCallbacks& callbacks, SubscriptionStats stats, - std::chrono::milliseconds init_fetch_timeout); + std::chrono::milliseconds init_fetch_timeout, bool is_aggregated); ~DeltaSubscriptionImpl(); void pause(); @@ -34,7 +43,7 @@ class DeltaSubscriptionImpl : public Subscription, public SubscriptionCallbacks // Config::Subscription void start(const std::set& resource_names) override; - void updateResources(const std::set& update_to_these_names) override; + void updateResourceInterest(const std::set& update_to_these_names) override; // Config::SubscriptionCallbacks (all pass through to callbacks_!) void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, @@ -55,7 +64,8 @@ class DeltaSubscriptionImpl : public Subscription, public SubscriptionCallbacks // NOTE: if another subscription of the same type_url has already been started, this value will be // ignored in favor of the other subscription's. std::chrono::milliseconds init_fetch_timeout_; - WatchPtr watch_; + WatchPtr watch_{}; + const bool is_aggregated_; }; } // namespace Config diff --git a/source/common/config/delta_subscription_state.cc b/source/common/config/delta_subscription_state.cc index f5629394cb082..f10cc5b30c317 100644 --- a/source/common/config/delta_subscription_state.cc +++ b/source/common/config/delta_subscription_state.cc @@ -13,10 +13,6 @@ DeltaSubscriptionState::DeltaSubscriptionState(const std::string& type_url, Event::Dispatcher& dispatcher) : type_url_(type_url), callbacks_(callbacks), local_info_(local_info), init_fetch_timeout_(init_fetch_timeout) { - setInitFetchTimeout(dispatcher); -} - -void DeltaSubscriptionState::setInitFetchTimeout(Event::Dispatcher& dispatcher) { if (init_fetch_timeout_.count() > 0 && !init_fetch_timeout_timer_) { init_fetch_timeout_timer_ = dispatcher.createTimer([this]() -> void { ENVOY_LOG(warn, "delta config: initial fetch timed out for {}", type_url_); @@ -77,6 +73,12 @@ void DeltaSubscriptionState::handleGoodResponse( throw EnvoyException( fmt::format("duplicate name {} found among added/updated resources", resource.name())); } + if (message.type_url() != resource.resource().type_url()) { + throw EnvoyException(fmt::format("type URL {} embedded in an individual Any does not match " + "the message-wide type URL {} in DeltaDiscoveryResponse {}", + resource.resource().type_url(), message.type_url(), + message.DebugString())); + } } for (const auto& name : message.removed_resources()) { if (!names_added_removed.insert(name).second) { diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index f251150f75d63..5659ce1649ff0 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -2,6 +2,7 @@ #include "envoy/api/v2/discovery.pb.h" #include "envoy/config/subscription.h" +#include "envoy/config/subscription_state.h" #include "envoy/event/dispatcher.h" #include "envoy/grpc/status.h" #include "envoy/local_info/local_info.h" @@ -12,14 +13,6 @@ namespace Envoy { namespace Config { -struct UpdateAck { - UpdateAck(absl::string_view nonce, absl::string_view type_url) - : nonce_(nonce), type_url_(type_url) {} - std::string nonce_; - std::string type_url_; - ::google::rpc::Status error_detail_; -}; - // Tracks the xDS protocol state of an individual ongoing delta xDS session, i.e. a single type_url. // There can be multiple DeltaSubscriptionStates active. They will always all be // blissfully unaware of each other's existence, even when their messages are @@ -31,8 +24,6 @@ class DeltaSubscriptionState : public Logger::Loggable { std::chrono::milliseconds init_fetch_timeout, Event::Dispatcher& dispatcher); - void setInitFetchTimeout(Event::Dispatcher& dispatcher); - // Update which resources we're interested in subscribing to. void updateSubscriptionInterest(const std::set& cur_added, const std::set& cur_removed); diff --git a/source/common/config/filesystem_subscription_impl.cc b/source/common/config/filesystem_subscription_impl.cc index c9bcaedd3d7d2..3ac76ed0b032e 100644 --- a/source/common/config/filesystem_subscription_impl.cc +++ b/source/common/config/filesystem_subscription_impl.cc @@ -27,7 +27,7 @@ void FilesystemSubscriptionImpl::start(const std::set&) { refresh(); } -void FilesystemSubscriptionImpl::updateResources(const std::set&) { +void FilesystemSubscriptionImpl::updateResourceInterest(const std::set&) { // Bump stats for consistence behavior with other xDS. stats_.update_attempt_.inc(); } diff --git a/source/common/config/filesystem_subscription_impl.h b/source/common/config/filesystem_subscription_impl.h index 6ff089208dc84..851935f1638e8 100644 --- a/source/common/config/filesystem_subscription_impl.h +++ b/source/common/config/filesystem_subscription_impl.h @@ -26,9 +26,9 @@ class FilesystemSubscriptionImpl : public Config::Subscription, // Config::Subscription // We report all discovered resources in the watched file, so the resource names arguments are - // unused, and updateResources is a no-op (other than updating a stat). + // unused, and updateResourceInterest is a no-op (other than updating a stat). void start(const std::set&) override; - void updateResources(const std::set&) override; + void updateResourceInterest(const std::set&) override; private: void refresh(); diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h index 085b648be5bf7..0437dd7939d50 100644 --- a/source/common/config/grpc_delta_xds_context.h +++ b/source/common/config/grpc_delta_xds_context.h @@ -12,6 +12,7 @@ #include "common/common/token_bucket_impl.h" #include "common/config/delta_subscription_state.h" #include "common/config/grpc_stream.h" +#include "common/config/pausable_ack_queue.h" #include "common/config/utility.h" #include "common/config/watch_map.h" #include "common/grpc/common.h" @@ -21,19 +22,6 @@ namespace Envoy { namespace Config { -// TODO TODO not actually needed; just an adapter to the old-style subscribe(). -/*class TokenizedGrpcMuxWatch : public GrpcMuxWatch { -public: - TokenizedGrpcMuxWatch(GrpcMux& context, const std::string& type_url, WatchMap::Token token) - : context_(context), type_url_(type_url), token_(token) {} - ~TokenizedGrpcMuxWatch() { context_.removeWatch(type_url_, token_); } - -private: - GrpcMux& context_; - std::string type_url_; - WatchMap::Token token_; -};*/ - // Manages subscriptions to one or more type of resource. The logical protocol // state of those subscription(s) is handled by DeltaSubscriptionState. // This class owns the GrpcStream used to talk to the server, maintains queuing @@ -50,72 +38,22 @@ class GrpcDeltaXdsContext : public GrpcMux, const LocalInfo::LocalInfo& local_info) : dispatcher_(dispatcher), local_info_(local_info), grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, - rate_limit_settings) { - std::cerr << "making a GrpcDeltaXdsContext" << std::endl; - } - - WatchPtr addWatch(const std::string& type_url, const std::set& resources, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) override { - auto entry = subscriptions_.find(type_url); - if (entry == subscriptions_.end()) { - // We don't yet have a subscription for type_url! Make one! - addSubscription(type_url, init_fetch_timeout); - return addWatch(type_url, resources, callbacks, init_fetch_timeout); + rate_limit_settings) {} + + void addOrUpdateWatch(const std::string& type_url, WatchPtr& watch, + const std::set& resources, SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) override { + if (watch.get() == nullptr) { + watch = addWatch(type_url, resources, callbacks, init_fetch_timeout); + } else { + updateWatch(type_url, watch.get(), resources); } - - WatchPtr watch = entry->second->watch_map_.addWatch(callbacks); - // updateWatch() queues a discovery request if any of 'resources' are not yet subscribed. - updateWatch(type_url, watch.get(), resources); - return watch; } - /* TODO TODO TRIM? void removeWatch(const std::string& type_url, Watch* watch) override { - if (watch == nullptr) { - return; - } - // updateWatch() queues a discovery request if any resources were watched only by this watch. - updateWatch(type_url, watch, {}); - - auto entry = subscriptions_.find(type_url); - if (entry == subscriptions_.end()) { - ENVOY_LOG(error, "Asked to remove watch of {}, but there is no subscription...", type_url); - return; - } - if (entry->second->watch_map_.removeWatch(watch)) { - // removeWatch() told us that watch was the last watch on the entire subscription. - removeSubscription(type_url); - } - }*/ - - // Updates the list of resource names watched by the given watch. If an added name is new across - // the whole subscription, or if a removed name has no other watch interested in it, then the - // subscription will enqueue and attempt to send an appropriate discovery request. - void updateWatch(const std::string& type_url, Watch* watch, - const std::set& resources) override { - auto sub = subscriptions_.find(type_url); - if (sub == subscriptions_.end()) { - ENVOY_LOG(error, "Watch of {} has no subscription to update.", type_url); - return; - } - auto added_removed = sub->second->watch_map_.updateWatchInterest(watch, resources); - sub->second->sub_state_.updateSubscriptionInterest(added_removed.added_, - added_removed.removed_); - // Tell the server about our new interests, if there are any. - trySendDiscoveryRequests(); - } - - void pause(const std::string& type_url) override { - // It's ok to pause a subscription that doesn't exist yet. - auto& pause_entry = paused_[type_url]; - ASSERT(!pause_entry); - pause_entry = true; - } + void pause(const std::string& type_url) override { pausable_ack_queue_.pause(type_url); } void resume(const std::string& type_url) override { - auto& pause_entry = paused_[type_url]; - ASSERT(pause_entry); - pause_entry = false; + pausable_ack_queue_.resume(type_url); trySendDiscoveryRequests(); } @@ -142,15 +80,29 @@ class GrpcDeltaXdsContext : public GrpcMux, } void onEstablishmentFailure() override { - for (auto& sub : subscriptions_) { - sub.second->sub_state_.handleEstablishmentFailure(); - } + // If this happens while Envoy is still initializing, the onConfigUpdateFailed() we ultimately + // call on CDS will cause LDS to start up, which adds to subscriptions_ here. So, to avoid a + // crash, the iteration needs to dance around a little: collect pointers to all + // SubscriptionStates, call on all those pointers we haven't yet called on, repeat if there are + // now more SubscriptionStates. + absl::flat_hash_map all_subscribed; + absl::flat_hash_map already_called; + do { + for (auto& sub : subscriptions_) { + all_subscribed[sub.first] = &sub.second->sub_state_; + } + for (auto& sub : all_subscribed) { + if (already_called.insert(sub).second) { // insert succeeded --> not already called + sub.second->handleEstablishmentFailure(); + } + } + } while (all_subscribed.size() != subscriptions_.size()); } void onWriteable() override { trySendDiscoveryRequests(); } void kickOffAck(UpdateAck ack) { - ack_queue_.push(ack); + pausable_ack_queue_.push(ack); trySendDiscoveryRequests(); } @@ -162,33 +114,49 @@ class GrpcDeltaXdsContext : public GrpcMux, // callbacks)); NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } - // TODO TODO needs to be idempotent. GrpcSubscription is special-purposed to non-ADS, and so knows - // that it needs to call start. My DeltaSubscriptionImpl can be either ADS or not, and so needs to - // (safely) call start regardless. void start() override { grpc_stream_.establishNewStream(); } private: + WatchPtr addWatch(const std::string& type_url, const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { + auto entry = subscriptions_.find(type_url); + if (entry == subscriptions_.end()) { + // We don't yet have a subscription for type_url! Make one! + addSubscription(type_url, init_fetch_timeout); + return addWatch(type_url, resources, callbacks, init_fetch_timeout); + } + + WatchPtr watch = entry->second->watch_map_.addWatch(callbacks); + // updateWatch() queues a discovery request if any of 'resources' are not yet subscribed. + updateWatch(type_url, watch.get(), resources); + return watch; + } + + // Updates the list of resource names watched by the given watch. If an added name is new across + // the whole subscription, or if a removed name has no other watch interested in it, then the + // subscription will enqueue and attempt to send an appropriate discovery request. + void updateWatch(const std::string& type_url, Watch* watch, + const std::set& resources) { + ASSERT(watch != nullptr); + auto sub = subscriptions_.find(type_url); + if (sub == subscriptions_.end()) { + ENVOY_LOG(error, "Watch of {} has no subscription to update.", type_url); + return; + } + auto added_removed = sub->second->watch_map_.updateWatchInterest(watch, resources); + sub->second->sub_state_.updateSubscriptionInterest(added_removed.added_, + added_removed.removed_); + // Tell the server about our new interests, if there are any. + trySendDiscoveryRequests(); + } + void addSubscription(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout) { subscriptions_.emplace(type_url, std::make_unique( type_url, init_fetch_timeout, dispatcher_, local_info_)); subscription_ordering_.emplace_back(type_url); } - // TODO do we need this? or should we just let subscriptions hang out even when nobody is - // interested in them? - void removeSubscription(const std::string& type_url) { - subscriptions_.erase(type_url); - // And remove from the subscription_ordering_ list. - auto it = subscription_ordering_.begin(); - while (it != subscription_ordering_.end()) { - if (*it == type_url) { - it = subscription_ordering_.erase(it); - } else { - ++it; - } - } - } - void trySendDiscoveryRequests() { while (true) { // Do any of our subscriptions even want to send a request? @@ -201,12 +169,12 @@ class GrpcDeltaXdsContext : public GrpcMux, // If we don't have a subscription object for this request's type_url, drop the request. auto sub = subscriptions_.find(next_request_type_url); if (sub == subscriptions_.end()) { - ENVOY_LOG(warn, "Not sending discovery request for non-existent subscription {}.", + ENVOY_LOG(error, "Not sending discovery request for non-existent subscription {}.", next_request_type_url); // It's safe to assume the front of the ACK queue is of this type, because that's the only // way whoWantsToSendDiscoveryRequest() could return something for a non-existent // subscription. - ack_queue_.pop(); + pausable_ack_queue_.pop(); continue; } // Try again later if paused/rate limited/stream down. @@ -214,23 +182,29 @@ class GrpcDeltaXdsContext : public GrpcMux, break; } // Get our subscription state to generate the appropriate DeltaDiscoveryRequest, and send. - if (!ack_queue_.empty()) { + if (!pausable_ack_queue_.empty()) { // Because ACKs take precedence over plain requests, if there is anything in the queue, it's - // safe to assume it's what we want to send here. - grpc_stream_.sendMessage(sub->second->sub_state_.getNextRequestWithAck(ack_queue_.front())); - ack_queue_.pop(); + // safe to assume it's of the type_url that we're wanting to send. + grpc_stream_.sendMessage( + sub->second->sub_state_.getNextRequestWithAck(pausable_ack_queue_.front())); + pausable_ack_queue_.pop(); } else { grpc_stream_.sendMessage(sub->second->sub_state_.getNextRequestAckless()); } } - grpc_stream_.maybeUpdateQueueSizeStat(ack_queue_.size()); + grpc_stream_.maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); } // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check // whether we *want* to send a DeltaDiscoveryRequest). - bool canSendDiscoveryRequest(absl::string_view type_url) { - if (paused_[type_url]) { - ENVOY_LOG(trace, "API {} paused; discovery request on hold for now.", type_url); + bool canSendDiscoveryRequest(const std::string& type_url) { + if (pausable_ack_queue_.paused(type_url)) { + ASSERT(false, + fmt::format("canSendDiscoveryRequest() called on paused type_url {}. Pausedness is " + "supposed to be filtered out by whoWantsToSendDiscoveryRequest(). " + "Returning false, but your xDS might be about to get head-of-line blocked " + "- permanently, if the pause is never undone.", + type_url)); return false; } else if (!grpc_stream_.grpcStreamAvailable()) { ENVOY_LOG(trace, "No stream available to send a discovery request for {}.", type_url); @@ -250,9 +224,9 @@ class GrpcDeltaXdsContext : public GrpcMux, // of subscriptions were activated. absl::optional whoWantsToSendDiscoveryRequest() { // All ACKs are sent before plain updates. trySendDiscoveryRequests() relies on this. So, choose - // type_url from ack_queue_ if possible, before looking at pending updates. - if (!ack_queue_.empty()) { - return ack_queue_.front().type_url_; + // type_url from pausable_ack_queue_ if possible, before looking at pending updates. + if (!pausable_ack_queue_.empty()) { + return pausable_ack_queue_.front().type_url_; } // If we're looking to send multiple non-ACK requests, send them in the order that their // subscriptions were initiated. @@ -261,7 +235,8 @@ class GrpcDeltaXdsContext : public GrpcMux, if (sub == subscriptions_.end()) { continue; } - if (sub->second->sub_state_.subscriptionUpdatePending()) { + if (sub->second->sub_state_.subscriptionUpdatePending() && + !pausable_ack_queue_.paused(sub_type)) { return sub->first; } } @@ -272,8 +247,9 @@ class GrpcDeltaXdsContext : public GrpcMux, const LocalInfo::LocalInfo& local_info_; // Resource (N)ACKs we're waiting to send, stored in the order that they should be sent in. All - // of our different resource types' ACKs are mixed together in this queue. - std::queue ack_queue_; + // of our different resource types' ACKs are mixed together in this queue. See class for + // description of how it interacts with pause() and resume(). + PausableAckQueue pausable_ack_queue_; struct SubscriptionStuff { SubscriptionStuff(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout, @@ -292,10 +268,6 @@ class GrpcDeltaXdsContext : public GrpcMux, // Map key is type_url. absl::flat_hash_map> subscriptions_; - // It's ok for non-existent subs to be paused/resumed. The cleanest way to support that is to give - // the pause state its own map. (Map key is type_url.) - absl::flat_hash_map paused_; - // Determines the order of initial discovery requests. (Assumes that subscriptions are added in // the order of Envoy's dependency ordering). std::list subscription_ordering_; diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index 8d309237c9216..f0ea2f5abb177 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -39,11 +39,8 @@ class GrpcMuxImpl : public GrpcMux, void pause(const std::string& type_url) override; void resume(const std::string& type_url) override; - WatchPtr addWatch(const std::string&, const std::set&, SubscriptionCallbacks&, - std::chrono::milliseconds) override { - NOT_IMPLEMENTED_GCOVR_EXCL_LINE; - } - virtual void updateWatch(const std::string&, Watch*, const std::set&) override { + void addOrUpdateWatch(const std::string&, WatchPtr&, const std::set&, + SubscriptionCallbacks&, std::chrono::milliseconds) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } @@ -135,11 +132,8 @@ class NullGrpcMuxImpl : public GrpcMux, GrpcStreamCallbacks&, SubscriptionCallbacks&, - std::chrono::milliseconds) override { - throw EnvoyException("ADS must be configured to support an ADS config source"); - } - virtual void updateWatch(const std::string&, Watch*, const std::set&) override { + void addOrUpdateWatch(const std::string&, WatchPtr&, const std::set&, + SubscriptionCallbacks&, std::chrono::milliseconds) override { throw EnvoyException("ADS must be configured to support an ADS config source"); } diff --git a/source/common/config/grpc_mux_subscription_impl.cc b/source/common/config/grpc_mux_subscription_impl.cc index 715eb416fc848..60aa8e3d28355 100644 --- a/source/common/config/grpc_mux_subscription_impl.cc +++ b/source/common/config/grpc_mux_subscription_impl.cc @@ -35,7 +35,8 @@ void GrpcMuxSubscriptionImpl::start(const std::set& resources) { stats_.update_attempt_.inc(); } -void GrpcMuxSubscriptionImpl::updateResources(const std::set& update_to_these_names) { +void GrpcMuxSubscriptionImpl::updateResourceInterest( + const std::set& update_to_these_names) { // First destroy the watch, so that this subscribe doesn't send a request for both the // previously watched resources and the new ones (we may have lost interest in some of the // previously watched ones). diff --git a/source/common/config/grpc_mux_subscription_impl.h b/source/common/config/grpc_mux_subscription_impl.h index 090ff2ffcd774..2a43f02140517 100644 --- a/source/common/config/grpc_mux_subscription_impl.h +++ b/source/common/config/grpc_mux_subscription_impl.h @@ -24,7 +24,7 @@ class GrpcMuxSubscriptionImpl : public Subscription, // Config::Subscription void start(const std::set& resource_names) override; - void updateResources(const std::set& update_to_these_names) override; + void updateResourceInterest(const std::set& update_to_these_names) override; // Config::GrpcMuxCallbacks void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, diff --git a/source/common/config/grpc_subscription_impl.h b/source/common/config/grpc_subscription_impl.h index b921fc9a70401..3625b75bbd9c1 100644 --- a/source/common/config/grpc_subscription_impl.h +++ b/source/common/config/grpc_subscription_impl.h @@ -34,8 +34,8 @@ class GrpcSubscriptionImpl : public Config::Subscription { grpc_mux_->start(); } - void updateResources(const std::set& update_to_these_names) override { - grpc_mux_subscription_.updateResources(update_to_these_names); + void updateResourceInterest(const std::set& update_to_these_names) override { + grpc_mux_subscription_.updateResourceInterest(update_to_these_names); } std::shared_ptr grpcMux() { return grpc_mux_; } diff --git a/source/common/config/http_subscription_impl.cc b/source/common/config/http_subscription_impl.cc index 78a5e6caf9c05..91bfcc8b782e9 100644 --- a/source/common/config/http_subscription_impl.cc +++ b/source/common/config/http_subscription_impl.cc @@ -50,7 +50,8 @@ void HttpSubscriptionImpl::start(const std::set& resource_names) { initialize(); } -void HttpSubscriptionImpl::updateResources(const std::set& update_to_these_names) { +void HttpSubscriptionImpl::updateResourceInterest( + const std::set& update_to_these_names) { Protobuf::RepeatedPtrField resources_vector(update_to_these_names.begin(), update_to_these_names.end()); request_.mutable_resource_names()->Swap(&resources_vector); diff --git a/source/common/config/http_subscription_impl.h b/source/common/config/http_subscription_impl.h index 452ba132582a1..2b334f0f29e0e 100644 --- a/source/common/config/http_subscription_impl.h +++ b/source/common/config/http_subscription_impl.h @@ -31,7 +31,7 @@ class HttpSubscriptionImpl : public Http::RestApiFetcher, // Config::Subscription void start(const std::set& resource_names) override; - void updateResources(const std::set& update_to_these_names) override; + void updateResourceInterest(const std::set& update_to_these_names) override; // Http::RestApiFetcher void createRequest(Http::Message& request) override; diff --git a/source/common/config/pausable_ack_queue.cc b/source/common/config/pausable_ack_queue.cc new file mode 100644 index 0000000000000..1f0d8fa66a193 --- /dev/null +++ b/source/common/config/pausable_ack_queue.cc @@ -0,0 +1,53 @@ +#include "common/config/pausable_ack_queue.h" + +#include + +#include "common/common/assert.h" + +namespace Envoy { +namespace Config { + +void PausableAckQueue::push(UpdateAck x) { storage_.push_back(x); } +size_t PausableAckQueue::size() const { return storage_.size(); } +bool PausableAckQueue::empty() { + for (auto entry : storage_) { + if (!paused_[entry.type_url_]) { + return false; + } + } + return true; +} +const UpdateAck& PausableAckQueue::front() { + for (const auto& entry : storage_) { + if (!paused_[entry.type_url_]) { + return entry; + } + } + RELEASE_ASSERT(!storage_.empty(), "front() on an empty queue is undefined behavior!"); + NOT_REACHED_GCOVR_EXCL_LINE; +} +void PausableAckQueue::pop() { + for (auto it = storage_.begin(); it != storage_.end(); ++it) { + if (!paused_[it->type_url_]) { + storage_.erase(it); + return; + } + } + RELEASE_ASSERT(!storage_.empty(), "pop() on an empty queue is undefined behavior!"); + NOT_REACHED_GCOVR_EXCL_LINE; +} +void PausableAckQueue::pause(const std::string& type_url) { + // It's ok to pause a subscription that doesn't exist yet. + auto& pause_entry = paused_[type_url]; + ASSERT(!pause_entry); + pause_entry = true; +} +void PausableAckQueue::resume(const std::string& type_url) { + auto& pause_entry = paused_[type_url]; + ASSERT(pause_entry); + pause_entry = false; +} +bool PausableAckQueue::paused(const std::string& type_url) { return paused_[type_url]; } + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/pausable_ack_queue.h b/source/common/config/pausable_ack_queue.h new file mode 100644 index 0000000000000..cb23fbafa3441 --- /dev/null +++ b/source/common/config/pausable_ack_queue.h @@ -0,0 +1,35 @@ +#pragma once + +#include + +#include "envoy/config/subscription_state.h" + +#include "absl/container/flat_hash_map.h" + +namespace Envoy { +namespace Config { + +// There is a head-of-line blocking issue resulting from the intersection of 1) ADS's need for +// subscription request ordering and 2) the ability to "pause" one of the resource types within ADS. +// We need a queue that understands ADS's resource type pausing. Specifically, we need front()/pop() +// to choose the first element whose type_url isn't paused. +class PausableAckQueue { +public: + void push(UpdateAck x); + size_t size() const; + bool empty(); + const UpdateAck& front(); + void pop(); + void pause(const std::string& type_url); + void resume(const std::string& type_url); + bool paused(const std::string& type_url); + +private: + // It's ok for non-existent subs to be paused/resumed. The cleanest way to support that is to give + // the pause state its own map. (Map key is type_url.) + absl::flat_hash_map paused_; + std::list storage_; +}; + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/subscription_factory.cc b/source/common/config/subscription_factory.cc index f9130e3f90337..eafadbbaa85f7 100644 --- a/source/common/config/subscription_factory.cc +++ b/source/common/config/subscription_factory.cc @@ -63,7 +63,7 @@ std::unique_ptr SubscriptionFactory::subscriptionFromConfigSource( ->create(), dispatcher, deltaGrpcMethod(type_url), random, scope, Utility::parseRateLimitSettings(api_config_source), local_info), - type_url, callbacks, stats, Utility::configSourceInitialFetchTimeout(config)); + type_url, callbacks, stats, Utility::configSourceInitialFetchTimeout(config), false); break; } default: @@ -73,12 +73,10 @@ std::unique_ptr SubscriptionFactory::subscriptionFromConfigSource( } case envoy::api::v2::core::ConfigSource::kAds: { if (is_delta) { - std::cerr << "making a delta " << type_url << std::endl; - result = - std::make_unique(cm.adsMux(), type_url, callbacks, stats, - Utility::configSourceInitialFetchTimeout(config)); + result = std::make_unique( + cm.adsMux(), type_url, callbacks, stats, Utility::configSourceInitialFetchTimeout(config), + true); } else { - std::cerr << "making a sotw" << type_url << std::endl; result = std::make_unique( cm.adsMux(), callbacks, stats, type_url, dispatcher, Utility::configSourceInitialFetchTimeout(config)); diff --git a/source/common/config/xDS_code_diagram_june2019.png b/source/common/config/xDS_code_diagram_june2019.png new file mode 100644 index 0000000000000000000000000000000000000000..f114c83026bb79b159fc4bb098b0dca9f254a78f GIT binary patch literal 92572 zcmYIwbwE>L+y2IY(Jde#9nwEH4F z-tYVUw>{fAJ5SyBbzj%@EK*Zl2_J_F2LuA)E5DF`1p-03Kp-?4EHt2{L0Xdw1VVt6 zfyL3pouNl?ug`9M{nPAeV2kfU``LQKEqL8DRSbf;dUd%)C`$IasDme;7H|KZl{!C{sF&w6hZ1+Qt_`{T9V zsI*Y4EGM-8eFrp4zx@d z-h4_q-IDwLqxY}Gd9qXH8)8LP0|U9Fsm12amro?A{_k$Aowpmjo%iR}fxAX4<)1&> zJ2+ULFx&J?%P|Ul%-IC^g z`cx#~wh21&m40e!%It3EwWM4eQb#X5m0Lh@C5Bqg!9nuZ($c0*K*trR{$lDO2$FZc zt2X=*C#k%!=l$Cww~d#iwH+O7ABNJ@a(Wm7hA(f(krp^G#c!L|Zm`j(bCZ_ej0{yp zIGS}eRh89ZBHP}(l(tDsPCa6ez%pt$V=gP_XNZa68!QTb+2Llx+#D)aWk6aM>rgtX zewxG#*VQ4@N^IcZ6yReY--Ojoyed$3`C3NsvL*N=beChc;nn7(PMK!pb1z<^N6hiA zOCL+=sd6Fwh2$-s=;ifG4Xz6NiX_Qf881<}G*4NOMou8^%2MQ7W1}zPf`4;kFopBx z@^IzLo9{JNL${nB4=?>ctq-RQtWGw&ZAlo!J`sI?v??$r;r9Cneqg3r2S^M#mdu{_ z=jTXwPb4OKm|8yLO}YRZnl6G^T0lT;9h-FccyDg={mJ?Z)w#Gg9nsM*HvQb5Q_XY1 zzMCFEoyoAu7mt54rMn?~kPI6q2~wql2I}3OC?m%M2fG5cUp)y_@`3BH@JvHra!Y-j7#w2Sv>7VXk?FyHD)(z5s2@wW_8L*Lc8{2x$#jM# z8HPkoY9aB0e`-40>FS~v_a@gNDud%+KNYo46;)8daps;e#CRI9_!rf%-UYszna;3# zB{M~Z@ow|lZJSJie$qtkrC=RP#}ln4j4~HEQ7BwYp<%jL*&Z$WXK&AMGu@`+vi;D*-TiN0TVRn?#jDq^ zFZZ1S7tDhBo>K4_n@0p}3_c1RQmaAq6A|uVFt+m0KQg9Na@aKVhcu=vsus{NnY z5jBq?Z}>TeUMy0>l2CX3)@6(2ToLl7wX7uC%TI#e%R3YLGoEg(SeriLejx1D6~Bf=DDOo8L{G8aprVN@rka=LV~Rc|lfIL~Rh3##!0S z-E{4=haNeT;R}!SVGMX!Qrr#p`<@?{L7r~4(ikWntNdCF`Gl<950$r8Z*xTj)_TF~ z1J~3<1}Fz&JJ zOljA7_s-4IFL+bOpq)_)CM=xEJk1mqf@o~~_xQ5Qey1b4pY=>y+y|tW`ZBs|@Rf71 zJDq>YKIB^r4;neO*L&_82F|F})HP7O?C>0+=sKAE3rf1Kht-8k#R`oePe5i$R}2=WcvnC=V~>{vPxl(xTaU%q80C?oEP z?C>Yz$NBvX^0ANK>Rs_P*7ZQ(Ups{IN#`k0j8H{j2Dg*oj@chuHq!?=9`#l@!)3^)1r%r9hKY(5^@ z$Rs?5b^R+IN$k`0uYagF{d|?2Mb4@&h{86p-N}3}CWA0-=3gtsef@|+Nut~r0#9tB z1H1SkrgdcJMD{0$WK}0wp_Al=hHb2o_0e`X=5>;$RRe-yl|tTV^;CrG8|feI-Sc+N zD3CT|&&|u5GuN?QzGX04S9&tq;K|s~AgsFzzJi;<*C+XU;yqyaBl!^YVWtMQ3WJ~8 zE*UT62<&jk7i^PVKvc181p9G5g2l#GrD^}>N1dF<`0uFr;a{a2%4D~#QU85=vs2JM zOS&xd$ZLsNMOSxh=dq!0cZ&=pq00V)&@L+$YNkrzUHf%c&A5u`+4D%v&Dp=8n10KH z7S^FpcJ`t3&$3!CbBsUy>W8~EIS_FR=dDlEQEhApz7^H4n>Rx=I0O@ayzfV(z3Ax74wF*-Z%&&M-cYB#* z$N@S2N9cRxYk zn$`p$;f;dc4nHZx);Z}?>VMR)CT(&F;SMI=k2kj`YsdOJz4oPc!^fUT|CzOGMufAM zq?B&=kLaHwsrD(#=1=fWw3=G273ke6hkn^9$o)ZUZf*|4bX{fB#WhC#eN>`JmBdU_ z+MSY`FFGyKBk6e>**sqdj4%+Z-e(-&jYI*j|1dxaC}%JpD_I0DIS&yXpnaNNTj=`t zNshH&;OQE8Ud1QkJo}63V_Kx-Ka3Of z{_``4i?!CQ|Aj{YpeT8~veNb*X#Kx$WGH(EihJ(OCTU1{ulQfa;Nn6tFfhKi2V5;U zb+$YI`sRDk{DI`K>31xY&d+(N#Us+GAY#;QbGWR${N*=~LV9@`Sqq{c ziweDwjjq)FhcWs2LH>UxiW*k}GL>BU(+U0ZP!|`Tah(=mWTV3Z>pT3}Oa^Jt^|J3_ z5h;Pvki_+N0qQhQ?!#gikg}1g!41onHS7-&4Gj(G7WpX{+v^XR60o?ph}dPNXXTI>wZspg331q{*N$tG9-X*JZTIuMC$%d&W zrl{f~Fb}s4y_P&)4_9GeOyF~HaBxxO5aO=;>`b!Dj)sRp0!ZGk_a`j-af=M;y4{ex z4jLtoz=kio^gSVzJc|#=Pic34j>f>RWrWydqaPjHHgx{zT4dy|!76Vv{_GiIJ+pjH z!UR!#y8ZX>@o)Jr%T%uj)leS;IhFgRSz(`><6%t>8Xs4=3|J0OYhbbTi8$0(;mw_!URO!|5^f^fD^%v5cGf6CbUhz3h1NR#_z~ z5>{7JL;Dcab+MFpL_1$2x3GU~XQ$fiheFjmCylN3vT`e+Kj&I({+T7qIg?m9IIni| z!YXiHo7te2b?wUethCJ4{XUcw-@ml!Ji!@k$gS-7=#e5mzmCZZ5&P80)TEz3eFoL6 zZ8I~^^4K*I>vlXfMVgF#@1RmNFEote9Rm5+*+v)vkSuL-bA|GFnBDS(Ec^F zRo~6cEo2EtGOVb)yqrlCl2p3Z@_k9kfl^x8D7Z?|G@KL(POhln%IUrY)(rq~8yprI z9GAq}Uw!aEli&+Qt67j`D9O*yhtj@B#P`wqa=B=L)(CUMAp(+muV2RjJC9jekDnWq zA%j)<)pcX=lT9;2(2Vcq>k35FYy?YE_ z^4`j(iTCabgPaB49{17tpJ`>Lc9wL#-Z99_#JJ`dKQoIhJIeZUG*i&`V{#7S{t7Q&pJ>6OYJ9Nu#&TGiji43DLAlI@QIHVO+%~Wh{wcZM4oDIQzrr`bAU; z;_mI;6D5b{#`zFQ>ibp}2d&5UC*xpz-fTj~tNdM}w*s%Nf>a{&|CFeYWZ8Du3_oM& z?QOJ@wd7{UcQ4O?_scjswx!BDID|#yn3yzPeeS2^YQ)Jj^7IVNOUV>u<_-$-jE7+U znn?fD@B|H`>`ulrcE90Q-6QGRnpc>}nF<4KLqi;vv38b4ZR?SYqTloL^XI!W(i@&h zIw^mCmeAFka7Fey;HkVna(6dg#P@qqBaB@QlYm{a#7Ue)n1Z2r)~#NLtXX{*&zc<& zR9|B4N#wxnNvli}1Mf1BP5m1tB4}v-tji)NAH$C*CnK9hG0(f+&gICwVnjtvdITwm zJ;LgzR8)Lx&^p!EbHH4pBJe(n^$$9j%AmGEAgPp34u4@!vuW2C|PfqZf(OOZW9wj`?k-kH?KIFy~`(LBf4T4WFDNvfi;{NU|%GaB%v%{w6^ZV(Xg8NSsd;C6tfvBZo>B zC-1Sukj#KMZk?qJ?Z7&D-qNFCp3YJN9Y1NTRSsAWDk2+6;x zFgE@tyzZavzB#mMAdVnpru9pQ1HN7I29dz6M_U3hs$!kp8 zmeQl_kEgF)CB0GPIw@{pS>Jp!<)0!v@%PKi_RXsZ>u^!7!d|-h_Iccu9F>zfj!?`s zY^~RqAjz6-ykFf3(~Gmbhz6%CE9@L;>9>{Z7GNdE!RMr<2VG!`nryy;zQgJBAVsrOa|%XUb|aM(H1Zz36UVLTIv@ zt}&ybriFqPSJ@WdEiucNBANu#q-tJ4%8rdDxMv&!ue3DHHj0bQVQ-UWX-aGC@;(iQ z{vZw>sv_hgU1`Zc_d`~)&wo^8x|r&Dfp(T5WdA-Nnxp`BcXZ^rtbQzD?L73kc_Z_C zHrW@}XPlg$*QVEp6B84&#l1-IEA|c!9Bpj}1_r=1R|aNgbd{jEhqg8e#wO9Ul$2xb zAPGGu6BP>rwosDsW^p^oM{C+*mwm&-sykLp@gc4S$WyIK^&DzpM=l;7R-=jhqf~CA z{-BowE=A4FqGK0*L?!~&qI=vd2+RmJjhu>ntKLRsEG?-9Q*varNvoo-tLNr$dh&91 zUHuyg*u$nUNS-E&cg)wGVLHTMmkr%i@D#C?P%-m!HOyYewB1Jt#s$f2%jyN6NV4Qb zVT{Y78_d6Dj0LqIk)V{b>r9EDaG2$2x%>Ox2@>Y7Kn*peV0&d2F_p}nfm;isqj=fF zsmXuNVnnaXL^L&j&I7xG3wW<`cUFY(JkLupHejxEyJdaJiy+U%0QoHLzwPFu;{1cR z^WKp{mkwpXqm(O#$^l@;M$OLrIJ9EMgAv4y>#|DMQ<+K7l~M#Hkqr;ZBW@3CHKT#% zA&gW&NXYU+G)Ew}yT;b=(be5B|4WMz^C1)(l<930Te zz2|X}m9VW?;`dqqI!sVpt=LNOcTa)G1AL#$Kxn$;$jCBh+fr zCI3B63T&uQhz|Si3^ndNFa>20Imfq>nB(wzhGG028XMbsjsv&0wA2ab{!4$J%&z@X zMTJ2DE7Kw|iRdHC8w-oy`yr!Mwm*wqs9DhLR6jifpRFt6!AX)}`l-JDHt*jB6U@!c zsi~=LD>jJlQ}%)3^q)d4W!v?_eW88;6j8Z?yNUa};wdpSGNRzpf9A3#zn%4x8z3hY zXNw}%ip42VR5>27hFa^9{AuP!9vU%vBF*re0c|cvFjX6QMFGdyP>Qla+Q#NcCYyTJ z>O_V9*SF*O3jf&2fi!?;bP71l47{5Oyn_ZFL7l(gjR90)*T{E5>30J;j?I<>Btg|8 z%KePth$`3ufRb&eUxSb-dZJFVU%Cx{Ue#`I9W1q4Il4#obo5d5k-qq_Rbf*9S^|0* zpDG@%58bA*;-Oijg~W4c{cHw3(H~TkYHhFl>#3oCE##$lKxAxha7_x{Mm#95Ue9+E z&Lng*w~unnEqFu^Y=992s(6}gmKBfakXGigf8JU$p&JfE&LRo~UQBKYa@w~g_do9H zmBOvAx#!V0KS@jfIl7ZnWH3YR{UTT z{w<;!>kQ0#^2NAWYC zVWT8YxlZi-2q|R0PP^lzvX5%NX`YH}v=kj(mT-hr(M!meh(>iBMT6zKUk=Ts@tWzm zObm`%xvRL0H=O8E$h8l=vAGA>N=J9=ci(!Z3mw4oJ0KfKYdKk=qpb)54B(-7U`cT- z=e=*fg77_1j-zMSh(I8Q%rJ}S@GHBz7G#PrU?AS zv&&G8oj^KF_66%3r`hcqg$b_vuyW{H%FSV?^r{lUhh{EXHu|UKKI#P%eMO! z{G!n5*zGipoo5rnZ2Ng8za4LqHL&i!j|PARg;`=G*DCtS>5!1uYQjX}G&F8nJ`lL0 zJ9C`~b$ep=A|(JS?nyy#`>IrDHbNq{nKYdp5@SbEL-^gFo{~~bkXU>wBbOks2fQVq z5^3u*d3@2(ZL$Oi6c2l#jGj0F{}HmZI&RhgGTWz3_O7xt;;0BOWRK)^zr^cLQ4WoNzvJc~!mpK@E+BM~}AQLcFqx$(xS;b^LMeFXds=}V? zNCI>lUL6icc=}T&7Zw(V-uyl@LQ{?6|7EyKG|UHw|Ay7M?SEo9X?bHoFMog^YGY&b zy}r;DJYPOf4O{32@kOr8&DnY+0U?B8wHj~S@QJaW9svXsE{yF^#E6diXsw%{!B_o_ z2#1B7oSdKQ7%955@huED@KiYPbb~kWs&(bAYM0jk_V2NXy-Wmy@$APx-+r+suvM8x z0VBT4OYJP!m`o~ZFF@s@m}qUBsi#0@_=|*Zd}J`GW}Suy8NPuD{EBc zyDH|IS{%RNl?!_EqSbzer+p)W5arrOxTnU{Nze{C24ypD50{wv#v%&8@oE)E`G}1U zM9yY%|K>=+OZ`@lf$Q@MdwxBSA^y>3*R-n=d1H{*H#L7y>$2W25IIJ5jp=LM^Ib!^*5-a9#xKdxJx2kT%%?I4@)JW}iK2)+ z2lfLFnkuQfkmy}I=#ZIU!~y}Hw597L<@YKZM(4&3T-TwmDiMWHSZiH5vz3)q?e-_W z<=Qil;APEX(<2tr8TqBfMb%MYtFeCR=(s=G7@A(qriTd&3C&gf_)y|#ZEdY6?u&f~r+`DwV_F!z8@LfNe z5u$^|8aRTW@R6XsP`C(aM5YcJj2uynQ2e0--u}EL2i*$wybIVa;teIZ>&&B`!Cd`z zoM|+4tCk_CjLTv4rK+?Ph2uUg^`utFIFvJl>x&!IoerSZ!gQD+F)Pr68)FO+aG9sBJ!IlSkO?8X(5NU zPbW2ZF*bj9)(x4lfGL zA-{p?Gomk-k)|_~sksX4{*|naB(x;$JGWbfPOl56j>8hL9)8Npn+w#$1CP7dB6x(|N9~@Mh$p2$_}RWue9s^G0RgEQIta+4zs+x} z*SGJr>R^AO_96h__x@Dqo1LodCvKhis-q(}@MR4vWk$|GmjdK}Pj4 z^i|}#r}H-02Yoypy5!6ymr|#8(iR=D?H_iYG|-R5(sw&641)w3RTu>Xf(!%3;;l?g zX+dHR^MWp4a8W0|Is8_G!5Fwe)r_HJN)SEsEge!X-#`N;ei zqX0(G$cT%nK1%6;Cb={}QbTXN>>3@emc~rupR5Z)K7NFJCfW*8D=H<9fzXWQN}HiO zu@T$BWkkGPoz2aEAEOsmRq;I2?i-pVj=$VBduTDoDkf&e@O#6+(c;4h#3tN~_&n82 z7m(pCh5j`a@+(o2E9h=aC@*&uUsn~kJ~>=!KYBLLDJ{t13E>5}1CO@rCFR+;5*_|u z>HYwkFK3^H05VcN#BJECOIqdtfBlDn^UwLjCD~@rEQ~jiAT!M8TTE_)x+fn$;^-&k zGO9lzeRH}wLKtRszB3i{d4QcHk*23)NFkRct5#=`$G_kmg!5OEM;b0vC}nekVO60r zTQ8PIlq8$F&yI(Ek7qC_JgA7djDO(fJa!b~zBaM=&X^!{^n2Z6GIV>K`}~}E%(l`x zyAF3r7#8B_8H^Ormze*v_t3voGWGR~+jI+^b9F^U1v=CLiw`2j5bFG)yt?|(;p_7Q zV|i!%@7&znh+?j$lU=Oj1*%k-R7^~aJd&PcL+phnPkPcDm5L1(?=Ha=L-7xWMww!w zvs&n)pyR>Remk1CJBV7t>yX@+j6_=i)=FtT!s_!pmF>^LizJdPrIdZ+cXAo{a7oM3 zR}9D@)DnN{^PWgsNx3s%`lJRbFoO1rH@bRVz;)Sno13gNQ(?FihW3r*XZ2FpWoRhj zR@tqK@Bl#p9U;y$Q>*RT_i@)j;Ipp6&PqP9!p_f8%mxxLw%F0fkF8xlnUzfqnWIOe zK>%S^P#Mnl4e0Nt$Y)-k$NjtI3yZ}{nxHgb!=MV*XaDIJ8=ISk6jQY}W6OaLcZfp( z598g~J+c2gw8YR>T`~{Rl)Mc~v56DTMIC9J0<@7Y+;j<3iOM1#}3!UiwBgZ%e26&G2a2r*TLepW4t~0qopY~sn1PY z*8^(VY4d&WFGYYJZ$F;(WH#yP;4b_8nflyG<&~bP@BTd9{VyA{tHDE7wd*$Is+WU9 zw%mF7dS&A>2QRN#jis`JLN6e>-TqwsCw~A6N+As_L=(8+bw);pqP@6#z@*C^#e(Dg zYDcED-)fRTYvAcP%Zf1bKOr0q*Q+orI=hjjj7$3N&lIm8gL~hT+=rP5pWl0PD+Ef! zfG&)Tk~$q0SS+}OEU9pyxf>Z7+1j$pd3SuUQ)x?01&aKJ_^su3-z6c*w=Z>db^C}M z988Ujr~}mtP4Eiy^4^G8YsfN8X2|?gX#b$0fRUZQeM^>qT0Qo#dpwo9pXu{^mFe{u z?U@N9m1ZiJ!HLGhnTBoQDW$hj;+pF=Y6_cFiqMJo^2#F4Uu?1ZWykeYwNLm(qASQm zM@R6@x6rH8E$Q<~Z9g~Ta_#{r0{~mrlQJ_^as+Hfhx_~CxbGs}T4#k)puC=DU7vtV z+D-^j(IbqEf9Q!Mwf)iEmguL`R0QOM6u=>TCRS!sviHT`m?`1>-J3F9WaB^k}+3R_FdIEgAsi~>5va+@H z=H6T_H=S9lXJu2jf%uq4mXEYbB^*NvC0 zr+_{dWi$LqW^{L^`Zc|~x{JW~d{}-0Arttmo{kQ(*7N2!;^T0paNO{Kk|XoHq36M3 zbFxb0$FhCGGFH*`tu1>SQpVD2j-dt;q=xIsdTU3=vV(A9eB?>VwPjRzp>9##W07A& zsoY6#q*h_FjuX>M!;8T7VFsO*{$aC@U0oTy-<$Fsq3Mnj^Mk&S=P_;jd-hYK1Q?tX z2W6ju-D(~?2ksj1{}ULO_dnI~@ng$!nCWmR-h7Jdto+4f%R ziJWIAdXM2ZfJ1|RAMtMKU(5rZ`SENoYUxc6RZhtVg_|p2MI~gxO1-*+F}B8XUylZx zps`npavkv?mC*;$B!`BEw7$1EJ0lYD^ucS3i^hyZFBIyuL?s+dLZ>jHu|-hDeZHVlJoq+u!pSf+Sj(gG|8fp)8 zh@hIhDE6&CfMz!sEP_$865c14Bxe$ydz4oATn-b-G)$nSx$%o?{ILwtV4=8c(O^J8 zY}Yc6?<&-(q0_QQhON9$tSL&JeGz68} z*4yM)K+EDfJeR`x8m(LGk@+;js1_H+6B1lWMvRYY5P8i>f~iyAyua9-RME99>b5B_ zP=&F*otl8g{Kp|lR-MlrfMv{^>jOzjILN$NfZ0~d7W2qrn!Jo;%h~1~zfb}86(meO zL(uknIDX_7j3in0n|~JP3j=IL%(Ptpi@9}9(H-#oFtJ1pGr!VNu7*$B5oq6v%Uy@i z)AUs@luFUK)>@5TQ*j%P$lL~pFZGKXtxldf;&H&7bJQz0j!qw#Q2 z(-sV38vKRZ04F==vLEbq8>;;qRJXT%;>xNHY-|8ZtM*O+LRz^nLeDW-#qi67$GIoi zu>Q;Lr84+c0=9vI8J3GE5qok880Zn5?;f9y|7sA_bAR4rXR-_sZf{n*L*k_GMS>8& zpY1n;tR_BPF1ithxvJ*fTBLh}TFt_B`P~3bEG|?iVOs`jb?-WN0 z0%DQ>m8&1~AyU0~k+vf(Xd8fe5U3LnjF;HksUnOL&ZK{91D>21vW_Jtl4NfEh@p=B zF%}*k4tR8MVh3Ixd_#lpa-wD6^2y+5o&xEmrnXiTY20kU#0*{o`nwowwU~nTvBEZZ zIt*f1q}iMNz!wd;Z@Sk8lGr#!`R%84YQq|rKr|U`VLO#E=`*u8ae=jhX6H8tz*><2MUTl!N-GpKbOnHo^ zyK`l43(E&^2LMiiG-3#xPtX~A`S?@ra|fzz>zN8Hke~~{fPjqUTu5&p5!P)|I1iF`}uZ#?SqRN85s#B$_P@Ba5Rq) z;)>CUIK!BqkRp{!ZPNtb>FI}C#_Zfny*p?<><~X}OJa?YQlPilM|*XmBL$=*iq5~i zeRP4wj@@HYH(VBR9=p?$w@2SYm403Ta=G~iQ2SyjvT=$(#yTom0_+9Zuv=jDySxTPZ&b{j}!IdgMo+CnJdTrJeX9X}BDFu$>~Vo72A z20Uz8)A`^k88ONduq%>X&sxMCe{h|$zyry-!S2yr&6()kSEoL|mOklcvQ#H<+@H)T zm601a?#X*|v#TH=2AUrw-n-W#$J^h+B7mk9Cj8yq_%{#9Aoy+)Xh9~Ohj?(Sn}ilN zV6ghB-|=K9jh7VbaGu?(=GD*!g^tH}(PA=qBIM+?i)BCf7~5;67(~&Z4zoX-x^08J z(Q5E(wSQrh9o7cG)b&-?G>>$rXG_zus;HGEqJ&qAc8Qgey?mnI`@l}`id<9jcwS94 zaHO9#+%XKoYz5P}HbM+O`kI6AeZC>-%^4HGBY+mY3JiQ|Cx3>VFU_+m1cgHWExMvZ zBET%i&`IA{wI1YSz1K?UMb-ETyc2O=B<@SN$lO!I)kiO-fMv<>8WU38dEYYX zxX=)TMJV-`{(a@dVdq1D$`R}t1ANpRmUG$g1Pk4CK(Aw!zIqp;_D8!!o#nIdVSAdP zeRfZ9Fq(SMd{5i&SaDP+#OHOFs=Af33i?S&UZJyxNBxKae(Ac7T9weux68Mta3JEuHCa+y@C*v`88%=5lXNAXr0)5YK)k0Yp6#z5yLzL8g=(%>rcBRR z`m*pT3rWFnbj^loNAo9D^)z_;tql&0hs*6oOwfMH@&*Zfu$vy2itOhS9UM@dI0-Dr z??n0a7~)^@DUWun!!Ft-OD9byY6okM5_S4BAX#7ff#XM2_U$`Q9tzoEnWDkxxBx!5 z4x{mPe}Tc%?0w1_7=uf}^UvG^EF0*ye9_mhUmG`=UTL44oYeUOE$9lNgN@Wpj!P|U zGzI{4!$nmM1tG@HH4yBFZKtD3wB9||Jmx78%_f~Th;u;L|KPJPOA{w`UD~@X52Et( zN-+ZjsUUvY?;D`cx|D;BqPS-pZy7{=FSu+Up-m<}_c>afo}X{tNcokK4mjF4);JDh z-w0xAj#34@sQc)>iq@MPw<3<_70}>hH~#bV!+HJR1STa)7jAz<-)LzOR}%mHT`HUaRqtxF`4k$A=6J_mYT4Pe4&<#mKdPml z|1mXLtpnCPNQFK6`({hXTAx8Gj}N)IAWVrZFZnNs8`5?wR?(%3{;lA- zjn_Dw&x;0e&^8%xx9*nPe%+Uwrh&Cjbvf{7A6v=^6T|}ea?`3+{>|wCm}lx_oJBzC zNXfuu6KjH$QXAu?WxTbS_!+x@!!M$jYqf^9eAgu=IV|ZfFQQbYtIae%oc*Vm`e$61 zfUQ`oMIS=AqZ^29<9;IxV~TBq`M^{@h))(duQ=u^0mTJahd=UI;QuBpSI{=GYZvhT zBqb2C9JcIMfg$ehhKWs8OnhL&kw-^;nlX!5LAGHoO*@T%C)P9o&|jr^?=v+5QkV(8qsHZ>ki&`mqECawOHos(RJ?`K6zea)PZhkFJlt5NI`3N}i$Qg7!;kefPfh zP)sCT(TjE{Mru^8n)cxjE^j3j7Xm=R7KFlk3dsy%X2i$GZ|CMA<{poF&1_I*(%Rs4 zXxh=*(z5sJCoMA2&AtqW_W8J%H~bZ6O)nIFY1NW?j!Gnx@PXVj0EoO3h&^FnE>{Aj zHPYZ8#`E(Tcem)NJWtaoLUB>0>(_)wO%pATK{yKNAS(dr$Swec1+=YEBS*qk<~cFK z^JjVSg@+$ttapk59ov7>GBB7kL)<&uEC5A%OJMTU`KGXR;GfvIF}lN<(_*tb`7SEd zqHZFN;TUPGEzORs__NYg!wTG?vB2B*okx%90xvHEn*4U`Tc4vViex@_NIV@2A(ZO> z+FqRTf$Hb9{RhUgc#vuVh#{lB{5dm$nwlQo(Z49ym=M|UZjRpvx2OP8-crCM#{FIN zM`%mMg`uX|M{{|lz|G`$N1?L^FLP+ztm;+qZ9P(mTiU~&EjcAqGH<*6AQ44gvUohROiqrznM|ywWy6}_v z;U+{%3vszZU~is&=#Ml)IvD1Rz+W)wn>kYj)h2UpyEqj2AaT43Ngkm;)?MiVUtg~{ zJ=Z>1o~Pe}daW%vgX$E8u*KMB>oMxL8FX*vbETY1w_{_S;>5pPbf=mArRHXb`#VL@ zTQDqzkZZDi{Zhqq+|Lw;dgF6i7yE;tk!32}n3#t%Y+|7qbvrqw#am=HPU!Gwx=dLm zPn+UXX6=M_wNi5BHRVc%2=ylwj@P7O%oK*U^+VPGD(9)YPrN05n!NsMHj%D9A)G~1 zCDQ&uU696*N7V1{GYF!twIo~8H#POw110{T-TEiq4oqGjQR@<8qjC2rktmN?EZF4? zAN%nfz+bh$r-;o%hlBik(!RTaGx{D~M>lCncz_I-F(md1R^GP=ECYHZ_nI|Dp<5+f zVtCQkX0@hOO*mbNV(|9}kMCvXkvy!juVTGM8OlohkNA7uycDVcqwCKduFT_cCJ(K_ z{03+@Rox{9aaGb)jP%Ynpz`^Jg{9DbIYLRxI~x)(-2VOj-&_rXP4A!80gOsp=($KA zHwUWq0+sBNA*cK8?h7Z4zk`8~8?T)1=SNUJ{@YI?S{FoTCaV%;qp+KVyy9_XI5BR6 zfjd=|IoO0nh*%kWwHBp@jpIJ{HaF64)Xxp*HyGL&)Vt#Vp$_V`4O?xsy28GFH@#=y(~x*|}(IjUbS7D<_a{k79=1Ey9S4~61jO=VRle(y;C55oRi;R*m zXsGxT<`I)b>bHX2E{6n7;zp{!a$ay`2yXgX9?T5>WcmFmOwmd8skk6vX;Wa<%$tg+~)7qsbEUg=B2RLTwv1Ehv93D84X8%H}nV3i~3ZYQGO|x zT^3U^2M~^V(SjR1kf(G3ZY-ZOGH;+L%dx-}334|td#sbMx$EBZjMbUDLQeYKPB^Ns zX^k8SA@76Zy4Qttu=f7|!$o(YEuX{7$`g{k7PPrh)$I1iHv{PcHcg@>za`apyGdN& z33OkcIJSv zE}{3)gTiBXt>dGvH>@4Hv-j+iIr1pq=a02fqMLyuKuFbr!bSL*m_nTHsjsfC7TJv{ z2oRrQM}_0604@^N?n;#d)R%$SiOFecq+oYXun)kCA>Ax2n94x#4coBw#R&GZ?eT)_ z?Cf=0uNx*-g(Pn8&GmKI@;r{wDuLklpXOhWp7h7Q-go)E=ZS!`;EN(fp3-!9aig$; zcm{nD>V2<$Eq1~-op?Lk01nm#Z|pk`swM+U>r0+}ZcIjgdto-}_}A%oRqE2OQDU9C zE=12fV(Afx)LUk(OOC}*`|UrVbN}?k6e0%N*@dR2Ie>xuo3jFj-T^Xhg%gvs@3AI8 zFF7B6q=}c8Xm7{ed7T@*qVe2mdC&1nhT^4Qp6MFtAGq=l4nt;Bir(zZ?jTmQ9k$sW z^`ebnv~15tR&UElVypI8Pkum?+sD*6lk-MArE4ers)zZqL}WpFBhvDra7Sxe$ZYNe zomq5c9JKFRsp#`ZUAo!?ZE#KU?ilBk8DqQG*)jnnvU@gKz;yP3= zQ8_~$QV1m~VF!+mSv}zSYf~sF<0bg+-$8^Dg-U7B_Pbht?cdZnA08n$jF%TChI~}3 zj3pkts->^4hLTMCsV`#`9)zgsAq}1?B;y1EJT51NanCDGvdHCzUS?y#pr&C^a`r8* zQihv!<0PwqsM={I)yp#wAeOeEXy}jyGs^CfHcjA+P2i}|A0U!UkRc&U*jNE;p5vc6 zgvxfs3!_LVFZv0VXvqzVAu|d_sq<+n)0XyWqwY7Rfa27l{F9nQG{kmtt7VE<`^i*{ zwH3f&fj*B+dsIbW!lQ;Rr0vDu&>#R6)y_PY_ZnrKA^f;jvL2)2ABhh|nY#o8NWH^p z16CWBmJ{p*nUDApNF<(bQGg=0rXLBUD)(OI%EGAlj3 zY>N}_SnEM7>VlY-5rmOGmXw4MzIyI3-?JoLywdwL)pFV}d<)Pjx_D~%01-4q)5*i4 zBQxOeaPTwk508##zs%yQ4{pz|;PR_)Xl9?iin@$60cw`+u8z1{4XgH~_CX9X&WxGT z4?cphR-0bvLF}iKunoTI9gt#!DJXt;pTm8kP#H|cgl7EB3mA+8=7&t9o0oPpBv51# ztm|B_hgW6Y8FKddjq*=l!#WJZFWVoxw0`l#?c_HR9DAPq_0iAw4_dpm>Nkr*Yqn1X ztajrTY$N1%YVUIx`)V=-mOq!4e#zmuE}1o;0gu!5oWF{~+(y_mh=1F|N1oHN`-Bci z8{P?Q8pgO_4*yeZ1Yt(veDtmFkKf+U-j=OF&PKDvZALPzAm02KpbXWDGLd3*R^s5^ z^Ye2*TyOS&H1@?iz(p_SY!>g$K5i!tY6r~iQeY|!L;#R*UKvPg8<3upMf@`llI$Oe@#_m zz2=9pom>H{ed7Ts;Hr8~6eOKs|H~Rt^0?}F#v_*YjUvRIzH*@efHgv|&%QA=1Oh}H ze>uod0IMFP0H_w(twlvFQZTo}oM&%>kCJWJ79WGQ_x9cml>v-Suxq0xtvV0vpA&_X zb0{3kp`J+SKlL;IFTl#P3w8bW*h?LhfhYjl64^3co0h4uT#*RR+}8B<4$67}qfK@M_uBw({I zE&B`@+^4@ezoiHBFypX<@Jsxa(Ptzro%uO^`$HNyteU~tZ$NB}D3;GTJL&_@!SrY0 zg&i86<1K-nthwpA^$UK86ETd7%fbeHwMbCi@@f*>#>ckGOSZNmwaqfJOBP|_ z48zZbLz8MJT{YA|@YYGUD?XG|;{;uVHv>s?HRb~k&1=-sX}`x;83hWNKQKm7)XXw5QRW&ifRy{^dU=Eg=)^SJ#&eM7^^Gy2%9 zPmvm<=5I{?rW^rqr5ryMV-~ zW&WPl5s|t#cQ3r5^3XUm7I!oL`iIq>mL+y4Kvp(8eXu|i46W?_(^Es!s*D7CHOqO~ zcCLh$2+m=kSD2OEv_Z{4+O3NWNJ3?F5EW75XL8roQT9;HB@BFwpBELL1aZsyJj96N zSDxXFNXSQw=XYOeF@hFLJ6J80;gaHpl#U4N_ioo`f6m%V#<2{cVm@V1bDc^mI^qX0@WZAd0J#yKY(uZKE9b{c2nAjCGUBSdQeA;VgjRLU&cN9iI!PZgR=|a=h1K{- z?IwhG=_*jZ)(u~1Ave>JN zixs03&kcCXzqcz!WoTu&P6y$5JnHZ7{{p1@3w|6R!{ejqa;5KWmq`EV=Uz&v|{>4ZyI^frl>3Eh6TX*fr;ym<9Bv}WebOF-{Umu>~1VhP8KRiU}jOv8v#;FcD92YI%?n@6DHv_Kk%ml zL)*_JjHnj?KjDXlTw_UJBEUg~2n9H2*k^duAN~0IVA(_#*0e5N&#Q;Jj*_696JHk$ zN{>s4LwqD%z~5iM*}Fes;SHOfedeAkPaP(As+81Sxj*i`G*4rRtNm5Cx#xxUNYLT*l&HpfPm`l@gWdr?rg&=e%G&E`)(S`8 ze=*M!EJk)gIW0!%2N?44Sw8~>%*ZFc?^NM(zgTui7E3fcCdVhL{hI;egz~J7H(aAPqZ-IA>AcX(%p@8gLFxEhja?k-3^MMba!_*Qc{vq(#^Z^`@8RRpUWTc z;heK)X3wlyvqsMFPBqKyp)E?IrzXUI9tZ)jb0}lxP=40n_IF&nG_?KmP;Anv19)u6 z0mp}jdbO5QV30>DS_L9txxr!Wsxn|UkZ21)_#dRM%YtCr!Fb*(2yEa*iZr9R#9mLN zIk2yY>ATjyPBkle!(Tfl<^E@FWy$C5&b<0k2V!_rQ=8x)`(E+-DSpDGX5DKF#%jgB z!Zn6A0NDsw)V(rrwdY8+mLS&r5Od_G$`o^bg**5X01!agB&|_l@EB?M(5<$g#wMT( zND=qr9_{SXwEL~I_E#9Cjo(_l#_K%j;Va&Yii%#y7pgF;^H$c|M?iCwLJLH1H(0YW z78M$7Rxl-GRLBB(4B%yZpISDq+ph)Di%f=3ZMSvT?&d+?2@Zv-<1w{HUn&*yBA3+5 z7NPDe;ZDyPu$8*Iy8-+TViLUNKhV*eAo2xH0b`v2OKc&jq|h&#YBZraW92Kf$0kykgE+*B-s z9{K6{51by+N-`OU5d?G#z`Hn2iVKAcr_Fo@<%$nypC*YbR(`PP7L9+Q{qx|;xKIsaxRkGsLN?EhYS+kFqbbV_ov zsHUHOj5@2p^LqP_lQtoBPj^>Wz?;}yUVC!)^Lqld7UmH_u6wACY=!N8aDE2m9@WvC z%9V{ALGP=RwcP3kHuT9C7~=)pEP45s!SLu$6VJE89e`@|*pwQdc*2l7bK4z~TR#3U zZ73iVve+&1XTI!5Sy`LYmZULB0p)BXTdjj^v5m9j591o$ZYaf zLS+xDregPC?nh=u#>1AUt*o!Fua#9vv!uR$wjj_!p_(&Dj@s0p_y?*}!NOQR?LU(& zaWc)u<%;P%s=eR4C+OevuAxuw6Px(-=@aO0a|kclh==VEaE{xujuSP6)X=p!CZ}$s zpb~NYqznF`QRW543e|2Wj<~}JYo?c$?xG!hdW8QQMK(+0spGpi%MQ))cxKL*Q!tr7 zJseIR=`grYy`88?TYG<-D-OtELVwUCCY{6lGWPMYC!30krPbgFY#=A;o}Qm}xI6aO zQlb=q?<_aF{qXPg1fRWcgk|tDM)?*-e8T7Ic}R;(KP0;Yunp}`kpu8(TG28e8PLEBMPqed%W8kh`s-o z^-oj=CARhP`IA8ozSpfkDj?3Y+I@Wbvf9XVl8JeD3$=CO6PbXFgyf~W+3@{4Qs|ZY z9_wq6Jl!2=c6fn;cF-n*UNePBe`#(mdGq-MFin`Xt9!4ftEltN0pnQB-R<4K5pEfO zJLEA755hR2SzvQkn~ zU|a=smGNbpMo5m=I&ux-ai5*@3{O-a=7{+#WcSKnbZH(b$6we~2BD(lJ7gB06dgDQ zG$&%y2y@*D{YWr+T)gdysrl7e8Cj%QmO>jm&v!u39MCJ1A0a6R{59~s-kO0gsCU=g z%m;Fl7|0~xQbi;Fk72?O6eveWlP040qcgm*#JrniPXD}>O+Yo*V$jWWJJ&``I4#Gv z=iFk|+}dc4;oR!S=Nw9WHK34#3?|T9es|LxVZ^%J)8(?9jk7;uhhh!HkPwadX9K2O zQc}{F=9C7H!;brw4&7?gA!O#(b#VOinpCcGPniH?aOilf_iz8t2u=q}F7po-DcMyf z2y*Pz1e{1P^3KW08O#p%i%fJE&!4&Wn(2>2wxl!2 zjX=vbcH4hqsm?|zn=ko5hfQaheX-T(cE88`drva^LWSY;kdhypJ6G5p*|W(;FuF!m zwkDPTrKbN}YD%xuV|wcYDiBTo;O@;!OlB@no& zG_kVU7cq}z)#LRizu0;vUb^(o$MY9V#=69~F-A<1#@^0x6TuA;pdCJS|c7PXw{KqWC0 zN-)_meSxMTA|ttNe|GZh?oCauG(E+f{P$pR%sTGDG{>K%6Hq6xuVZWZr zSI4NgWu^Z@b>B!{pT0ZgP=b8V`4|&4*7`gD9R#$)z&7ydH4S|)>lg5WSzuakb}S(5Z>Fa58bG z2O4h4&UhPUe#m0QhYT9|Hc#~9JUL~92&t&z6u$KPcfjDBdV0L)MayD*i%%Vmi=Z-07AF9nfzk1fJ=+fD@yu~!Jh zNDzT=)=Qq7ilOaJ$s4*Wr_e78&zH-fXbsCxi1O$o+~YX0A8kR1e$F85b90(@pyAMd zYfi`zCg3V&R^r@|=%&vFJ?Jq!>JO<`G!ZZ&JkA!}#;!ZM_U6%dGf_A}yUMr^%lxzZ0zFj=%LlsVroSp)QTwEJE7(JkpXV+-@ieFf4~;Oa(VK>N>}pqeAC?@oj;T) zPC$@vHRGj6{&`f1+i`n%GJ}(vn!?Twv8n4eFKmp5KrLb6ms`m2tv0}zMFOSV1d_aN+( z9;8|C3er$_e7KR;mwvY;?IL@)ce*BMMepDoyShw#NKK#wbCS5X^A>P z3{irEvvUKCHw3YtzWHn4X@OBy^)v+4^sMPPsUTs#K@77Ymn~8_Cc;*ZutT+~5b+>c zhREyB^<}U+thbW7%I~k7KjMvz48Q)~U6FNk%q6okim2i8*SKf6d7C9IIrIyaxrfIW zhUix?le4hIc!GsopK{RA*Q{+&?cnY8%V9I-3t7WE;RCKlGXG`5wQ^^vs}LDsbEiHv zBc&6UJ@UboC-13;SZh7((IB>gJAQ1A#VC>h6y!SK;cg7GN!F^1MtgJT{UIm>Jp;;Y zr6{aDV;9daX4AEGro0FVuIzSSZ&A^%sX{5NzpY^yfpS7|hVS3^UNP*W=2?s%hO_&( zSJ6h=h)&=RFr&N<(=OxSz%L3>prdE2^#IJi<94@5ohG*K>M01+dvlBH-%VzbteA4| z`N=T)quq-kKKETuvb@)5+TO;XZ2qIKAauS%;8l-B&(E)LG#No5Di3B-W)f?%m8I)d zm;dS8NtV3vTttlsCJLQ6eX#mg61jswZ9%Ya$Sv7OU~rP1S(w8omCB@_0mnawiZTDj zG*#i+U-jp_-}d8{Im35B*&B;C0oLjtRN24D+%tYnZ!a+5!%=o#{z1{lWzgcIFXc+GJlaCxmXW9*N>7Bu6jdv2z zuATIRW=WIpIg6y&4&~Z@cd%M0e$k5~G5b6^plece}Y%0 z+@mX_1xd}gm|q+YN)9VE_I+2$%&Hw!S~?c00{}{ z<;EL-Vz_cY%J?AL&d>KcvKNt~uYxH#CHWGfHmW^gMGrdZH#t8&IogxhhVoXm8HRXu z-)4Qi0i@S(QchZ5U;ixywTRmM!>u6e6w;^yOdcZL+lbkC|NP;1y6ImDSRCEELl9I) zK@OHrNLng!!-6zXHw`1Gzu`y~3n?4$UzxuoM@|VnrAUmPfjN9~Mc^Vf(3k;1* z%z9qaV)75oij<@z0YFd8&MqXbslC?CNYv466_j0iJ3KEq=iEHCrk+3t#>^SOfT+?- zHv3dm_MNRUYBYUjtEw|lD_!mO7GoSPgHfbs02Oto@hjCPG-?9WW{^rdjUoreZzhI| zH)yP8Vljx5#Dg{Izh{cx#o#f=kT=3wo>hHy=}F^dPtXN;)&4M+1hrB!a*Qt7UPi`e zqVQ}9X`!Td;m$i6?<^ytqCS@aQ3QQ+QdkAnX=O% z?X`Z7UXo^vjL<%-wML?*n50n%u2B9EWNTw%7)-UuHVnG+(38D^dKP1B7Rkhrb&!*# z3|N-hddT*)Mby-wRpzPCIJDvYDy5o1B&-|R_@}^-!p|id5r-AaS?E+V&G|1Pud>9x z$%mJU6ny1fF*_`-Bj!~O|1IH;TnBZ5#e}M>s{ma9K{oAmUh|UlOBTMdSd9L_CIHR! zTsc#D7+Vwd&hcgk!EeQUG#Xn1M-dy<&?Os;mjRI=H#fJ!#sLx`pO4G+c68tMjAAkS zy_X8V2r+jF$`&u}wfmG%0z`$PY%J~kXO#)`ILsmF#Nr8`8XEQMYNjGK;cnZ*iAshg zCKu1*czWnX{;4$A4=YSOXZK1Jwwhbh)c7s_-E!A)iNgG&i1GaR`=RV`>Cr3qH6=i# zzG8bz#XDk;61LIwry7zlsbYknpEtw?sGRz9SK7M^?SrT4fAEb>Cg;IZzJOG{o=bNDay+uk!p`O5Z zDl*hH7W3`9xrZ_{{1aMk``OvqfbrV%^#P+ZKIwQnAst;QqXE(Xls%{Y369OULsw28 z{z#fQ{NnDr$w3PnxRQ2pIlY@*fJQ2GzW)QbZ`ioFXeZw(-D49%AATRIh`9kjhhqsJ zlM8YN6nG$@vzzPhR}7c(?p=Jg8hH4BJl9@Rt<+U~=brOh==|JKf^y!V+}ccA|12~7 zKR-7oD-IHC*JPp2p6*YeGr+hDnJ@iHnPJBh$W6slvae`)Yx?xM(!7&Uo(o3QEkJka zX`YSv3=kvB(U9TAlPzS^bN|aM<@g3GnOaywLM`~EUcDha;|OvISrp+Zh;mTn6UZe*9wid_3mw+F3(Sopqd4g%BSM}Q0I@0p!Gubw^jvc`X8bMG&NDLwu}2j$)k=Qy7EGcL z#84?Tw~QJ0yZ<$(R}wVCHJ-Wd45@Xe+DWr0{Dz3>|EqPd;XjQtnX%pJQ-&7E3sB8K z^|(lho5qlza;P6u>8Fa9ZJQ6lety_R9v$2;C#*od2H&wRY6+lFP& zI^)svCj&nVLxf({Ny!|b$*Ea5RP=CcocdWU& z-eJu^PtOO$hYO;PcSkDDRs$U;#c4KhZou~vsuQMld%ncB0@%&<5M=o1e&+AFlU2&5WM3gO$7 zT{zG2bd#vHt^Uiy+l5(EE-fot6#HZ-Eu~z0XX>-N%OZ&gjc{%4__oL9X0fIw>bO|c zBzwzu>!az^7MYpJt*wLPfx%`Y@}@`iBxNq6&iWL&u*jpu(1Jr!x{RUSM6}~+A5<~M zwWFC-qMDU)1E&?rMCJ8ejyxRB{6RAQ9s_TTI)EfBJyK8vUX zIUTtlCM`O7MMg6y5w_4sb0nF$;lcc;J@}Xw<5@ELIMBeHm4JaS^eq;`n1h3U-)tDe zv79Av`5OSJ{gUyi#6R z)5!~mRv1ogZvSPQf!Yl9`cq()yx*dF9o}xZ!Y3HWQ z!13C-6vlW8n7X2}L5ce?H6}c>Ujo5{Z-WL^b5=gf+b1W7*1u;$IQG>S5cw7^QoM~d zTHS&dn_)qkg=EP=cxg1sutf5=}ZA>gEEdVF;S*DC$pjAQ+EpAy7}6VKJnB%^~Cde0CtaM?><=bkNqM z#VcCi+VYu%7bq)cK|!)?<~N7V3(PAUdOF|HY z-EfsqjaM=)qF*~v*!|8{P+`EW{Tg24%KVyjw1$`ZL*X0Pce~_?X^fFlLTk*#3ty?gV8{gSgP=Fg;TE>bNwn})7M$1?Gt z^GurgTLK{qj!^R$*_`E_X1-nfeE%MGgnRKjt;SMU4J&;AN~+6-7;q3A_y!obAtvqL z{&0~AXANOT5M@PU5Lw<_3Bk=9h>g`>NntcMA|HqymVG_};tizp$&CHZo+bNSehCmU zyH~&Z$M`ujwuyrQ7@5p0|KqYYq7i^`Wup= z%w2V47U=>*AqWuk6IPq26c%wwX~us%hUIg-id|%vRAB<%Q442S$&bGlRtHnZ8RNl* z6f8_;Dzs!57#0ICmT!ODsTPzu*m%OwS)yL3KnHmM^x$e%5`EJgd2;si*0_fF)gIz7 z{Za)HWwgH@AwC$I+G!r6`eVg9skPLPng?Vc1(7_Vt7ED3Wj)hZI zpslds0+{SwIcP><476V&gx1=}$_p<|J*y<2(241iLSsk^N=Aq^g)*aQpEf4PF)l(t z+VmWi(z0DtF$r~63g%rJ?YM~5twQj(lzv{!P8@qWYTgT|I@Pf@ZK((YBM z2ISHcOO0kgOgXbc2;n^Y#;K1yJv?M>MBFNxDz8Atx4r&TWUlmsG*@6EqSf3r;E5^b-g#S~G=2R0 zTJLN#lKHBK(E6J8!{@nWQYsmOuX#S854dzBCZ@)2PtZO$FPZ+*2bd7NUWve=R|FXK zsB;(3M-QV+G_eWXvm&LFzeeRHtfh$G#xu-b0pZrq%YXZ$L&?p&;qE_+!`z;P!tHiL zc8_P`Y1=`^q-GPv%&H5(%EB8DTV)n(i!%h)Luy+G)|=v>PC{na<-j$K<&rTpCb3fD#dy`BA8H3CBK>5n^U zopB`U7{(a*N!CBeF3>);IC{>O25;itbVMw{28(zGG-mX0d`XBM<~M_Gbr-Kb?C(UG zK&fhM%%G&jOA2&i*EgH^jlhjQu4Kd@t*N;H@GsDECAqI_yrTW^Y8iagEE3*AAxk$< zBV1UY0vopb)w<}>D)gc{56Tx>28Pm#3Z*9&MA_h9&0$`F62*!ss{yXA7cg(f>+

m2LR1t0uf)U1PNEbHx&R1;^nv6aav$ruDI%YpuPDR)RJRCRJ8f z)*J8yFfnQ4ey=CNN;%5Tyn>#=@(eMw+Q-Q&gO?s;)d$0bW=%Za=5oCqY14slGARDX2JlzDAt3GZpCY_?hW z8^TmNWmP2li7SD_6~Kx)$EJa7ASqX)D+Ed2ha5TH(@8Xv2OttnOs$|~jvH2vcI z^fj18gV*=2_Qd&b==4Ix2~ZOUI%7UG7C@1Y%l`1M#pz31Veieb@L`nvJ#^Gfz?b=! zR2tS+1RKopt;Ng0lvp-&hcQ@~44JRtt^)PE)QIrzqUe1gSAgAwhOi(;?0!c=91G?6 z9oF(gu~zy+M=nPC10Ggh{0vcaGPa#eq-;_QA+a)@`T3ni;vE-aci%6Z6!3Ci04YSq@#`UqeYSF3-W zmfN%K_CI}1d4Q0nH$%ejuGi#jCgECK%tDGxLh@w&Qb2l1NZ7YSP<El=TW!AYsEsaW-It4tnHp0oR>A=KT-x;2B9qa))Y=VXk@Qb6ynnc zV;^><>g!gyoZi`C()`M~m7u)rK1-ZJ*UjM8^~O(%-E};gm$r!-YDB!sI@jdQ?Quv; z2^4&tNscZG?r&PHq6r|BF~mX!e^b)Hk7G*1P{C6TB~H6({0J(Jsjo9x_JM0Vgo_p- zqPBeLAC&m{?;<2zXPB4l`)ShSzzaTDAWj|rWmR2&(=^^$NrViQ10-hOxR1Jt!-qM!FuAs!=f z8D)_-JG?Hfn~O6{J)a-137X14B5&F-R9bp*arm!oM4E?{+VxUsop0RwZXI5tZD}J9 zwZ(fc6K>>W!ZRO}Qt7y+?*wJSgHqt7^bZOO%GchozcB+q%uBNBD+oriZ$N0h`*ko3 z%3V;Cm5*~(BY#gy$=Kc%5Cl5gkY1xBbLWD7BnF#<0jaB%p}RNbbNR}q1vKg&1;u18 zF#LqCS5WeD7dM-3>)Y+FX>Xl0Ok)!=4)d1^C7gi`fp!{@V&=bG2)D7Z5ud}>R6SnW zH(pTc&pWxhTrlvq14RpvAfRmjhH{aW&fD2X_8%fK@-q>Vxg{QSdDAG-K6-eW6wm_z z+L22qUbqCV$?2N*6+lpY4NPM(2`$pDY2yJKAUP!kG-SQL6uD3__4V~p-d}Z`1RngO zV_Y^6M^mAZ38CU0iGlI3#q{?h#y7 z=l?q)1I)xg#+IpX8`>@~b>lLx(__u->%EgW%J$u26K1tsZ^8UABM7YJ<%)~-vsZpq z*FiFnQ$^vF$HgqwFH8qe`{Lo@&19>J<{lp}yJx*fGQjU$ya@*l8*J9jAJpFKktOlr zH-$}aa(uDfVNRg@kS{$2M|LmiBw`#IkK~Qs%Idx|4NEFiqQds4bt-%wvA?i-v45*9 zp6z`!^dphDkxjR`bzX1*yo zkiDLmd1(TI{5+B@Y*SVao|ChnzW(ZL+pi_<1dxXS+G27A_Y5G0>BFclRSO+!sj$Vj zRNMA_0l>4(of5b>UvJ88OLgeLe#9V ziU%%`aZ3CO#GV!I94qyMu9#msM(lgJ1#Du-_k0S98~Uca&LC3*&7IDTxa?${rV2(H zu4q{ontg(-#E(?OcI9scWsRd;9Dc#TL`Fn}9&F!(ZpA<}kw^Q4T@Y4LEin52hBMU_ zBX=fSOH2HKZcvcWrri1c?x8?wF!0>Q`ECfvAbfB)MbtHVtNqawFS~XE6lL~pIcW*( z)S*fAA}K@u_KYqI@GrkbvHMw~LL87bg^WzQLS5_q3QmF_quD4GK?FGf>Z#}r1wET9 zqz?0@{_gA^&PDA+J+eGaC6(dJoN`jyq-h#;xXMJR@&FH;>w)q#*YUulI6c z2;_&4u?Z?_3*2iyHF zRNb(bTevyQWCu^ow_a>!mV@6F-zy;H29CPckX&nMB6pnAo#&{HA@5Dl>cfOs=4C}d z!{>)YUh}eYlSoRL&Y4_h7Ik>f;H9Iwel{Ie#%hZV;rExe$EZ!1Plf5!twPM*b$=lV z`wpC=&ah=MSd4#f+^U;KiduICLS$59OJi6{n{wn!GDpYSLQ2yuIm9}MaX9DUesO*z z){#}nmx=RZY=OK|sk7{1zhfWV;8V3BEaS#)Sds6O!s4FsYCBTtc02S1FHQ;gQM7O0 zLPcR3Zj_gn+Je0wzz~6W88yQ4>Z+aRMQ0#1@VuFlHt*}^T0Z}$hy4%fg5K_CX4{Mq zOLF_B(mWUlz>syEoSc9Z3UIl{0sjZk2|yJL{##!UG(5}A`f0izWo{mq-v zp&<&aPinBvA_T5`dwXg89`?Szf`PrZ_V(0-h+kPEI62=vt%D3+7NcU#G^RR*+3*Rp zUHor0!b%Epvh5eyYSu#yUt}uLCrjRPF2Zrcwv=tdG2830k3!ZDk0S*I^bunz=6#wC z5b6Jtd>Qk`F)hVCStb#3iwFIL!`Vwfk}l?pv4Txz;&OopitFbtqcV=oOm;vtsWD#42`klhL^IsPX7D2Jii#`1X zko;lgS&&T|fs#8-m}&yDodKdrQmR66Q2uom*4ZpjZ>B@sE2go00Ha1Mtm`9x`%nM4 z0D0G;#0J|%s2s882U}Z?mdTz%!2a<+DEUUNuGm%hV2c`z4g<8{AiqG36>I=1wwTB- zN~suu(@yA)=lhO|7k~sY%wq_WL^=UM1bV6J4C(9*Rh8c8sX6W*F66KT0a|jFQ;-j^{Uv7hL>X zb!TdMB3vh=Yfy5h6oeM+WZNxPFhltddzbJH^RJ&qs7#^8dSNUFLtl+>S1JZU9O?9K zEP+Nm$_G*UxUX{r^DqSYGCT$uS^+7#T#K09VLT6}0wM@eenhc)G5OPR!A3zN_JgPL zYAvA_3EbsTqZm=)>xIFY0A+K$wBjf~j5}6gv;GbAGyux%TkZ%xRy9yc*3QO|*4#0L z)stCydN*FztJBF)gg-9~*l# zg}NHkG61UO0T~VPScebiP6r1Cb$uY0U_`2?L=63t1SljDBI|7BgchPLIg+ zg6$?98em}8-y=xcq|L7!9;>{bc+I4zdPPgX2_@Jw83$;BB8-lR;^6k`X~B)`D`fIl z$`x9L)q|+b&xb4+o0y;}A{Ky|b5lnQAuH8FtshMqps8K|F3iONO3WT7B;KtYl8ATx@p^1imW9{!}*_4N)vULn0Z zZuD9E1&q$VQx(dG|6=b>z;yfGw*q1{j!Z9DY@9TXfOyLbE|-DJ9zPtKc6RPirO%Oa$dgso8wi4RbLiW!^F7AL-W13n1L9u@GfJketK+b_7 zojT1Cu`w(nZ$+N^3MC{|WKZ>#T;6Hx$&RLdOdr{;8tnVNlNReazB(ZDap>}W{8w(< zp9X(npWv&Bu>*W6{3GS4g!3GFqQYKY7d4M8V)o}D?T;4E| z1~P*oYN)ekVc{BZv8@9nxjdyE9a$>f8f5+NzWFx;k<-J1*aK@Vh&x+|GA1(6T`F)c zg62@2ZH)CY&Z;z5H97`OriTX~nx})nkrzFs`1nRu`aAC06qbmt>(JE8`I2fa1~|)1 zq*6p|Xf!BXQO$sN>m#-h3O|lJIeybae-4f{hb1fbG`B=1!8X0cUB>9SY#h~u|m5=tF{P~&e zimv*ykZ;ux_;7PSJ)$so(3nMJZNZWZu^y= z)6-`mAWN0-yFnAxZuF=karWK zqrJa6ZX!GO_V=CU9a@2i<39zjk#Y?r+2oXn@ZAenu4SVt{5%pX17od8oX=RZ&Wmfr z252bXbFBL7_59!?V7CKehoU82hOtw8)#&!ZroXP%+-ceFL5{zEhk{-_%n-N|%LKgDQ>x$SFO0)gpr>Gl|cHwa)`H2W?ll zL(V??>QivO)9X&Y)d<-RIuCPltIO%3OsS%ILLJQs*OvU8kEEPw3up!34=UN*$H%Y> zVM>Liq7o2~-`O#}dT(m_+ZF34AOk~WY)f^1jf@;QJv}9cXanjf0Ra$~6g7|>x8@ox zC!I||@nH?z#5y_=`qM;`pxSz{%ZpDJI=J+j)v&7-q!dXEtW&h`D;Ax$x3_cXP zavUR;t-pDGb448t3C*%$SNQc4nJjK3WJVp&l&aYDk*#RN1h#TD-OvsWbnJIKF_EkK z(=)DyWz@@F_E!XyNVX;)$tV+X?@i zloYrKDhGTzIyxRCB3jcN2!;7+o>x-&uQGJtcoK+OfE1%*ZeWqiMlrZE!OA+ahjn0vsr|*_=HgniK>hn^eF1}{4F8sl`roU4gsikwAs2CMR;$N>u}oBvPm=BU zh#Y~;qcnEpF0Lrxe_pj%!Ud7BVvi3GlM|G==7H1FUZN*M`p(SD+9FydsCm_4H@8XO z%G4C*6e$K!at?R?{Q1+>B{B+*WdA!9igi5dXL)HPqrUp*FCw;s&^#$AGc#KWd86Re z@_mdrSUhZP83Y7cD=N^pzkK^v7}wd^2`0Y=&vFqb!PdxL3Z7SC#Yf`GsHc3d2;Pfr z&s_)$2t<81&I+(zKLjl4td3mUd%W^%fPsR@01B%sPyL&e%}t5VT?f{FNZKGW%wuwL z*}BVUkg@(~utJXSi8Gt@;2sU7Wh|NZh~WxbM&M%fwp>t9Nsw3UL&mHkHCmAiw)Wg8Wy4|56=@IcH~#4h2tnGK5t(GJ6NH5@ zC~Z0>Nj%XPUcRQLrrSuc3dbiW=;-LVQ$KV-nR~?0UH%oCgBGjNO7rgzJetIz5bq#n zRvH@k>lq<>L}N5$EUby`?T67SEtE-yc~y?g+M}t zmPMx9OUv7$QF`@b3H(cq%>Mp-vuq{6`a&t!)TU%Qf zmzq_L>bmzlAFuwZYulT=`5b2o&Qesx>hhYF20r=`kD=u%51xf zPsLCIj2wi5ZI9KwDa(|_Il?@bcg3r335khE$HtQ+OO5jFK2{9hKWEN0YLgVgn91tx;8T_Ix?LZyrlqBU%xU{g4bVpe zs;JwoqqVghQMkkvfAI+bo@{JsS*-D+ZaFz9PVw7EFC^}X#Hwj+23%;4$T|s`2BLt4 z`Rk;F=i;nK@$CyTTD&KvZGT**oJ_N*1R^Km!jL*wSvioVDJp8Tl-FDmKzhR$(`OuDUR6q?pLh1BL#3Ql{p?AZ zck6Xg)`Z>rJ3?Fygg#n|ZzDB*v>&QcsLM2v^?)Vm=FDtwZ!=~}HG%g5M$A9yF$Kn& zrYwONul)o)EIeEBr1bAEZL7o15Lx15tMj9Fm0)Z70L9be{mvi2tq^xWVOsPHduPrQq;U*jc0&PuA zi-{`C$jDD!k|F`q<6V;fj>D$ppbU?_*T_&mkJs4PBxvzSGT7XDlYm7KTrm!AFGsOS zNJKOOOeQuKetPjj0u*&Xb*SNT+epCi&pe}d`v%a)q@tqQ*x1l1o6kY5v$4T(aQN$` z@zV_?r4VDrV~kq?0b9t+eaJ*E1H_OrBluK?S)0H@y*0qlc-(jqdoOh&9UI$pQB3jj z^0JAE$K2aY=jXfc2{o0K z9RY~|L_|a&A5rD+swk_9vNB%vl4{KxbFeb47(RIAra70#zEb4UDxIT99FibCw#DXW zVP{wSHnUjpySlm>=nb|sr-MnD#$Rb^_}(!G-Fu_PApg*on-8|f$oTS8y!7^d6MLZD zyiAdkn>#ElY&ppF(#zM!rSfUW#pWTzj){DFp{p`noj}^OKptya5_vD z0983WylZZ8iH5w8{omn~g+L4&CcjPgGakN`B!(X_uQE?m{DzEk4Y0?a6tk?&4yZuf^Y}-3~<@%p5&mdt=E_l zsj1a2-&B=5{j&da8BxO7NVmXG$Hvh{7EQc^T#pC)fzwg@bkWW=|vM%GU13%FZuY9w#9=!Ir(A4>?NL;!n!*3bfd3^{pzOC zS4%im8ZfR(DJ?200`VKH0;5VXVeBBNKiy}&rY$HhN83ZOZUE2L)+Vc$#o~&Ff-O10 z@bC088txn-_GoVHUt`L8a4PZEXwBnvvI+?aadUgnxg%@*UGV{vUm+nOW-X3w2C*|# zG&IY!!=tJaHZZFz*wK>vhrh|_{+;5OS|J>U-Tky8hGz@adx;J6Dy{nXDN6v_Kq_2a zUo$tyeH}mp^-ZOUo;dPqn>W~4@V;=01MDqW#LG`gc`GE;ZA8hLynyhp3YmYH)2*0= zi=?yA&zn{oxeo5rAqBP+5*7w@s%?O18cOKo*e`G2~a|R|NafQg6}>yf@=blCHlQG1O5HoJw0}{QRu;e;9$TY@XE?cSC?o@ z(?oYe|J6}9`)Y~WVG6j1UY@Wd-r&QEk9}mpCFag}q3!X2@!}_uEbu&_W&uh%`g>I% zjzqFR0pN@8ZR#|2tY5-N`GDwV(Zt6 z`zis(yyWGhaDGJB%*b{vE$Mh`(P`=0T*=_QB7dEK3MKlV^@UOQCs)+Hb~MUwL3qtm zybFGeHzG>XL_ObZPHJ3^oTXbb) zR0IUyogQ7d<~ReJ1Fqp|U94s$uPpSl_`$m2%q-A&(~ONFMI59ZZ`s(KbNa^w1qBBM zCS?8qZ~n^Dk;s-V5Axkw)yc|&r^KPunvM>~QVa|Vh8*C#;9}*>EgvoRdH-GgE}U+c z$SERFkiAL^t)jl3>&%fmcNAn6Af{z1jYmX80KIF5Dtq9?gwb#``o^@qR7j=7Bp}V2 zP-5ZWpa1VnZV^jSjN!NqukcXD)W#W7*ag?%#8p)>c`nnj`hn7@R4l3QmDaLpM0oiA z-kz3gGQg%(zJ`RrwE6Yf_Q1h~i1RTsSMjmX$!OVEYP}>#Hl$Fr{@Qsyzv)Xk39&-9 z!JX$kY_Vzzs@7>s4j1oFnKT5z-NwRdEYv}02Ok$C$ol%L5vAC~q#6;CeWo>;nV8sb zG?0Mrz5Jfv{~fNhK%!cer4OQ{M7hiF^jJHP<-U#3*;@nwP*zq}Ik38#O*iZ*hpAA7 zj#=~Sh~0Y*LcFHrzl(QF$>jeHQyPgN%>=i~`PrFy=A-Y_eKpkUmmP>4CImeWg3|Y31{UI&;p3mE&RE>TBl1f(}W_*=HvWhP^R(s zvTm-V!8Vx(Hs~$w?D`f0nYX_QKU#rg z9dOG{ION{a$t<>c&$q$CLb=$@WtF9mWXuy1N_-I+}e3;!}R>T z`^BD0+#OgFmvYpsqnk`A6@q8cs;cV9_@D($0B&zvRbQWk4`jFzfq{_y8ZC2R)9@YG zll+E?s;u;0D}`}(u|&G|&v8!1N}r~}^2LIWS96-Pql`9bXG0Nm&gR!R_67IT?IWGdGo!2E|O) zZ~&4CvjIInBT{*EM8u2MdAm(L{wiNK6V}2N18B^F5M%V2U+LRNIlOjCNA`AKbad9g zQ6L0^1fs30ZTt{U$Hzp+AxGETa~1VzWMZtj)|$;Gw5|n1>yu^ zTyE~>7G)Y8i}GQcriPnf2}9%l?-TjUPTgq##u`oJT(A@g+&r!4ie% zFEjf$P0oaY3HtiUg5Dur(Aus>nQ0*Par};5&j_JYfj3WxDIZIWft4}=vv+q#7eGpe z*GmmaxFXr?Bt8qH4$p>$%ua)blBU2TD>nv5`UVAp0V2a$b~#~QgOy0oo;xrK+?SG) zaF&flMdp&42^16*!go`#g9U}(zR@r;qK>Zq{Aqcaad8!KlCha%G~4U92V&`nALJd4 z@a+&FPNYkasCPUbIaugj^*ZZ$dLysAL4)fv&ZI@kLMs1Wj+;k}LGc+E9|^KXKwuQN zKaUB5J5+t0#T4#F2rJL&H8u_&O$-teW(oynULff3Z$>RQWKMii3qFCIAB=L@5J?KH zfml%yjAM->A%KE@w=*nTrEr0MC`qxsUBID7j!~~q$IQ%3OS^ZuU^vi{k%Lam^9l#& z5-bpSc-R#d1`7EfKQ_)iN*h}S-tTOgw7WI#5?_b(8CM;WP0&$TsS?ytU2x4U3+ zR#a70RbC#2FM+IDL+4`R45o+^>cSl;O)ub9j`J56*$^^lYr#*c>;dfQj z@wnX4=kDm}=;hU1r-8J&V-yhR1j~rjo88-B*@8m5Q*nO#&>Ojqc zR%Wa@_5O4YEIqyis^PGAH(-#xnjbC2jY<=%91{fBuSG?nAt9|`g*H%1M*Gz@_A*le zre~+5H?vc4m@x8MT0W)ofTeQf#tdEkV<6~mhd}rO{t$>zPL??Q3qVeUosI36-@TnX z@|bae0iWdV&W@y{Bo_zA!&O5EthmWN+4EmAdV2axug8M@ruJXNe|uN;l3U;^0$aOFz?O6ip?a;SC&cHYj(62f zvjvxi&o;BU&$s)+w~3)-4@Rb|&E>-PHRz8I30V_y40LoQV1u1o^M_X;@hlSy0Gd1- zlW$`NCtpVBO%|O|(9sK_ug=bvz*g#Z-+LYEU~wSd0}l`1Ab^Ms=0{O42k!3fDl02N zyaq`*Kz4%=oEm{sV0!1?-~y=?6eOf2Av)XIisOzUR5E{VWDF+4QHQ`#l7YH<2!*zu-b%;QpAKg-P!(u!ijeU- zhlu~+DPUt`Bc`oIBLBIt;IY*o4R&vW+y;QqK!gOA$s{!_Y#^TI$8JGkAz0*`C?T{Q zA{g%|CYx0)ryEU>kRA z_@aBW^Y&Af3A(TEz(SdKl!q;)8IBhWl%$E4+|v^{7n?aqS8?&4WrvQj?1JDWaeigX zWS~qQ_jiouDX^MbYt-_-2y8q#!K|;D)BqXJzgrU2nJ0yng?(ULCx)G2DDM4Bg z7%Ay)L8K9h|Mq@<@4vj-Geq-S^wx=Q-_jtj?*>5d@pH@Z zQpv4}2^IU`5ti4RJEChr z%?EHi1H%9g4XRpWMMZ@WiUrmc3IQ1S-y+Q@@?y;xcY3ly-LayltSc*%K!;9$Qh(tzcOKITT;3a@VgU9k6S(2eRRKx)6k z1e#;F4i9Im!Ll7yNl8hQp!>^yAiDzY6@Gr=wMHsYEO6r6B4*%XiSm;t0Cu6*q}1ET z2O-qR&=5Ln!KIuHsHB|m#OUaGkYDx&x=MR6ySrQ0*oZF{COys59Q4R+Jb!c4nEqCvMBvk;WL-{0Ub*n}Uf z|A2J=o|W(4Q<9SlubP3C`V_qH{Cs@aI5>-6ou0qARJgb%R9#zk5F4SDB6>?1@;|1%6} zdP*iaTI5C!lF+oh6XQFY>xnoX^;E2;lJxMH&l!_PwHrLlz$m)SQ2^d^`rv%w<|ktl zv(wWAXLF1uGs)!Tf16WQ4t_RNdU_o8DVB+?ezGW(L!I%E>Tui1~t$J zblK0H9Tc?;=+VYT0|={Wv_gI5w9tt1r%X6;0YZ+VpQR15naUA)sKh_f#@ILpuC4L8$A%v|>CautO@WHU6eFG{_yK8B(Tv$1^( zDHeegYGpD*av1kd1GKeYWM_w#{`g1)!G5>;+31^+tQ2}*dB{`l$EBdZ-sig~kkBa= zulw8Kx-RZWLSL{1?dIYD?MK}j8uzb7OGMF7^F1e|1SvuZh%!YQDVgA4XRnL-4jF>) zFb+Y=Yp}cq0|TCruq#wzA=HU&pZm*L@FXtLEj0s{lVgedTy zX5A73^ME+y*rO7^?VnVg_kgt^=A=|AEkd8UnHF+gLFpG{9xo~?Bqcs0l5~7x9w=<4 zLm=mD- zp7~uT7K`%c(z-cE?y>}|1wFnW+7K>#mcT@9B*?4Z6x4j%a>{Muo2}rbMCe3ZSFAjG zz~}usFo>!w3uqZcN@#jsBz{b0xd&jJg~&U6$=gZfGQ7QfPf~(w;TpKvC`eN8Mq2 zUtCIBIwC5nqO2^%?J16!6??F!2az`A?5ue=t&5v`GXS++&&#u7 zudYeRAfw&g=FeE`>g5RaPnQb6JEwty*pF6IKd|$GI zK%bq0cR~(%0`Vn#pvd}>QK!-fKTYt^s>e_2E!?z>T(%l1*gL-yV|BDzPXTsiQh5TX zNlzdRS#e1MlWL-AT?>SKcz5Tufcrk)2jJ4KRCPJ!Vz;&sUpIxGduN|)b?ps{`o`$` zeA4o(tNtouJ)k>bkBY5EJ;S|+I^$^aJPomrriO{vPdoU_5 z8X9I;!i9U2TgQgka}JfG#WvNdI+6_d&JI1592FtM6cSQ8(`ThLvB&di&K?n&oB35o zD_Pnj*1Bk^-(|TrsE%(`Jc@H0Xu+3YfIAiVC-=ZRfr=PoNsNya6jdFZuZSN9Uu!{uc0ZZ7;vI zw~wcJ@BwiIVn4L~ijS3{tEo(~r5=rOaom&g=B6*zgxMXCE4$xDWm#RFlA%SvWuWcW zXe#}6|8ZQG;{KEA{Y}q}NLRjhkDF!uFJiL61M#*gBX+je%-`on% zOBP3lPwM)GTXkBwwK2nlY}I^i2gBVR>H#d@G?VH1q6504-H}ynWnc>c;zdz}TM*3# z7T{7}E8%?jx8l6#;j&KWw>3XM|J?WQ>KYng86pJ@P0;W4e^?SR5fNy66Y~5jXQ$|u zvhyekgTxLLaddN2&!Oagm0m<7m}}oSKMS-vu58sBNTX+g){Dc0nqF+WrX7p_^Ur1) z_f~GG0nFS`SFw#Gj^$86tPF0{_2G*7)#kw20NToO)<4$S69Eox62Oy}#RMyX1m9Mt z=w_r{7AU{0tjc9d0EF^7@XlLaKE_Cz+)GW+khZj(aU+Xt;O=bf?av=#LzomXF;5j0 zeeT!#c>OChbp=fXEtvN04TAGIpy#jB&7aeAk#iXx(XoASDr4=wv_JU*o|-yRXb%8P zjt&-FJpNue_~Bsib90AokO^L+7ijA1lZ(=)GpmvMr@_fd;4j%ppp(BQ-!YmWH^4{(EdRlDx+`-@DJ*rId; z3~>2<|3GrK{DhmiZoc4-s8hcK?GK>A04+}=7AV5Y%ge~f$ivgr*mwpKoyW**auF6^ z9RA{0tNe|G79RF52eRvJ)IrOSa4i|V1>DJ(oJNu+0f_F@wF^(#F{;4v@R#y#vctEK)8DW@P}k%e`y`?T zC9dbxt0d;0MTh)cM|gCaI?^lJf4Dpn@0dK~wkoFdtK1fN%zb=(Kzp-p)3vv^m+JP^ z(IkTgxc(z}cEJN9AwliqmsDqP?cE<4lW9yIm&m`nGhQ!~Y$=^^m((0ntbE;Lvz=jm znnOyGI?Y~XuA;B+3rLRm3)i=O;70O5n{ts0*`+34Gf#Q@`}@nu$pL6ZU0ofFlOGaz z9j9emnx=n$Ws(G?nNowVmk%d5D1WXn`fV+M=k$?ljM}@sqVYLp7I_g`LXL)?5)QYNq?wBx5W;cAIVc zPQ#eY0CShZD@JrN>fnPM$c`sJg;Eo!N&i08a)dUyxqdKS^55=2Rt#R5w{J`G1g^Qr z)919y8sB`K(B9lk3x-(ksaPKCTW-nhBt3pQ?hTOzv8-%t{vf|BAYb1TKVZ$Nqu#Fc z6uC_#5gv+#F6-@K)@Rx=FG;G@M$+_!_usn-uq-3`J!(Q5kryvw=@U}jk}RWnR~M>g zez2mTD7<-xPgr_$`vUMdadC|Sotr0hQc}zZ2MRvz#Xng4`+3B^voYbqeQM5#9DOKN zua_defWhc5+(63dpw^n=>AxpzaAp2O>K9Xp5K>{tcR(j@FP8u$Ah{d1N4>SB;MZ9^ z&0;@R)?D;NQ-Nv?h5qXjF z5a7MA-e!*EdvAf#NTKw6{m9Lqj0`KiMtmAZ5r#sJ&TJ ziNcO|^m9OLQc``VvkTxwQNWjBq}ka?@1DA_ihO62{Q9y{PAIJ^5*Z4hlPTb`!F#s zA8&L(Vut1O@DSB%gLnWko>6db4J9AX&-RDwT*vySr`YmCtb(nATQ#k%t)-<0tqTII zQUZ5IiAvjA^n696={@xt5C!-~kB*P?*d+%`LH3o^@)AfVA2$9ql&b`K^R~)bt=Kpe zWG04ao3~coHn~yRC|G%?!+;if!HU<<>o_V4H zx4=&dz*K+_xQTRW`=#W9daHJ>vSE6f)Kf`iDys{u>S4iK?aaQ^+0E7X?4(y@ zj1>US<9GclPqhJH_~%uS4zo1Dy!Em&HN;74E&OVAHa86tS5 z%=D+HPD_Zr_L5swF${&B^!wcHtmB5rc7TrAM>P`0H(mYlJDTFGkMjq)KHDG8Nn6LHltqqDRruB`V}=ZMqtm5{O$ zfBQmTp1HkeYuRs?gCkvtM`drx&S8Y=R#)djQTzIdB;JSY_`|f;bC% z>EK8;);Oa~PLpyYEcjL3gi1&|@^G^ur_f@T`Q+GG7i@BT{M&G_RUywAq@+c^NxhLs zk~lFwp7+%&P(OalBOsjO#VES`+E|7d>F7kcT!IV@V5f+3z-j_YG!PK$bu=Gh{TtYG zT9IJ6geTss?4Ih1mT|=E#*+@yWh8-*0b)oOhqGahWtTMzF=%o)ygoev*%0z9bUD7} zoCMa+Fkn?Z8>m zLOZ#RKg{fc13~04Jd9evxz`*>vKVj+5EfO+6T~~VPnAnv8ff3#pmT$<(K$XLPk20a z*vyRHz^dG5aCQ8BXp)XvSXkIiUV&K9!_DayD40Ok0{C_Sjst2Vhki$z)jtL!AQiC38N_RBJjCFoaQVkMVtuvOx(x~QKS0#tr%En(cj$yDs{ z7qPe`TX_PYC(*(pUyC456wLERqquyEmwfcjC1=g(h~-iMFS!DxE;434`;pH@mT9xd5@X%!deW)QxE zq>LmfC;ps$v{Z5z6?VzNSERqrtp)uoIu8^54ZJ3|!9Uas7Krac^2NVGe4TkXu>Tx5 zBkntSsLqB&KAA?i9YcJXn(#hp1K|afXNQ-{c$Y`(6&6==pW;)pLI;N5&wXia8sm-C zTzP!ixGEPRckHt8+B~q&e)I!LJJ^a_4gxVY`YeRuJDs!Z_#5SoOj)H#3Q){(uCY}J< zPx43M{9v*&yW34G2NgLB(<>VsyhFHsKZ=6Hn@kukGs>axi`u8-fTXtuzdmhD= zbnWUg8*2Np1noe%@W#N! zl!B`E8#D&uo5rRp#hZlH99{s?HzFu4d{*E{X&&czwB$5MdHghxw%2`JNld0h9nL#C&gpg%v%Hgp{On_xxSdwEfEkG6Lc)HVh0aySLnx&TyNpo$vGWy0Wrp(&R}= zNy9KMz+eK!EvRyJb(1j&K@49TWYBv1PAOKlf`4D+@O-Ub_I$e8KQ zGt5L_Fn2XiAARYMgGNPz1jVB81;}Dw zI8tUzpNnx?P3uK{7!+c-1 zlI=WvJAeo=Gs8?xd#NG%V12g+odz0LrG-gOJsnKu$8$4sNUYHzezbBKRWx~Fm1K+a zzPhfi+_~yZcU47(jlLZ5b3fS&|Ebt8?KLJwMt%2!{idqcH_T{cYOnR=4Qqt@<2XN~ z^AC z)z3}ob>ka=%nygX0)a7mFp#x^$AqC#fmsn zsN+Bk*;4?haNv_)aQHJ-5m=rKtQQw{IZ3|lA@wa-AX5URi9O_-& zL0GL%;i|EjJSAn#-QOUm{E!S}pz=tn;2AlvE}R`hKD}@VEC!nrOgZx{G2p;rMGQ$m zs2AEvRTUL)1#vWH6ts2-%fxZ$7=^$|%3do_pHKqx^%IA%KYG2H^if>-Y0I6rw9C6j z9+(WUs42f^yY9(g_2~>r-NAxg3TIEml zD9ApJao%VG@5nH`@q9`tglEKE>s_GE{MpJSN}E5(5cER@+8ZS4rhZ092XJZ9O5A_8 z+`X+tsgy<;`9jG(JxKpZ4e_$hm+ z`@eoU@@iNNBCH3V4w6)~LCz6ixj93#jpC`?b2gMK*XNY%PkkhTlpI)9(oydLcP0L0 zDna{#;A>9D0a71oW-gh|Erwkb+N8cP_mpth5A-L7XDxGBoo`HsS8ub!BEON!Mf@gD zjx`cUi8m_#Mwln%gD+?4OV=n$9L@Igd?i6|FW~D!E7n8kyDA(_GCpjB|6WLtvO`Ox$XJ?@bk@ zfRr<0UyLSBZyN=nd62T{Hi-qxYsXtx#i?YxXtvw7ONI_~I`r={SUF~tk#iU^gIjOn zFlv#9?gXCUh|{TNJ1cR}@cOU)tbsVtaqs^)aQ{EZ_W%@N*Y(ODKQlLR^Z2HZ^Lf)h zlQUQ;jroUT44iqVTQyKp6*e_B9jNx7ow<7aCBnliYTeWo4;BR*1S->VJFBKKf>J)> zv`P?w9R}MfDpxtS!R4ze4iDl4OBNG=2xVmbr|jFXQusl}jdZz8tf41RhXU;ZIWexB zhmk)U_q)gn!IU>Goyb={xriu_@n#PdU>%2MFY+}+1}!2`*klNbJR>EyjB|c#i_KUJ zivN7MEnu=DmE4BgQX=%Bb@3}BAEy8xmoMlar#N;(KErh@T*~Y0i`Z3lbVvl98KU+< zr6VIF1J;=i1Yw;{X3fhMSd-2{8K;m%LoR6{@C~vUW>iN`O&N!xHghYTrl6; zco4XpZOiNT0hTU$3VqrVQ*v(dOt+CEHUIUHsPBRChli6#u$nm>p$s~o{E>?o89ucsaogWG z0yV0uA)T_B%8^0Q4I7Bz2Zdj@KW@Q$nNqZD#qKV z1n*0mNDGOc3kYQCpOfj>6q5LWBwm|xPfG;y*K_f92k=wc6?zqbS4`|*mEzIiRESx~ z@0USph*=ips~)S%`euz-6l7xWf{2$6?|9v6K7PBv;(f(f689Age>??k7RrxFQV*PL zsQcDWR9Pe85ve)9P;vZXw1I$zfFP8AEZt&TQPCU82s(3a6=?hpaS{Q7ksW6|}|0*`F#{yr=Y{sIfGO{nj7vDs?OJp$>j_aQFH*7*4o zrAJ#QE_&0utRfDw+s9W$n#Ic{JN9UQ!;GwddCBjv(&sa?XUA=e zOG&%`6QYO=5#ujyS{4EIZoXjX#GE< zE|D5e0N7Q*_N+Jhv2>B0=_jx?)=Tu9;Y z@bC^|4pgZ?{rc~bJwko-Qz_8a5m&2+E zLSxXc_F;h0(^@`0d)E{_!n>kf83xTWo`E`P7DcJN=ib*6D?x~c-F$Ni5 znUBTHeF2jCbia_|j*yZtRk}>(SLvBy3{HvU-6r*2#Ai<)}>)(M~3ZV{2gYNFmlT5t7H2+5|JdKM((K?U6R5%cKn*f09 z@>xv{FcDt(zLuuSRKy3+yN8ewm?LDV3iP($bcjlLf0_1uDc_V!Txb)T%~%AmK{81gnz@*)k#1ujx0Vz`uWf86IxF<0=FU8K2i%HC>{|IHrUS87(7ZIJ%qeAy1(z~3A_&k zngPfKkd^=e4XMceqFo7)k$8BBh&f0q6FPAJ!^huF0&8g}$Q1zs58%xJCO{i`dTA=) z6x^-c;z{L=5WM>w;e-KrvW~K{2nhMs0T~Z?F@aATocE}(Uqic$TerT_6$GX4vu%v+ zWm%7*{nXT<@n436-_~>2dh~O0@Xd-?$G=ra1$ zV*oI!e0>wrSo|w!THt|G6oBcuK)X&310&2IHW9|3)wJ2*UyUq{s?&Bwz6d*pRmu$;spCg6lH>V{%f&T8Wx2IzGVvo8m86 z_sT-kZo21obi&R)q*mLyqNv%Ol&jljdH`^V-@gyV#?s|y7mjs;2^h{|mZ@G9ma_>8Prz0z2j3lb*Gur7_4c@(L+T z_n0ifo{25DB!vpacsp`)1TSQ;LIoph?f{ugc&ASobwA<|@RdNj=f8o{Knj2#8S$yR zXm2=GGhfk22Q)7w~|1B=EvU)Rf3aIYgy}i1t%B^>$lMZ;P)7_WLuqI0I$`5SE zN!A5LCWu97Bgg>mO#GFM49VGGD*+Z<5l9KIr>11U)pl62Qcww;U!C}gL&65gM0=6l z3kmDRxj7m2;hlFEQhCSBAUWj)MvM{IPEJm>+QPCAfGGQSzX9vy zwhwfSS0^+^ZNFZwHjJw-i|}2{GL>XryN)$2GL$3&`gH8y1#_8|rm}u#C5Q_Joo-!r z`KM0;!bjW>rf|z!8V)IvUUqU&h^74S!FKVeZ+uwhPtYohQh;d}dRoNbW zmX@$HIiC`dJl)$CNv@VSJ^j5ARxZcg2obD=>scH!T70Edo;nBg$bbqd}%Q_euh9$a@J4oHofF7-M(vGtFg%{L{X zJiyaO@p1&v77aj;XjzO5J4%lcusz?60hgwW>B90GMJ|v?>nZ5-p z*1_US6Mh5CSfRj&yFVZ$0Nm>kae4W#AEJdsD@hznTa1;a*!+-TEuds(OG%POPi8Uk z^Irkm#@_0LI1t8Nf=Tz1E>cajF8dmS##eoa?Z?Lt3Ov%tngPX3HQm*MDV41^dk*62NDXP`lU>@lp*_klE=Jk z`m?FDETP8Wdj5wOC7^@6f>;O>PtsbO8fCbI&aS9sze3#g)lZIU{)gzN+0SFO&F zKulEh7w}{5>_i-cX942R@>)QE)D#=NJjrzDpAK{#>aixvrs6igPj;Fz1sI5styggW zns6*sNjEnhX{w}{7@!dlZvjDj;EBowhu^&?C`Lv|yqbD?@({4kacDI=m(7$ienZ&H z8|TB_-JN14hu$|1KR?Vi_o}LW(r{}b_vaWGeLg(*Pw_~Y&TjraGEc=Q zG=!);i71Q{N43AXiN^Nwim>v5j57-g>8C|X(bhza;89bdn69*I5V~$h&(F81T98r` z`-Fx{6<1WWlaIulZ!(Xz`?Zlp%xdg5PRv8CT)9H0)Clai6kKff<<-+fI9(?Kj8XTc zD{GIbeLla9OC-Z8TzUZ}g0IBSk6yigKdGV@hq++&=ObZ@F5;Pwl9QXjI zf{O6@s314Dg!(X8iwx>{S63IvM%#-QnqU}jGzgBw$H(7AKV1Qh9v^Yg;Xx;!sjvEhXR#3#fid8@1z)QP+`J^izu-li}@pEiVa;; z-oO95>UZK&u;&aS(7;mC@SUeF3F{L0rmwivf*HG)m&1MfMG*8S#?;cpZ4?@;&N2hf z%YSnnAA{tW(*jj@gF=UxChxDdkhoJvF7)IagNUrwtcDOoqkH1btIrXlG-q|H{}^LPy;)|;F0?@V+IZ^X?9BMSh@z3+9b_$^baabiJPvG2aNyxC#Bxp-^PRA>)SG=kh|5C%mlu2{qjc7P&sq?!#aLlQ>Ax_rs z1RYyj<`S8V6|)fKaD~*-b2M1(oJcly5ruw}6-O|taW%$!#@bWg#zyE*u^z zVq)jpujF*DivC`WL*TQs#giG+U(L+=J`8Mjv9Ro(kNSn4e>D2Zg(D9vx2UM7M^~>F zmh9#`y=I23DY?2wX(s)Ql*^~6=0bPDtiX{}bRK%4^eEPdP*p={hYz1E9MlbO5}=_e z06Qk%z3KpW|1w8{+be^Z3Px0A@-xtUjBgn00;tBiySjSeH@b#~@#d2X)30Rb*4Ise zrcb*XnM zanvc5ULAkJ$qT=(KI79Wr4*$9AeGOXj2i1*Tn9u{z0pS+QGnF!kg;3T$o0gMkVREhgit`}Lo_ zFSc}Q-^^6kC2m?qo!4N%qN1S@F#CamTJ6xq zc<#Ok8z3}+8tINBV*HFp!-kxl#!gQPmIWUp0ba-GIi8)Gx zxGHve@HpYLGq?!JtH!eeaA60JoIFoO$Jkhui-0dniBZG`WP6b|1>T(w(;lSfo!;L5 ztRu@TP?EY_1CB4SAP`u=uEg&8#r!X4b(#VOO%*P^Fr|PEIH)2ttIUl4H%T7Z0yEO9 zFXbTaXSww4=^GNnhm5&zg~nfNdL7-hm<2f;QBkFf-IHaputR07C^=SE5zJhS{A3T;h!m5GY94+WpiATUW&n>Jr8Fipl?w@=zVBGv(sAy^r5d114` ze0y~~EBy4neq+90H8s-QXiZr!bV@Du_IGz+C_p8 zdKhxZ4+wvjEfvt6agm>j_@!W!byk}w50r5n;p#TPr_UMmXAHbhg@uJQjYm+}LEzmM zP08$EkZt-O-?o}^Bn=QFl#gku<`V#u#k|gogK^JC&PETdU`$}pJ!P29q1t;-3olS_F)3N z)4jvM&Bw#bdvWsc>*OZCLAw@AJFC7w*=0lIdQB|0OJ*_P<(QpV-N!2aAz(KBC&rx( zs01Qhn}cA;=_G#Yg#sTp2uUdUcXr$WKRp%g;Z2yMbU~H>wjJYv(+eVk$4Jt`Jl*a~ zg)coCqyA};1TIyQ3 zE=jAaD-HjnVf$krCZk_|AZUf_qt`5+6``G95VF4GBab@G*ct!>WO3ydkVB9sL>4E4 zj@~a`2X@3E8t@i`ZWLn5AliY! zaiI}dMj{N2UYH+i{uxT3NWsnq-sK94hdR9`)Pp6a=Zcd23>}@Fe{XKqyZv2_NI#K9 zfM7C;sG521ZIk;T^HOi$gm>3nt~-|tpXdYx{;4ju!B0#72||o;wjIy5zor1`zUO|# zxaU9BswPEy8a{w%Kp%T_g$TB4yiZSW$?fs!HohrL+t81UTIlWP;03uGAifp^aKsAX zUwP~lwoz>I$Yt*>*BtwP-|WO{HyVk%R5_=A~M4yhpJEXK`Akg-Y& zcJHS*bl3kmeE%g#ZQ`R=^J5#=UT^i|?uDUiA5#DFzwj>kk>~Gi9*m2%H}c55wxt=t z0}Mha3IeXOOCUQB_a)x7^dyv*QL;HI9Jjv#a#m}=bTk)U0>(>5Mk~u_a&|3N8V8%^ z1`)W{>=Mlkco(^unVCSJsXqL24zYV)aptta?B&z@p8=tztoLb43+5ok9S*6TJ^(o| zr6?$Yw!!-AY@g=V)=VV$eS(MgX_}4xN&%t`3{JT?aCTL!VSssm8J(a193ad+hNLcM zW5a@Q5nFUlI5Wd~@|eOs!_wNrhRZ?y^O==_d9;DLrY3S^Z$MM$WBie%-xa#fFptc< zac)N^k(K}=E9RwUyKPH5v(h5BUtK>barfsIppK;^BHCfb0%n!@hJN?k3)`2G)b}^D zMRS9X(#~hwa>n_2tJMIY8X1v;PQ|`*K|<5+*diTZ zM3<&A*?1nqSW=(x5u2D5N9=+)Pz3ezE?tn}ABd9j6NeA3yoe8KVtO|~)O&2FsKb?@Si|rk2{gcFah_JIO%TQH3k6|VD zpbY-5!RoyQ%m`f-uFn#~ngmFp(+mx~q#Zuz<^PM%jE4ZlQcd|W@Fh@8Tal-n@21M( z69%Yd#~b=8@afmp)|P2SULnG#Bk(q4KbyPXtkh^!aVNjqlGbd@o)KAGd@)bb6LH~= z=PBjTQ8*nVRXb;Y@O@R3=sFxHh3EBr24rQbxRRuNv&46nRg@fy{sX^)cHPYv7H4{f zQRn3*EmFRk8bSobQx3a08i66CN-+hs$9X?L>7W=Iz85j>HjNTrI^yCgHc_{x)KqEN zK7^Rh(qUDoMs0zB)T*fw0|$jsjgV+{>s(&pr6c;_GhNN3k0>z$owEq!h@${L1DMVx zOKZ@b@8#@wpfAtO&&OW776q;5SB@t=%^>#(j0*Yr)SV93zk7>g(4XK>qpjSvbv=VQ zJ11u_j8<-$QpXB?=2pb-aZ=BD_l={h4wc0u{j69L4YGfF>Y}PT2>@_=QY=)^h=WK= zmQ;TDGy*vpcu9cnoVf{@3S~als_iT{SwD@YBm;`{O-Ab_XY`TIfGO+O%>KuVEqSSp zCc*qq@nX}T@;_-yY%;jtQaHWQ($uV%jzUS=+-nNxjBZN6#wytkBHFhCnL+M$K|%2~ z$d3por7XA}5(9OnH|SPV$SBgH z$A+t_l%%tX3SB$|ND>G6Ha~B)t~+-)RpP&hl=>j@3-v=RNvs5QF8XYLh(PY#E&cm~ z0s$8MQ@|Jp1ed#jxOfst35mdm^8$d~q7;UyHv~xN0y`QwW=as;TuCTReq?iW*Xy1T z_lUWV7>ihkrT<&2aCm52$TLz72xWDB9l;A}zfe!4THNzyNf>Z@)&0-`4lqkgvu4JU zM6f`E&1=TRbtJ8)teV^d`TsphU^K%1X7AOz<<=D4x@f!EjSa;8v zyg0EUUOpJ0_Fhi|_snLpP(b^trIIIBTICP=X5x1}^gk5Coo^hK}C7x}f-2chb+SV49Z#m;~D{jsf zWgM8D!H|d@7tH2iW^Od+qq?KtJY_UKJ>`i8*Zw$~!`Cvg$1@FoF%(MUi%C$X=MI`P z-s|c=I^O5yrfOhD7XIpqABs={ohoHG$IMbJ#T*z2VP`K!E@biysrY_fNb_b1`u8Sl z;rm+96Z+@SF^#uj^`z+ed9Tyao-6(a2`Plk;@P|JFx(n+76a8HV$PU`gU?=ZzKFRs zwWsC8t~@@l`TX?oop`gHs=Ut!@=C4`LPjG>bIpQoHM&$luAIVPs%Go<97!pt49O{jU@A1_)rh z5J>s|zPv0_qQk-ZXnhtCH}&oVi0xVkLu>8XBNfAfCzjPI4C?;2O$KrYuOZNRYQadA z=U@26ks$~P=_~zGSnrRUPb9@VadGL>Ce4k_k1PiPQ~`%+rm!5_mi;IzWLU1^cp`a1 z*`IJ^80AOLuh9-U9Ye-{+%h1?zdkD6x`L=ZGjlD0H#2#MY=LC$tVx*o$JvO2LQqhQIDyy9($Y)rO3Bd2dt*_Wi zqvP&X(I*O1xW7lqBF73%YEYi9K!q)$j4-waGD6W%MTxRVemlIpWJMrfn4z|G_T(vX0|C)l$p**8BCEIVgDJ->=|(U zuw)t}(p;~wzrMuoi~&JJl8_R5JN zCfwY4KJ>KE%sp9vL@>Vr0>iz#kPdfeL#N&wogmJQYT9~sLrTSt9}qA|R>f!=JQM)c za$#z~9Q5^$7LNCewtXuXJg^UQp+(}wFv*UWQzl9s{DzinsYff!-*0Oj6 z0~RX_i@dTj7%`q=W8c%owS-$f1sY$%_uWNBX4y#c+|au4&vQ?%6PgU%0biB=l%UBK zg8d;?I~{5!p8(aZXKQa&Gr%VyQ4J~gH3(p%E9vZPuRypAHHl%=(5QGI8jQ^CL^%qT zauF)rzUYxr3;l+nIT!ne75ay z#q!fzwB%G&T4!yyp7O_%VzbE-P+K?SKRjqWiZ2MtgSJXWS{m$^z5%;U!70&u(q2)q zcU=QkHhFY|s2VWW12Oa`_XB+AWTy##$`cT*@^@NQY}U2~q}YH7LC(?fSEJ8Ii(q5A zqRtKRR~S^3GfCsm$1jnbncvt-!Y zU|Wqp$Bo&Kg~>x*fw(V?{i1v@)v~sfi}Ct5E_NQ(;%u-@BK7OoUp0UxY;cmfyt?9k z$r*s_`fIrzL}&sS+%hY{7#|9RoNcWif71kk*w89h*U1g{9wTpuuKEBRqyZ490D>_= z%EDIW#(=U|;Nf=0Jh2#aqXndP)JOZ`p8IgrHTGLxSm=O7edk||MA2R1z^xHgtF;;; zyg;sPD$BC8)Km=+vU~3uB@}@%+^1#qdgn03lF8PIBuHx`%2p@L;lkYM^B*f|X%qr# zJF<(;wR;X89yhX3W_FB1Bt*n_Y|F@OooXQ~7RHC44%tYEsXCw==mCt`N{j$T(bC#k zRu7MP)4y%Odem@|b8@2s=m7cot=wCHaa?b?cu5o%d=(&um*aXnrXGf#++1z;vXlGVI|+^7Zz;r z_#^PqGIr-0J3HUgM+u8SSykEoA5&i)RaN&zduap-rQ;H!NP~290SQGKq(Qp78ziKq zkp?MgB?Y8G8ZO=4-5~uo-|vk#p2I&KcsOV6wbz}fI=brNUxq&rmU5nPeNCa55Uhp~{0BR2$L%$OB zz;Xac7d4nt^=^JZq(&P^#-yfdX=`hL`t&Jqhmrn|?zG8D}_< z&4`jK%E<}1K_#^08=q%>Y97H@H76%PznfG%gzv9H$Wk>IwSKG5g(;$W>#~+cz(2Bn zb+F)lIS-VYX!*KE?#j#N9`c9z%3ksDX_!{}+Lm%40rw&-QrbA{F#?b^N!9#Pd!$L1 zFlWHy0SzK2dwXhZ(8Ghb3_x~0WEfJqy_>CVxiUgkT~Zn0GLPuU(zeE1%VcYIXlI4z z`9&FV`!D#^51kyQ0x&&nc_HtE)ClBr%upYEFBU{-2Uiy^r8Zv>%sUVq)Xt%wfA>+8 zcckheuyz+aXQ>NhUZAMi-= zz8J;@2C{bgUoYWV=zn(Pc@p|W2W@YJSedlNy{UY9h+R;yfvK;8sd=Mxd}rEyBRP3< zdNbL5Krr*_X*J~{=7ln2*T;i1D{(qR|ggr3_^+adkj@|^I(dD6at8CKt9dvkN*Z0*%(rRto#p2&dDe38v-^vq9mE7wbr}uGDdpZhETt-`3PB1tS-jEdZi0?Cc1Uk~1Tk3WAHFT8O< z6Mtw2pycaJuL^GjQqX{`t*;OC_bZe(2#!`4306pz6&A+mZ+mmE&jKvKZazPWFYCz$ZYv#6Cb^Se5PiHVGj7!7n^2k-9`Py93lwE(eRBSskLi@DN2uim zebjkRWR;0YJm2=;eA_^NC9f-o#saOljEs!rjFM>Jr{H;v{awB)*;ot9p0mleJthB)hRC@GX;N zkSzBqaKc``ehnHi9!}2O8Luxu;}9FGw4T%xXyHM#fX3P-AW%_hqIL05b*r%kZU|bAhK0~M0=6Lh!SHln<4>oC5UZB^_y>o05MJ6qx z$hy;B%sYkCxuqos-JF~S>x)<)*@2Vw@6u16SZou9{`{F`5@5fgqr(VEk$IbyOt*ns zt$B2h*8T%jWS~D%wUm;;nmyF~3UtdX#7>oEiS7ocJa8CT7S3D+uC69_BHOQ#chPi} zmDO0QtF>ybH&Qkmri)1x3wxc!8xuz*osL&J_3n1mg5SFS*tl<2+D-mZp3+={X~oB) z{ac}#&sHUedm?e6i81EW2dfcnr#XtxJTF0gUY6FI;*g~Cu1gq)9jq@xd z3=ICr1~I*n{Tktkyo8sj0oVPQ$m1XxJQGa@7tZYsp2L5MY0v+02@f?E-;C{hPGbb! zz48+y)>0tK5G=qVth+9j4~2$tg|;tO4OoTfFfrqMMVho`Y1)@(+FO2?@8tggFA4;HBzLR|YN`l}y{QDGno4b9lxw-CpDq5mM zgQ5N(Fu5@Fw5Bt6dXCp($QUilCMClxWkuZ0p}E>DlYq#BlI7jd3vmjOlSnn3IJ;eC zENd~jQC&mODB9p*u=xi`+!fru7D_O@8j@x>x>7=#l8ob4Jo{BW%2Y0-=YORT_0xS-#& z?y`}2`*p_ayqD`d9?bJi-kxab$+M+1hLOm1b-FK*Zy*A7qQAYVbW->=c}brau1t6O zS2Shm`CVAHiAs{^b$zSz?!d)e8I!}dut>`ht+IZZHxFq~@gP=WeJ7@Nb0tl6$+3Nuix+q1K9%R8ACP$_@?LQ1x<-Eg@)DLq#*-U=q%-97P8hDs{A zwg7rX7`8h)PO$yKJq^fF($NqBdNvsKfI?TL60DyB@lcKIQVciK59d8J|GqhHZ~e4q z9NCO_eLwf*;+N_`j1`~taaTE1}-B+pbqQi6&!6tDYKjF(J7_(+OK zadF7XE(_mJ=VT|8>393I1Y!N)GWGYkw2U&3*pB!zE|gVu;g@1jxra+?QbFP=C-HnI z2^mp;KCE&UI!cF~#^M#+>kIIxLB%4+=1Vi|ql{CXrt+=Q($5^||C+EKAaz*gU=Jpp zAg>;;4@*(z^_faLNpYOrUL7uaA9l!z#O!0kfnXk}#sE0&60os^P!AyZ&dd)2`+5e$ z$6{iT@F>8-2Zan08up`V>g+?{k1TVMe|B-BZf2>-Xt2?ds_H@*iNnNdxj`bXAS`_N zvygyy<_Nc6Cp|sJJ7~MROH{lOeK4fTE0_68YG+L#dK(%wgmWMPO7E05uAUg2h??=i z{C%fCDsyp_yI0p=547`E6g|KECzsx*wh;cG78WJrvP0a8j{>M?v(54G@pNB-HG`#v z#fBJJ*sWKK2eZQk-CX8Uj!(hMK_f&+)d8|Y^*R?OPfd-<+Ev1rc(jkrCiHX)-d65L z9cj^+s}M7K$dy>pPfbvVy6O2lU(cL1T@!c z4Il}9yY^aSx(R}{1gL1WWoaI8Abz*TD4|~03tlf&y3%IdcPH=VZhH;n(98QZnYH+K z;)mRrs6=s3jqm4(7E)5`PrD`L3_q*N0~L!3D7?!}0l zk=h^nPl$78OQZ6rbKi^xXOm)@ot%etRo%5hIDJW97e{`8=*=8Lm@tL9y9r-+N-QBh z*IRfyDrUKq<++OqkJrwm|Kdne*NvJri7dKZV@&|NDQajCh*wMurC1L#Os`1oDXT5+ zPw$yG&h!A!a6a0B*D5QWFF0cmKQ#JU2bBWPg>Uc0+OLtIDx|86o9=lz^5oTBh*<{I zy&zWtMS@b5R?IA~K){q76=LSuBsdR$J+JA>6T!4cOFm~vE~B#vF>}9r6$icQ?pQ%D zACq;#Z#n$Kq*G4nw*al5ye?%G_VO&ql<-S&+(no%>svdh)O}aYUsC+tFrb|=&o`$e}I)=R9>XnZUJ=152t zCwJ~ATfLiW6(m$i8JAotj!kLGsaLGef2H!?tV?TZIAqHvRD^c2QDOdcc!3<7;#b&_ z%lZy2?m0h1GsR+s_tq!+lVmVM=r_l1$gBku=81#kN~U^>-JNiz(?7d>{Ll+se(I5n zz7k(v)RT?(vApxB5Y2^3GCyppEiZcltCgbW#0;7qr=xZofDT?(||Ed@Dq3OwrSRDQxXoZ%Rjmufo zk(-P56@pb@VvFp60d7xD*qL8JJ zG1-Se%qPPty?AfGd_<~>`XH^Hc6lCi(b&D;D=WhjY?JNNQg_-ZvT6Y-O=?P5;KB)E zh0TU%FY+D-qCSCp;3ppZKsD2$ISzSe>X-7qS$`E*PJvcx#=FrmY6Pd&wayr#RUk!tTXJ(ydEfQm*1Dg;4XMXu%5L_!Z%P;b3WroF!a^maEDb>V z1+>Y`qi$_xnj9<)im*Sf&47a2HZVFm3Z%OJw=)D{h=VMw|mDHyPTKFgK|%^Dq1d5Dzvs3)4hQRtt z1o*V4R3c!0&QmJ0Fy-g5->vot%9GfK`Pg8mY6?EY>4-;I+ZKKkSH1M=6pVKDxM{ps zh`sFOEG{2A@DKOc@k<+U=jV4u);EhMm6f>V{KG3{&zkR(82;SfCOtTmms_*TNhLCsw5#pZy zekKM6z^~rj+arfUv0h1RU0KY7ik?+ZAN?1Lxj5}l`R3OMU3K4jdKPR#`z6ruQd0@) zo?{F3{Vb!R`Eu%?aV|sQmV^t9GrL$vI1^(yNkO@;qGb8aL_6t0P4JwW{=EdR3bS|{ ze*Xcd>W5!Y#1MK8sb56#N$6{R@{3+|h}vj~b-Cc5+Do&(sT&l#vg4SNmqzYk6D9aOb;zqXj zdrtEQK6f2xPNv&RC{Qv6)Zi3L6C?c3D=RMsSmZ_S&er>qn2mkd7RJZNfq&Y^kBRB& zK=zPHdVXS}rU{=_jgB!&h-UH4>U_HGOO%OOQxsydt@+$eHHtJ6bB@QCYinlnKAF2I z`cvIUorH5hdjgWIK!bVv@Q{b7zuT;_cc6Nrub4~)GWTqwwb zA{)UB3BH^N(nD^;Vfr_b2X7=eYSv|}Fr@+^wA@&;R?XRD)1l6zt@=}0%&@MKcVji* z!Ij#0p>jc$=_ODeNJ&WnQ$)a{1a#+#i3tU5Q6xCtuFq{J*GU2-~Cs9dxv=S@iB|F&jKu83RLYX0Cra03yI6AV^3{OG{0~ly`M<`YlGhhMZL} zDRcw>hr+`SX`DRUJQnd9KZ)Gdvia|QLG2Ix1KZTWVbm^R)T`PIf zn>Rv)gelD16R2be+QenYsdrP4N{RS2%5N!lk8+44ON)Qx{WX~5&p4lL^fcL(>Gt_W1|nCJ>KW%1T^qkb zQTTSDMfg!#RuvNq*b`45ANL9|SW9ZB)#`WmDbvRdj>TK>g|u+nOHX`rA-2|@lS%zT z)kI1{vK({PU}<7v;^b6SP+;oaxVUNP+C>?YMjn=mxkI91tW4N}E$j0&uWdT_Tp)n8 zh^cyNJdgXwlUB1G^_#cK9$LHmb|iiatM!*FwTE8+^8b6Ey0yXa0a1R0Q1HFwh0e$D z|aY2@(Bm+-&wm6L9^BUYmdXDuk(QcxfT|GW|5F~ zT1s^uCd`O_RF5|dN(3T!0Cz1Q$_tpMKvx~Ci-DHZ;+=Pt#4^%uGP4b!>RM zHJ@>!Olz&S`VBpA2Z`c6$kC7;dHg$-iWyCPZB~)9wYy_wY zwMPnRx$38rlNl=1-T5G-Aw~dCH5z`~@I#*NiNyEYU%wVqQTd`zq+@usb>p{FzRdL` zape6_Vtuky(dur`f**<=OCPPNkj0PmR!;heHi(o|#ib*DoX{H~Wpvphn)ZnUZ|y6} zGB#r>g|Y;us4Z8vR9lT#igMY#ruMFKU%c|Ft}8&q2c+D4?GWJP72L{gXSk;u*SmqOf0F4MQBn7J;%b~UfZg5HiVKzKUqjh7*>JNU|>Hb8$@AP{_c()CZv`SYa-jf$xK)OOv`4nprxlvsH100$j!rkFPuBw!$Y# z_pcu_0(Zss#m|Z@H!dCK=8Afto#*rKXt`LeJQ$jF&IqUm>xu7hxcipiM?Wom*x3`0 zgEj&a#dn<25eG`kpP*TPu-tX9SYMCP{W5K)#=xaI?-Mf&hBrB1g-NNzR8_~DmX5hs zAXs{BpV{U5znChhM{*))Q<%Q#U1>#jad)7W34PLXjHi@qVt4e*DF6U2=o?7l2zm^- zt)hB<2RljxK`fz2czL%SR+d@|#I06?ww3Qj@UkEm$JpS;lGXG2chNXud$&E2 z`SiTxa3Wl0Z8H1_UmEWXEQbnpX1H$IR+S`szv|LP~IUwrFEkf3H$$c-7|< zhq&&y(HH7Rq&30W=2W3nhHUJS!T7vk`Qdh?%t~=Q-I9`IM3ZDX;Mg=DXQzX*oYy=$A_|>uYjB z*lwc_LW)n2N|C#}gVRyZi!4B6P_Lf z8j9qbnl~2F_QHnE!~5AN*md{W9E1+RavDH!0(6Lg4k*~+;^E-|lU7g~9(c(yoz@hI%d@oLEc1OtdPU>5huDXHHQQwdMV z;#C{^`v||&6a2m~OLPGb#1sQz>exCD_SoJU?X__TXn%3?20nH?GFQB4b!so<{9snucu`;aU-d10 q#nnNcuBVMTLt9Y*fH|BiZ!g zaPgt>(EAWgJOyryldN;BWP_1&rV z@!Wf@hJlMGXZ&106=aBlmK2?FU9iFf5=1+YKLtCdoDl1(N#vEO#hRV5l4q^Qpf>c#7wV>e&H#OhJP>a0{R7h$#8pW6i>G^^cIz3X+mVm5Hx7r(6kn2_) zKBJkyE0UUm|2`~;zX3fy_tWTS#6v|NA|-#(eU5~EsNac$Ek9c$FX}JK;z{{oGDP^= zAoYc!^+j7pV2I;WZz#;@$(mOAu(-v_XC$}thU=cO_)ASi?)N_msCNcYHU3@qOuRcT zy}Z^z3{KUabFwq0Bbfi{@2`Ui2eMs{s}kEOK6kr%4}-2vr!7c0Z>6M8@yei-l#96^ z+!Fy=;{Grz;cEE z58v*qCy@~~r5##g9V$9HLhS6{eMb(W+$o+**Z*h1+rMr{MQ-DVIyiCG@-ycZHVhT5 zJuU@ORbF+IHD3h9fG^Qcc|6=*ihzLhC`mH4c(}jA-Cw+$U$jVCP{8yUT0TVL%GQBDFkfsdwjVuK!S@FSF4ine6bHnZ;H@UIM}K9?5hgYKMnqp2}<2u9$q$F?$<5~ zJFIns<(f596O#>qPiJdkyD#~pJzB=BJp0CoN_@HKD{{O~+s7U&%j-#P^1?C|*T-j9 zYw@JombMyD?UlK^gODCSc`uBYI^MtWCHh%h(iy@O75y(%cW_i50E1@A6CLP-;AuTO zi?sDxjgg`lHjYN#rWQPdi=Xwsi>owbOpsF)9SeYn=tl>VJ%>rT zjpW84Y=u1ej^z`ghD(xpL%LmHxuM-d@=F$cR!+ z^QBNZE2a+yInsK{Ftp~C$iG&ckkeJlhku(Q4}8=jH`7D^o-l%Z9q_05cAfP}VCgGt z4X~FLaDuPIQAPaSuXFDe&3l_#nHc0ydE~~VIK%^bjx}cD`H7mn7MtTdw1fYubef!V zX(oQ3I0NsAM$LHOwLt=LA+N!#`!+QRR&753@#A zmUTp6wjGh_;1}CxvP<0T7B(Uy&Uy^5X?>!Jmq$P#dX$=7nE4~6LbV~Q%KL8M&cbZi zo>kNT5uXY5vzC_DzTpTq9`>?Mg9f*m47!dPw5?7-Iuxn3vNT2qxndOU=j$J6OFf#w zBV2Dron0f~L%R1@NksR9;K&YNpPLt)&@?1UpglUly4^{VkyulXkNvp3k3ri4S9V)l z8<-@3{lk|-4p4x+SJ{D2CYpCH8a}RUb}vkW1b0DnfjEu5Z>{FP1&P#w9Q)slI3Q2R zCX;r%NJ<7vssc%}sfNR*U6+P^AMTK+tB*-0X%8|7*cE6Hvl}-yW15b2hiS*FRvaIg z#f%VT@(d=dkhdcv%NwHmcMPf~u7|Kbba48sJfrz=$oG#8WTxBfPBX1?p;(a+>cu)( zO1KLaaR5Xq=zfx0f&gB6>e$GNb2uTw`Ptc)1PmB50mvXPyY!s*$AM!&q20CZ>NBlvI=AmV*3g6`_*GbcLgFRE24LYJ))#zJX-_Ev@vs<@479 zTC&zpy^Oiv-AYv;k5eF+{qnWzr@B3k?Z|D#hP%Y$&%M3CRZlZ}Oii;ttJOV_`O=R8 z{b(lS1q3$xn(IBkJ zZ{RY?Hk1ehK|w`RC(90sODQSLbniZX+$s}uc>$voaCr(_n?2OcUdDxJ_GeKmcIh&! zlenE*@}I!Pabi9{6}v$(Lp%K3Cvy6l#EtO914xdtYRla*G+^Q46zZ?b3iCa-KWyY! zyd5ol2TDyzGTgJL;^HH&AFZbeiHM91eS8)`N}i{dmJ66S*#*`})lHb@f%(x8(%jiv z|8X@VYimpMP5}XA92~|kejaXa`l8cXcp5hyZ~z9BMWJ{Z`&&Xsg|#uRrV(b!1_zDr z^Et9~a2US94GW?fGjO@pC)Gh_gd>zhGscwzDkL zjG2#&1ji9P@-y6^95xzU48RfIVq5+Vym76tghWpJIc7%Msgo~WuXb4?RvrCn4_rN2 z?{oOAF1`CM@_Qr_r>5I0ZqApd&yE1K{xlVu!zNc|k!ES;W&v;kC+Y;CdO`fGWGt~OrL?n+mkowTr@L7t zi~g2H+GDNz*#)2@?=}i{2c`+Q+E`ib34h(|!nkRhgEY74T31$% zg2+l|Su&&v22S=sGcG1UxdtOkN{T?m`Z>(eL~L^@o!j-`;Nwdkj>@(`aY`-Srv8|> zAyL>ES;nudvEa9^9na!{7p$L*3lb8B-8l^Ok1oq<7rhDxOIf`(ZtB*MQFe4$Z&Yio zjKVx8h*3luXDC&8SX|5>u}?%sC77A%js0hBa4(o+#j3gnKUw#LE9rF1GioaZ?n0^Q zU_T@XCuMI*VY-bUQ{-TEY+*qf=^QJ~0hz|W@fXA@w{~0{vc-q{%RV6kpnxDGq%%ui zjtZwb1RHrWY-&tJT!U>DSx1x2_s+ksfN}IoOs!q>kOKM43Cv(ebf%l5cAPkRf83gy zGpaTaoa&mIJ4vqeewE2Dk?z2#gm|ITaxwbbT3!{h1+XdQI2*4EHV!$rv+(Ew&-Gj%%})J1;; zQc^-cR1yRM>)-wIJ5QP+%r7xgU7YCV#!CLM_|c+#Qeh&jszybA`ko)3(stUCi4WPo zw6t_#*#6B=6C$@Fd4Jt6b{s*&A78LBK4~jqd-IdUs01N}u|*f6sbv7^(Dq8FolQgG zdF<83I%)%*nR&3*P8FzY-)KP$#_O z!;N)OH`Z?Ufff-GzQ)eAsU>(6tAZvY9svH8i}3w2j;}PYz4XpweK5_JCK>k$^bCMv z_d3*TQ(*C#2ciGiO0t}b5IEm~(IIU0wx8oC-W(5KmPj}@8vE|kzj@5z!t+`RyQOJS zYL?~5Ra>w}a}I)z3y}h_YI52|dWD#5w`{TtK<>x-N6c@*EDjGB=)yH}6Ym+2-H%eX z46?$)&&4!TlC%Smw*rx65-rCfx(x@Gx3?MC7e_}KA!D_-*SZ6K`qAcgBC(@GL{B_c_r0%?HOaOdL)^vEWO~u`X(x zAhzQ0qoDXWfx_N|xz^)Xw*H0zf@d6?#tO5 z^U@I|u1{$xMHq&@r&MTivh-&qr8?ZWXdywe&iU*$`@EJRHCZq$HOtEYY_hFjR8`I7 zSt*jesP{j<@ETKT&1S0i68&R`?RFb6U&UgoeHfyRkGZMi<6>u?ppCA2Y6WJTt&vsj zOZxKWO{Zv;YCh(V-Gj?)_TnWlH)f~V_|eS%C`{&}`60P58_Uk8TvXr@mCyTG`|WDa4qy&acSI(jmb-=$bq0 zHC5tkaycqorbIGqKagb>idMiFa@ICd zcPMNqUiO5*kN}oufHx>X7=Xw9n;p4cyC{w-j;1H+8+Pr>M?mCeyx;6*#kR@v@fUY= zvTY?TtJRhM$)cuy%2C`oQ&YwV7Ol@$dL$)XM5&*hj(LJ}enc!VT;MH)|Amyyp6@%t z%&%ySK(wRn7)9<+Tk-fPt?+ENG;c6_I-MKNhP)A)w`3LVeVd4;f>dQl4g4PoB>TZ+ z3h@>6NKyP~DeHY=OGJcm9~N;6FS3$J!?LGqt?NZLp=oYk{-UNLYL}Pt#SD=i(W-4j zt}PXPbU6N#+=+lQBMeSFy1p)Sm?M_H0Ur%Z{9qEhvO`t4hW|OcSys#&CEMEwVU2tpu=oyIAOMq{zajoa!LJUF?yr>-Ih3rRl|GD(3_LC zMH0|%&KT9*#7jt-CUOapexkbuEh#8X_YMyiPfoabctF9{cjO3YEWo!jK7-7*YBSH& zMgj(gX21hW3mxuf-u;mrQk5ZBTj=}cX@!mB_+;_&T1kmrJ_t;+V$Qp(DYc=+%S{no z9*||KdcgS`J@#kj2hE+0UW1zqaW>wN`a?AraB?ekcx>b$DkF7VJ>=uKMOz^B>QS$CmyrCy((t zA$>Tu+HQ)y$n{nZs7AY!K$Fi)LnHnWn2QILNH#X8wt*55Sa3EZZtNN#HI%S&*bc>e z&7xCZ6$BpA2n1Gn6S3#zLbC1se|RHJj`a2^!tS`!ZuN&<7pmug7{N+U{{%;<#PF>t z9oTBM>v^yERdT4me{Z%HCzgeY>4TgcOYK0lzM8S|`rKUFmEmQ`-@h+R+2H?m%#Wc9 zMrqt*ZMGZ-*qLBdYlLr8r zVQ>G2T|!q~eJfGbMhN(Z#|>|djEsOqz$8-&;f}PFlp;0Nk{>^oHrv`hU$tL` zlt1);hlEjIg|E$I20yvEsY~VnZNV(?v)cfTKR=&Vo#bcP$!}XwrW+^yie9h`3sbS4 zIZz~ie%+Qw(SFVtD_0%~e)>M!e99l!Yasm92pACEbY44*H48wF($ZcIOQO66cV;ci znZ-~x9NRw>P`pDTCBeqW_{4-}??&}&Vc~_7h!Eh10pLlX+Qj3QNZH-n%RooR_Hd!A z>{LaNtcFd*sw)WAqlur_bL3B;ae=o9nV~C5*=2m#oxK4FDoK!h*~i_hDdT46Rh@}K zzw$<@)NuvQsm6MVE+AzApq(gkz7>bM7aj~OY9BsCTo_FPJJE~`VIiRbNp0=4*-8V1 zMMy38TKCYcUNUCw_4%%=hP8Wx5DbGNZ! z0Xo=CO&R-uQ3=Mo0OW~}$2V(eY}5&;25oob7AVT+48h?uqDZ{`?*)JKzADEV(!I)i zge_7Ep{q4G7Fx+bDu#eIJ#-TKlBcN>2&_JYVG=f0PW!}9NRnY*_ zsBBev6OLubN!g~{uua7w#15bq2RIOj6C09KoxR;5(GT;3D= zwP5Xc)5Z`^vq#jSj*QknEk51IVZxw^>7BRqb0%MUc*zz^Hv=oJCwP-&^I)EJu+ zpzR!qcW1ePyTjYNv2PumgRsa*fS|f|Z13(G5@~!J$H0Ppk$epzEk@gA_%US-j_WNh zJR~(`KHeM{Uz*ZBN)8!#QzMs@W{T{q5GLDkd+N!m)eO3JP0E!ve;81BYBu~vP!No0 zF3v{f<>lm1<>XW8!LAcxQw1?bL~(5Z{+70aLUaNk;)O4oHGn}h$Y*p)zzxhlMi_sf<8T;7-a!lk-8T$-fLD_p4$+F<*VwIn{6}xA*`!{95X$wX-_FIsgQV z>$cK%#W_dmziA`O?)H?N0gF#1LIc;bVWe@ojy}u#`qi|3nuJh5z51eoq$L3q_~H{2 zd%0t{c#t}%KOlfS%D}*Y;q?jV&Uty=Mre?@Z!*R%!~n>~4Jb&ky*yJIsR-{mr=xDY zU6%A$QBZ)Pn4yoDEo*PgFOO=e`{4-xqq5?*1wBYjI4#<^7=zkw{436vSg?R$J-Om% z#{1`222swhcp0vK%3eJJ^#VNh1Q9M)yzxm%Xgs@_EmiaGj^5sPa5x-Qo-{WqEDSNF zudffR_W{|Ss>a7GKqcY%b7MDN76qL6mTN!@-f^gVDoiRm#CAqVxOCb4;61*fGVu5W zDz#q{3MFl0JiyHLJ8J@i*#v2sT27y+h0ou5?d*T4Ifx$L9>ujYJ$4F#_?eU&*k1r; zWQ`2CXZ)Nxi9PJ>*l1~kZ5PKt0tsNEy2WCN3;8BhkedtHI&lG0mCzD^`vN7k0dIXk zDQ&Acd+f6Y6oDJ}q0gm2z*kA%dMt! zKP6D=J7FB^V)&-($eggPNU)!v!i8r~z|6v;3*?Ji{E+~GzxuyR7E%pvb@u(CeHZXT zTwIDyjEs!%iHKN}iW{`7I3%T{?&R>!7+)a4`5nNfMTA|z5}*`=r6M5ffuBWhP*Wwj ztn65+3f$B9r?Uzi%iTWM1f3`a3Fjo=Tj0Y$P&|y6`xtKxqu=Enn)-TsK73r)RaGUZ zkYNGGPC#nuq>$ILr2PgD(I!d}>$YpZSJ2Sry>(g?&Zpf!B{;KVTbdy0{Y+>owEg7c zBg~r$+^0!^H6#L1C?5d49go~P02QONBYy*l=pZuy-!ARjH+5ZI7VZg<8xu`4Uc6); z#cJIGihPc&d%V?E>Zu&conJG6v=F0OSs-=xJe4FZO#m=Ro zzbc4f;c5aTicQ+tO4yR&j=fR~m`^NhRx~YxzeYp=NeJ-=T@%@Nb1EYrgy*FUisj{baEwpvsVcm|88mLmqGBh$qo&?!BDrw zDzsg4wqb~WyU?wg2&eQc@fR^u{Dg%eLRn0Feesj)K;sCDpK@101ZfvD$zlCCHGaLi zYGVMbxQsAjhBN1NnN}J!np|KUv|%P-*9sS2M>!U zTi#-uV89f`)dFtT8v>pkuyp+E@tFf`bD%|!45atY1_o_QIm79@n?$osc*`GjSSIi9 zdYwQ4I$Sf^h{EN@G|~Tp%I$|sPLD9&x{jiRws816R4c5)%svG^v02OddZcQ-_e!)c zigCgM27$@-Tx$RtwA>RE09i)ANrEZJ$s-~o364rjNn0+Nw?T`TD4_V%n0 z&H%jK7NYn~QC*_zH6x>#d$lKil-MStVZnKE^?1<_wuic5QV(D`1o(WB)L)(jG z7;m~XJRDO#Ji@!FuAX4dgpmoDW(t<3)2$M1_s*_yO}U6}oHG@2^K zUOT-OPQVtS-k_LeD|K;3lblt1@M-R`+|ws|#_Po#j!m*A;C`}}F5tT74?jKn$xhlc zI9P+j=j!U(q54GO3+ai76?La4KUhbb8>dB z{*{yF`0=j) z99X51Qt~|1DJp!w_W~gRjQV`R5lhM5YAk7PYV<8D_fkg7s2hD8Fwa<)ODz2bg$({j z(O;~u3Wp7vu0DJ%8mb)bi`o?4{rIgo`QNzHc^u9U=pSZUdV%1?*0T%F^*eD|s?RfT z(f8a9?d2lXGsBb@!;7zfCxase#0w4A#WF@fF$-&lKfr@ z7Wy}$*dUE0fO}S0R)#GurdIS>XWKC^zo7It?`bAksW_N^JmvCnnkxH!E3B@wo95$gF z6xIkBE~s{NP1Xpa=I@7NyEr+$q@{i2keir*6~10^Ek(s=L_|D0in_WPbNH!0c!5=!he=)-pqjuJ9$G-nE^*rLPL zO)$J1J;bMdyjPt1v&FHFck(_MWw#EY;csU|xz;;okgIqGI6?2s3)oToJ4H>E;(CI< zdtzFA(n|CH>H5=Qe=N&1L@*2NK)T!dEVUq5_%` zO-xV-dreL1IQjTA)r1XpcjGou<2q3g6BAQV1pj1=*kuK@S5RB8Ws84li^j!~(X%%L z?Z0Mw`-UwK`^;N#obHal`p&5qyj7MF_x0o%gMu^oV(M@#XSn=OnFG^%T$wn6K1kW` za4pMKD@#K%hc%%n+l)kof==yesdHW1F3J#q*Lyh*WZp6UUX%iKnHbxwzI6~!U@y(u z+*}`MQI!D!bI`b3nSaBMh&r8$jQaX@R!3Ie)5r!F3Emohv-zuw0#jU!va00gPk^t? z$bkhJA1B~PJa9q#%)C(z&!Kr91X>2ZY<0#VPPR?#DbwSVY%d8@Hdw_T2gyu-QKvZF zDt7Lp8k&pxt_`Xau$Zl^T>0~7bY`X=pwT0hywQ01 zih~1h(8kgdFQV+nkM6K3406c^#H>8y$IiBD7vUP1I38CH2{#lEhW$;V{nrN-v3l%~ z7|*VoRFi}?(kdlW6LdB#D7Fx{7CwJBC5fcPY>4EF_v&5c`cxv$cSn^X*od@#ADJ6a zx%DkAO%S*m1BX}8DtTR+n40o%@p%VU*VKT3|Hb&`jiTVYfO!v5LU4nN?i_bfURKj+ z)(TrGmuriS{QLdO9{@;BK)P7)q}zrgIDe*Y;?rY`k36xZ(B3|SfuWgzuzf|s)d-tF z`eH9oIbkP31i{QQYW%?)8oigNRE_offmXu1;YI}ku|QtxDjkta_uf9sIPS}rFJCEy zcGMkEfS&^xc89y8z7+cJ%0c3p9Q!dCFuPAdM%LHUv%R&&<{F5wb-n}WK4**EDZOvM z?pOrpZkQDR=)T|Q(Lu!R&v2c8sm0T39 z`jmpi8r#oz<=pM%s1Fku&Igp=S2RjkP%O{Iv)kK59nV-D#eE1M*`$q8Qz96UFBg(~ zDmlBK3V0DKtE&K8TH0u8Y64AswC2rSCVyK9h+udv~JZo@@o)f7J;8+lda8SHpZ5|4?}-z4?bB8sq| z{kIJqHBql4bGByM7X8@2(wSRWZX9^+t|+DAs9Kuxw>T}K{C$TNF9L1eP+EIrEBpuL zO;Upb9b{G5DL{&xoH&V@m)S{m^vXTZD;)H#t-5 zqK#2i=;u^(nnz#^b7=oApMc|ka0-h?_$^5bPO-&{Gk~S;HoPqe0oSuNx@KW&@KbD z0?6i|{E3Ko+E6U8bM$Pxf0~a+$;AOI(ZEH#5vtc_DV4gmvVgpgW6+7}lGtlHSHMX1`eD1jxP|HorvkMFt? zbD6?dYjHUg(@A7o3 z5F2WPKg%XM8T}^RXfp8R;#S{wEWS_B0 z7j$=+9Au^3Ju34yc}!wyDO(zE$^^B`CDYfPhqT;x+PPoOD{uKuETvqRXhl+nKRHKZ z#z=VrE@sDVJn)M6nryY6qP$1l@;4DsMKc}FCSo!In>Y4Axs{eSA1r@XRIqb#RcB>U z0l!)i5np|Meb8G2QKtNmI&GGQLu`;_Uy!P#j01$>!&VRrLxmu#G@w#(D*}a3NjHiM zYdTGN(&?qH=Eo2#)nZqn)yCXt`(5$j(rxm@#cWFD3C+B;nIkYJ0=%udx`l&QAdM~~6HZ~+-Q7(GVC8+# zX#7eH&IOC0AhHH3f>5&_Mg76dVFJ-fHKAI|l!pW`Z@kZjrCbg_G{}lq7i~XML8>(B zH8-8r8H34J!UL=|KH_@M6b2XT>s?->SsJfo{k7j$-^5IMu`z@eugaMmn?&oZK(TRg zE+@qILsxp6pi>QOr1*mGrK6L6xS|Alc5v7X_NZxxZE90nK$-xmAw|Q1fyj#lZIKtG zu;$AJPr&M$)$}{qtJ>M=e@t0*VA0tNJlDRA(>}xSeQ%Y|V{oOua&OrSCRaF@d1s+D z!4;TmJVO1n^!0_@*^fdW(ITON@YU~NPJCRsu93FxbjxlU?&sEsA1)5gCU2nb`A)*; z65%ba*T89@r57Nmd!U#A3o8faP7G!-pPT^60EBm2o0zzGfYlbdpR>ID*L9Mz zjFgm_d0#i5rjrX?!h{43W`upqszScTgv7*Bge|TiHH<4TSqj`NedM;$q=F{vhfYCl zvTrhA>}daSvu~8Le^H!#`;M_vNAB`goJb}+{L;6JU*WJX3zfiUh7?NxWlfc6?I<+tq%CB zPX@JDf2HGuKU`cBi;j&3g_Jc=LElh-GDCgG*bB0AQi^Jcu!6c#Gf-pWcu1VxYRQ6U z+N)697YVSy-B_p~8-#AOKDgoV5@WndSUc6cCj~SA_fT(Sq}HkGK(6CIN;T zHJT@eA79vgbeeC=&)@!q-X<|5QgJ=-7_s_Y>$syI_(_4|r^v}WpAP+1x^sT+w^N3a zhjYO-SZ_zxBePwVD_0+sCsN!1XLQ9g9+n90O30 znM{z0YnIS9vwdXn6-!D<=~)wf#~LfP_X|}z6~v!}clgNrT zNXDJ$eqTu1KHDf`>~27>OEOeq{@(C_gYU^bIBJJxmxSm0&_TarnbIflLv3Sy9Ys_~ zQilfPwTXeOcjC%p5aIY*`Rp#qr@y0JsiL#oTyhc%eD&FuE=q**8paK|ZM1us{B*~R zqutRhdg>mix~LUAy}0b$J%kX-ZREHn)FY5&X)7#}A@+nR_<=X#X9mevwEp13b&;-(npG z6!~QR87?$6*VSK$UwxZSEeVGMm;y2Kf3kdLJmLHwluyD%KYClmY#l)emzT*&eW^1x zf%W2CHn;{Hbh|e*C>6zwaZta@lMvEx-m(=nbha5>E%Vr5p?Z?==dY#ktID1@dwSNt z^{QkC2ZE+Wjg2SM)#d=>88UstWH+;G2VWIPG=2s9?T4CahdtjxVCwSH6137V@{7cC zK%c~&c@a-Ej?Pd3@z5!p-IOSZgLi*#y7uss6Y_7Iq-8Y5+dO*ntX2TC4k7KsMjoHc zNJ%+`iqu5e50kX~s@upzjfgDlefi@@`Lym%pAcX7=EB!#im>mdPP`~UJV^gFgG4;y zmK!%R+K{HQzY!});>NwtoMLp&{%`H-Z~xVq)QJVn53IvGgvbo_CcGEXp1JEEv?h?% zMK=?uhL5uX;FN+nR=MJ~fW`goqFYcQGXWS&LoA$OUQ#F=ivQ}CjR{L}cV0;e8@q%i z2P-Sl#VlmD*eS?@fym)UJ7s^H6;(9ZL$eQ%toVW?n>xX~gmK#kwN0MB_6Qe;jh#Iq zm2^6bvpHT6)5^)krTW2NQ@Oxol9HM;->r>}gnVJl-lZDDn3fh%?mR$R&%wxq0a1jK=>3Hv8-wm$g=f=aHoG@D+Yox4si zq5t%(u^1uYbGdZY*Rw~4k@Zrbue_QRT3Q^Z7oWUWm5R|HC19fb%!ZqT^vh7ThhGkr zs1(^+=HNr2^D|wZwvZ_T6WR~j-Fv89qCg>pQEb3Md(b-~229E$H!_Lk(JR(jHMJ0`MxVqTX zB4;YDT>>yH>VJpPA?jJAxyf8?9KcD5@^+UaHTwSOxMr32%;CtP`!jEXsTD#L0aI5&n^d%u(r1d>N>=PB*fyd#{&c@@5tZfTBUWKgEwVdL~B9pYkA z==}iC#vrGo%S=dM!o$}c>wi5Rc)yy@ys}{_c|8djqV^TXQ@+DE$){5{#f_^&TD*uKJG}jCJT8HONv?aS8&qT zM(OK}zV$yNH%;ceD_N|D{Z4_Upn{)Ogs+)8ztgzpKq3qjy;)*jADWxfjC@B>O8>FO zH7%(8`}^S4P3g_rRZX|Hzxlkbg|16=cYY}pCgi=W8JZMWuUm)`>oLfNSC)=j`uIgn zO`i8>Wxle8y!Ee|eVg9Gs3v-EFhe~mJ~!f9#}C$H@i-rnvYSEaG{S2`AZo7?_s8FI z{J9S^1&Q#sx%v$UH|%80k_yKW7nlk4^eA3@{Zha;r8!&IXX9Sbix&(wIX@SvA7>~# zHD2L6hHq$Tk{(-Hb6^voAgM<%R6FF?YwMrk8=)_O-GJ%jgfyt*UsAoGIy}O(!Kr@F zszEsbh69uSw0EUvX5_0Ukp-?9!b5fL_ie_7wQfkOko)+!g>F7(og}3}x5<;>u*;H< zJ48BJTeA=`JN20IQ|iQRjn3Tg$9Ws{{JG&vPGua&)nC6Ju4Owf2#bjspx3E?-{t;my7@zHi@0`H?)fNV6$^TNSkm>(E3Q-)fIy14bS= z^UhQ;SqBjtt>3L3ENx#j(EQ(oAithT9-Z}>ujPt${MSOb=1g- zyzb0QJ6$0Z=}oUJ{-jOM3sf6FKJh&H4QmsVDv&l=YidDMk!^+weu|1UqG?*6dQ|hZ zb1=V%=A-&?%bKn(BkQ5F%c1$7erGFj9(ERd_iHbu@&sM1ztd4WkicN)Nwu({9Ds1d z>Xw+Z8_C)V(AA~i#^f%TabhuToej>s9d>*uCX^bNeOxl3NtrWYijrPB>NuT}on6VH zEGDg`->|KI@;zn6R77cdiqAs~7%ep9VsB)u168-ZAE5$SAU_fKknG1~p0wF|XX4k1 z4=^@M@m)z)50FFaM!`CtqJMy}mQZYCmY%7NuRgA>$Q+YS+_>17{*OC@DH<|ZRU-Tb zpyWV~J01Tl-&Ax*-uT;6$k8%!*O_y@CL~fDp&CuBUW7#;j2lebb#tB+*3nylq6t%of)LGM?!P<-5sHr-lw2KJc#An5o0` z3@ogsq2&|fB()oMi;u>{;755sgU23s(rgZa~2EQGzvByQD(yn zgH6E|p1vyk566rlaMMnD>me^?py$`@+D>qL=+}u;t#rM5?BVY2%(kG@q#Z2*+R)!9 z$S@Aftw;VF4pbB@1gbyB$A5X1l!L3Ay1F_jb|Z$c>eT#Qi_qf}5^^g)Hhdt!L&mCX zTu{6SvXM4nC??xf@AUR#<5o+#iv)kIQ{zMQ-`?}_gFC3$0rA^#m zeEC|SL8GGLrmC%2yW}=oeVZ|r$6c1P=Ra!rD?4NU`U~O`x!$AoO+lrWQ6yMHml(9& z!{51;Ykte2|58JlC!fian2L&nfWnYG(f(8X&B%Dt0;Y_VhvA`YsaaF3{hl5bC^jme zs-Iahhkio%>%0>(d3C-zNVXtKq-)&vG5#jTaO6YPDq~g#N32#HmR6I$xqH1*S@FVK z9<6fcEDx}dX2E84H^8f4%CE30Z)b+L+KB)kLDxsa7f7&BnhU+NsAzhhvoml6plh|W z+pt3!YB*$|g@Pdgn^(YypgM*eooqfi;d)XEnx;WrWqbRw1%4@1$7WyBe#vii5H(%Y zz=fj!tg?PnSNy+C|7$$0{)z?Me;!3kE`lP8cJ=`B){c)I1(IV>-FdAY>NxHpCnxH2)i?kA0dSn?i`y>4(;K9# z-u{2&DG?=mvgkI|tM4($-QAO@wFsSveEj`09+bnc!^kKTXAngQ!wfNEgzclws&C(WQ1rK;QAhkr ze)rOYFio5FU;|1UEiJ8W&mQNZtd_r(eDfEbW_9Cud%n)rE@hKlK>2d&L#~px)U}mx(GS z=9=DQDk;wR`c}qNzMo%}v&epRxSHM-XUz1L2S;bpH zp}U7-{s-H}hQ4Pug>oQ)Q`E>@?O~<}B0}UCsX%uvOh#UQn`F zY}?Pn{7xTFYKzqI^CN!71eLqmLNn9|DF)q%y3y;EB7gYTwLMR#bex7sQaraYoOHRS z!iayKO6%c>3AwK#ufP`WyF?1TX}{4V8IZPIRyoA1JnhPrTNV580p{TO;70**DaXR{d*0m{pj63e+{n)b8f@Nu{9RrWL8AKyVGm*Q)N2bo4Hn zrIl)iv-e@^vQS_@W7S^0a&bvWOx#va&#`dn;arlw5LYQlJ2D=1FRx<~^*`mU|Kz=U zJKerSi92@pkV9#{48u+3{CC#x9#{#kVcLJ~7=(xxQoQJ4;*f0SmuI2gTH~ ziB@V{mxb9mCWV%h$>xOYg;R}+T#}g2QEqRFy7%1t(!G>rC4*kK^oou@kV0`+q1}9@L-}6T7>-$;$A4 zS1x0hQhe#qdwu8j2CejvWKr6y*7E97xgLoG2bA%*!f&_pYTIxgO_~-;5cd)q(DffRJJ@T~a(t&~ZAUXQkd<@zPY9&G3PF0e33Vk`C0rC zzU~btMm5nL(&9G@XS&J!qy*0fsksp%AHUrQ#(6Qwm@EFDFfEX$0cPdU0hX|2d4Kt{ zLBRa2zn^&cY(Ish6lg5y>A)lc9-Ku)3j)!;hlR8eLqCM@7N9Vtpu1ZZSo%3R0csuT z+HBbNXWO#~Yv28PoA{Nnnl(zCMz0PsAU`&aa6 z@WR-ci+HH4-0T{{wz0=pee+t(^xQDqm)a-r1_xOc8r7G2xq7?Pv!j=cz9kv&HI--e zUANih{=mdP2~YWR!iJG+fwiq`w5r@+h|k8XUKb{7EQA3wRlH=t<;R9~qSh8L910Fb z8@w+l33KtSxzVvP9B5Smd^Bs#$?5n>JVT3MVPPSp6Rlf8si5%T128lY2z8R@>v!Df zQjRneN_&|ccb>xo_9;zKlXh?9%zxvo{1?!If80VC+4{YvUZ(HcRPk4!fffr9 zGQM8CQfpQF_8RyD4iIcifvJw`lka%!6XvF7iP>0|tRU$u?XqAyG748et3N>cfeY3^ zrZ&0##$^1|=ib^8yLi9JPaikltRKA`pu%1p?)^}`dou$+%%qRtqYcxw^~9>a$OLZm zT%!{MrG%g$83l#4*Jl$T2CPv%y-N@}^g(3#x7P*gAynsauMYF@8+%+D+hzierPR^pn4s2f=CqvV z-OWbvyRx^HzIXPkXy$nw&^6|Y(=tWf5JLePl%4e1PkQ5{rT9C&30p;(n~ojV?B@zb zdfqHuU2T8a16I4QbmfO1*n}lYzvjK1;>>r|J4pMbtE{KZYAB=Vwz2DVla%U)mI}c{ zX^zXI`WA)PvsT}pO94NUl#Z_A-7g@AzAl3Nv#t+$;KL&&B?TV+FA8HC2&MCQQ3SkV zVX4230HvCMLGq*L#nGeEFOSZU-Y=_udC>fM5xW;&o1=z-DQCNXD*3+L_)SgDyz^+= zuGZ=mM%tG*P)cGg9WdAG~8f{^KqQA=jdsqzf6*mFP zk^hz}?MC}=4RT^|;<6ysa1UM?@x3!;pRVeZ`ud~b6OVUcCO&`NKW8}d>8eiL7ilUC zs?oXI*I6D@YKg8%abRO7!8dWri|`VuUPoN&+n4MDSw-$Ofl;%o!t3bRSP(vd7QMyr zLrzW(%p~W#Kh6C3;s36i|A%WJkh8z9e-T>TMW3Ct8v2@Q(B5|`9DP)t)Y)7k2iUf_ z;%32qG}<~42o5rt+hl8Qb|kZhCD+{kV~~}0^Dl-Eqk7e|9>-HQb$!f$fW+KbAn8)c zc%mdn9?{bK-}F?rd-rH!ae=4As873sWt&bJB&&X7qa(Zhebp!eCB*7>XsrovRaG}Q zw7GcZYod~DJ--8hF(dG^W;D3c_C|%sh(&~c!#u9#+{f!F^1`8hp)aS;58!af5um(t z%5u(%+&|RhK`OnQGJeW^Y>AC-gD{LC*XZ8mJ!p3mo*%z6`kqC+?~<=wO%%|4 zq$C6rG&M9FgL#fq9AwaP!d_LodqGngp>1WyXU+FN9DM^{&um7BYP=gXRI1F~;t#VGGV;qS$faK}ZJ>G8Jx zKtxf*=^i@V2rLnYtdga))&6XKx zX}s@S;vBaxd;b<74dAA_2p+q(&ep*o-C=xWd>m42;_Fm7uC*BD<@p_MocHaqiW)Hx z*JjEiXQzuO>n|_L3zFkvqEk>?K=WdGo$vr$`z4OCworN_cQkN#xi~+^F8Y>ppXZa8 zR#|x=g^B+jh$Vx}G|oC(Jri-N+X6|q$qDv}{SPzWu!OYMDVb3)7E;?6E>IX)vhn%( zJj|FJaIzb9yeVxzW^E)cJMG=CJ@Z55o9)1KqW?pSZ)@Bzi;js<=CiN>7R%cwciru6 zzFWLuNz4)$zF$u^Ts19V-_h3cp8jLk)a&x_>9l(BaJ8wA5^58|*&b!sK~u&#PoMhy z_;wco=?`P`^E-$|U>4}@k>TUv?Yco{uo|PyB_{U1ffq)u{)>89f~Gh-TU!DsbkCpZ z=m4ekg^2XpD!Vl(?g;|_?EThx;z9Hw^DhjD+6SI8$Jzj&PX10!bHXc+v;7tD=J4J2 zuG8u~dSe*IX8Wy!Q>YHtxy7VPG{8V`KW9P|7Nf!0ZEU#@jne zHoJ~Ln{v<%^!0C%kZ`cGV?ziA4~JHfa_zhK@5>A%5qehSH9{bU_3M~<@7mgkf{3FN zvl6nNXAg9AY{0sq_Nl6I97^RCUYm>M-?k)7gXz!#VH;Y#6UNM~{MPW{+1ATbP~CuZ zX~LCd1vU)?e!=@TF*P+@Ue{4{V4|--V60ilUaO-fU-;|r80fD5=bo^Z&TKa@K*OPc z$TDj8A=cWj@#9Ds%fEZQS!14t5YzBy0|Ur=*Azi1pjt;UiZa66A1WJo3d6tM?OB(J z$&cv}0OTBP@;A98)Zaa7`Qf5r4>el^76)0F3R9BO0%c%|h(bKKuBvdwgp=mS`xH9p zHu4u77OJQi>y}f!+w5$&-ZtneBIzo4N4n53d{>>RiUp5%;pnfP#Q zwK19j3A&iAu|eQ|foAgfgy>PR3LOTy7G__a0{=vEm2T4OLY4rr7*& zhWq*=A|r{-?E|O1irLPUY?IQnxvdhRG<yjx~%8eiYAhaFF`=NC0ux1O4rIblz3C-YTJ%})L!wp-#@tWpblHo-5HKvi<&uQ0>C4O6v3EK1%FiHJ4>iBCe91^*dCQbTMrPOP^JN zklEncab6gcfMaG5-Ec@PZ*@rD?`;G6Y*|wD;t968#>U#lni@@p!N+>ikHXQhQI|aI zU?3!4e8fr6^}7$Hnc1pTM(m$TwPpTKJ`va2`r{dv(tq~&rTu{#bR-v0i3V-uDj=76 z7#hpomBqS{r<3jd+|t-(n^wwfcOhsttX^UrJg9l9N#kJb*w@qg)UKu4?XEguko#QO zUds`(^j0w6Vz<* z8rUwz*PW*`LGM+5O>n*~m|+Wg>wgg78T;L~P8CQ9{9XL_@898(u&ski&W705$)?T* zyI$(IL_sxQy5`QB)Manq%FLgU6!f}5%kBQpvwed@8a%W(0-L)iC~CJ`Z{ogV(bXyg z(>^FH>`Z9K*-`Njz!*aqf8r-CFJF9Ccvn>Xsq_qOE^2z$wj!P8)0fz6>pBjk2}T6y zaRB^`k3YpnD!<(k;*q!jj8)KUfgD)IVu-pQlj?s@M2w@~$+ z{zW$7ZQnKb0T~C;c@r$(8L;Z{0|H|r%j3Cbx8Au?q&_f+M_<8(95ll3F7KC^olU5^ znZle<$=?{6koXDZ-~6S)FMbICn68DUId`+-X1)1D#VRsrwTQPApw#XtG-F;=DR_n2 zQd_G<>wfwt_fgIbg4oU`U&?76Gwq<_hy^3jT~Yh$8dk#+kTVQ*jGM-aL@niMg%>No z7F84b@)$@bk&!O>%*k0;>ls`vNArF=pI}lq@H6Q8xAGL0wzqam=^a=K3vJIzl^frF ze)Qqdx{o7z?K9zq#)o91i<@((+qJdU-!fb{Q6zbJHMF$GARJ9dNQvYjIu^$m^Y5TF z7^S9{O;79Uvu*d9^Q<4*a)|1qx+^RH;xEV^o1SihZx%D>&DVZ+hQ=dL;%)@Vz~M?C z3JTXLhZW(jkc&V5gTPtuIPJlAH;WJmX%q57AJkB|MxB*&u(3dwgqhHtH+ZJkv-fCSaV^Jb@8=d8TZf<{p5o$C)o7$gQ zj!6;<{gvLo8JSTtdsgPSDp!@LMqEfbV_OyykgW?i&a-K^;k@&LjZ!9|>T)cg#>}jI za*_@W`QzEfLA8CQID>|JQp%hy%{JHGM_h>muEcwj7&d8wtVel}JOZtD40H6$NZpZp(ri0qeo z#ygCmW4A#mQn#a7)UmQ+has9UDCqhONKkU(O)FP>XovP!ZXw;N={2RLs$^Abcj=e9 zIWXfM5H|hc1;+^fzj(BRkJAbuFMh>2SmgeFIfb_w8C&n!{_TPoQi4rG-h%II{2|@9 z)#8Xl44cm=_Ou(6Y5gYkGZ+Q<68&rmR&|O-Y>YMr>3S%THO18zP{d7(UX;W zjlP(lWvqP=_piw{4{k&#(6}9MeNRZZHJ>bI6A!E>n7wydRhO4XAHQ&SC;d3Kxp#|o z&bY(T$Drjx;DJ?lkD`FmWfT(i+~i7ok`bk=`LTv>6GEE+c&2-i^;B0l#p6>)Eh zX*0dBvqSUgMi)4>Khb5mGX#pMuHSnR_)Dj*Rpi*XzaQPm@>R}&DA9V(jV@!~-$|je z!QGyT_McH=4qcZ#hbMu#c@^Mi%|O!KeEITfA@Z)^TY2;7-(vzj%!1XCcmH-UFh8hP zRufq5I<$SEa#pRm&h7Mpj0m@-F0kM!DG`|okwU3PmByQo*kmQSx5N>_nBVBb&|fcf zXnyPHWc*yiB0j%TGxnRC6Za2ymA@3@`|oKNZEJJiGt-8D8oJ@9%Z4l=*Bi34vqiUO zw{8kFNW)}5;0>vbY+rQMp6-tA$h-}`Os~o33MDhJG_vsao;F*BCUF1YU~gk(<(}KG z9=anbO&7F8R^<#O>&-R~3f?TbL65`Z<3HdF9G?9CR8xyM(F!lQFpVu^2I^auB8U>JuI4JX-^_UU?FToUo0ehlh4 zPI{-8&RUhY2^DSsTs$#bK7EpaskKx$8*fai)~pEQhXgb=>?hgjTW{8hJb*5_zO$pV zx`1)KKggpJ{xqZ0+3xq&R+p`9XE?sAXIHazf5JWCMm5U)b^7S&CU=kNO9)}RU;ej| z{{Y*4^U3SOyIe71!(VdUXmQDUXmK1tx3l&4PJ`1c@8c~!h_1uaSw7|EpNI|h$f3Ns zJUY8{jHeJ`S<>yhH_616c}}8s%1dEs)TCc!<50u?LG(xT>BWTs%MMF`xXwg6q5J$} zkN34d4IT$Ehkm6kuDyFJNcHh$bf-zczj}~~^70Z@roQ?x@8AHe=*`IrgJ#zP@j;YF z+iPpD$D%RBT2@`=10eepSk&jh`P%x~d?3M(({VYY6zga6_cB{f-=NM$^&hzb{md1R zPfod;REU@E0tNS6&Y-o z8jCo)dXgHav&lj{K!Q4v^K$fbc!=cVv&kr_avhwEb{xzY_Y~brCz2sE$x>Be-H>|o zx2^=~&I+s{Rtyw+$F$u@{z-dAr0!es=H1G(&ralQcL>?E&ZB#t;4uUwD2Zwya2OLq z?kt2QuLivM{?1dMZiMf_<&|2!fS<3gUHuP1`$^&7*;yb>t5q)o!ix08XHGm2y-A)f zpvNeNJeCOA2$4DwQAUvMCiZ>%Mu3BpBS*3X&LVaTND@ws^1ZEF2VL6_YfVx<9r^7K zn9Z*#9R*PJ?i*jed=)#j8A;^2@aLcYh_lKHlxR@gFDxuTW}af?Iweg2H+Qs% zJuMFpk8qu+Ub*@AMkf<>^=~}5c>Wu}wX7^DvGr^vCVyaNZu|T>#DW!;Ag+D6AEpMLtM72whu+r@2N!&)5r%_|1^TyU|5wcjO`p?uRWm#>l3V`Sp;6eeAZB4#+zr{+!CK|KH1l zO&c&cm(HJS?(Dn^TOBYR5cmyz1wER>Y|Bwa<6N-v=8-RpM-^b{?Cf@n(GnbgL9L96 zit2V1vOM!#T!1mQ+k*e>;^MLMaKY(4JY5{A_~%ws%3+^Zz0=;M6J|0;K*?k$-`myU>wv|RxqzeyyXO!63gO2;j&n{_qVWY2q2@9&S7 zk1zd0%m2<4iXP^oXs>=Lhs%TvHsmt3#}?k}zflAXK&w8JgCV6w`&SKGXdPKxW+K;k z@X_X!Sc!dhz@-nE_VVB+fSrHyd_Ty^`;oHPF1JDQphKwpw?mMp^j3Ij9%+&kZU15z zQ^bIq^1V_SGs-_b2Bp0E4Ma9nd-W0J8bp=#HN#R}F{}aP-CH<|8u7XWPv7!+tmIg2 zB%FQ}4(Lw~lrK_cMPgPyMhex*T^~sqCfR8>*uU`z#XbJ(^(KiL3jBB6Mw(l`zq#LSBRQ z4zlOtS|j_w-_KL3KeKT#&nwRoW2N(QZTLv_ddA-;vJ~HQjIl@8o3%jWnnBH0nH4HC zz+pp%#{a%QZ6(=7`l0q8B?a~&cmlm;HJ-#Et?6L?;N#n(~;;_pkr-r|zRnL<|mTf;oktth* zDd5%pEMjfpQRnI5^t#M>&ELjd|0`UUl=6bBu2U9qL;ovB^686m02IA|$Tw8^UXrJPzOiXYMa z26?Z9JFDW?iN+eulc|KqaIrDf>4HhPjb8uiNP?B8Wu(qq{9O9sLU<}#bOMPOm(}I{ ze{v0XM3Z_gOWfR>R1xN8X5^;91VV_aDk2?|pTpSIGNz^rAarN2l6idZb#?j`oxoaB z6P4AZjEC`qCP9J)p}N%w>!<+sfg#Vv!__w%q~B7*ZRTSAvjX+>rb0Irf*S6x9zD2s znI)l59i*D)5_LxIQO%TeW`D_`ou7LCIeOKN=o0x1Gb6f6ah!iOZ7TQAEz#(+(~s-y zp)^rBYr*|?) z*~=k-&PZ+*OmK1=7zumGP%s*0v+dZv<0)x^OuN2E2y$z?{+d*!)2&w3_W%C#2d1@g zF)%t#}J`4Z*v5A+>Ny@`itrgm9(kmCou{8Uw#!YSre zLt&bev$!Y>`~T4+t&>0~mRtxVl2=u;d2KgI&)LCO3FO_md3i3OnMOfYZ)}86NcAx7nlw1l++*5to9SR@|GHpI=ktCnhFl*z4Ekt3XWw?;pv< z&Z-we20A)k;N1Pm@8taKET;sp8lJY(C79|!5t04!7;;ld&m$!SKr#UVsMU=AP;y52 zJ*VcIH_lwPd{sXnTeF_FBPt3~Qs~GV8yeVOw?b@u-bh5+^h==a=T6vFlRW=% ztR89w4a-PSU|?Wn+j(hIQ&V#@)xN{BDTwy&P_e^aHXS|i(oMf!DSo-kRhM-YdrmhR zJ-P@mK{VvKexeVy`NNrF6x7s@UNawH(_(>70=PSOkV)eT zcCBn0{|&JVK+t4S5g3Td<*hB##b2+NN)=D8`3vz<9#K*1q8n4#0$TE|eDkj^xW*nP z`Ya9s=~z`wZNf$%8%Q%y%T*e+hMOb013FEs!R?~BqGD)pFe)xiX|bzI{v#bN?MJU2 zdhwZ-6&Tv?%zYALWPIKuq?t!Z!EG4G3RzrHt6%y)U;GjK8f2XJkK(5?%!&XagU;&y zPOvB|GL%SXZ~lp!XY9e?(C>2xMfZiTIa8Z1qSaCaPF83`q$7^QbD#XE2-Q|OVWtOb zA(}}LYhY|VH#;j)5l{=5j07k+VQ2k@>6wTKdCb)KxU?JmL?-@H!K`v_C)+bOh+i*3 zQi6;>T~l^$Sw~05w^ycI@NV#DpLxZJ=p1(I4Ef|a5cGrL2Yl32$^SnU#E+?p>0f13 znYHxvsLJ<1w*Uw!59N6P0ML-;3*X+7_5@5F32|}#I#Re`w!M3Bz`@P!srh99;*UY| z2gP!z>bH6OuRebUgx5J(mt?Xz@>|pwdrxTWU}rWATbbi z-kHTXM({IvVO=!y5BLCln=u%9F6=HRhYk-d%d8Z+ZWCP#rKo@ajNoDt@xO`2lGzkS z7-Rd0z!M9nz~Jig@|EF)+|kC<1qaY7Bn4TsVhw@p*rSKA4T96*<^|Iku!|+S6R;Pt z#ukJyUxq6o6!t7*!?40j^Qpgu(d)cvsvx|55d4c3+11nd^WMMxRHZZ(N z*YNo>!xYN&H`w>s)gf20vGw_BcSNbpdS1PJ&|P)3dtHosc?n^rYak-c!SyKULe?Ml znVai%i1k5mr0?w|u*A;nFg*nyN|S2?6V%dI8g+l3*F=1ks z_T*Yv&*i@hx1R6y_Bu8;>tG?Z6~KuC8>J@jY-fG?vEFJ#?95y9D=}e9mzx4P(urw& zyu7BAy(J<)IKTM2?3w85_RYHSgE0g^j}{Q@d9?MtRz^k!eoi-Q>FZ-GtutDjuI$2Q z>&pP2IsJbt)oE*I2kFpofdAB5B?Zr`YlcVD#_92l)1D>UTfhHSyI;yN?&aqFH>F8}2R5w~@yO zmlr!?8jr&O4O!C9EhF#rTs%IGIFb5=;!QJf!VMD!SE()h%<{)-j4C*)f63MOdj{4T8S(lcO5Nul# zL2!D0&TY_u7xn~m#Zp#}ZUc0qx!SU_Xp^~fXvI)@3X3Gf6V&@BC z8+-#4vg8p*cN|Ql&(|)^0z%eZP%pDdB&DUs4+kHW`l$pl-xmozJTVut_!UkgT&G?3 z@^qK+`as(%+>Sn5{X(N3x$fK`Ek_G2!_8_qM|-||ec_LPc$(`=kVGt7jxA7{_ts&i zp~9$@-069)CMURJb#^Yt2+alF&W-*t4ptA^75ew;;XJCs_AJAD1!Aw4Bf>-)>&(pZ zcGa5Ltm(nyZD13ds(7ac`%8;i_ZxVF^U8?KqJo|aaR2Mt(AX+?XRPT@@lvj_+6Fd2 zaTIyg(@5JMO@9n`ISK5F6xZM4rFp|lUF)|*8!QJ?8s}fpsO5A((25!H&4mMY3DSZd zgBQmB+3&WZ_n@m}4TQJ0Xl|zKcaY%IwuA zvx9gEy6ig2o0*!3#i(0(c$gH+q$Nkk$8$|Cd78MgIL%46-bJD{dX-aobmcl2`s5z5 z=8VYHNgIIy=lbGO5A4_a2p5My+k2{*?9p){ah%J@;)3;7ytJ}T9Sy|~H+OT8)`w$D zk2=4MKkr{p;Qta+P+aV;OZO#+zslO(SiiGOKD$h*mkrtVikgN7Zl&TZM>%cnNss|v z-_iUe5cSSNL+NQsE%wmh#f$Rj&jU!FE_J?UHq0NUysS~T+xT5M(PCO2d57O}WUNoJ zD0bzbrzN!~4bm2}VMOjeIXbv)5nCsoCJnVfD{`f+O|oDTUPVN7Qj~PrjnolrD|V-Q z^%c_`K5%?l;~~?nNayd&-mu$m-j2`lGseE2IF5wz2;vwy3@{};s1WxTi0$@J?@F! z3#*w_?ly3{`j(uT9HP?rlO3^Lt|M*sh?#1cE~@=b0Bv&ouTXjqSc}bvqoE@LVjCBE6BCnqIk}sQK2W$PH^Si! z-!=J`9H-ZQ2`2RfnI$Ptd1YlUZ8OJ{myJy1%4lyU%5*vhll8$*AtmS+8>YU)9Fg{+ z7LH(05xWpu>8E30zmq!*WX7)NL&blBe9YP^8pY5Mf;U`7a;E(5?xjJdl!53K++c$< zaLTZ4CU7ypX-S$A6Q7w0m;9KSv$ICo#HiqG0@VEj-NVnWx7#We%se9E;w=i+)1Ba}2=syVbVo4E*9KFw|B6gzFdVtW$Yf_@ zOREa(w*Yaiu=7IT=g(Ju%l^9^H|{?~@_qm0;w;u2hfi>Wr2b(c*1fP0GqyKWF^Z|# zthWmn(={Ae@1*2LD=;sEaNN5{05@Bkr8V$0mz%ufHX(x~XEx@SH)$L4SGBznV@|eDY`U?rYoGeTms0nmHk~oku*2IHl%WBMs14zL zu4M#zjZ6OB2Al_X8c=>gRM*}ER#xoyvfm#~N7uZpB8Em|tI_(Vz!MugyBY!krav8F zxYq_q|Gk!b)3v{p3~DSQ|D70iZdk=5%#H7$4Y{#nrWgC1?&gRd+;L%XUfEgd_0@v@ z{Q#>%n*+^APtP`hi!@6nnBb-WW_lA9)IP0VJFE1IDe{Vn(3nSDU)3Z;L`1jIAS}i# zfGe@T4tDTw=5>U$79x^{S6tlx#v81Nv7D2_`|9=xPFfECRkB>h* z1V1RRannn~->Ds=9n&p}k(^3IM_g_3K$Pw)0j z5ucTGw{D)B_TEJ`s`c-=*#v+QO%6`dkcfR4xz+D82dxhv0|i`?q>$ zAgd-|QmS&q<|j}>W9KHXoSR<#^M&f{zr`tNZq~yP1>>Qfn5WO4H7|^-Sm8d5%jbgs z`v(BhZR>x)0cm}e(%>+IWt;UL4S?$H^lx8@gL)={J=8VBj9NYG8yl~;sD&ofKF-pF z*x0EqxXg*ll~X3Io-rSOr({U@Jz!)E@_R$$cDVk8hI>WYwAf7>VKkX0!h~!9X@Q8x z2I+qfvoa(!--QaUI+qI608$YU5L|+EWC zU{>Hn(F4a1uglJyM7JK`V%NUHUORK3a)N9GgK^n@n9p(!_4TE4R%DGz-UKulsJeJ7 za0-Y&x7Wo@$yEh2-&;pfE$fsmEHSJSq)39*VuMz{6YkaGdG?`FF*Yu91@Pq4)i*YF z;8n2DV^281Pcj0#}oN_k{>+9g)A`tkKU~Dc#TP15uSQjPpxtGAhWMsi2SGAPUDuH; z2mAXk>aG@FGkh@qME&f#eyI2djo_mom_BlTQLPKdsbf~e)zLn|FO21fOG=Y{iTc)o zi@z3OLEa31%vlY$EjdXOy@o$s6p*#@SDuw=>tPFs2JVBC+~fzT&4n5}8=L>mj6?z{ zzvvJg1_p+VPgXzE;m8H=)wnnvDf6(eN&?hsnwqz0(=Q@sw18h+(|c3@wevz%E!-|t zc-&lEVfU6<@2a?Pei?^XnH+mO*|goKtiXF2zD!wMW8M>ta^Cnng?Vj5Yo zlHbYLD@q&|Wv80-<_GLLro9cybrHwmc2iz)|I_&Y&bmYhzx??#+HD$u#1tMA zZi39z)IKh`K3PP0`B%%|*%g@7SS8#jwOxBkh0jMTKOO(5mO@+94ll02{ISF>ZRrvJ zluEY2M(yjgKH|w-XQ!Om0}Z?}*x0O`sCVPdae8c=AQL*a9V(2xwKvinu^$deB!VP3 z$R&lqPt)gco%nhU3JVJg6l8N{EG_Q|VKX|$jo1KdIEyQ&LBqISnkOB@1K0CrokCJA zb-^12>j#cMS@~!nYno#4b?e~5NsZ+oAb89N=9@?QxB|P!$0^>)A{V5k;857IkTwSg zTEqpOQvV(xf63rk`SW&YlMqyM!op2?dFBK+4`F8(dL%Acpv6RC+$;a`O~CWk&!2O1 z1V-fOiq?}Ki~unEAnygaZh-09Y}8K~8$gYXAC%G;1I{U`%N_(j+;ns(Fn(Z4Nx{L} z^%4keE`4{};Mxm@+Tif;u$dVNTQ*+DBUGTU5l*7|%n#CoEu3bE=My&DFhL{nRFuuW z*BSov?{!;>Gp*a=UNTDXgH)%XI3mRXyH5Tk;;qn}w>fm)z~ppOL%3wdr>8#x@Fd~7c}>pZz3XAav9nBt6odUto70W|^-Vq6UBW3YB*L$fnLDCmN52sc3tI{-m$q$`;z z(lmHjR9q>~@`eB?V0}LHNL}Kq&M#f2D-NF>nF&~RjNS?-R&f)@)nIs{=s zX-iK}?}_C~N=j;BZM`hKg%$)_%(hqmMF0C0Z>XaW;DN z$O7s9H~4$c<}F#5hwRq0g`yN}$Q!tnH^+SturLXVKeD6=@k4cZRKS`^!{ z2hR@dn>Wvg11_&eYw@;_^v2PjGqFHA$R!tHFp5N?IJCH}#U6acYhjDSNm7bgc=m6a zG?KQ8(zMO@h<%__m@$3-XJWs!Le%80wohT?sN`MxibXQ6G789)qLuWg+)OyN!s(HI zzadYh8!9pxxrXFyd2O}Wv7tnndv`!Z-Nr*|f(x=$t1}r#Wc>smNC%M`a0WX4(5OPcn%O;?LNu!qNwRLnP zz{Lf3H6o56{x^)5&VJMMo#Czn#1m@c}5fR`7Z};Lwe(5(oUq3%E$e=x9 z%>Zo*&*#6R$)5$TKZ{8AXDGOd!gvXAMd(t#(6+}QF0p{j3Jzoa~ z9+hC9DDc?_9UmMd{-#@wj*&vr@N@M?1JiaSB3dzzSHIm~;M-0xk(ZgN>1|=*|J8Nn z@lft-{In>0${l5!M$$F*GRbx`4VvmGOE;y3?lCcCDMT3@$x=?oL}W=qjxA@RTSN_g z99s*+oZ}>A3E7h+3R%wgm3#j4#UIt@bMC|u)B zl8H!-&nrJ@gG+L2<%mZg%*o9i9UHUTpnkkI`f#5X-|u5j-A^p%O@W^ws8}%cx*cN& zk3RHFDWt=zUW}vY_V#(=c~r1%59MCo37*9=Emsg$tcIN*^hI{0zzI2A+*MWr^PYu; zMIa=w=O=5-Tt5GE!OW!%NAvQ_?(j*uRhr9z zx1M!u-^12hr{&`3>wDn90UYukYAMl&D8JKa%ha>cvta*~ae&P2<*#BA5*3wb?tm6) zyMJ%Wz0q}SeEkw#-F(m))c0Lo8BK#c5e}M%^t}HuGBJV9xJs*khxorpmy`R@-`{K` zP5i25hEVTbHGc)u{isNJqN1WA=qtnp?y~KCqpe#fYSsroC=0!bz1iW=gTlhniI+!i zb(_FEksebcvn`0z=`}XPFQH5jZFizm#_|ZkCrS=y7j`$2wu%1FAKTL<8}K{SRt~C ziUB~Pr~3H&DnSHpP8Do#85AW-AW!+5wrzU|WlaU+eCITt>~%Y!=1S_@3vRQL48~JN z28z?h#?!s2!C8@l{W?DrkynOTJe+00sj^J`Cmac#OGFRJ!Uy0*jZ?VK-uI|N0qijE zQt3}KCs1%}Nu;{`g&5Fj2SIY29&&lO3a<*e-i(f@ml;mO zAq5<6DKXbf(FMT?;Qzlg=y~egc(%inKNXbEW6IOiF`3SIRE8?j!AHy!FxR@*5KKfX zcwn9LJ+1Yqi;H?u^_r+692SchAl7-0xO+DbWU7)<;Jc?W zxK*WXId^v%;N%Ha~&Nb3J`#deqRCfX-;L?hYJoG$v z-$iQq+A2sBv8aOZ#R%d?r(ZDjI>SJ5F8jw$N|u@}ZIg|c@U|tGYp~h>rPCB1n|=BW z${G2A+ydKr?(A7Bwl|;&pKq$BCXzpXI2>RDVbm~p6d9VLWW<#3FolSoCTQ<~YZY|F z#H_Y5Pf=-b?kXpZxQJb=PFhve3Q5c6=KhWj74i%=mi6mX@i=w#v~)pETT>G}n6(3( z+1an@H;!o_bAtL$Krv_4{?#;TX{`d+wovx!G$j(L`{$=o4<0-~gRqeZLEcw7hLDw& z!)Xa|amQcR*X!!&>|5}EDj$iQ=w|MHxK{40)Xky5pdeRD!}DVVVv{pO5TDg}TgNFa z%geh`?Y_gyYgCxQRDR>aWUg#9QdM=Hd#z7AmD^G&B%f_?NTkF59F9WNPTM0uTH=Ia zV!?Q_hTr>Fo0o;ylcx!`=@5irtuGa=V7|C>M}t`y;YM=rg-avto|+3KQX-eVX|S#| z0YrZl4cu)SU)I-KQ~54+Hvr{oYioT6UsCmL^GZsZs~v2#8@`X%A})HrgsEsa4o<4S zZqLz4DCCBjkK+Y7n#L<6AIv7uFb?t*m*9)D_I8iKMY^bp zL=sAm9TcCbu@g;YKzCT&{GN zVIrH&zOWgFfC&Q@mX;@JHyRrofhW?_RknRWp;Jud{%(v)czr_KbHhgX5RO94H@l|F zOGZ5NZZnDo%7?U5Nlyb%JPJo;zJu`-)!-eGMc5*Aqm5Q;>SMjxRhyXT=nF|8w4+@D z)nlK%yKA$bjqBG784gY4*2AbynMgM1Vr6o zcRZfQ2hnkG$QVl@MckYZpk^^#x!ab?@bKVmEJ|Ke>^);~reY04sR~wvwx=|qp`Fp` z=wkH9!_Kq+fZA@oNWk`>>2UzS)_+Y zO8hOqbLZNz*Pl>iRkqML=Rr#FrVpa-CPReBpZ2$eX?4N#T(H&fs=5! z3efZ*Dj^Yi@i9H+@uNo#e*$HqAU!6rxGB^ggr%TxB|{<|0M^{-`(bDZ-4qhy;ur*> zOeXgj5LG8EZfyWq7&GXp!dL*Kz?Lv`ecaF$tM8g7ApBu?xYv*94Z`0RlTfWP!@Ojy zJaz=MYZy4kLMs{+8aFBm*_O73#%<%pHE&76QUkRDyfHF_C~0A2G$sX`ogev*?5& route_config_providers, SubscriptionFactoryFunction factory_function) : config_update_info_(config_update_info), - init_target_(fmt::format("VhdsConfigSubscription {}", config_update_info_->routeConfigName()), - [this]() { subscription_->start({}); }), scope_(factory_context.scope().createScope(stat_prefix + "vhds." + config_update_info_->routeConfigName() + ".")), stats_({ALL_VHDS_STATS(POOL_COUNTER(*scope_))}), + init_target_(fmt::format("VhdsConfigSubscription {}", config_update_info_->routeConfigName()), + [this]() { subscription_->start({}); }), route_config_providers_(route_config_providers), validation_visitor_(factory_context.messageValidationVisitor()) { Envoy::Config::Utility::checkLocalInfo("vhds", factory_context.localInfo()); diff --git a/source/common/router/vhds.h b/source/common/router/vhds.h index 147b4c1d3ea74..d435b07017d99 100644 --- a/source/common/router/vhds.h +++ b/source/common/router/vhds.h @@ -71,10 +71,10 @@ class VhdsSubscription : Envoy::Config::SubscriptionCallbacks, void registerInitTargetWithInitManager(Init::Manager& m) { m.add(init_target_); } RouteConfigUpdatePtr& config_update_info_; - std::unique_ptr subscription_; - Init::TargetImpl init_target_; Stats::ScopePtr scope_; VhdsStats stats_; + std::unique_ptr subscription_; + Init::TargetImpl init_target_; std::unordered_set& route_config_providers_; ProtobufMessage::ValidationVisitor& validation_visitor_; }; diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 8fce7e0d34a96..33576e59395be 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -204,6 +204,33 @@ ClusterManagerImpl::ClusterManagerImpl( } } + // TODO(fredlas) HACK to support + // loadCluster->clusterFromProto->ClusterFactoryImplBase::create->EdsClusterFactory::createClusterImpl(), + // which wants to call xdsIsDelta() on us. So, we need to get our xds_is_delta_ defined before + // then. Once SotW and delta are unified, that is_delta bool will be gone from everywhere, and the + // xds_is_delta_ variable can be removed. + const auto& dyn_resources = bootstrap.dynamic_resources(); + if (dyn_resources.has_ads_config()) { + xds_is_delta_ = + dyn_resources.ads_config().api_type() == envoy::api::v2::core::ApiConfigSource::DELTA_GRPC; + } else if (dyn_resources.has_cds_config()) { + const auto& cds_config = dyn_resources.cds_config(); + xds_is_delta_ = + cds_config.api_config_source().api_type() == + envoy::api::v2::core::ApiConfigSource::DELTA_GRPC || + (dyn_resources.has_ads_config() && dyn_resources.ads_config().api_type() == + envoy::api::v2::core::ApiConfigSource::DELTA_GRPC); + } else if (dyn_resources.has_lds_config()) { + const auto& lds_config = dyn_resources.lds_config(); + xds_is_delta_ = + lds_config.api_config_source().api_type() == + envoy::api::v2::core::ApiConfigSource::DELTA_GRPC || + (dyn_resources.has_ads_config() && dyn_resources.ads_config().api_type() == + envoy::api::v2::core::ApiConfigSource::DELTA_GRPC); + } else { + xds_is_delta_ = false; + } + // Cluster loading happens in two phases: first all the primary clusters are loaded, and then all // the secondary clusters are loaded. As it currently stands all non-EDS clusters are primary and // only EDS clusters are secondary. This two phase loading is done because in v2 configuration @@ -220,14 +247,12 @@ ClusterManagerImpl::ClusterManagerImpl( // This is the only point where distinction between delta ADS and state-of-the-world ADS is made. // After here, we just have a GrpcMux interface held in ads_mux_, which hides // whether the backing implementation is delta or SotW. - if (bootstrap.dynamic_resources().has_ads_config()) { - if (bootstrap.dynamic_resources().ads_config().api_type() == + if (dyn_resources.has_ads_config()) { + if (dyn_resources.ads_config().api_type() == envoy::api::v2::core::ApiConfigSource::DELTA_GRPC) { - auto& api_config_source = - bootstrap.dynamic_resources().has_ads_config() - ? bootstrap.dynamic_resources().ads_config() - : bootstrap.dynamic_resources().cds_config().api_config_source(); - std::cerr << "MAKING DELTA ADS" << std::endl; + auto& api_config_source = dyn_resources.has_ads_config() + ? dyn_resources.ads_config() + : dyn_resources.cds_config().api_config_source(); ads_mux_ = std::make_shared( Config::Utility::factoryForGrpcApiConfigSource(*async_client_manager_, api_config_source, stats) @@ -236,25 +261,20 @@ ClusterManagerImpl::ClusterManagerImpl( *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.DeltaAggregatedResources"), random_, stats_, - Envoy::Config::Utility::parseRateLimitSettings( - bootstrap.dynamic_resources().ads_config()), - local_info); + Envoy::Config::Utility::parseRateLimitSettings(dyn_resources.ads_config()), local_info); } else { - std::cerr << "MAKING SOTW ADS" << std::endl; ads_mux_ = std::make_shared( local_info, - Config::Utility::factoryForGrpcApiConfigSource( - *async_client_manager_, bootstrap.dynamic_resources().ads_config(), stats) + Config::Utility::factoryForGrpcApiConfigSource(*async_client_manager_, + dyn_resources.ads_config(), stats) ->create(), main_thread_dispatcher, *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), random_, stats_, - Envoy::Config::Utility::parseRateLimitSettings( - bootstrap.dynamic_resources().ads_config())); + Envoy::Config::Utility::parseRateLimitSettings(dyn_resources.ads_config())); } } else { - std::cerr << "MAKING NULLLLLLLLLL ADS" << std::endl; ads_mux_ = std::make_unique(); } @@ -287,15 +307,8 @@ ClusterManagerImpl::ClusterManagerImpl( }); // We can now potentially create the CDS API once the backing cluster exists. - const auto& dyn_resources = bootstrap.dynamic_resources(); if (dyn_resources.has_cds_config()) { - const auto& cds_config = dyn_resources.cds_config(); - xds_is_delta_ = - cds_config.api_config_source().api_type() == - envoy::api::v2::core::ApiConfigSource::DELTA_GRPC || - (dyn_resources.has_ads_config() && dyn_resources.ads_config().api_type() == - envoy::api::v2::core::ApiConfigSource::DELTA_GRPC); - cds_api_ = factory_.createCds(cds_config, xds_is_delta_, *this); + cds_api_ = factory_.createCds(dyn_resources.cds_config(), xds_is_delta_, *this); init_helper_.setCds(cds_api_.get()); } else { init_helper_.setCds(nullptr); diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index 1c7b26e5f8754..d106c19b98586 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -207,8 +207,7 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable, } // Server::ListenerComponentFactory - LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config) override { + LdsApiPtr createLdsApi(const envoy::api::v2::core::ConfigSource& lds_config, + bool is_delta) override { return std::make_unique(lds_config, clusterManager(), dispatcher(), random(), initManager(), localInfo(), stats(), listenerManager(), - messageValidationVisitor(), api()); + messageValidationVisitor(), api(), is_delta); } std::vector createNetworkFilterFactoryList( const Protobuf::RepeatedPtrField& filters, diff --git a/source/server/server.cc b/source/server/server.cc index 170fb5ec8ef0e..4600114f70292 100644 --- a/source/server/server.cc +++ b/source/server/server.cc @@ -472,14 +472,18 @@ RunHelper::RunHelper(Instance& instance, const Options& options, Event::Dispatch // Pause RDS to ensure that we don't send any requests until we've // subscribed to all the RDS resources. The subscriptions happen in the init callbacks, // so we pause RDS until we've completed all the callbacks. - cm.adsMux()->pause(Config::TypeUrl::get().RouteConfiguration); + if (cm.adsMux()) { + cm.adsMux()->pause(Config::TypeUrl::get().RouteConfiguration); + } ENVOY_LOG(info, "all clusters initialized. initializing init manager"); init_manager.initialize(init_watcher_); // Now that we're execute all the init callbacks we can resume RDS // as we've subscribed to all the statically defined RDS resources. - cm.adsMux()->resume(Config::TypeUrl::get().RouteConfiguration); + if (cm.adsMux()) { + cm.adsMux()->resume(Config::TypeUrl::get().RouteConfiguration); + } }); } diff --git a/test/common/config/delta_subscription_impl_test.cc b/test/common/config/delta_subscription_impl_test.cc index 921537c9568e3..34ace0973be1d 100644 --- a/test/common/config/delta_subscription_impl_test.cc +++ b/test/common/config/delta_subscription_impl_test.cc @@ -25,15 +25,15 @@ class DeltaSubscriptionImplTest : public DeltaSubscriptionTestHarness, public te TEST_F(DeltaSubscriptionImplTest, UpdateResourcesCausesRequest) { startSubscription({"name1", "name2", "name3"}); expectSendMessage({"name4"}, {"name1", "name2"}, Grpc::Status::GrpcStatus::Ok, "", {}); - subscription_->updateResources({"name3", "name4"}); + subscription_->updateResourceInterest({"name3", "name4"}); expectSendMessage({"name1", "name2"}, {}, Grpc::Status::GrpcStatus::Ok, "", {}); - subscription_->updateResources({"name1", "name2", "name3", "name4"}); + subscription_->updateResourceInterest({"name1", "name2", "name3", "name4"}); expectSendMessage({}, {"name1", "name2"}, Grpc::Status::GrpcStatus::Ok, "", {}); - subscription_->updateResources({"name3", "name4"}); + subscription_->updateResourceInterest({"name3", "name4"}); expectSendMessage({"name1", "name2"}, {}, Grpc::Status::GrpcStatus::Ok, "", {}); - subscription_->updateResources({"name1", "name2", "name3", "name4"}); + subscription_->updateResourceInterest({"name1", "name2", "name3", "name4"}); expectSendMessage({}, {"name1", "name2", "name3"}, Grpc::Status::GrpcStatus::Ok, "", {}); - subscription_->updateResources({"name4"}); + subscription_->updateResourceInterest({"name4"}); } // Checks that after a pause(), no requests are sent until resume(). @@ -47,11 +47,11 @@ TEST_F(DeltaSubscriptionImplTest, PauseHoldsRequest) { expectSendMessage({"name4"}, {"name1", "name2"}, Grpc::Status::GrpcStatus::Ok, "", {}); // If not for the pause, these updates would make the expectSendMessage fail due to too many // messages being sent. - subscription_->updateResources({"name3", "name4"}); - subscription_->updateResources({"name1", "name2", "name3", "name4"}); - subscription_->updateResources({"name3", "name4"}); - subscription_->updateResources({"name1", "name2", "name3", "name4"}); - subscription_->updateResources({"name3", "name4"}); + subscription_->updateResourceInterest({"name3", "name4"}); + subscription_->updateResourceInterest({"name1", "name2", "name3", "name4"}); + subscription_->updateResourceInterest({"name3", "name4"}); + subscription_->updateResourceInterest({"name1", "name2", "name3", "name4"}); + subscription_->updateResourceInterest({"name3", "name4"}); subscription_->resume(); } @@ -147,12 +147,12 @@ TEST(DeltaSubscriptionImplFixturelessTest, NoGrpcStream) { std::unique_ptr subscription = std::make_unique( xds_context, Config::TypeUrl::get().ClusterLoadAssignment, callbacks, stats, - std::chrono::milliseconds(12345)); + std::chrono::milliseconds(12345), false); EXPECT_CALL(*async_client, startRaw(_, _, _)).WillOnce(Return(nullptr)); subscription->start({"name1"}); - subscription->updateResources({"name1", "name2"}); + subscription->updateResourceInterest({"name1", "name2"}); } } // namespace diff --git a/test/common/config/delta_subscription_test_harness.h b/test/common/config/delta_subscription_test_harness.h index 8c80769dd5d46..977c5de78ae51 100644 --- a/test/common/config/delta_subscription_test_harness.h +++ b/test/common/config/delta_subscription_test_harness.h @@ -39,7 +39,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { random_, stats_store_, rate_limit_settings_, local_info_); subscription_ = std::make_unique( xds_context_, Config::TypeUrl::get().ClusterLoadAssignment, callbacks_, stats_, - init_fetch_timeout); + init_fetch_timeout, false); EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); } @@ -70,7 +70,6 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { last_cluster_names_ = cluster_names; expectSendMessage(last_cluster_names_, ""); subscription_->start(cluster_names); - xds_context_->start(); } void expectSendMessage(const std::set& cluster_names, @@ -152,7 +151,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { Mock::VerifyAndClearExpectations(&async_stream_); } - void updateResources(const std::set& cluster_names) override { + void updateResourceInterest(const std::set& cluster_names) override { std::set sub; std::set unsub; @@ -163,7 +162,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { std::inserter(unsub, unsub.begin())); expectSendMessage(sub, unsub, Grpc::Status::GrpcStatus::Ok, "", {}); - subscription_->updateResources(cluster_names); + subscription_->updateResourceInterest(cluster_names); last_cluster_names_ = cluster_names; } diff --git a/test/common/config/filesystem_subscription_test_harness.h b/test/common/config/filesystem_subscription_test_harness.h index aee1270ae4109..047cc700d9ea9 100644 --- a/test/common/config/filesystem_subscription_test_harness.h +++ b/test/common/config/filesystem_subscription_test_harness.h @@ -44,8 +44,8 @@ class FilesystemSubscriptionTestHarness : public SubscriptionTestHarness { subscription_.start(cluster_names); } - void updateResources(const std::set& cluster_names) override { - subscription_.updateResources(cluster_names); + void updateResourceInterest(const std::set& cluster_names) override { + subscription_.updateResourceInterest(cluster_names); } void updateFile(const std::string json, bool run_dispatcher = true) { diff --git a/test/common/config/grpc_subscription_impl_test.cc b/test/common/config/grpc_subscription_impl_test.cc index 199449a6ea083..50064de63cb7c 100644 --- a/test/common/config/grpc_subscription_impl_test.cc +++ b/test/common/config/grpc_subscription_impl_test.cc @@ -22,7 +22,7 @@ TEST_F(GrpcSubscriptionImplTest, StreamCreationFailure) { verifyStats(2, 0, 0, 1, 0); // Ensure this doesn't cause an issue by sending a request, since we don't // have a gRPC stream. - subscription_->updateResources({"cluster2"}); + subscription_->updateResourceInterest({"cluster2"}); // Retry and succeed. EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); @@ -59,14 +59,14 @@ TEST_F(GrpcSubscriptionImplTest, RepeatedNonce) { startSubscription({"cluster0", "cluster1"}); verifyStats(1, 0, 0, 0, 0); // First with the initial, empty version update to "0". - updateResources({"cluster2"}); + updateResourceInterest({"cluster2"}); verifyStats(2, 0, 0, 0, 0); deliverConfigUpdate({"cluster0", "cluster2"}, "0", false); verifyStats(3, 0, 1, 0, 0); deliverConfigUpdate({"cluster0", "cluster2"}, "0", true); verifyStats(4, 1, 1, 0, 7148434200721666028); // Now with version "0" update to "1". - updateResources({"cluster3"}); + updateResourceInterest({"cluster3"}); verifyStats(5, 1, 1, 0, 7148434200721666028); deliverConfigUpdate({"cluster3"}, "1", false); verifyStats(6, 1, 2, 0, 7148434200721666028); diff --git a/test/common/config/grpc_subscription_test_harness.h b/test/common/config/grpc_subscription_test_harness.h index cb98b0795b1d5..e58394fffdcf0 100644 --- a/test/common/config/grpc_subscription_test_harness.h +++ b/test/common/config/grpc_subscription_test_harness.h @@ -113,7 +113,7 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { Mock::VerifyAndClearExpectations(&async_stream_); } - void updateResources(const std::set& cluster_names) override { + void updateResourceInterest(const std::set& cluster_names) override { // The "watch" mechanism means that updates that lose interest in a resource // will first generate a request for [still watched resources, i.e. without newly unwatched // ones] before generating the request for all of cluster_names. @@ -127,7 +127,7 @@ class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { } expectSendMessage(both, version_); expectSendMessage(cluster_names, version_); - subscription_->updateResources(cluster_names); + subscription_->updateResourceInterest(cluster_names); last_cluster_names_ = cluster_names; } diff --git a/test/common/config/http_subscription_test_harness.h b/test/common/config/http_subscription_test_harness.h index feeef1185de0c..d125ff01437f1 100644 --- a/test/common/config/http_subscription_test_harness.h +++ b/test/common/config/http_subscription_test_harness.h @@ -105,10 +105,10 @@ class HttpSubscriptionTestHarness : public SubscriptionTestHarness { subscription_->start(cluster_names); } - void updateResources(const std::set& cluster_names) override { + void updateResourceInterest(const std::set& cluster_names) override { cluster_names_ = cluster_names; expectSendMessage(cluster_names, version_); - subscription_->updateResources(cluster_names); + subscription_->updateResourceInterest(cluster_names); timer_cb_(); } diff --git a/test/common/config/subscription_impl_test.cc b/test/common/config/subscription_impl_test.cc index 2297b820fd7ca..bdc0e6fd6a48e 100644 --- a/test/common/config/subscription_impl_test.cc +++ b/test/common/config/subscription_impl_test.cc @@ -45,8 +45,8 @@ class SubscriptionImplTest : public testing::TestWithParam { test_harness_->startSubscription(cluster_names); } - void updateResources(const std::set& cluster_names) { - test_harness_->updateResources(cluster_names); + void updateResourceInterest(const std::set& cluster_names) { + test_harness_->updateResourceInterest(cluster_names); } void expectSendMessage(const std::set& cluster_names, const std::string& version) { @@ -139,7 +139,7 @@ TEST_P(SubscriptionImplTest, UpdateResources) { verifyStats(1, 0, 0, 0, 0); deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); verifyStats(2, 1, 0, 0, 7148434200721666028); - updateResources({"cluster2"}); + updateResourceInterest({"cluster2"}); verifyStats(3, 1, 0, 0, 7148434200721666028); } diff --git a/test/common/config/subscription_test_harness.h b/test/common/config/subscription_test_harness.h index 75423403c0f9c..02b9c2f76d4ed 100644 --- a/test/common/config/subscription_test_harness.h +++ b/test/common/config/subscription_test_harness.h @@ -30,7 +30,7 @@ class SubscriptionTestHarness { * Update cluster names to be delivered via EDS. * @param cluster_names cluster names. */ - virtual void updateResources(const std::set& cluster_names) PURE; + virtual void updateResourceInterest(const std::set& cluster_names) PURE; /** * Expect that an update request is sent by the Subscription implementation. diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index 9d3b73be829bf..521f3acaca411 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -324,10 +324,8 @@ TEST_P(AdsIntegrationTest, Failure) { EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "", {}, {}, {})); - EXPECT_TRUE(compareDiscoveryRequest( - Config::TypeUrl::get().Cluster, "", {}, {}, {}, Grpc::Status::GrpcStatus::Internal, - fmt::format("{} does not match {}", Config::TypeUrl::get().ClusterLoadAssignment, - Config::TypeUrl::get().Cluster))); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {}, {}, {}, + Grpc::Status::GrpcStatus::Internal, "does not match")); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {buildCluster("cluster_0")}, {buildCluster("cluster_0")}, {}, "1"); @@ -339,11 +337,9 @@ TEST_P(AdsIntegrationTest, Failure) { {buildCluster("cluster_0")}, {}, "1"); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "1", {}, {}, {})); - EXPECT_TRUE( - compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "", {"cluster_0"}, {}, - {}, Grpc::Status::GrpcStatus::Internal, - fmt::format("{} does not match {}", Config::TypeUrl::get().Cluster, - Config::TypeUrl::get().ClusterLoadAssignment))); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "", + {"cluster_0"}, {}, {}, Grpc::Status::GrpcStatus::Internal, + "does not match")); sendDiscoveryResponse( Config::TypeUrl::get().ClusterLoadAssignment, {buildClusterLoadAssignment("cluster_0")}, {buildClusterLoadAssignment("cluster_0")}, {}, "1"); @@ -354,10 +350,8 @@ TEST_P(AdsIntegrationTest, Failure) { Config::TypeUrl::get().Listener, {buildRouteConfig("listener_0", "route_config_0")}, {buildRouteConfig("listener_0", "route_config_0")}, {}, "1"); - EXPECT_TRUE(compareDiscoveryRequest( - Config::TypeUrl::get().Listener, "", {}, {}, {}, Grpc::Status::GrpcStatus::Internal, - fmt::format("{} does not match {}", Config::TypeUrl::get().RouteConfiguration, - Config::TypeUrl::get().Listener))); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "", {}, {}, {}, + Grpc::Status::GrpcStatus::Internal, "does not match")); sendDiscoveryResponse( Config::TypeUrl::get().Listener, {buildListener("listener_0", "route_config_0")}, {buildListener("listener_0", "route_config_0")}, {}, "1"); @@ -369,11 +363,9 @@ TEST_P(AdsIntegrationTest, Failure) { {buildListener("route_config_0", "cluster_0")}, {}, "1"); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "1", {}, {}, {})); - EXPECT_TRUE( - compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "", {"route_config_0"}, {}, - {}, Grpc::Status::GrpcStatus::Internal, - fmt::format("{} does not match {}", Config::TypeUrl::get().Listener, - Config::TypeUrl::get().RouteConfiguration))); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "", + {"route_config_0"}, {}, {}, + Grpc::Status::GrpcStatus::Internal, "does not match")); sendDiscoveryResponse( Config::TypeUrl::get().RouteConfiguration, {buildRouteConfig("route_config_0", "cluster_0")}, {buildRouteConfig("route_config_0", "cluster_0")}, {}, "1"); @@ -382,6 +374,7 @@ TEST_P(AdsIntegrationTest, Failure) { {"route_config_0"}, {}, {})); test_server_->waitForCounterGe("listener_manager.listener_create_success", 1); + makeSingleRequest(); } @@ -492,7 +485,7 @@ TEST_P(AdsIntegrationTest, DuplicateWarmingClusters) { {buildListener("listener_0", "route_config_0")}, {}, "1"); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1", - {"cluster_0"}, {"cluster_0"}, {})); + {"cluster_0"}, {}, {})); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "", {"route_config_0"}, {"route_config_0"}, {})); sendDiscoveryResponse( @@ -555,17 +548,19 @@ TEST_P(AdsIntegrationTest, CdsPausedDuringWarming) { sendDiscoveryResponse( Config::TypeUrl::get().Cluster, {buildCluster("warming_cluster_1")}, {buildCluster("warming_cluster_1")}, {"cluster_0"}, "2"); + test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 1); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1", {"warming_cluster_1"}, {"warming_cluster_1"}, {"cluster_0"})); // Send the second warming cluster. - sendDiscoveryResponse( - Config::TypeUrl::get().Cluster, {buildCluster("warming_cluster_2")}, - {buildCluster("warming_cluster_2")}, {"warming_cluster_1"}, "3"); + sendDiscoveryResponse(Config::TypeUrl::get().Cluster, + {buildCluster("warming_cluster_2")}, + {buildCluster("warming_cluster_2")}, {}, "3"); test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 2); // We would've got a Cluster discovery request with version 2 here, had the CDS not been paused. + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1", {"warming_cluster_2", "warming_cluster_1"}, {"warming_cluster_2"}, {})); @@ -583,6 +578,13 @@ TEST_P(AdsIntegrationTest, CdsPausedDuringWarming) { test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0); // CDS is resumed and EDS response was acknowledged. + if (sotwOrDelta() == Grpc::SotwOrDelta::Delta) { + // The delta implementation sends a separate ACK for every cluster it got while paused, whereas + // SotW just ACKs the most recent. + // TODO(fredlas) when the implementations are unified, this if can become if(true). + EXPECT_TRUE(compareDeltaDiscoveryRequest(Config::TypeUrl::get().Cluster, {}, {}, + Grpc::Status::GrpcStatus::Ok, "")); + } EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "3", {}, {}, {})); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "2", {"warming_cluster_2", "warming_cluster_1"}, {}, {})); @@ -635,9 +637,9 @@ TEST_P(AdsIntegrationTest, ClusterWarmingOnNamedResponse) { {"warming_cluster_1"}, {"warming_cluster_1"}, {"cluster_0"})); // Send the second warming cluster. - sendDiscoveryResponse( - Config::TypeUrl::get().Cluster, {buildCluster("warming_cluster_2")}, - {buildCluster("warming_cluster_2")}, {"warming_cluster_1"}, "3"); + sendDiscoveryResponse(Config::TypeUrl::get().Cluster, + {buildCluster("warming_cluster_2")}, + {buildCluster("warming_cluster_2")}, {}, "3"); test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 2); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1", @@ -648,7 +650,7 @@ TEST_P(AdsIntegrationTest, ClusterWarmingOnNamedResponse) { sendDiscoveryResponse( Config::TypeUrl::get().ClusterLoadAssignment, {buildClusterLoadAssignment("warming_cluster_1")}, - {buildClusterLoadAssignment("warming_cluster_1")}, {"cluster_0"}, "2"); + {buildClusterLoadAssignment("warming_cluster_1")}, {}, "2"); // Envoy will not finish warming of the second cluster because of the missing load assignments // i,e. no named EDS response. @@ -658,7 +660,7 @@ TEST_P(AdsIntegrationTest, ClusterWarmingOnNamedResponse) { sendDiscoveryResponse( Config::TypeUrl::get().ClusterLoadAssignment, {buildClusterLoadAssignment("warming_cluster_2")}, - {buildClusterLoadAssignment("warming_cluster_2")}, {"warming_cluster_1"}, "3"); + {buildClusterLoadAssignment("warming_cluster_2")}, {}, "3"); test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0); } diff --git a/test/integration/integration.cc b/test/integration/integration.cc index 8cbbf958e6cab..3a601502a3397 100644 --- a/test/integration/integration.cc +++ b/test/integration/integration.cc @@ -532,21 +532,21 @@ AssertionResult BaseIntegrationTest::compareDiscoveryRequest( const std::vector& expected_resource_names, const std::vector& expected_resource_names_added, const std::vector& expected_resource_names_removed, - const Protobuf::int32 expected_error_code, const std::string& expected_error_message) { + const Protobuf::int32 expected_error_code, const std::string& expected_error_substring) { if (sotw_or_delta_ == Grpc::SotwOrDelta::Sotw) { return compareSotwDiscoveryRequest(expected_type_url, expected_version, expected_resource_names, - expected_error_code, expected_error_message); + expected_error_code, expected_error_substring); } else { return compareDeltaDiscoveryRequest(expected_type_url, expected_resource_names_added, expected_resource_names_removed, expected_error_code, - expected_error_message); + expected_error_substring); } } AssertionResult BaseIntegrationTest::compareSotwDiscoveryRequest( const std::string& expected_type_url, const std::string& expected_version, const std::vector& expected_resource_names, - const Protobuf::int32 expected_error_code, const std::string& expected_error_message) { + const Protobuf::int32 expected_error_code, const std::string& expected_error_substring) { envoy::api::v2::DiscoveryRequest discovery_request; VERIFY_ASSERTION(xds_stream_->waitForGrpcMessage(*dispatcher_, discovery_request)); @@ -564,7 +564,7 @@ AssertionResult BaseIntegrationTest::compareSotwDiscoveryRequest( expected_error_code); } EXPECT_TRUE( - IsSubstring("", "", expected_error_message, discovery_request.error_detail().message())); + IsSubstring("", "", expected_error_substring, discovery_request.error_detail().message())); const std::vector resource_names(discovery_request.resource_names().cbegin(), discovery_request.resource_names().cend()); if (expected_resource_names != resource_names) { @@ -582,47 +582,69 @@ AssertionResult BaseIntegrationTest::compareSotwDiscoveryRequest( return AssertionSuccess(); } +AssertionResult compareSets(std::set set1, std::set set2, + absl::string_view name) { + if (set1 == set2) { + return AssertionSuccess(); + } + auto failure = AssertionFailure() << name << " field not as expected.\nExpected: {"; + for (auto x : set1) { + failure << x << ", "; + } + failure << "}\nActual: {"; + for (auto x : set2) { + failure << x << ", "; + } + return failure << "}"; +} + AssertionResult BaseIntegrationTest::compareDeltaDiscoveryRequest( const std::string& expected_type_url, const std::vector& expected_resource_subscriptions, const std::vector& expected_resource_unsubscriptions, FakeStreamPtr& xds_stream, - const Protobuf::int32 expected_error_code, const std::string& expected_error_message) { + const Protobuf::int32 expected_error_code, const std::string& expected_error_substring) { envoy::api::v2::DeltaDiscoveryRequest request; VERIFY_ASSERTION(xds_stream->waitForGrpcMessage(*dispatcher_, request)); - EXPECT_TRUE(request.has_node()); - EXPECT_FALSE(request.node().id().empty()); - EXPECT_FALSE(request.node().cluster().empty()); - - if (expected_type_url != request.type_url()) { - return AssertionFailure() << fmt::format("type_url {} does not match expected {}", - request.type_url(), expected_type_url); + // Verify all we care about node. + if (!request.has_node() || request.node().id().empty() || request.node().cluster().empty()) { + return AssertionFailure() << "Weird node field"; } - if (!(expected_error_code == request.error_detail().code())) { - return AssertionFailure() << fmt::format("error_code {} does not match expected {}", - request.error_detail().code(), expected_error_code); + if (request.type_url() != expected_type_url) { + return AssertionFailure() << fmt::format("type_url {} does not match expected {}.", + request.type_url(), expected_type_url); } - EXPECT_TRUE(IsSubstring("", "", expected_error_message, request.error_detail().message())); - - const std::vector resource_subscriptions(request.resource_names_subscribe().cbegin(), - request.resource_names_subscribe().cend()); - if (expected_resource_subscriptions != resource_subscriptions) { - return AssertionFailure() << fmt::format( - "newly subscribed resources {} do not match expected {} in {}", - fmt::join(resource_subscriptions.begin(), resource_subscriptions.end(), ","), - fmt::join(expected_resource_subscriptions.begin(), - expected_resource_subscriptions.end(), ","), - request.DebugString()); - } - const std::vector resource_unsubscriptions( - request.resource_names_unsubscribe().cbegin(), request.resource_names_unsubscribe().cend()); - if (expected_resource_unsubscriptions != resource_unsubscriptions) { + // Sort to ignore ordering. + std::set expected_sub{expected_resource_subscriptions.begin(), + expected_resource_subscriptions.end()}; + std::set expected_unsub{expected_resource_unsubscriptions.begin(), + expected_resource_unsubscriptions.end()}; + std::set actual_sub{request.resource_names_subscribe().begin(), + request.resource_names_subscribe().end()}; + std::set actual_unsub{request.resource_names_unsubscribe().begin(), + request.resource_names_unsubscribe().end()}; + auto sub_result = compareSets(expected_sub, actual_sub, "expected_resource_subscriptions"); + if (!sub_result) { + return sub_result; + } + auto unsub_result = + compareSets(expected_unsub, actual_unsub, "expected_resource_unsubscriptions"); + if (!unsub_result) { + return unsub_result; + } + // (We don't care about response_nonce or initial_resource_versions.) + + if (request.error_detail().code() != expected_error_code) { return AssertionFailure() << fmt::format( - "newly UNsubscribed resources {} do not match expected {} in {}", - fmt::join(resource_unsubscriptions.begin(), resource_unsubscriptions.end(), ","), - fmt::join(expected_resource_unsubscriptions.begin(), - expected_resource_unsubscriptions.end(), ","), - request.DebugString()); + "error code {} does not match expected {}. (Error message is {}).", + request.error_detail().code(), expected_error_code, + request.error_detail().message()); + } + if (expected_error_code != Grpc::Status::GrpcStatus::Ok && + request.error_detail().message().find(expected_error_substring) == std::string::npos) { + return AssertionFailure() << "\"" << expected_error_substring + << "\" is not a substring of error message \"" + << request.error_detail().message() << "\""; } return AssertionSuccess(); } diff --git a/test/integration/integration.h b/test/integration/integration.h index fb542490fa10b..1c661c5dd9c28 100644 --- a/test/integration/integration.h +++ b/test/integration/integration.h @@ -258,6 +258,8 @@ class BaseIntegrationTest : Logger::Loggable { for (const auto& message : messages) { discovery_response.add_resources()->PackFrom(message); } + static int next_nonce_counter = 0; + discovery_response.set_nonce(absl::StrCat("nonce", next_nonce_counter++)); xds_stream_->sendGrpcMessage(discovery_response); } @@ -284,8 +286,8 @@ class BaseIntegrationTest : Logger::Loggable { resource->mutable_resource()->PackFrom(message); } *response.mutable_removed_resources() = {removed.begin(), removed.end()}; - response.set_nonce("noncense"); - response.set_type_url(Grpc::Common::typeUrl(T().GetDescriptor()->full_name())); + static int next_nonce_counter = 0; + response.set_nonce(absl::StrCat("nonce", next_nonce_counter++)); stream->sendGrpcMessage(response); } diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index 540dfee5544dd..3436c3a435deb 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -43,7 +43,7 @@ template class MockSubscriptionCallbacks : public Subscript class MockSubscription : public Subscription { public: MOCK_METHOD1(start, void(const std::set& resources)); - MOCK_METHOD1(updateResources, void(const std::set& update_to_these_names)); + MOCK_METHOD1(updateResourceInterest, void(const std::set& update_to_these_names)); }; class MockGrpcMuxWatch : public GrpcMuxWatch { @@ -71,15 +71,13 @@ class MockGrpcMux : public GrpcMux { void(const std::set& resources, const std::string& type_url, SubscriptionCallbacks& callbacks, SubscriptionStats& stats, std::chrono::milliseconds init_fetch_timeout)); - MOCK_METHOD2(updateResources, + MOCK_METHOD2(updateResourceInterest, void(const std::set& resources, const std::string& type_url)); - MOCK_METHOD4(addWatch, WatchPtr(const std::string&, const std::set&, - SubscriptionCallbacks&, std::chrono::milliseconds)); - MOCK_METHOD2(removeWatch, void(const std::string&, Watch*)); - MOCK_METHOD3(updateWatch, void(const std::string&, Watch*, const std::set&)); - - MOCK_METHOD1(removeSubscription, void(const std::string& type_url)); + MOCK_METHOD5(addOrUpdateWatch, + void(const std::string& type_url, WatchPtr& watch, + const std::set& resources, SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout)); }; class MockGrpcMuxCallbacks : public GrpcMuxCallbacks { diff --git a/test/server/lds_api_test.cc b/test/server/lds_api_test.cc index ecdc6506f5ffb..8dae4748f4c94 100644 --- a/test/server/lds_api_test.cc +++ b/test/server/lds_api_test.cc @@ -64,7 +64,7 @@ class LdsApiTest : public testing::Test { EXPECT_CALL(init_manager_, add(_)); lds_ = std::make_unique(lds_config, cluster_manager_, dispatcher_, random_, init_manager_, local_info_, store_, listener_manager_, - validation_visitor_, *api_); + validation_visitor_, *api_, false); expectRequest(); init_target_handle_->initialize(init_watcher_); @@ -185,7 +185,7 @@ TEST_F(LdsApiTest, UnknownCluster) { EXPECT_CALL(cluster_manager_, clusters()).WillOnce(Return(cluster_map)); EXPECT_THROW_WITH_MESSAGE( LdsApiImpl(lds_config, cluster_manager_, dispatcher_, random_, init_manager_, local_info_, - store_, listener_manager_, validation_visitor_, *api_), + store_, listener_manager_, validation_visitor_, *api_, false), EnvoyException, "envoy::api::v2::core::ConfigSource must have a statically defined non-EDS " "cluster: 'foo_cluster' does not exist, was added via api, or is an " @@ -312,7 +312,7 @@ TEST_F(LdsApiTest, BadLocalInfo) { ON_CALL(local_info_, clusterName()).WillByDefault(ReturnRef(EMPTY_STRING)); EXPECT_THROW_WITH_MESSAGE( LdsApiImpl(lds_config, cluster_manager_, dispatcher_, random_, init_manager_, local_info_, - store_, listener_manager_, validation_visitor_, *api_), + store_, listener_manager_, validation_visitor_, *api_, false), EnvoyException, "lds: node 'id' and 'cluster' are required. Set it either in 'node' config or via " "--service-node and --service-cluster options."); diff --git a/test/server/listener_manager_impl_test.cc b/test/server/listener_manager_impl_test.cc index e9c4c5fdc47f3..89e7376866b7e 100644 --- a/test/server/listener_manager_impl_test.cc +++ b/test/server/listener_manager_impl_test.cc @@ -731,9 +731,9 @@ TEST_F(ListenerManagerImplTest, AddOrUpdateListener) { InSequence s; MockLdsApi* lds_api = new MockLdsApi(); - EXPECT_CALL(listener_factory_, createLdsApi_(_)).WillOnce(Return(lds_api)); + EXPECT_CALL(listener_factory_, createLdsApi_(_, _)).WillOnce(Return(lds_api)); envoy::api::v2::core::ConfigSource lds_config; - manager_->createLdsApi(lds_config); + manager_->createLdsApi(lds_config, false); EXPECT_CALL(*lds_api, versionInfo()).WillOnce(Return("")); checkConfigDump(R"EOF( From b3c98e7198d0f5ea65e1eb339757fce5bb84e222 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Sat, 15 Jun 2019 17:59:49 -0400 Subject: [PATCH 29/56] rename GrpcDeltaXdsContext to NewGrpcMuxImpl Signed-off-by: Fred Douglas --- source/common/config/BUILD | 8 +- source/common/config/README.md | 10 +- .../common/config/delta_subscription_impl.h | 10 +- source/common/config/grpc_delta_xds_context.h | 286 ------------------ source/common/config/new_grpc_mux_impl.cc | 231 ++++++++++++++ source/common/config/new_grpc_mux_impl.h | 117 +++++++ .../config/subscription_factory_impl.cc | 4 +- .../common/upstream/cluster_manager_impl.cc | 4 +- .../config/delta_subscription_impl_test.cc | 10 +- .../config/delta_subscription_test_harness.h | 6 +- 10 files changed, 375 insertions(+), 311 deletions(-) delete mode 100644 source/common/config/grpc_delta_xds_context.h create mode 100644 source/common/config/new_grpc_mux_impl.cc create mode 100644 source/common/config/new_grpc_mux_impl.h diff --git a/source/common/config/BUILD b/source/common/config/BUILD index fbebc3605ac98..4cf792d20b720 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -83,8 +83,8 @@ envoy_cc_library( srcs = ["delta_subscription_impl.cc"], hdrs = ["delta_subscription_impl.h"], deps = [ - ":grpc_delta_xds_context_lib", ":grpc_stream_lib", + ":new_grpc_mux_lib", ":utility_lib", "//include/envoy/config:subscription_interface", "//include/envoy/grpc:async_client_interface", @@ -214,10 +214,12 @@ envoy_cc_library( ) envoy_cc_library( - name = "grpc_delta_xds_context_lib", - hdrs = ["grpc_delta_xds_context.h"], + name = "new_grpc_mux_lib", + srcs = ["new_grpc_mux_impl.cc"], + hdrs = ["new_grpc_mux_impl.h"], deps = [ ":delta_subscription_state_lib", + ":grpc_stream_lib", ":pausable_ack_queue_lib", ":watch_map_lib", "//include/envoy/event:dispatcher_interface", diff --git a/source/common/config/README.md b/source/common/config/README.md index 7c0b860aaea5f..77dd614a29025 100644 --- a/source/common/config/README.md +++ b/source/common/config/README.md @@ -1,4 +1,4 @@ -xDS +# xDS xDS stands for [fill in the blank] Discovery Service. It provides dynamic config discovery/updates. @@ -16,7 +16,7 @@ subscription logic will maintain a subscription to the union: X Y and Z. Updates to X will be delivered to the first object, Y to both, Z to the second. This logic is implemented by WatchMap. -If you are working on Envoy's gRPC xDS client logic itself, read on. +### If you are working on Envoy's gRPC xDS client logic itself, read on. When using gRPC, xDS has two pairs of options: aggregated/non-aggregated, and delta/state-of-the-world updates. All four combinations of these are usable. @@ -25,7 +25,7 @@ delta/state-of-the-world updates. All four combinations of these are usable. For Envoy's implementation of xDS client logic, there is effectively no difference between aggregated xDS and non-aggregated: they both use the same request/response protos. The non-aggregated case is handled by running the aggregated logic, and just happening to only have 1 -xDS subscription type to "aggregate", i.e., GrpcDeltaXdsContext only has one +xDS subscription type to "aggregate", i.e., NewGrpcMuxImpl only has one DeltaSubscriptionState entry in its map. However, to the config server, there is a huge difference: when using ADS (caused @@ -36,8 +36,8 @@ and having identical client code, they're actually different gRPC services. Delta vs state-of-the-world is a question of wire format: the protos in question are named [Delta]Discovery{Request,Response}. That is what the GrpcMux interface is useful for: its -GrpcDeltaXdsContext (TODO may be renamed) implementation works with DeltaDiscovery{Request,Response} and has -delta-specific logic; its GrpxMuxImpl implementation (TODO will be merged into GrpcDeltaXdsContext) works with Discovery{Request,Response} +NewGrpcMuxImpl (TODO may be renamed) implementation works with DeltaDiscovery{Request,Response} and has +delta-specific logic; its GrpxMuxImpl implementation (TODO will be merged into NewGrpcMuxImpl) works with Discovery{Request,Response} and has SotW-specific logic. Both the delta and SotW Subscription implementations (TODO will be merged) hold a shared_ptr. The shared_ptr allows for both non- and aggregated: if non-aggregated, you'll be the only holder of that shared_ptr. diff --git a/source/common/config/delta_subscription_impl.h b/source/common/config/delta_subscription_impl.h index a2d16c04b2eef..99981aa8e5642 100644 --- a/source/common/config/delta_subscription_impl.h +++ b/source/common/config/delta_subscription_impl.h @@ -2,7 +2,7 @@ #include "envoy/config/subscription.h" -#include "common/config/grpc_delta_xds_context.h" +#include "common/config/new_grpc_mux_impl.h" #include "common/config/utility.h" namespace Envoy { @@ -10,16 +10,16 @@ namespace Config { // DeltaSubscriptionImpl provides a top-level interface to the Envoy's communication with an xDS // server, for use by the various xDS users within Envoy. It is built around a (shared) -// GrpcDeltaXdsContext, and the further machinery underlying that. An xDS user indicates interest in +// NewGrpcMuxImpl, and the further machinery underlying that. An xDS user indicates interest in // various resources via start() and updateResourceInterest(). It receives updates to those // resources via the SubscriptionCallbacks it provides. Multiple users can each have their own -// Subscription object for the same type_url; GrpcDeltaXdsContext maintains a subscription to the +// Subscription object for the same type_url; NewGrpcMuxImpl maintains a subscription to the // union of interested resources, and delivers to the users just the resource updates that they are // "watching" for. // -// DeltaSubscriptionImpl and GrpcDeltaXdsContext are both built to provide both regular xDS and ADS, +// DeltaSubscriptionImpl and NewGrpcMuxImpl are both built to provide both regular xDS and ADS, // distinguished by whether multiple DeltaSubscriptionImpls are sharing a single -// GrpcDeltaXdsContext. (And by the gRPC method string, but that's taken care of over in +// NewGrpcMuxImpl. (And by the gRPC method string, but that's taken care of over in // SubscriptionFactory). // // Why does DeltaSubscriptionImpl itself implement the SubscriptionCallbacks interface? So that it diff --git a/source/common/config/grpc_delta_xds_context.h b/source/common/config/grpc_delta_xds_context.h deleted file mode 100644 index 1f4810fe3a600..0000000000000 --- a/source/common/config/grpc_delta_xds_context.h +++ /dev/null @@ -1,286 +0,0 @@ -#pragma once - -#include - -#include "envoy/api/v2/discovery.pb.h" -#include "envoy/common/token_bucket.h" -#include "envoy/config/subscription.h" - -#include "common/common/assert.h" -#include "common/common/backoff_strategy.h" -#include "common/common/logger.h" -#include "common/common/token_bucket_impl.h" -#include "common/config/delta_subscription_state.h" -#include "common/config/grpc_stream.h" -#include "common/config/pausable_ack_queue.h" -#include "common/config/utility.h" -#include "common/config/watch_map.h" -#include "common/grpc/common.h" -#include "common/protobuf/protobuf.h" -#include "common/protobuf/utility.h" - -namespace Envoy { -namespace Config { - -// Manages subscriptions to one or more type of resource. The logical protocol -// state of those subscription(s) is handled by DeltaSubscriptionState. -// This class owns the GrpcStream used to talk to the server, maintains queuing -// logic to properly order the subscription(s)' various messages, and allows -// starting/stopping/pausing of the subscriptions. -class GrpcDeltaXdsContext : public GrpcMux, - public GrpcStreamCallbacks, - Logger::Loggable { -public: - GrpcDeltaXdsContext(Grpc::RawAsyncClientPtr async_client, Event::Dispatcher& dispatcher, - const Protobuf::MethodDescriptor& service_method, - Runtime::RandomGenerator& random, Stats::Scope& scope, - const RateLimitSettings& rate_limit_settings, - const LocalInfo::LocalInfo& local_info) - : dispatcher_(dispatcher), local_info_(local_info), - grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, - rate_limit_settings) {} - - void addOrUpdateWatch(const std::string& type_url, WatchPtr& watch, - const std::set& resources, SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) override { - if (watch.get() == nullptr) { - watch = addWatch(type_url, resources, callbacks, init_fetch_timeout); - } else { - updateWatch(type_url, watch.get(), resources); - } - } - - void pause(const std::string& type_url) override { pausable_ack_queue_.pause(type_url); } - - void resume(const std::string& type_url) override { - pausable_ack_queue_.resume(type_url); - trySendDiscoveryRequests(); - } - - bool paused(const std::string& type_url) const override { - return pausable_ack_queue_.paused(type_url); - } - - void - onDiscoveryResponse(std::unique_ptr&& message) override { - ENVOY_LOG(debug, "Received DeltaDiscoveryResponse for {} at version {}", message->type_url(), - message->system_version_info()); - auto sub = subscriptions_.find(message->type_url()); - if (sub == subscriptions_.end()) { - ENVOY_LOG(warn, - "Dropping received DeltaDiscoveryResponse (with version {}) for non-existent " - "subscription {}.", - message->system_version_info(), message->type_url()); - return; - } - kickOffAck(sub->second->sub_state_.handleResponse(*message)); - } - - void onStreamEstablished() override { - for (auto& sub : subscriptions_) { - sub.second->sub_state_.markStreamFresh(); - } - trySendDiscoveryRequests(); - } - - void onEstablishmentFailure() override { - // If this happens while Envoy is still initializing, the onConfigUpdateFailed() we ultimately - // call on CDS will cause LDS to start up, which adds to subscriptions_ here. So, to avoid a - // crash, the iteration needs to dance around a little: collect pointers to all - // SubscriptionStates, call on all those pointers we haven't yet called on, repeat if there are - // now more SubscriptionStates. - absl::flat_hash_map all_subscribed; - absl::flat_hash_map already_called; - do { - for (auto& sub : subscriptions_) { - all_subscribed[sub.first] = &sub.second->sub_state_; - } - for (auto& sub : all_subscribed) { - if (already_called.insert(sub).second) { // insert succeeded --> not already called - sub.second->handleEstablishmentFailure(); - } - } - } while (all_subscribed.size() != subscriptions_.size()); - } - - void onWriteable() override { trySendDiscoveryRequests(); } - - void kickOffAck(UpdateAck ack) { - pausable_ack_queue_.push(ack); - trySendDiscoveryRequests(); - } - - // TODO TODO but yeah this should just be gone!!!!!!!!! - GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, - GrpcMuxCallbacks&) override { - // not sure what GrpcMuxCallbacks is for, but we need a SubscriptionCallbacks here, so...... - // return std::make_unique(*this, addWatch(type_url, resources, - // callbacks)); - NOT_IMPLEMENTED_GCOVR_EXCL_LINE; - } - void start() override { grpc_stream_.establishNewStream(); } - -private: - WatchPtr addWatch(const std::string& type_url, const std::set& resources, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) { - auto entry = subscriptions_.find(type_url); - if (entry == subscriptions_.end()) { - // We don't yet have a subscription for type_url! Make one! - addSubscription(type_url, init_fetch_timeout); - return addWatch(type_url, resources, callbacks, init_fetch_timeout); - } - - WatchPtr watch = entry->second->watch_map_.addWatch(callbacks); - // updateWatch() queues a discovery request if any of 'resources' are not yet subscribed. - updateWatch(type_url, watch.get(), resources); - return watch; - } - - // Updates the list of resource names watched by the given watch. If an added name is new across - // the whole subscription, or if a removed name has no other watch interested in it, then the - // subscription will enqueue and attempt to send an appropriate discovery request. - void updateWatch(const std::string& type_url, Watch* watch, - const std::set& resources) { - ASSERT(watch != nullptr); - auto sub = subscriptions_.find(type_url); - if (sub == subscriptions_.end()) { - ENVOY_LOG(error, "Watch of {} has no subscription to update.", type_url); - return; - } - auto added_removed = sub->second->watch_map_.updateWatchInterest(watch, resources); - sub->second->sub_state_.updateSubscriptionInterest(added_removed.added_, - added_removed.removed_); - // Tell the server about our new interests, if there are any. - trySendDiscoveryRequests(); - } - - void addSubscription(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout) { - subscriptions_.emplace(type_url, std::make_unique( - type_url, init_fetch_timeout, dispatcher_, local_info_)); - subscription_ordering_.emplace_back(type_url); - } - - void trySendDiscoveryRequests() { - while (true) { - // Do any of our subscriptions even want to send a request? - absl::optional maybe_request_type = whoWantsToSendDiscoveryRequest(); - if (!maybe_request_type.has_value()) { - break; - } - // If so, which one (by type_url)? - std::string next_request_type_url = maybe_request_type.value(); - // If we don't have a subscription object for this request's type_url, drop the request. - auto sub = subscriptions_.find(next_request_type_url); - if (sub == subscriptions_.end()) { - ENVOY_LOG(error, "Not sending discovery request for non-existent subscription {}.", - next_request_type_url); - // It's safe to assume the front of the ACK queue is of this type, because that's the only - // way whoWantsToSendDiscoveryRequest() could return something for a non-existent - // subscription. - pausable_ack_queue_.pop(); - continue; - } - // Try again later if paused/rate limited/stream down. - if (!canSendDiscoveryRequest(next_request_type_url)) { - break; - } - // Get our subscription state to generate the appropriate DeltaDiscoveryRequest, and send. - if (!pausable_ack_queue_.empty()) { - // Because ACKs take precedence over plain requests, if there is anything in the queue, it's - // safe to assume it's of the type_url that we're wanting to send. - grpc_stream_.sendMessage( - sub->second->sub_state_.getNextRequestWithAck(pausable_ack_queue_.front())); - pausable_ack_queue_.pop(); - } else { - grpc_stream_.sendMessage(sub->second->sub_state_.getNextRequestAckless()); - } - } - grpc_stream_.maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); - } - - // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check - // whether we *want* to send a DeltaDiscoveryRequest). - bool canSendDiscoveryRequest(const std::string& type_url) { - if (pausable_ack_queue_.paused(type_url)) { - ASSERT(false, - fmt::format("canSendDiscoveryRequest() called on paused type_url {}. Pausedness is " - "supposed to be filtered out by whoWantsToSendDiscoveryRequest(). " - "Returning false, but your xDS might be about to get head-of-line blocked " - "- permanently, if the pause is never undone.", - type_url)); - return false; - } else if (!grpc_stream_.grpcStreamAvailable()) { - ENVOY_LOG(trace, "No stream available to send a discovery request for {}.", type_url); - return false; - } else if (!grpc_stream_.checkRateLimitAllowsDrain()) { - ENVOY_LOG(trace, "{} discovery request hit rate limit; will try later.", type_url); - return false; - } - return true; - } - - // Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or - // a subscription update. (Does not check whether we *can* send that DeltaDiscoveryRequest). - // Returns the type_url we should send the DeltaDiscoveryRequest for (if any). - // First, prioritizes ACKs over non-ACK subscription interest updates. - // Then, prioritizes non-ACK updates in the order the various types - // of subscriptions were activated. - absl::optional whoWantsToSendDiscoveryRequest() { - // All ACKs are sent before plain updates. trySendDiscoveryRequests() relies on this. So, choose - // type_url from pausable_ack_queue_ if possible, before looking at pending updates. - if (!pausable_ack_queue_.empty()) { - return pausable_ack_queue_.front().type_url_; - } - // If we're looking to send multiple non-ACK requests, send them in the order that their - // subscriptions were initiated. - for (const auto& sub_type : subscription_ordering_) { - auto sub = subscriptions_.find(sub_type); - if (sub == subscriptions_.end()) { - continue; - } - if (sub->second->sub_state_.subscriptionUpdatePending() && - !pausable_ack_queue_.paused(sub_type)) { - return sub->first; - } - } - return absl::nullopt; - } - - Event::Dispatcher& dispatcher_; - const LocalInfo::LocalInfo& local_info_; - - // Resource (N)ACKs we're waiting to send, stored in the order that they should be sent in. All - // of our different resource types' ACKs are mixed together in this queue. See class for - // description of how it interacts with pause() and resume(). - PausableAckQueue pausable_ack_queue_; - - struct SubscriptionStuff { - SubscriptionStuff(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout, - Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info) - : sub_state_(type_url, watch_map_, local_info, init_fetch_timeout, dispatcher), - init_fetch_timeout_(init_fetch_timeout) {} - - WatchMapImpl watch_map_; - DeltaSubscriptionState sub_state_; - const std::chrono::milliseconds init_fetch_timeout_; - - private: - SubscriptionStuff(const SubscriptionStuff&) = delete; - SubscriptionStuff& operator=(const SubscriptionStuff&) = delete; - }; - // Map key is type_url. - absl::flat_hash_map> subscriptions_; - - // Determines the order of initial discovery requests. (Assumes that subscriptions are added in - // the order of Envoy's dependency ordering). - std::list subscription_ordering_; - - GrpcStream - grpc_stream_; -}; - -typedef std::shared_ptr GrpcDeltaXdsContextSharedPtr; - -} // namespace Config -} // namespace Envoy diff --git a/source/common/config/new_grpc_mux_impl.cc b/source/common/config/new_grpc_mux_impl.cc new file mode 100644 index 0000000000000..6adeb069dc92a --- /dev/null +++ b/source/common/config/new_grpc_mux_impl.cc @@ -0,0 +1,231 @@ +#include "common/config/new_grpc_mux_impl.h" + +#include "common/common/assert.h" +#include "common/common/backoff_strategy.h" +#include "common/common/token_bucket_impl.h" +#include "common/config/utility.h" +#include "common/protobuf/protobuf.h" +#include "common/protobuf/utility.h" + +namespace Envoy { +namespace Config { + +NewGrpcMuxImpl::NewGrpcMuxImpl(Grpc::RawAsyncClientPtr async_client, Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, + Runtime::RandomGenerator& random, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info) + : dispatcher_(dispatcher), local_info_(local_info), + grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, + rate_limit_settings) {} + +void NewGrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, WatchPtr& watch, + const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { + if (watch.get() == nullptr) { + watch = addWatch(type_url, resources, callbacks, init_fetch_timeout); + } else { + updateWatch(type_url, watch.get(), resources); + } +} + +void NewGrpcMuxImpl::pause(const std::string& type_url) { pausable_ack_queue_.pause(type_url); } + +void NewGrpcMuxImpl::resume(const std::string& type_url) { + pausable_ack_queue_.resume(type_url); + trySendDiscoveryRequests(); +} + +bool NewGrpcMuxImpl::paused(const std::string& type_url) const { + return pausable_ack_queue_.paused(type_url); +} + +void NewGrpcMuxImpl::onDiscoveryResponse( + std::unique_ptr&& message) { + ENVOY_LOG(debug, "Received DeltaDiscoveryResponse for {} at version {}", message->type_url(), + message->system_version_info()); + auto sub = subscriptions_.find(message->type_url()); + if (sub == subscriptions_.end()) { + ENVOY_LOG(warn, + "Dropping received DeltaDiscoveryResponse (with version {}) for non-existent " + "subscription {}.", + message->system_version_info(), message->type_url()); + return; + } + kickOffAck(sub->second->sub_state_.handleResponse(*message)); +} + +void NewGrpcMuxImpl::onStreamEstablished() { + for (auto& sub : subscriptions_) { + sub.second->sub_state_.markStreamFresh(); + } + trySendDiscoveryRequests(); +} + +void NewGrpcMuxImpl::onEstablishmentFailure() { + // If this happens while Envoy is still initializing, the onConfigUpdateFailed() we ultimately + // call on CDS will cause LDS to start up, which adds to subscriptions_ here. So, to avoid a + // crash, the iteration needs to dance around a little: collect pointers to all + // SubscriptionStates, call on all those pointers we haven't yet called on, repeat if there are + // now more SubscriptionStates. + absl::flat_hash_map all_subscribed; + absl::flat_hash_map already_called; + do { + for (auto& sub : subscriptions_) { + all_subscribed[sub.first] = &sub.second->sub_state_; + } + for (auto& sub : all_subscribed) { + if (already_called.insert(sub).second) { // insert succeeded --> not already called + sub.second->handleEstablishmentFailure(); + } + } + } while (all_subscribed.size() != subscriptions_.size()); +} + +void NewGrpcMuxImpl::onWriteable() { trySendDiscoveryRequests(); } + +void NewGrpcMuxImpl::kickOffAck(UpdateAck ack) { + pausable_ack_queue_.push(ack); + trySendDiscoveryRequests(); +} + +// TODO TODO but yeah this should just be gone!!!!!!!!! +GrpcMuxWatchPtr NewGrpcMuxImpl::subscribe(const std::string&, const std::set&, + GrpcMuxCallbacks&) { + // not sure what GrpcMuxCallbacks is for, but we need a SubscriptionCallbacks here, so...... + // return std::make_unique(*this, addWatch(type_url, resources, + // callbacks)); + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; +} +void NewGrpcMuxImpl::start() { grpc_stream_.establishNewStream(); } + +WatchPtr NewGrpcMuxImpl::addWatch(const std::string& type_url, + const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { + auto entry = subscriptions_.find(type_url); + if (entry == subscriptions_.end()) { + // We don't yet have a subscription for type_url! Make one! + addSubscription(type_url, init_fetch_timeout); + return addWatch(type_url, resources, callbacks, init_fetch_timeout); + } + + WatchPtr watch = entry->second->watch_map_.addWatch(callbacks); + // updateWatch() queues a discovery request if any of 'resources' are not yet subscribed. + updateWatch(type_url, watch.get(), resources); + return watch; +} + +// Updates the list of resource names watched by the given watch. If an added name is new across +// the whole subscription, or if a removed name has no other watch interested in it, then the +// subscription will enqueue and attempt to send an appropriate discovery request. +void NewGrpcMuxImpl::updateWatch(const std::string& type_url, Watch* watch, + const std::set& resources) { + ASSERT(watch != nullptr); + auto sub = subscriptions_.find(type_url); + if (sub == subscriptions_.end()) { + ENVOY_LOG(error, "Watch of {} has no subscription to update.", type_url); + return; + } + auto added_removed = sub->second->watch_map_.updateWatchInterest(watch, resources); + sub->second->sub_state_.updateSubscriptionInterest(added_removed.added_, added_removed.removed_); + // Tell the server about our new interests, if there are any. + trySendDiscoveryRequests(); +} + +void NewGrpcMuxImpl::addSubscription(const std::string& type_url, + std::chrono::milliseconds init_fetch_timeout) { + subscriptions_.emplace(type_url, std::make_unique(type_url, init_fetch_timeout, + dispatcher_, local_info_)); + subscription_ordering_.emplace_back(type_url); +} + +void NewGrpcMuxImpl::trySendDiscoveryRequests() { + while (true) { + // Do any of our subscriptions even want to send a request? + absl::optional maybe_request_type = whoWantsToSendDiscoveryRequest(); + if (!maybe_request_type.has_value()) { + break; + } + // If so, which one (by type_url)? + std::string next_request_type_url = maybe_request_type.value(); + // If we don't have a subscription object for this request's type_url, drop the request. + auto sub = subscriptions_.find(next_request_type_url); + if (sub == subscriptions_.end()) { + ENVOY_LOG(error, "Not sending discovery request for non-existent subscription {}.", + next_request_type_url); + // It's safe to assume the front of the ACK queue is of this type, because that's the only + // way whoWantsToSendDiscoveryRequest() could return something for a non-existent + // subscription. + pausable_ack_queue_.pop(); + continue; + } + // Try again later if paused/rate limited/stream down. + if (!canSendDiscoveryRequest(next_request_type_url)) { + break; + } + // Get our subscription state to generate the appropriate DeltaDiscoveryRequest, and send. + if (!pausable_ack_queue_.empty()) { + // Because ACKs take precedence over plain requests, if there is anything in the queue, it's + // safe to assume it's of the type_url that we're wanting to send. + grpc_stream_.sendMessage( + sub->second->sub_state_.getNextRequestWithAck(pausable_ack_queue_.front())); + pausable_ack_queue_.pop(); + } else { + grpc_stream_.sendMessage(sub->second->sub_state_.getNextRequestAckless()); + } + } + grpc_stream_.maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); +} + +// Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check +// whether we *want* to send a DeltaDiscoveryRequest). +bool NewGrpcMuxImpl::canSendDiscoveryRequest(const std::string& type_url) { + if (pausable_ack_queue_.paused(type_url)) { + ASSERT(false, + fmt::format("canSendDiscoveryRequest() called on paused type_url {}. Pausedness is " + "supposed to be filtered out by whoWantsToSendDiscoveryRequest(). " + "Returning false, but your xDS might be about to get head-of-line blocked " + "- permanently, if the pause is never undone.", + type_url)); + return false; + } else if (!grpc_stream_.grpcStreamAvailable()) { + ENVOY_LOG(trace, "No stream available to send a discovery request for {}.", type_url); + return false; + } else if (!grpc_stream_.checkRateLimitAllowsDrain()) { + ENVOY_LOG(trace, "{} discovery request hit rate limit; will try later.", type_url); + return false; + } + return true; +} + +// Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or +// a subscription update. (Does not check whether we *can* send that DeltaDiscoveryRequest). +// Returns the type_url we should send the DeltaDiscoveryRequest for (if any). +// First, prioritizes ACKs over non-ACK subscription interest updates. +// Then, prioritizes non-ACK updates in the order the various types +// of subscriptions were activated. +absl::optional NewGrpcMuxImpl::whoWantsToSendDiscoveryRequest() { + // All ACKs are sent before plain updates. trySendDiscoveryRequests() relies on this. So, choose + // type_url from pausable_ack_queue_ if possible, before looking at pending updates. + if (!pausable_ack_queue_.empty()) { + return pausable_ack_queue_.front().type_url_; + } + // If we're looking to send multiple non-ACK requests, send them in the order that their + // subscriptions were initiated. + for (const auto& sub_type : subscription_ordering_) { + auto sub = subscriptions_.find(sub_type); + if (sub == subscriptions_.end()) { + continue; + } + if (sub->second->sub_state_.subscriptionUpdatePending() && + !pausable_ack_queue_.paused(sub_type)) { + return sub->first; + } + } + return absl::nullopt; +} + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/new_grpc_mux_impl.h b/source/common/config/new_grpc_mux_impl.h new file mode 100644 index 0000000000000..dfc0858b99596 --- /dev/null +++ b/source/common/config/new_grpc_mux_impl.h @@ -0,0 +1,117 @@ +#pragma once + +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/common/token_bucket.h" +#include "envoy/config/grpc_mux.h" +#include "envoy/config/subscription.h" + +#include "common/common/logger.h" +#include "common/config/delta_subscription_state.h" +#include "common/config/grpc_stream.h" +#include "common/config/pausable_ack_queue.h" +#include "common/config/watch_map.h" +#include "common/grpc/common.h" + +namespace Envoy { +namespace Config { + +// Manages subscriptions to one or more type of resource. The logical protocol +// state of those subscription(s) is handled by DeltaSubscriptionState. +// This class owns the GrpcStream used to talk to the server, maintains queuing +// logic to properly order the subscription(s)' various messages, and allows +// starting/stopping/pausing of the subscriptions. +class NewGrpcMuxImpl : public GrpcMux, + public GrpcStreamCallbacks, + Logger::Loggable { +public: + NewGrpcMuxImpl(Grpc::RawAsyncClientPtr async_client, Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, + Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info); + + void addOrUpdateWatch(const std::string& type_url, WatchPtr& watch, + const std::set& resources, SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) override; + + void pause(const std::string& type_url) override; + void resume(const std::string& type_url) override; + bool paused(const std::string& type_url) const override; + void + onDiscoveryResponse(std::unique_ptr&& message) override; + + void onStreamEstablished() override; + + void onEstablishmentFailure() override; + + void onWriteable() override; + + void kickOffAck(UpdateAck ack); + + // TODO(fredlas) remove these two from the GrpcMux interface. + GrpcMuxWatchPtr subscribe(const std::string&, const std::set&, + GrpcMuxCallbacks&) override; + void start() override; + +private: + WatchPtr addWatch(const std::string& type_url, const std::set& resources, + SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout); + + // Updates the list of resource names watched by the given watch. If an added name is new across + // the whole subscription, or if a removed name has no other watch interested in it, then the + // subscription will enqueue and attempt to send an appropriate discovery request. + void updateWatch(const std::string& type_url, Watch* watch, + const std::set& resources); + + void addSubscription(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout); + + void trySendDiscoveryRequests(); + + // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check + // whether we *want* to send a DeltaDiscoveryRequest). + bool canSendDiscoveryRequest(const std::string& type_url); + + // Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or + // a subscription update. (Does not check whether we *can* send that DeltaDiscoveryRequest). + // Returns the type_url we should send the DeltaDiscoveryRequest for (if any). + // First, prioritizes ACKs over non-ACK subscription interest updates. + // Then, prioritizes non-ACK updates in the order the various types + // of subscriptions were activated. + absl::optional whoWantsToSendDiscoveryRequest(); + + Event::Dispatcher& dispatcher_; + const LocalInfo::LocalInfo& local_info_; + + // Resource (N)ACKs we're waiting to send, stored in the order that they should be sent in. All + // of our different resource types' ACKs are mixed together in this queue. See class for + // description of how it interacts with pause() and resume(). + PausableAckQueue pausable_ack_queue_; + + struct SubscriptionStuff { + SubscriptionStuff(const std::string& type_url, std::chrono::milliseconds init_fetch_timeout, + Event::Dispatcher& dispatcher, const LocalInfo::LocalInfo& local_info) + : sub_state_(type_url, watch_map_, local_info, init_fetch_timeout, dispatcher), + init_fetch_timeout_(init_fetch_timeout) {} + + WatchMapImpl watch_map_; + DeltaSubscriptionState sub_state_; + const std::chrono::milliseconds init_fetch_timeout_; + + private: + SubscriptionStuff(const SubscriptionStuff&) = delete; + SubscriptionStuff& operator=(const SubscriptionStuff&) = delete; + }; + // Map key is type_url. + absl::flat_hash_map> subscriptions_; + + // Determines the order of initial discovery requests. (Assumes that subscriptions are added in + // the order of Envoy's dependency ordering). + std::list subscription_ordering_; + + GrpcStream + grpc_stream_; +}; + +using NewGrpcMuxImplSharedPtr = std::shared_ptr; + +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/subscription_factory_impl.cc b/source/common/config/subscription_factory_impl.cc index 2f3e175b1ee5f..9d0d160fdae91 100644 --- a/source/common/config/subscription_factory_impl.cc +++ b/source/common/config/subscription_factory_impl.cc @@ -2,10 +2,10 @@ #include "common/config/delta_subscription_impl.h" #include "common/config/filesystem_subscription_impl.h" -#include "common/config/grpc_delta_xds_context.h" #include "common/config/grpc_mux_subscription_impl.h" #include "common/config/grpc_subscription_impl.h" #include "common/config/http_subscription_impl.h" +#include "common/config/new_grpc_mux_impl.h" #include "common/config/type_to_endpoint.h" #include "common/config/utility.h" #include "common/protobuf/protobuf.h" @@ -62,7 +62,7 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( case envoy::api::v2::core::ApiConfigSource::DELTA_GRPC: { Utility::checkApiConfigSourceSubscriptionBackingCluster(cm_.clusters(), api_config_source); result = std::make_unique( - std::make_shared( + std::make_shared( Config::Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), api_config_source, scope) ->create(), diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 67901d10097dc..17447dde515b7 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -19,7 +19,7 @@ #include "common/common/fmt.h" #include "common/common/utility.h" #include "common/config/cds_json.h" -#include "common/config/grpc_delta_xds_context.h" +#include "common/config/new_grpc_mux_impl.h" #include "common/config/resources.h" #include "common/config/utility.h" #include "common/grpc/async_client_manager_impl.h" @@ -256,7 +256,7 @@ ClusterManagerImpl::ClusterManagerImpl( auto& api_config_source = dyn_resources.has_ads_config() ? dyn_resources.ads_config() : dyn_resources.cds_config().api_config_source(); - ads_mux_ = std::make_shared( + ads_mux_ = std::make_shared( Config::Utility::factoryForGrpcApiConfigSource(*async_client_manager_, api_config_source, stats) ->create(), diff --git a/test/common/config/delta_subscription_impl_test.cc b/test/common/config/delta_subscription_impl_test.cc index 34ace0973be1d..de7c8e6448b13 100644 --- a/test/common/config/delta_subscription_impl_test.cc +++ b/test/common/config/delta_subscription_impl_test.cc @@ -11,7 +11,7 @@ class DeltaSubscriptionImplTest : public DeltaSubscriptionTestHarness, public te DeltaSubscriptionImplTest() : DeltaSubscriptionTestHarness() {} // We need to destroy the subscription before the test's destruction, because the subscription's - // destructor removes its watch from the GrpcDeltaXdsContext, and that removal process involves + // destructor removes its watch from the NewGrpcMuxImpl, and that removal process involves // some things held by the test fixture. void TearDown() override { // if (subscription_started_) { @@ -77,7 +77,7 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { message->set_nonce(nonce); message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); nonce_acks_required_.push(nonce); - static_cast(subscription_->getContextForTest().get()) + static_cast(subscription_->getContextForTest().get()) ->onDiscoveryResponse(std::move(message)); } // The server gives us our first version of resource name2. @@ -91,7 +91,7 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { message->set_nonce(nonce); message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); nonce_acks_required_.push(nonce); - static_cast(subscription_->getContextForTest().get()) + static_cast(subscription_->getContextForTest().get()) ->onDiscoveryResponse(std::move(message)); } // The server gives us an updated version of resource name1. @@ -105,7 +105,7 @@ TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { message->set_nonce(nonce); message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); nonce_acks_required_.push(nonce); - static_cast(subscription_->getContextForTest().get()) + static_cast(subscription_->getContextForTest().get()) ->onDiscoveryResponse(std::move(message)); } // All ACK sendMessage()s will happen upon calling resume(). @@ -141,7 +141,7 @@ TEST(DeltaSubscriptionImplFixturelessTest, NoGrpcStream) { const Protobuf::MethodDescriptor* method_descriptor = Protobuf::DescriptorPool::generated_pool()->FindMethodByName( "envoy.api.v2.EndpointDiscoveryService.StreamEndpoints"); - std::shared_ptr xds_context = std::make_shared( + std::shared_ptr xds_context = std::make_shared( std::unique_ptr(async_client), dispatcher, *method_descriptor, random, stats_store, rate_limit_settings, local_info); diff --git a/test/common/config/delta_subscription_test_harness.h b/test/common/config/delta_subscription_test_harness.h index 977c5de78ae51..112193a092bf5 100644 --- a/test/common/config/delta_subscription_test_harness.h +++ b/test/common/config/delta_subscription_test_harness.h @@ -34,7 +34,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { node_.set_id("fo0"); EXPECT_CALL(local_info_, node()).WillRepeatedly(testing::ReturnRef(node_)); EXPECT_CALL(dispatcher_, createTimer_(_)); - xds_context_ = std::make_shared( + xds_context_ = std::make_shared( std::unique_ptr(async_client_), dispatcher_, *method_descriptor_, random_, stats_store_, rate_limit_settings_, local_info_); subscription_ = std::make_unique( @@ -146,7 +146,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { EXPECT_CALL(callbacks_, onConfigUpdateFailed(_)); expectSendMessage({}, {}, Grpc::Status::GrpcStatus::Internal, "bad config", {}); } - static_cast(subscription_->getContextForTest().get()) + static_cast(subscription_->getContextForTest().get()) ->onDiscoveryResponse(std::move(response)); Mock::VerifyAndClearExpectations(&async_stream_); } @@ -187,7 +187,7 @@ class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { NiceMock random_; NiceMock local_info_; Grpc::MockAsyncStream async_stream_; - std::shared_ptr xds_context_; + std::shared_ptr xds_context_; std::unique_ptr subscription_; std::string last_response_nonce_; std::set last_cluster_names_; From 7bee783d1b241712835c92e5dd00cef96f9f2be4 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Sat, 15 Jun 2019 18:21:06 -0400 Subject: [PATCH 30/56] add to version_history Signed-off-by: Fred Douglas --- docs/root/intro/version_history.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index ca2608b014f64..78c17dd687a66 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -12,6 +12,7 @@ Version history * admin: the :ref:`/listener endpoint ` now returns :ref:`listeners.proto` which includes listener names and ports. * api: track and report requests issued since last load report. * build: releases are built with Clang and linked with LLD. +* config: added support for delta xDS (including ADS) delivery * control-plane: management servers can respond with HTTP 304 to indicate that config is up to date for Envoy proxies polling a :ref:`REST API Config Type ` * dubbo_proxy: support the :ref:`Dubbo proxy filter `. * eds: added support to specify max time for which endpoints can be used :ref:`gRPC filter `. From e1743c8c9a975d45259470583b38e90198bf89a0 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Mon, 17 Jun 2019 15:30:43 -0400 Subject: [PATCH 31/56] appease spellcheck Signed-off-by: Fred Douglas --- source/common/config/new_grpc_mux_impl.cc | 3 --- tools/spelling_dictionary.txt | 2 ++ 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/source/common/config/new_grpc_mux_impl.cc b/source/common/config/new_grpc_mux_impl.cc index 6adeb069dc92a..ef9b891d66d31 100644 --- a/source/common/config/new_grpc_mux_impl.cc +++ b/source/common/config/new_grpc_mux_impl.cc @@ -93,9 +93,6 @@ void NewGrpcMuxImpl::kickOffAck(UpdateAck ack) { // TODO TODO but yeah this should just be gone!!!!!!!!! GrpcMuxWatchPtr NewGrpcMuxImpl::subscribe(const std::string&, const std::set&, GrpcMuxCallbacks&) { - // not sure what GrpcMuxCallbacks is for, but we need a SubscriptionCallbacks here, so...... - // return std::make_unique(*this, addWatch(type_url, resources, - // callbacks)); NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } void NewGrpcMuxImpl::start() { grpc_stream_.establishNewStream(); } diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index ced4cb24a3d71..694d9738d4cf8 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -300,6 +300,7 @@ abcd absl accessor accessors +acks acls addr agg @@ -604,6 +605,7 @@ params paren parentid parsers +pausable pcall pcap pclose From 06cf88d05237cb3be1b0e2a6e782793598f3541a Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Mon, 17 Jun 2019 15:57:35 -0400 Subject: [PATCH 32/56] remove not-yet-used SubscriptionState interface Signed-off-by: Fred Douglas --- include/envoy/config/BUILD | 10 --- include/envoy/config/subscription_state.h | 65 ------------------- source/common/config/BUILD | 4 +- .../common/config/delta_subscription_state.h | 2 +- source/common/config/pausable_ack_queue.h | 10 ++- 5 files changed, 12 insertions(+), 79 deletions(-) delete mode 100644 include/envoy/config/subscription_state.h diff --git a/include/envoy/config/BUILD b/include/envoy/config/BUILD index 337f9d6fa1aa2..5889414922ead 100644 --- a/include/envoy/config/BUILD +++ b/include/envoy/config/BUILD @@ -59,16 +59,6 @@ envoy_cc_library( ], ) -envoy_cc_library( - name = "subscription_state_interface", - hdrs = ["subscription_state.h"], - deps = [ - ":subscription_interface", - "//source/common/protobuf", - "@envoy_api//envoy/api/v2:discovery_cc", - ], -) - envoy_cc_library( name = "typed_metadata_interface", hdrs = ["typed_metadata.h"], diff --git a/include/envoy/config/subscription_state.h b/include/envoy/config/subscription_state.h deleted file mode 100644 index b844899772fd4..0000000000000 --- a/include/envoy/config/subscription_state.h +++ /dev/null @@ -1,65 +0,0 @@ -#pragma once - -#include -#include - -#include "envoy/api/v2/discovery.pb.h" -#include "envoy/common/pure.h" -#include "envoy/config/subscription.h" - -#include "common/protobuf/protobuf.h" - -#include "absl/strings/string_view.h" - -namespace Envoy { -namespace Config { - -struct UpdateAck { - UpdateAck(absl::string_view nonce, absl::string_view type_url) - : nonce_(nonce), type_url_(type_url) {} - std::string nonce_; - std::string type_url_; - ::google::rpc::Status error_detail_; -}; - -class SubscriptionState { -public: - virtual ~SubscriptionState() = default; - - virtual void pause() PURE; - virtual void resume() PURE; - virtual bool paused() const PURE; - - // Update which resources we're interested in subscribing to. - virtual void updateResourceInterest(const std::set& update_to_these_names) PURE; - - // Whether there was a change in our subscription interest we have yet to inform the server of. - virtual bool subscriptionUpdatePending() const PURE; - - virtual void markStreamFresh() PURE; - - // Argument should have been static_cast from GrpcStream's ResponseProto type. - virtual UpdateAck handleResponse(const Protobuf::Message& message) PURE; - - virtual void handleEstablishmentFailure() PURE; - - // Will have been static_cast from GrpcStream's RequestProto type, and should be static_cast back - // before handing to GrpcStream::sendMessage. - virtual Protobuf::Message getNextRequestAckless() PURE; - // Will have been static_cast from GrpcStream's RequestProto type, and should be static_cast back - // before handing to GrpcStream::sendMessage. - virtual Protobuf::Message getNextRequestWithAck(const UpdateAck& ack) PURE; -}; - -class SubscriptionStateFactory { -public: - virtual ~SubscriptionStateFactory() = default; - virtual SubscriptionState makeSubscriptionState(const std::string& type_url, - const std::set& resource_names, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout, - SubscriptionStats& stats) PURE; -}; - -} // namespace Config -} // namespace Envoy diff --git a/source/common/config/BUILD b/source/common/config/BUILD index 4cf792d20b720..fd930570a932a 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -102,8 +102,8 @@ envoy_cc_library( srcs = ["delta_subscription_state.cc"], hdrs = ["delta_subscription_state.h"], deps = [ + ":pausable_ack_queue_lib", "//include/envoy/config:subscription_interface", - "//include/envoy/config:subscription_state_interface", "//include/envoy/event:dispatcher_interface", "//source/common/common:assert_lib", "//source/common/common:backoff_lib", @@ -276,8 +276,8 @@ envoy_cc_library( srcs = ["pausable_ack_queue.cc"], hdrs = ["pausable_ack_queue.h"], deps = [ - "//include/envoy/config:subscription_state_interface", "//source/common/common:assert_lib", + "@envoy_api//envoy/api/v2:discovery_cc", ], ) diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index 5659ce1649ff0..080c12bc1776e 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -2,13 +2,13 @@ #include "envoy/api/v2/discovery.pb.h" #include "envoy/config/subscription.h" -#include "envoy/config/subscription_state.h" #include "envoy/event/dispatcher.h" #include "envoy/grpc/status.h" #include "envoy/local_info/local_info.h" #include "common/common/assert.h" #include "common/common/logger.h" +#include "common/config/pausable_ack_queue.h" namespace Envoy { namespace Config { diff --git a/source/common/config/pausable_ack_queue.h b/source/common/config/pausable_ack_queue.h index cd8f3b970211d..46222a8b2e3ce 100644 --- a/source/common/config/pausable_ack_queue.h +++ b/source/common/config/pausable_ack_queue.h @@ -2,13 +2,21 @@ #include -#include "envoy/config/subscription_state.h" +#include "envoy/api/v2/discovery.pb.h" #include "absl/container/flat_hash_map.h" namespace Envoy { namespace Config { +struct UpdateAck { + UpdateAck(absl::string_view nonce, absl::string_view type_url) + : nonce_(nonce), type_url_(type_url) {} + std::string nonce_; + std::string type_url_; + ::google::rpc::Status error_detail_; +}; + // There is a head-of-line blocking issue resulting from the intersection of 1) ADS's need for // subscription request ordering and 2) the ability to "pause" one of the resource types within ADS. // We need a queue that understands ADS's resource type pausing. Specifically, we need front()/pop() From 540110ccbb7fea9abba624eebc62cba72c9ca6a4 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Mon, 17 Jun 2019 17:15:45 -0400 Subject: [PATCH 33/56] add to arch_overview and config overview docs Signed-off-by: Fred Douglas --- docs/root/configuration/overview/v2_overview.rst | 12 ++++++++++++ .../operations/dynamic_configuration.rst | 10 ++++++++++ 2 files changed, 22 insertions(+) diff --git a/docs/root/configuration/overview/v2_overview.rst b/docs/root/configuration/overview/v2_overview.rst index 7802030f434ce..9e626adebd1f8 100644 --- a/docs/root/configuration/overview/v2_overview.rst +++ b/docs/root/configuration/overview/v2_overview.rst @@ -570,6 +570,18 @@ to with the effect that the LDS stream will be directed to *some_ads_cluster* over the shared ADS channel. +.. _config_overview_v2_delta: + +Delta endpoints +--------------- + +The REST, filesystem, and original gRPC xDS implementations all deliver "state of the world" updates: every CDS update must contain every cluster, with the absence of a cluster from an update implying that the cluster is gone. For Envoy deployments with huge amounts of resources and even a trickle of churn, these state-of-the-world updates can be cumbersome. + +As of 1.11.0, Envoy supports a "delta" variant of xDS (including ADS), where updates only contain resources added/changed/removed. Delta xDS is a gRPC (only) protocol. Delta uses different request/response protos than SotW (DeltaDiscovery{Request,Response}); see :repo:`discovery.proto +`. Conceptually, delta should be viewed as a new xDS transport type: there is static, filesystem, REST, gRPC-SotW, and now gRPC-delta. (Envoy's implementation of the gRPC-SotW/delta client happens to share most of its code between the two, and something similar is likely possible on the server side. However, they are in fact incompatible protocols.) + +To use delta, simply set the api_type field of your :ref:`ApiConfigSource ` proto(s) to DELTA_GRPC. That works for both xDS and ADS; for ADS, it's the api_type field of :ref:`DynamicResources.ads_config `, as described in the previous section. + .. _config_overview_v2_mgmt_con_issues: Management Server Unreachability diff --git a/docs/root/intro/arch_overview/operations/dynamic_configuration.rst b/docs/root/intro/arch_overview/operations/dynamic_configuration.rst index bb51c6ebdf56f..e5729d04699cc 100644 --- a/docs/root/intro/arch_overview/operations/dynamic_configuration.rst +++ b/docs/root/intro/arch_overview/operations/dynamic_configuration.rst @@ -88,3 +88,13 @@ The :ref:`secret discovery service (SDS) ` laye by which Envoy can discover cryptographic secrets (certificate plus private key, TLS session ticket keys) for its listeners, as well as configuration of peer certificate validation logic (trusted root certs, revocations, etc). + +Aggregated xDS ("ADS") +----------------------------- + +EDS, CDS, etc. are each separate services, with different REST/gRPC service names, e.g. StreamListeners, StreamSecrets. For users looking to enforce the order in which resources of different types reach Envoy, there is aggregated xDS, a single gRPC service that carries all resource types in a single gRPC stream. (ADS is only supported by gRPC). :ref:`More details about ADS `. + +Delta gRPC xDS +----------------------------- + +Standard xDS is "state-of-the-world": every CDS update must contain every cluster, with the absence of a cluster from an update implying that the cluster is gone. As of 1.11.0, Envoy supports a "delta" variant of xDS (including ADS), where updates only contain resources added/changed/removed. Delta xDS is a new protocol, with request/response protos different from SotW; existing control plane servers will need support added. :ref:`More details about delta `. From 9ea226ebb584bdc1aff23d90b8b3d76e890c0ae6 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Mon, 17 Jun 2019 17:23:14 -0400 Subject: [PATCH 34/56] add link to xds_protocol.rst Signed-off-by: Fred Douglas --- api/xds_protocol.rst | 2 ++ docs/root/configuration/overview/v2_overview.rst | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/api/xds_protocol.rst b/api/xds_protocol.rst index 3906d64f1e7a4..1cf248b564aca 100644 --- a/api/xds_protocol.rst +++ b/api/xds_protocol.rst @@ -358,6 +358,8 @@ An example minimal ``bootstrap.yaml`` fragment for ADS configuration is: admin: ... +.. _xds_protocol_delta: + Incremental xDS ~~~~~~~~~~~~~~~ diff --git a/docs/root/configuration/overview/v2_overview.rst b/docs/root/configuration/overview/v2_overview.rst index 9e626adebd1f8..42d2db8f5b59f 100644 --- a/docs/root/configuration/overview/v2_overview.rst +++ b/docs/root/configuration/overview/v2_overview.rst @@ -578,7 +578,7 @@ Delta endpoints The REST, filesystem, and original gRPC xDS implementations all deliver "state of the world" updates: every CDS update must contain every cluster, with the absence of a cluster from an update implying that the cluster is gone. For Envoy deployments with huge amounts of resources and even a trickle of churn, these state-of-the-world updates can be cumbersome. As of 1.11.0, Envoy supports a "delta" variant of xDS (including ADS), where updates only contain resources added/changed/removed. Delta xDS is a gRPC (only) protocol. Delta uses different request/response protos than SotW (DeltaDiscovery{Request,Response}); see :repo:`discovery.proto -`. Conceptually, delta should be viewed as a new xDS transport type: there is static, filesystem, REST, gRPC-SotW, and now gRPC-delta. (Envoy's implementation of the gRPC-SotW/delta client happens to share most of its code between the two, and something similar is likely possible on the server side. However, they are in fact incompatible protocols.) +`. Conceptually, delta should be viewed as a new xDS transport type: there is static, filesystem, REST, gRPC-SotW, and now gRPC-delta. (Envoy's implementation of the gRPC-SotW/delta client happens to share most of its code between the two, and something similar is likely possible on the server side. However, they are in fact incompatible protocols. :ref:`The specification of the delta xDS protocol's behavior is here `.) To use delta, simply set the api_type field of your :ref:`ApiConfigSource ` proto(s) to DELTA_GRPC. That works for both xDS and ADS; for ADS, it's the api_type field of :ref:`DynamicResources.ads_config `, as described in the previous section. From eee089aeae66a510dbea9667a47a49685e7bba02 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Wed, 10 Jul 2019 12:19:28 -0400 Subject: [PATCH 35/56] move arguments Signed-off-by: Fred Douglas --- include/envoy/config/watch_map.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/envoy/config/watch_map.h b/include/envoy/config/watch_map.h index bbee092b942d2..6050e03a52dcf 100644 --- a/include/envoy/config/watch_map.h +++ b/include/envoy/config/watch_map.h @@ -13,8 +13,8 @@ struct Watch; using WatchPtr = std::unique_ptr; struct AddedRemoved { - AddedRemoved(std::set added, std::set removed) - : added_(added), removed_(removed) {} + AddedRemoved(std::set&& added, std::set&& removed) + : added_(std::move(added)), removed_(std::move(removed)) {} std::set added_; std::set removed_; }; From 7c298bda3a677a01d2470901003f4efc2ac9d04f Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 11 Jul 2019 17:31:33 -0400 Subject: [PATCH 36/56] move release note to 1.12 Signed-off-by: Fred Douglas --- docs/root/intro/version_history.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 0defafc80806d..a12aa557f803d 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -3,6 +3,7 @@ Version history 1.12.0 (pending) ================ +* config: added support for delta xDS (including ADS) delivery 1.11.0 (July 11, 2019) ====================== @@ -19,7 +20,6 @@ Version history for each host, useful for DNS based clusters. * api: track and report requests issued since last load report. * build: releases are built with Clang and linked with LLD. -* config: added support for delta xDS (including ADS) delivery * control-plane: management servers can respond with HTTP 304 to indicate that config is up to date for Envoy proxies polling a :ref:`REST API Config Type ` * csrf: added support for whitelisting additional source origins. * dns: added support for getting DNS record TTL which is used by STRICT_DNS/LOGICAL_DNS cluster as DNS refresh rate. From 6f465a81bbe2fe53de01da89a3fddffe7fa54885 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Wed, 17 Jul 2019 18:40:03 -0400 Subject: [PATCH 37/56] clean up WatchMap, DeltaSubscriptionImpl becomes more simply RAIIy Signed-off-by: Fred Douglas --- include/envoy/config/grpc_mux.h | 11 ++-- include/envoy/config/watch_map.h | 55 ------------------- .../common/config/delta_subscription_impl.cc | 15 ++--- .../common/config/delta_subscription_impl.h | 2 +- source/common/config/grpc_mux_impl.h | 12 ++-- source/common/config/new_grpc_mux_impl.cc | 36 +++++++----- source/common/config/new_grpc_mux_impl.h | 14 ++--- source/common/config/watch_map.cc | 37 +++++++------ source/common/config/watch_map.h | 39 ++++++++----- test/mocks/config/mocks.h | 10 ++-- 10 files changed, 101 insertions(+), 130 deletions(-) delete mode 100644 include/envoy/config/watch_map.h diff --git a/include/envoy/config/grpc_mux.h b/include/envoy/config/grpc_mux.h index 353c7cf89025c..ba3e62f5abfe5 100644 --- a/include/envoy/config/grpc_mux.h +++ b/include/envoy/config/grpc_mux.h @@ -65,6 +65,8 @@ class GrpcMuxWatch { using GrpcMuxWatchPtr = std::unique_ptr; +struct Watch; + /** * Manage one or more gRPC subscriptions on a single stream to management server. This can be used * for a single xDS API, e.g. EDS, or to combined multiple xDS APIs for ADS. @@ -110,10 +112,11 @@ class GrpcMux { virtual void resume(const std::string& type_url) PURE; // For delta - virtual void addOrUpdateWatch(const std::string& type_url, WatchPtr& watch, - const std::set& resources, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) PURE; + virtual Watch* addOrUpdateWatch(const std::string& type_url, Watch* watch, + const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) PURE; + virtual void removeWatch(const std::string& type_url, Watch* watch) PURE; /** * Retrieves the current pause state as set by pause()/resume(). diff --git a/include/envoy/config/watch_map.h b/include/envoy/config/watch_map.h deleted file mode 100644 index 6050e03a52dcf..0000000000000 --- a/include/envoy/config/watch_map.h +++ /dev/null @@ -1,55 +0,0 @@ -#pragma once - -#include -#include - -#include "envoy/common/pure.h" -#include "envoy/config/subscription.h" - -namespace Envoy { -namespace Config { - -struct Watch; -using WatchPtr = std::unique_ptr; - -struct AddedRemoved { - AddedRemoved(std::set&& added, std::set&& removed) - : added_(std::move(added)), removed_(std::move(removed)) {} - std::set added_; - std::set removed_; -}; - -class WatchMap { -public: - virtual ~WatchMap() = default; - - // Adds 'callbacks' to the WatchMap, with no resource names being watched. - // (Use updateWatchInterest() to add some names). - // Returns the newly added watch, to be used for updateWatchInterest. Destroy to remove from map. - virtual WatchPtr addWatch(SubscriptionCallbacks& callbacks) PURE; - - // Updates the set of resource names that the given watch should watch. - // Returns any resource name additions/removals that are unique across all watches. That is: - // 1) if 'resources' contains X and no other watch cares about X, X will be in added_. - // 2) if 'resources' does not contain Y, and this watch was the only one that cared about Y, - // Y will be in removed_. - virtual AddedRemoved updateWatchInterest(Watch* watch, - const std::set& update_to_these_names) PURE; - - // Intended to be called by the Watch's destructor. - // Expects that the watch to be removed has already had all of its resource names removed via - // updateWatchInterest(). - virtual void removeWatch(Watch* watch) PURE; -}; - -struct Watch { - Watch(WatchMap& owning_map, SubscriptionCallbacks& callbacks) - : owning_map_(owning_map), callbacks_(callbacks) {} - ~Watch() { owning_map_.removeWatch(this); } - WatchMap& owning_map_; - SubscriptionCallbacks& callbacks_; - std::set resource_names_; // must be sorted set, for set_difference. -}; - -} // namespace Config -} // namespace Envoy diff --git a/source/common/config/delta_subscription_impl.cc b/source/common/config/delta_subscription_impl.cc index 9222f655dc918..2a7d1403177ad 100644 --- a/source/common/config/delta_subscription_impl.cc +++ b/source/common/config/delta_subscription_impl.cc @@ -11,14 +11,8 @@ DeltaSubscriptionImpl::DeltaSubscriptionImpl( DeltaSubscriptionImpl::~DeltaSubscriptionImpl() { if (watch_) { - // The Watch destruction process assumes all resource interest has already been removed. Doing - // that cleanly needs the context's involvement. So, do that now, just before destroying the - // watch. - context_->addOrUpdateWatch(type_url_, watch_, {}, *this, init_fetch_timeout_); - // A Watch must not outlive its owning WatchMap. Since DeltaSubscriptionImpl holds a shared_ptr - // to the context, which owns the WatchMap, the watch must be destroyed first in case this - // subscription holds the last shared_ptr. - watch_.reset(); + context_->removeWatch(type_url_, watch_); + watch_ = nullptr; } } @@ -33,13 +27,14 @@ void DeltaSubscriptionImpl::start(const std::set& resources) { if (!is_aggregated_) { context_->start(); } - context_->addOrUpdateWatch(type_url_, watch_, resources, *this, init_fetch_timeout_); + watch_ = context_->addOrUpdateWatch(type_url_, watch_, resources, *this, init_fetch_timeout_); stats_.update_attempt_.inc(); } void DeltaSubscriptionImpl::updateResourceInterest( const std::set& update_to_these_names) { - context_->addOrUpdateWatch(type_url_, watch_, update_to_these_names, *this, init_fetch_timeout_); + watch_ = context_->addOrUpdateWatch(type_url_, watch_, update_to_these_names, *this, + init_fetch_timeout_); stats_.update_attempt_.inc(); } diff --git a/source/common/config/delta_subscription_impl.h b/source/common/config/delta_subscription_impl.h index 99981aa8e5642..910681a5867a5 100644 --- a/source/common/config/delta_subscription_impl.h +++ b/source/common/config/delta_subscription_impl.h @@ -64,7 +64,7 @@ class DeltaSubscriptionImpl : public Subscription, public SubscriptionCallbacks // NOTE: if another subscription of the same type_url has already been started, this value will be // ignored in favor of the other subscription's. std::chrono::milliseconds init_fetch_timeout_; - WatchPtr watch_{}; + Watch* watch_{}; const bool is_aggregated_; }; diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index a30f27c086ff0..e9deb6521850e 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -40,10 +40,11 @@ class GrpcMuxImpl : public GrpcMux, void resume(const std::string& type_url) override; bool paused(const std::string& type_url) const override; - void addOrUpdateWatch(const std::string&, WatchPtr&, const std::set&, - SubscriptionCallbacks&, std::chrono::milliseconds) override { + Watch* addOrUpdateWatch(const std::string&, Watch*, const std::set&, + SubscriptionCallbacks&, std::chrono::milliseconds) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + void removeWatch(const std::string&, Watch*) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } void handleDiscoveryResponse(std::unique_ptr&& message); @@ -134,8 +135,11 @@ class NullGrpcMuxImpl : public GrpcMux, GrpcStreamCallbacks&, - SubscriptionCallbacks&, std::chrono::milliseconds) override { + Watch* addOrUpdateWatch(const std::string&, Watch*, const std::set&, + SubscriptionCallbacks&, std::chrono::milliseconds) override { + throw EnvoyException("ADS must be configured to support an ADS config source"); + } + void removeWatch(const std::string&, Watch*) override { throw EnvoyException("ADS must be configured to support an ADS config source"); } diff --git a/source/common/config/new_grpc_mux_impl.cc b/source/common/config/new_grpc_mux_impl.cc index ef9b891d66d31..187bbe182e407 100644 --- a/source/common/config/new_grpc_mux_impl.cc +++ b/source/common/config/new_grpc_mux_impl.cc @@ -19,15 +19,26 @@ NewGrpcMuxImpl::NewGrpcMuxImpl(Grpc::RawAsyncClientPtr async_client, Event::Disp grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, rate_limit_settings) {} -void NewGrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, WatchPtr& watch, - const std::set& resources, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) { - if (watch.get() == nullptr) { - watch = addWatch(type_url, resources, callbacks, init_fetch_timeout); +Watch* NewGrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, Watch* watch, + const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { + if (watch == nullptr) { + return addWatch(type_url, resources, callbacks, init_fetch_timeout); } else { - updateWatch(type_url, watch.get(), resources); + updateWatch(type_url, watch, resources); + return watch; + } +} + +void NewGrpcMuxImpl::removeWatch(const std::string& type_url, Watch* watch) { + updateWatch(type_url, watch, {}); + auto entry = subscriptions_.find(type_url); + if (entry == subscriptions_.end()) { + ENVOY_LOG(error, "removeWatch() called for non-existent subscription {}.", type_url); + return; } + entry->second->watch_map_.removeWatch(watch); } void NewGrpcMuxImpl::pause(const std::string& type_url) { pausable_ack_queue_.pause(type_url); } @@ -97,10 +108,9 @@ GrpcMuxWatchPtr NewGrpcMuxImpl::subscribe(const std::string&, const std::set& resources, - SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) { +Watch* NewGrpcMuxImpl::addWatch(const std::string& type_url, const std::set& resources, + SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { auto entry = subscriptions_.find(type_url); if (entry == subscriptions_.end()) { // We don't yet have a subscription for type_url! Make one! @@ -108,9 +118,9 @@ WatchPtr NewGrpcMuxImpl::addWatch(const std::string& type_url, return addWatch(type_url, resources, callbacks, init_fetch_timeout); } - WatchPtr watch = entry->second->watch_map_.addWatch(callbacks); + Watch* watch = entry->second->watch_map_.addWatch(callbacks); // updateWatch() queues a discovery request if any of 'resources' are not yet subscribed. - updateWatch(type_url, watch.get(), resources); + updateWatch(type_url, watch, resources); return watch; } diff --git a/source/common/config/new_grpc_mux_impl.h b/source/common/config/new_grpc_mux_impl.h index dfc0858b99596..c63b2f9de02f8 100644 --- a/source/common/config/new_grpc_mux_impl.h +++ b/source/common/config/new_grpc_mux_impl.h @@ -29,9 +29,10 @@ class NewGrpcMuxImpl : public GrpcMux, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, const LocalInfo::LocalInfo& local_info); - void addOrUpdateWatch(const std::string& type_url, WatchPtr& watch, - const std::set& resources, SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout) override; + Watch* addOrUpdateWatch(const std::string& type_url, Watch* watch, + const std::set& resources, SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) override; + void removeWatch(const std::string& type_url, Watch* watch) override; void pause(const std::string& type_url) override; void resume(const std::string& type_url) override; @@ -53,8 +54,8 @@ class NewGrpcMuxImpl : public GrpcMux, void start() override; private: - WatchPtr addWatch(const std::string& type_url, const std::set& resources, - SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout); + Watch* addWatch(const std::string& type_url, const std::set& resources, + SubscriptionCallbacks& callbacks, std::chrono::milliseconds init_fetch_timeout); // Updates the list of resource names watched by the given watch. If an added name is new across // the whole subscription, or if a removed name has no other watch interested in it, then the @@ -92,11 +93,10 @@ class NewGrpcMuxImpl : public GrpcMux, : sub_state_(type_url, watch_map_, local_info, init_fetch_timeout, dispatcher), init_fetch_timeout_(init_fetch_timeout) {} - WatchMapImpl watch_map_; + WatchMap watch_map_; DeltaSubscriptionState sub_state_; const std::chrono::milliseconds init_fetch_timeout_; - private: SubscriptionStuff(const SubscriptionStuff&) = delete; SubscriptionStuff& operator=(const SubscriptionStuff&) = delete; }; diff --git a/source/common/config/watch_map.cc b/source/common/config/watch_map.cc index e24c6f81b6f5e..dfd022966f95a 100644 --- a/source/common/config/watch_map.cc +++ b/source/common/config/watch_map.cc @@ -3,20 +3,21 @@ namespace Envoy { namespace Config { -WatchPtr WatchMapImpl::addWatch(SubscriptionCallbacks& callbacks) { - auto watch = std::make_unique(*this, callbacks); - wildcard_watches_.insert(watch.get()); - watches_.insert(watch.get()); - return watch; +Watch* WatchMap::addWatch(SubscriptionCallbacks& callbacks) { + auto watch = std::make_unique(callbacks); + Watch* watch_ptr = watch.get(); + wildcard_watches_.insert(watch_ptr); + watches_.insert(std::move(watch)); + return watch_ptr; } -void WatchMapImpl::removeWatch(Watch* watch) { +void WatchMap::removeWatch(Watch* watch) { wildcard_watches_.erase(watch); // may or may not be in there, but we want it gone. watches_.erase(watch); } -AddedRemoved WatchMapImpl::updateWatchInterest(Watch* watch, - const std::set& update_to_these_names) { +AddedRemoved WatchMap::updateWatchInterest(Watch* watch, + const std::set& update_to_these_names) { if (update_to_these_names.empty()) { wildcard_watches_.insert(watch); } else { @@ -39,7 +40,7 @@ AddedRemoved WatchMapImpl::updateWatchInterest(Watch* watch, findRemovals(newly_removed_from_watch, watch)); } -absl::flat_hash_set WatchMapImpl::watchesInterestedIn(const std::string& resource_name) { +absl::flat_hash_set WatchMap::watchesInterestedIn(const std::string& resource_name) { // Note that std::set_union needs sorted sets. Better to do it ourselves with insert(). absl::flat_hash_set ret = wildcard_watches_; auto watches_interested = watch_interest_.find(resource_name); @@ -51,10 +52,10 @@ absl::flat_hash_set WatchMapImpl::watchesInterestedIn(const std::string& return ret; } -void WatchMapImpl::onConfigUpdate(const Protobuf::RepeatedPtrField& resources, - const std::string& version_info) { +void WatchMap::onConfigUpdate(const Protobuf::RepeatedPtrField& resources, + const std::string& version_info) { if (watches_.empty()) { - ENVOY_LOG(warn, "WatchMapImpl::onConfigUpdate: there are no watches!"); + ENVOY_LOG(warn, "WatchMap::onConfigUpdate: there are no watches!"); return; } SubscriptionCallbacks& name_getter = (*watches_.begin())->callbacks_; @@ -85,7 +86,7 @@ void WatchMapImpl::onConfigUpdate(const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info) { @@ -127,14 +128,14 @@ void WatchMapImpl::onConfigUpdate( } } -void WatchMapImpl::onConfigUpdateFailed(const EnvoyException* e) { +void WatchMap::onConfigUpdateFailed(const EnvoyException* e) { for (auto& watch : watches_) { watch->callbacks_.onConfigUpdateFailed(e); } } -std::set -WatchMapImpl::findAdditions(const std::vector& newly_added_to_watch, Watch* watch) { +std::set WatchMap::findAdditions(const std::vector& newly_added_to_watch, + Watch* watch) { std::set newly_added_to_subscription; for (const auto& name : newly_added_to_watch) { auto entry = watch_interest_.find(name); @@ -149,12 +150,12 @@ WatchMapImpl::findAdditions(const std::vector& newly_added_to_watch } std::set -WatchMapImpl::findRemovals(const std::vector& newly_removed_from_watch, Watch* watch) { +WatchMap::findRemovals(const std::vector& newly_removed_from_watch, Watch* watch) { std::set newly_removed_from_subscription; for (const auto& name : newly_removed_from_watch) { auto entry = watch_interest_.find(name); if (entry == watch_interest_.end()) { - ENVOY_LOG(warn, "WatchMapImpl: tried to remove a watch from untracked resource {}", name); + ENVOY_LOG(warn, "WatchMap: tried to remove a watch from untracked resource {}", name); continue; } diff --git a/source/common/config/watch_map.h b/source/common/config/watch_map.h index 268193165dc3d..136aed6650df2 100644 --- a/source/common/config/watch_map.h +++ b/source/common/config/watch_map.h @@ -15,6 +15,19 @@ namespace Envoy { namespace Config { +struct AddedRemoved { + AddedRemoved(std::set&& added, std::set&& removed) + : added_(std::move(added)), removed_(std::move(removed)) {} + std::set added_; + std::set removed_; +}; + +struct Watch { + Watch(SubscriptionCallbacks& callbacks) : callbacks_(callbacks) {} + SubscriptionCallbacks& callbacks_; + std::set resource_names_; // must be sorted set, for set_difference. +}; + // Manages "watches" of xDS resources. Several xDS callers might ask for a subscription to the same // resource name "X". The xDS machinery must return to each their very own subscription to X. // The xDS machinery's "watch" concept accomplishes that, while avoiding parallel redundant xDS @@ -34,16 +47,14 @@ namespace Config { // update the subscription accordingly. // // A WatchMap is assumed to be dedicated to a single type_url type of resource (EDS, CDS, etc). -class WatchMapImpl : public WatchMap, - public SubscriptionCallbacks, - public Logger::Loggable { +class WatchMap : public SubscriptionCallbacks, public Logger::Loggable { public: - WatchMapImpl() = default; + WatchMap() = default; - // Adds 'callbacks' to the WatchMap, with no resource names being watched. - // (Use updateWatchInterest() to add some names). - // Returns the newly added watch, to be used for updateWatchInterest. Destroy to remove from map. - WatchPtr addWatch(SubscriptionCallbacks& callbacks) override; + // Adds 'callbacks' to the WatchMap, with every possible resource being watched. + // (Use updateWatchInterest() to narrow it down to some specific names). + // Returns the newly added watch, to be used with updateWatchInterest and removeWatch. + Watch* addWatch(SubscriptionCallbacks& callbacks); // Updates the set of resource names that the given watch should watch. // Returns any resource name additions/removals that are unique across all watches. That is: @@ -51,11 +62,11 @@ class WatchMapImpl : public WatchMap, // 2) if 'resources' does not contain Y, and this watch was the only one that cared about Y, // Y will be in removed_. AddedRemoved updateWatchInterest(Watch* watch, - const std::set& update_to_these_names) override; + const std::set& update_to_these_names); // Expects that the watch to be removed has already had all of its resource names removed via // updateWatchInterest(). - void removeWatch(Watch* watch) override; + void removeWatch(Watch* watch); // SubscriptionCallbacks virtual void onConfigUpdate(const Protobuf::RepeatedPtrField& resources, @@ -71,6 +82,9 @@ class WatchMapImpl : public WatchMap, NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + WatchMap(const WatchMap&) = delete; + WatchMap& operator=(const WatchMap&) = delete; + private: // Given a list of names that are new to an individual watch, returns those names that are in fact // new to the entire subscription. @@ -85,7 +99,7 @@ class WatchMapImpl : public WatchMap, // Returns the union of watch_interest_[resource_name] and wildcard_watches_. absl::flat_hash_set watchesInterestedIn(const std::string& resource_name); - absl::flat_hash_set watches_; + absl::flat_hash_set> watches_; // Watches whose interest set is currently empty, which is interpreted as "everything". absl::flat_hash_set wildcard_watches_; @@ -94,9 +108,6 @@ class WatchMapImpl : public WatchMap, // 1) Acts as a reference count; no watches care anymore ==> the resource can be removed. // 2) Enables efficient lookup of all interested watches when a resource has been updated. absl::flat_hash_map> watch_interest_; - - WatchMapImpl(const WatchMapImpl&) = delete; - WatchMapImpl& operator=(const WatchMapImpl&) = delete; }; } // namespace Config diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index dd33cb0cbd376..cc287dc9e7703 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -82,6 +82,8 @@ class MockGrpcMux : public GrpcMux { GrpcMuxCallbacks& callbacks) override; MOCK_METHOD1(pause, void(const std::string& type_url)); MOCK_METHOD1(resume, void(const std::string& type_url)); + MOCK_CONST_METHOD1(paused, bool(const std::string& type_url)); + MOCK_METHOD5(addSubscription, void(const std::set& resources, const std::string& type_url, SubscriptionCallbacks& callbacks, SubscriptionStats& stats, @@ -90,10 +92,10 @@ class MockGrpcMux : public GrpcMux { void(const std::set& resources, const std::string& type_url)); MOCK_METHOD5(addOrUpdateWatch, - void(const std::string& type_url, WatchPtr& watch, - const std::set& resources, SubscriptionCallbacks& callbacks, - std::chrono::milliseconds init_fetch_timeout)); - MOCK_CONST_METHOD1(paused, bool(const std::string& type_url)); + Watch*(const std::string& type_url, Watch* watch, + const std::set& resources, SubscriptionCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout)); + MOCK_METHOD2(removeWatch, void(const std::string& type_url, Watch* watch)); }; class MockGrpcMuxCallbacks : public GrpcMuxCallbacks { From c3628c414b4d06df71727dbc940537d3a7d59083 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 19 Jul 2019 16:08:59 -0400 Subject: [PATCH 38/56] add watch_map_test Signed-off-by: Fred Douglas --- source/common/config/watch_map.cc | 6 +- test/common/config/BUILD | 11 + test/common/config/watch_map_test.cc | 396 +++++++++++++++++++++++++++ 3 files changed, 409 insertions(+), 4 deletions(-) create mode 100644 test/common/config/watch_map_test.cc diff --git a/source/common/config/watch_map.cc b/source/common/config/watch_map.cc index dfd022966f95a..a40eb439c69ae 100644 --- a/source/common/config/watch_map.cc +++ b/source/common/config/watch_map.cc @@ -154,10 +154,8 @@ WatchMap::findRemovals(const std::vector& newly_removed_from_watch, std::set newly_removed_from_subscription; for (const auto& name : newly_removed_from_watch) { auto entry = watch_interest_.find(name); - if (entry == watch_interest_.end()) { - ENVOY_LOG(warn, "WatchMap: tried to remove a watch from untracked resource {}", name); - continue; - } + ASSERT(entry != watch_interest_.end(), + fmt::format("WatchMap: tried to remove a watch from untracked resource {}", name)); entry->second.erase(watch); if (entry->second.empty()) { diff --git a/test/common/config/BUILD b/test/common/config/BUILD index ebc6d79e5330d..24a8b07fcf166 100644 --- a/test/common/config/BUILD +++ b/test/common/config/BUILD @@ -276,6 +276,17 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "watch_map_test", + srcs = ["watch_map_test.cc"], + deps = [ + "//source/common/config:watch_map_lib", + "//test/mocks/config:config_mocks", + "//test/test_common:utility_lib", + "@envoy_api//envoy/api/v2:eds_cc", + ], +) + envoy_cc_test( name = "filter_json_test", srcs = ["filter_json_test.cc"], diff --git a/test/common/config/watch_map_test.cc b/test/common/config/watch_map_test.cc new file mode 100644 index 0000000000000..b7ab15ac7e131 --- /dev/null +++ b/test/common/config/watch_map_test.cc @@ -0,0 +1,396 @@ +#include + +#include "envoy/api/v2/eds.pb.h" +#include "envoy/common/exception.h" +#include "envoy/stats/scope.h" + +#include "common/config/watch_map.h" + +#include "test/mocks/config/mocks.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using ::testing::_; +using ::testing::Invoke; + +namespace Envoy { +namespace Config { +namespace { + +class NamedMockSubscriptionCallbacks + : public MockSubscriptionCallbacks { +public: + std::string resourceName(const ProtobufWkt::Any& resource) override { + return TestUtility::anyConvert(resource).cluster_name(); + } +}; + +// expectDeltaAndSotwUpdate() EXPECTs two birds with one function call: we want to cover both SotW +// and delta, which, while mechanically different, can behave identically for our testing purposes. +// Specifically, as a simplification for these tests, every still-present resource is updated in +// every update. Therefore, a resource can never show up in the SotW update but not the delta +// update. We can therefore use the same expected_resources for both. +void expectDeltaAndSotwUpdate( + NamedMockSubscriptionCallbacks& callbacks, + const std::vector& expected_resources, + const std::vector& expected_removals, const std::string& version) { + EXPECT_CALL(callbacks, onConfigUpdate(_, version)) + .WillOnce(Invoke( + [expected_resources](const Protobuf::RepeatedPtrField& gotten_resources, + const std::string&) { + EXPECT_EQ(expected_resources.size(), gotten_resources.size()); + for (size_t i = 0; i < expected_resources.size(); i++) { + envoy::api::v2::ClusterLoadAssignment cur_gotten_resource; + gotten_resources[i].UnpackTo(&cur_gotten_resource); + EXPECT_TRUE(TestUtility::protoEqual(cur_gotten_resource, expected_resources[i])); + } + })); + EXPECT_CALL(callbacks, onConfigUpdate(_, _, _)) + .WillOnce( + Invoke([expected_resources, expected_removals, version]( + const Protobuf::RepeatedPtrField& gotten_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string&) { + EXPECT_EQ(expected_resources.size(), gotten_resources.size()); + for (size_t i = 0; i < expected_resources.size(); i++) { + EXPECT_EQ(gotten_resources[i].version(), version); + envoy::api::v2::ClusterLoadAssignment cur_gotten_resource; + gotten_resources[i].resource().UnpackTo(&cur_gotten_resource); + EXPECT_TRUE(TestUtility::protoEqual(cur_gotten_resource, expected_resources[i])); + } + EXPECT_EQ(expected_removals.size(), removed_resources.size()); + for (size_t i = 0; i < expected_removals.size(); i++) { + EXPECT_EQ(expected_removals[i], removed_resources[i]); + } + })); +} + +// Sometimes we want to verify that a delta onConfigUpdate simply doesn't happen. However, for SotW, +// every update triggers all onConfigUpdate()s, so we should still expect empty calls for that. +void expectNoDeltaUpdate(NamedMockSubscriptionCallbacks& callbacks, const std::string& version) { + EXPECT_CALL(callbacks, onConfigUpdate(_, version)) + .WillOnce(Invoke([](const Protobuf::RepeatedPtrField& gotten_resources, + const std::string&) { EXPECT_EQ(0, gotten_resources.size()); })); + EXPECT_CALL(callbacks, onConfigUpdate(_, _, _)).Times(0); +} + +Protobuf::RepeatedPtrField +wrapInResource(const Protobuf::RepeatedPtrField& anys, + const std::string& version) { + Protobuf::RepeatedPtrField ret; + for (const auto& a : anys) { + envoy::api::v2::ClusterLoadAssignment cur_endpoint; + a.UnpackTo(&cur_endpoint); + auto* cur_resource = ret.Add(); + cur_resource->set_name(cur_endpoint.cluster_name()); + cur_resource->mutable_resource()->CopyFrom(a); + cur_resource->set_version(version); + } + return ret; +} + +// Similar to expectDeltaAndSotwUpdate(), but making the onConfigUpdate() happen, rather than +// EXPECTing it. +void doDeltaAndSotwUpdate(SubscriptionCallbacks& watch_map, + const Protobuf::RepeatedPtrField& sotw_resources, + const std::vector& removed_names, + const std::string& version) { + watch_map.onConfigUpdate(sotw_resources, version); + + Protobuf::RepeatedPtrField delta_resources = + wrapInResource(sotw_resources, version); + Protobuf::RepeatedPtrField removed_names_proto; + for (const auto& n : removed_names) { + *removed_names_proto.Add() = n; + } + watch_map.onConfigUpdate(delta_resources, removed_names_proto, "version1"); +} + +// Tests the simple case of a single watch. Checks that the watch will not be told of updates to +// resources it doesn't care about. Checks that the watch can later decide it does care about them, +// and then receive subsequent updates to them. +TEST(WatchMapTest, Basic) { + NamedMockSubscriptionCallbacks callbacks; + WatchMap watch_map; + Watch* watch = watch_map.addWatch(callbacks); + + { + // The watch is interested in Alice and Bob... + std::set update_to({"alice", "bob"}); + AddedRemoved added_removed = watch_map.updateWatchInterest(watch, update_to); + EXPECT_EQ(update_to, added_removed.added_); + EXPECT_TRUE(added_removed.removed_.empty()); + + // ...the update is going to contain Bob and Carol... + Protobuf::RepeatedPtrField updated_resources; + envoy::api::v2::ClusterLoadAssignment bob; + bob.set_cluster_name("bob"); + updated_resources.Add()->PackFrom(bob); + envoy::api::v2::ClusterLoadAssignment carol; + carol.set_cluster_name("carol"); + updated_resources.Add()->PackFrom(carol); + + // ...so the watch should receive only Bob. + std::vector expected_resources; + expected_resources.push_back(bob); + + expectDeltaAndSotwUpdate(callbacks, expected_resources, {}, "version1"); + doDeltaAndSotwUpdate(watch_map, updated_resources, {}, "version1"); + } + { + // The watch is now interested in Bob, Carol, Dave, Eve... + std::set update_to({"bob", "carol", "dave", "eve"}); + AddedRemoved added_removed = watch_map.updateWatchInterest(watch, update_to); + EXPECT_EQ(std::set({"carol", "dave", "eve"}), added_removed.added_); + EXPECT_EQ(std::set({"alice"}), added_removed.removed_); + + // ...the update is going to contain Alice, Carol, Dave... + Protobuf::RepeatedPtrField updated_resources; + envoy::api::v2::ClusterLoadAssignment alice; + alice.set_cluster_name("alice"); + updated_resources.Add()->PackFrom(alice); + envoy::api::v2::ClusterLoadAssignment carol; + carol.set_cluster_name("carol"); + updated_resources.Add()->PackFrom(carol); + envoy::api::v2::ClusterLoadAssignment dave; + dave.set_cluster_name("dave"); + updated_resources.Add()->PackFrom(dave); + + // ...so the watch should receive only Carol and Dave. + std::vector expected_resources; + expected_resources.push_back(carol); + expected_resources.push_back(dave); + + expectDeltaAndSotwUpdate(callbacks, expected_resources, {"bob"}, "version2"); + doDeltaAndSotwUpdate(watch_map, updated_resources, {"bob"}, "version2"); + } +} + +// Checks the following: +// First watch on a resource name ==> updateWatchInterest() returns "add it to subscription" +// Second watch on that name ==> updateWatchInterest() returns nothing about that name +// Original watch loses interest ==> nothing +// Second watch also loses interest ==> "remove it from subscription" +// NOTE: we need the resource name "dummy" to keep either watch from ever having no names watched, +// which is treated as interest in all names. +TEST(WatchMapTest, Overlap) { + NamedMockSubscriptionCallbacks callbacks1; + NamedMockSubscriptionCallbacks callbacks2; + WatchMap watch_map; + Watch* watch1 = watch_map.addWatch(callbacks1); + Watch* watch2 = watch_map.addWatch(callbacks2); + + Protobuf::RepeatedPtrField updated_resources; + envoy::api::v2::ClusterLoadAssignment alice; + alice.set_cluster_name("alice"); + updated_resources.Add()->PackFrom(alice); + + // First watch becomes interested. + { + std::set update_to({"alice", "dummy"}); + AddedRemoved added_removed = watch_map.updateWatchInterest(watch1, update_to); + EXPECT_EQ(update_to, added_removed.added_); // add to subscription + EXPECT_TRUE(added_removed.removed_.empty()); + watch_map.updateWatchInterest(watch2, {"dummy"}); + + // First watch receives update. + expectDeltaAndSotwUpdate(callbacks1, {alice}, {}, "version1"); + expectNoDeltaUpdate(callbacks2, "version1"); + doDeltaAndSotwUpdate(watch_map, updated_resources, {}, "version1"); + } + // Second watch becomes interested. + { + std::set update_to({"alice", "dummy"}); + AddedRemoved added_removed = watch_map.updateWatchInterest(watch2, update_to); + EXPECT_TRUE(added_removed.added_.empty()); // nothing happens + EXPECT_TRUE(added_removed.removed_.empty()); + + // Both watches receive update. + expectDeltaAndSotwUpdate(callbacks1, {alice}, {}, "version2"); + expectDeltaAndSotwUpdate(callbacks2, {alice}, {}, "version2"); + doDeltaAndSotwUpdate(watch_map, updated_resources, {}, "version2"); + } + // First watch loses interest. + { + AddedRemoved added_removed = watch_map.updateWatchInterest(watch1, {"dummy"}); + EXPECT_TRUE(added_removed.added_.empty()); // nothing happens + EXPECT_TRUE(added_removed.removed_.empty()); + + // *Only* second watch receives update. + expectNoDeltaUpdate(callbacks1, "version3"); + expectDeltaAndSotwUpdate(callbacks2, {alice}, {}, "version3"); + doDeltaAndSotwUpdate(watch_map, updated_resources, {}, "version3"); + } + // Second watch loses interest. + { + AddedRemoved added_removed = watch_map.updateWatchInterest(watch2, {"dummy"}); + EXPECT_TRUE(added_removed.added_.empty()); + EXPECT_EQ(std::set({"alice"}), added_removed.removed_); // remove from subscription + } +} + +// Checks the following: +// First watch on a resource name ==> updateWatchInterest() returns "add it to subscription" +// Watch loses interest ==> "remove it from subscription" +// Second watch on that name ==> "add it to subscription" +// NOTE: we need the resource name "dummy" to keep either watch from ever having no names watched, +// which is treated as interest in all names. +TEST(WatchMapTest, AddRemoveAdd) { + NamedMockSubscriptionCallbacks callbacks1; + NamedMockSubscriptionCallbacks callbacks2; + WatchMap watch_map; + Watch* watch1 = watch_map.addWatch(callbacks1); + Watch* watch2 = watch_map.addWatch(callbacks2); + + Protobuf::RepeatedPtrField updated_resources; + envoy::api::v2::ClusterLoadAssignment alice; + alice.set_cluster_name("alice"); + updated_resources.Add()->PackFrom(alice); + + // First watch becomes interested. + { + std::set update_to({"alice", "dummy"}); + AddedRemoved added_removed = watch_map.updateWatchInterest(watch1, update_to); + EXPECT_EQ(update_to, added_removed.added_); // add to subscription + EXPECT_TRUE(added_removed.removed_.empty()); + watch_map.updateWatchInterest(watch2, {"dummy"}); + + // First watch receives update. + expectDeltaAndSotwUpdate(callbacks1, {alice}, {}, "version1"); + expectNoDeltaUpdate(callbacks2, "version1"); + doDeltaAndSotwUpdate(watch_map, updated_resources, {}, "version1"); + } + // First watch loses interest. + { + AddedRemoved added_removed = watch_map.updateWatchInterest(watch1, {"dummy"}); + EXPECT_TRUE(added_removed.added_.empty()); + EXPECT_EQ(std::set({"alice"}), added_removed.removed_); // remove from subscription + + // (The xDS client should have responded to updateWatchInterest()'s return value by removing + // Alice from the subscription, so onConfigUpdate() calls should be impossible right now.) + } + // Second watch becomes interested. + { + std::set update_to({"alice", "dummy"}); + AddedRemoved added_removed = watch_map.updateWatchInterest(watch2, update_to); + EXPECT_EQ(std::set({"alice"}), added_removed.added_); // add to subscription + EXPECT_TRUE(added_removed.removed_.empty()); + + // *Only* second watch receives update. + expectNoDeltaUpdate(callbacks1, "version2"); + expectDeltaAndSotwUpdate(callbacks2, {alice}, {}, "version2"); + doDeltaAndSotwUpdate(watch_map, updated_resources, {}, "version2"); + } +} + +// Tests that nothing breaks if an update arrives that we entirely do not care about. +TEST(WatchMapTest, UninterestingUpdate) { + NamedMockSubscriptionCallbacks callbacks; + WatchMap watch_map; + Watch* watch = watch_map.addWatch(callbacks); + watch_map.updateWatchInterest(watch, {"alice"}); + + Protobuf::RepeatedPtrField alice_update; + envoy::api::v2::ClusterLoadAssignment alice; + alice.set_cluster_name("alice"); + alice_update.Add()->PackFrom(alice); + + Protobuf::RepeatedPtrField bob_update; + envoy::api::v2::ClusterLoadAssignment bob; + bob.set_cluster_name("bob"); + bob_update.Add()->PackFrom(bob); + + expectNoDeltaUpdate(callbacks, "version1"); + doDeltaAndSotwUpdate(watch_map, bob_update, {}, "version1"); + + expectDeltaAndSotwUpdate(callbacks, {alice}, {}, "version2"); + doDeltaAndSotwUpdate(watch_map, alice_update, {}, "version2"); + + expectNoDeltaUpdate(callbacks, "version3"); + doDeltaAndSotwUpdate(watch_map, bob_update, {}, "version3"); + + // Clean removal of the watch: first update to "interested in nothing", then remove. + watch_map.updateWatchInterest(watch, {}); + watch_map.removeWatch(watch); + + // Finally, test that calling onConfigUpdate on a map with no watches doesn't break. + doDeltaAndSotwUpdate(watch_map, bob_update, {}, "version4"); +} + +// Tests that a watch that specifies no particular resource interest is treated as interested in +// everything. +TEST(WatchMapTest, WatchingEverything) { + NamedMockSubscriptionCallbacks callbacks1; + NamedMockSubscriptionCallbacks callbacks2; + WatchMap watch_map; + /*Watch* watch1 = */ watch_map.addWatch(callbacks1); + Watch* watch2 = watch_map.addWatch(callbacks2); + // watch1 never specifies any names, and so is treated as interested in everything. + watch_map.updateWatchInterest(watch2, {"alice"}); + + Protobuf::RepeatedPtrField updated_resources; + envoy::api::v2::ClusterLoadAssignment alice; + alice.set_cluster_name("alice"); + updated_resources.Add()->PackFrom(alice); + envoy::api::v2::ClusterLoadAssignment bob; + bob.set_cluster_name("bob"); + updated_resources.Add()->PackFrom(bob); + + std::vector expected_resources1; + expected_resources1.push_back(alice); + expected_resources1.push_back(bob); + std::vector expected_resources2; + expected_resources2.push_back(alice); + + expectDeltaAndSotwUpdate(callbacks1, expected_resources1, {}, "version1"); + expectDeltaAndSotwUpdate(callbacks2, expected_resources2, {}, "version1"); + doDeltaAndSotwUpdate(watch_map, updated_resources, {}, "version1"); +} + +// Delta onConfigUpdate has some slightly subtle details with how it handles the three cases where a +// watch receives {only updates, updates+removals, only removals} to its resources. This test +// exercise those cases. Also, the removal-only case tests that SotW does call a watch's +// onConfigUpdate even if none of the watch's interested resources are among the updated resources. +// (Which ensures we deliver empty config updates when a resource is dropped.) +TEST(WatchMapTest, DeltaOnConfigUpdate) { + NamedMockSubscriptionCallbacks callbacks1; + NamedMockSubscriptionCallbacks callbacks2; + NamedMockSubscriptionCallbacks callbacks3; + WatchMap watch_map; + Watch* watch1 = watch_map.addWatch(callbacks1); + Watch* watch2 = watch_map.addWatch(callbacks2); + Watch* watch3 = watch_map.addWatch(callbacks3); + watch_map.updateWatchInterest(watch1, {"updated"}); + watch_map.updateWatchInterest(watch2, {"updated", "removed"}); + watch_map.updateWatchInterest(watch3, {"removed"}); + + Protobuf::RepeatedPtrField update; + envoy::api::v2::ClusterLoadAssignment updated; + updated.set_cluster_name("updated"); + update.Add()->PackFrom(updated); + + expectDeltaAndSotwUpdate(callbacks1, {updated}, {}, "version1"); // only update + expectDeltaAndSotwUpdate(callbacks2, {updated}, {"removed"}, "version1"); // update+remove + expectDeltaAndSotwUpdate(callbacks3, {}, {"removed"}, "version1"); // only remove + doDeltaAndSotwUpdate(watch_map, update, {"removed"}, "version1"); +} + +TEST(WatchMapTest, OnConfigUpdateFailed) { + WatchMap watch_map; + watch_map.onConfigUpdateFailed(nullptr); // calling on empty map doesn't break + + NamedMockSubscriptionCallbacks callbacks1; + NamedMockSubscriptionCallbacks callbacks2; + watch_map.addWatch(callbacks1); + watch_map.addWatch(callbacks2); + + EXPECT_CALL(callbacks1, onConfigUpdateFailed(nullptr)); + EXPECT_CALL(callbacks2, onConfigUpdateFailed(nullptr)); + watch_map.onConfigUpdateFailed(nullptr); +} + +} // namespace +} // namespace Config +} // namespace Envoy From 512f4f9b4ffd8051e9348dc23fc3b4d0fdce6383 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Mon, 22 Jul 2019 11:01:34 -0400 Subject: [PATCH 39/56] RELEASE_ASSERT for impossible broken case Signed-off-by: Fred Douglas --- source/common/config/watch_map.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/source/common/config/watch_map.cc b/source/common/config/watch_map.cc index a40eb439c69ae..e29455f3d4cdf 100644 --- a/source/common/config/watch_map.cc +++ b/source/common/config/watch_map.cc @@ -154,8 +154,9 @@ WatchMap::findRemovals(const std::vector& newly_removed_from_watch, std::set newly_removed_from_subscription; for (const auto& name : newly_removed_from_watch) { auto entry = watch_interest_.find(name); - ASSERT(entry != watch_interest_.end(), - fmt::format("WatchMap: tried to remove a watch from untracked resource {}", name)); + RELEASE_ASSERT( + entry != watch_interest_.end(), + fmt::format("WatchMap: tried to remove a watch from untracked resource {}", name)); entry->second.erase(watch); if (entry->second.empty()) { From 4f50d89fd7379c4bb14ca0c8774cdc30838ff636 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Mon, 22 Jul 2019 11:41:10 -0400 Subject: [PATCH 40/56] fix test failed by merge conflict Signed-off-by: Fred Douglas --- source/common/config/delta_subscription_impl.cc | 1 - source/common/config/watch_map.h | 4 ++++ test/common/runtime/runtime_impl_test.cc | 4 ++-- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/source/common/config/delta_subscription_impl.cc b/source/common/config/delta_subscription_impl.cc index 2a7d1403177ad..268b4a1a86239 100644 --- a/source/common/config/delta_subscription_impl.cc +++ b/source/common/config/delta_subscription_impl.cc @@ -12,7 +12,6 @@ DeltaSubscriptionImpl::DeltaSubscriptionImpl( DeltaSubscriptionImpl::~DeltaSubscriptionImpl() { if (watch_) { context_->removeWatch(type_url_, watch_); - watch_ = nullptr; } } diff --git a/source/common/config/watch_map.h b/source/common/config/watch_map.h index 4c1d83a7d6a46..a5417f9493398 100644 --- a/source/common/config/watch_map.h +++ b/source/common/config/watch_map.h @@ -28,6 +28,10 @@ struct Watch { std::set resource_names_; // must be sorted set, for set_difference. }; +// NOTE: Users are responsible for eventually calling removeWatch() on the Watch* returned +// by addWatch(). We don't expect there to be new users of this class beyond +// NewGrpcMuxImpl and DeltaSubscriptionImpl (to be renamed). +// // Manages "watches" of xDS resources. Several xDS callers might ask for a subscription to the same // resource name "X". The xDS machinery must return to each their very own subscription to X. // The xDS machinery's "watch" concept accomplishes that, while avoiding parallel redundant xDS diff --git a/test/common/runtime/runtime_impl_test.cc b/test/common/runtime/runtime_impl_test.cc index cb73ae6e73bf3..042210451e283 100644 --- a/test/common/runtime/runtime_impl_test.cc +++ b/test/common/runtime/runtime_impl_test.cc @@ -675,10 +675,10 @@ class RtdsLoaderImplTest : public LoaderImplTest { EXPECT_CALL(init_manager_, add(_)).WillRepeatedly(Invoke([this](const Init::Target& target) { init_target_handles_.emplace_back(target.createHandle("test")); })); - ON_CALL(cm_.subscription_factory_, subscriptionFromConfigSource(_, _, _, _)) + ON_CALL(cm_.subscription_factory_, subscriptionFromConfigSource(_, _, _, _, _)) .WillByDefault(testing::Invoke( [this](const envoy::api::v2::core::ConfigSource&, absl::string_view, Stats::Scope&, - Config::SubscriptionCallbacks& callbacks) -> Config::SubscriptionPtr { + Config::SubscriptionCallbacks& callbacks, bool) -> Config::SubscriptionPtr { auto ret = std::make_unique>(); rtds_subscriptions_.push_back(ret.get()); rtds_callbacks_.push_back(&callbacks); From 861e434b6a0d16a3b14ee26ec106c95166159608 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Mon, 22 Jul 2019 16:30:31 -0400 Subject: [PATCH 41/56] fix CI complaints Signed-off-by: Fred Douglas --- .../configuration/overview/v2_overview.rst | 26 ++++++++++++++----- tools/spelling_dictionary.txt | 2 ++ 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/docs/root/configuration/overview/v2_overview.rst b/docs/root/configuration/overview/v2_overview.rst index 42d2db8f5b59f..40fe89ece5ba5 100644 --- a/docs/root/configuration/overview/v2_overview.rst +++ b/docs/root/configuration/overview/v2_overview.rst @@ -575,12 +575,26 @@ the shared ADS channel. Delta endpoints --------------- -The REST, filesystem, and original gRPC xDS implementations all deliver "state of the world" updates: every CDS update must contain every cluster, with the absence of a cluster from an update implying that the cluster is gone. For Envoy deployments with huge amounts of resources and even a trickle of churn, these state-of-the-world updates can be cumbersome. - -As of 1.11.0, Envoy supports a "delta" variant of xDS (including ADS), where updates only contain resources added/changed/removed. Delta xDS is a gRPC (only) protocol. Delta uses different request/response protos than SotW (DeltaDiscovery{Request,Response}); see :repo:`discovery.proto -`. Conceptually, delta should be viewed as a new xDS transport type: there is static, filesystem, REST, gRPC-SotW, and now gRPC-delta. (Envoy's implementation of the gRPC-SotW/delta client happens to share most of its code between the two, and something similar is likely possible on the server side. However, they are in fact incompatible protocols. :ref:`The specification of the delta xDS protocol's behavior is here `.) - -To use delta, simply set the api_type field of your :ref:`ApiConfigSource ` proto(s) to DELTA_GRPC. That works for both xDS and ADS; for ADS, it's the api_type field of :ref:`DynamicResources.ads_config `, as described in the previous section. +The REST, filesystem, and original gRPC xDS implementations all deliver "state of the world" updates: +every CDS update must contain every cluster, with the absence of a cluster from an update implying +that the cluster is gone. For Envoy deployments with huge amounts of resources and even a trickle of +churn, these state-of-the-world updates can be cumbersome. + +As of 1.12.0, Envoy supports a "delta" variant of xDS (including ADS), where updates only contain +resources added/changed/removed. Delta xDS is a gRPC (only) protocol. Delta uses different +request/response protos than SotW (DeltaDiscovery{Request,Response}); see +:repo:`discovery.proto `. Conceptually, delta should be viewed as +a new xDS transport type: there is static, filesystem, REST, gRPC-SotW, and now gRPC-delta. +(Envoy's implementation of the gRPC-SotW/delta client happens to share most of its code between the +two, and something similar is likely possible on the server side. However, they are in fact +incompatible protocols. +:ref:`The specification of the delta xDS protocol's behavior is here `.) + +To use delta, simply set the api_type field of your +:ref:`ApiConfigSource ` proto(s) to DELTA_GRPC. +That works for both xDS and ADS; for ADS, it's the api_type field of +:ref:`DynamicResources.ads_config `, +as described in the previous section. .. _config_overview_v2_mgmt_con_issues: diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 475a1f17093b9..36797edf8b7a8 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -85,6 +85,7 @@ EVAL EVLOOP EVP EWOULDBLOCK +EXPECTs EXPR FAQ FDs @@ -468,6 +469,7 @@ evthread evwatch exe execlp +expectDeltaAndSotwUpdate facto favicon fd From a972cb035ab0909ba74a559cb457e863f7f8bd14 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 23 Aug 2019 14:59:57 -0400 Subject: [PATCH 42/56] fix scoped_rds_test Signed-off-by: Fred Douglas --- test/common/router/scoped_rds_test.cc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/common/router/scoped_rds_test.cc b/test/common/router/scoped_rds_test.cc index 999c0e7fbe735..2db53486f160f 100644 --- a/test/common/router/scoped_rds_test.cc +++ b/test/common/router/scoped_rds_test.cc @@ -6,6 +6,7 @@ #include "envoy/init/manager.h" #include "envoy/stats/scope.h" +#include "common/config/grpc_mux_impl.h" #include "common/router/scoped_rds.h" #include "test/mocks/config/mocks.h" @@ -97,8 +98,10 @@ class ScopedRoutesTestBase : public testing::Test { class ScopedRdsTest : public ScopedRoutesTestBase { protected: void setup() { - InSequence s; + ON_CALL(factory_context_.cluster_manager_, adsMux()) + .WillByDefault(Return(std::make_shared<::Envoy::Config::NullGrpcMuxImpl>())); + InSequence s; // Since factory_context_.cluster_manager_.subscription_factory_.callbacks_ is taken by the SRDS // subscription. We need to return a different MockSubscription here for each RDS subscription. // To build the map from RDS route_config_name to the RDS subscription, we need to get the From 0ca1b0a6f71855c243a61070109f00c4f0c114c1 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 29 Aug 2019 18:17:10 -0400 Subject: [PATCH 43/56] snapshot Signed-off-by: Fred Douglas --- .../arch_overview/operations/dynamic_configuration.rst | 4 +++- docs/root/intro/version_history.rst | 2 +- include/envoy/config/grpc_mux.h | 3 ++- include/envoy/upstream/cluster_manager.h | 2 +- source/common/config/delta_subscription_impl.cc | 10 ++++++---- source/common/config/delta_subscription_impl.h | 7 +++---- source/common/config/delta_subscription_state.h | 2 +- source/common/config/grpc_mux_subscription_impl.cc | 2 +- source/common/config/grpc_mux_subscription_impl.h | 4 ++-- source/common/config/new_grpc_mux_impl.h | 2 +- source/common/upstream/cluster_manager_impl.h | 4 ++-- test/mocks/upstream/mocks.h | 2 +- 12 files changed, 24 insertions(+), 20 deletions(-) diff --git a/docs/root/intro/arch_overview/operations/dynamic_configuration.rst b/docs/root/intro/arch_overview/operations/dynamic_configuration.rst index e5729d04699cc..676284d7594a9 100644 --- a/docs/root/intro/arch_overview/operations/dynamic_configuration.rst +++ b/docs/root/intro/arch_overview/operations/dynamic_configuration.rst @@ -94,7 +94,9 @@ Aggregated xDS ("ADS") EDS, CDS, etc. are each separate services, with different REST/gRPC service names, e.g. StreamListeners, StreamSecrets. For users looking to enforce the order in which resources of different types reach Envoy, there is aggregated xDS, a single gRPC service that carries all resource types in a single gRPC stream. (ADS is only supported by gRPC). :ref:`More details about ADS `. +.. _arch_overview_dynamic_config_deta: + Delta gRPC xDS ----------------------------- -Standard xDS is "state-of-the-world": every CDS update must contain every cluster, with the absence of a cluster from an update implying that the cluster is gone. As of 1.11.0, Envoy supports a "delta" variant of xDS (including ADS), where updates only contain resources added/changed/removed. Delta xDS is a new protocol, with request/response protos different from SotW; existing control plane servers will need support added. :ref:`More details about delta `. +Standard xDS is "state-of-the-world": every CDS update must contain every cluster, with the absence of a cluster from an update implying that the cluster is gone. As of 1.12.0, Envoy supports a "delta" variant of xDS (including ADS), where updates only contain resources added/changed/removed. Delta xDS is a new protocol, with request/response protos different from SotW; existing control plane servers will need support added. :ref:`More details about delta `. diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index 748e4732e600e..a19d1361c2d49 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -9,7 +9,7 @@ Version history * api: added ::ref:`set_node_on_first_message_only ` option to omit the node identifier from the subsequent discovery requests on the same stream. * config: enforcing that terminal filters (e.g. HttpConnectionManager for L4, router for L7) be the last in their respective filter chains. * buffer filter: the buffer filter populates content-length header if not present, behavior can be disabled using the runtime feature `envoy.reloadable_features.buffer_filter_populate_content_length`. -* config: added support for delta xDS (including ADS) delivery +* config: added support for :ref:`delta xDS ` (including ADS) delivery * config: added access log :ref:`extension filter`. * config: added support for :option:`--reject-unknown-dynamic-fields`, providing independent control over whether unknown fields are rejected in static and dynamic configuration. By default, unknown diff --git a/include/envoy/config/grpc_mux.h b/include/envoy/config/grpc_mux.h index f6d2da539209e..1742ea60e0601 100644 --- a/include/envoy/config/grpc_mux.h +++ b/include/envoy/config/grpc_mux.h @@ -130,13 +130,14 @@ class GrpcMux { }; using GrpcMuxPtr = std::unique_ptr; +using GrpcMuxSharedPtr = std::shared_ptr; /** * A grouping of callbacks that a GrpcMux should provide to its GrpcStream. */ template class GrpcStreamCallbacks { public: - virtual ~GrpcStreamCallbacks() {} + virtual ~GrpcStreamCallbacks() = default; /** * For the GrpcStream to prompt the context to take appropriate action in response to the diff --git a/include/envoy/upstream/cluster_manager.h b/include/envoy/upstream/cluster_manager.h index 67a5b3dfa8856..df3dd11c2d3ce 100644 --- a/include/envoy/upstream/cluster_manager.h +++ b/include/envoy/upstream/cluster_manager.h @@ -187,7 +187,7 @@ class ClusterManager { * * @return GrpcMux& ADS API provider referencee. */ - virtual std::shared_ptr adsMux() PURE; + virtual GrpcMuxSharedPtr adsMux() PURE; /** * @return Grpc::AsyncClientManager& the cluster manager's gRPC client manager. diff --git a/source/common/config/delta_subscription_impl.cc b/source/common/config/delta_subscription_impl.cc index 961c185e4ce86..ddfd22d51173a 100644 --- a/source/common/config/delta_subscription_impl.cc +++ b/source/common/config/delta_subscription_impl.cc @@ -3,10 +3,12 @@ namespace Envoy { namespace Config { -DeltaSubscriptionImpl::DeltaSubscriptionImpl( - std::shared_ptr context, absl::string_view type_url, SubscriptionCallbacks& callbacks, - SubscriptionStats stats, std::chrono::milliseconds init_fetch_timeout, bool is_aggregated) - : context_(context), type_url_(type_url), callbacks_(callbacks), stats_(stats), +DeltaSubscriptionImpl::DeltaSubscriptionImpl(GrpcMuxSharedPtr context, absl::string_view type_url, + SubscriptionCallbacks& callbacks, + SubscriptionStats stats, + std::chrono::milliseconds init_fetch_timeout, + bool is_aggregated) + : context_(std::move(context)), type_url_(type_url), callbacks_(callbacks), stats_(stats), init_fetch_timeout_(init_fetch_timeout), is_aggregated_(is_aggregated) {} DeltaSubscriptionImpl::~DeltaSubscriptionImpl() { diff --git a/source/common/config/delta_subscription_impl.h b/source/common/config/delta_subscription_impl.h index d2f81a5a960d8..1a1ad64489f52 100644 --- a/source/common/config/delta_subscription_impl.h +++ b/source/common/config/delta_subscription_impl.h @@ -32,13 +32,12 @@ class DeltaSubscriptionImpl : public Subscription, public SubscriptionCallbacks // is_aggregated: whether the underlying mux/context is providing ADS to us and others, or whether // it's all ours. The practical difference is that we ourselves must call start() on it only in // the latter case. - DeltaSubscriptionImpl(std::shared_ptr context, absl::string_view type_url, + DeltaSubscriptionImpl(GrpcMuxSharedPtr context, absl::string_view type_url, SubscriptionCallbacks& callbacks, SubscriptionStats stats, std::chrono::milliseconds init_fetch_timeout, bool is_aggregated); ~DeltaSubscriptionImpl(); void pause(); - void resume(); // Config::Subscription @@ -54,10 +53,10 @@ class DeltaSubscriptionImpl : public Subscription, public SubscriptionCallbacks void onConfigUpdateFailed(ConfigUpdateFailureReason reason, const EnvoyException* e) override; std::string resourceName(const ProtobufWkt::Any& resource) override; - std::shared_ptr getContextForTest() { return context_; } + GrpcMuxSharedPtr getContextForTest() { return context_; } private: - std::shared_ptr context_; + GrpcMuxSharedPtr context_; const std::string type_url_; SubscriptionCallbacks& callbacks_; SubscriptionStats stats_; diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index 94c639dd219b7..3475fca2b55aa 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -83,7 +83,7 @@ class DeltaSubscriptionState : public Logger::Loggable { SubscriptionCallbacks& callbacks_; const LocalInfo::LocalInfo& local_info_; std::chrono::milliseconds init_fetch_timeout_; - Event::TimerPtr init_fetch_timeout_timer_{}; + Event::TimerPtr init_fetch_timeout_timer_; bool any_request_sent_yet_in_current_stream_{}; diff --git a/source/common/config/grpc_mux_subscription_impl.cc b/source/common/config/grpc_mux_subscription_impl.cc index 5669138dd523d..6c34276b9aadd 100644 --- a/source/common/config/grpc_mux_subscription_impl.cc +++ b/source/common/config/grpc_mux_subscription_impl.cc @@ -9,7 +9,7 @@ namespace Envoy { namespace Config { -GrpcMuxSubscriptionImpl::GrpcMuxSubscriptionImpl(std::shared_ptr grpc_mux, +GrpcMuxSubscriptionImpl::GrpcMuxSubscriptionImpl(GrpcMuxSharedPtr grpc_mux, SubscriptionCallbacks& callbacks, SubscriptionStats stats, absl::string_view type_url, diff --git a/source/common/config/grpc_mux_subscription_impl.h b/source/common/config/grpc_mux_subscription_impl.h index 2bfdd260f4c20..299aa4f1480ee 100644 --- a/source/common/config/grpc_mux_subscription_impl.h +++ b/source/common/config/grpc_mux_subscription_impl.h @@ -17,7 +17,7 @@ class GrpcMuxSubscriptionImpl : public Subscription, GrpcMuxCallbacks, Logger::Loggable { public: - GrpcMuxSubscriptionImpl(std::shared_ptr grpc_mux, SubscriptionCallbacks& callbacks, + GrpcMuxSubscriptionImpl(GrpcMuxSharedPtr grpc_mux, SubscriptionCallbacks& callbacks, SubscriptionStats stats, absl::string_view type_url, Event::Dispatcher& dispatcher, std::chrono::milliseconds init_fetch_timeout); @@ -36,7 +36,7 @@ class GrpcMuxSubscriptionImpl : public Subscription, private: void disableInitFetchTimeoutTimer(); - std::shared_ptr grpc_mux_; + GrpcMuxSharedPtr grpc_mux_; SubscriptionCallbacks& callbacks_; SubscriptionStats stats_; const std::string type_url_; diff --git a/source/common/config/new_grpc_mux_impl.h b/source/common/config/new_grpc_mux_impl.h index c63b2f9de02f8..1bd3b5a1f1742 100644 --- a/source/common/config/new_grpc_mux_impl.h +++ b/source/common/config/new_grpc_mux_impl.h @@ -24,7 +24,7 @@ class NewGrpcMuxImpl : public GrpcMux, public GrpcStreamCallbacks, Logger::Loggable { public: - NewGrpcMuxImpl(Grpc::RawAsyncClientPtr async_client, Event::Dispatcher& dispatcher, + NewGrpcMuxImpl(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, const LocalInfo::LocalInfo& local_info); diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index 7049271c7d95d..fdd51658b7116 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -217,7 +217,7 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable adsMux() override { return ads_mux_; } + GrpcMuxSharedPtr adsMux() override { return ads_mux_; } Grpc::AsyncClientManager& grpcAsyncClientManager() override { return *async_client_manager_; } const std::string& localClusterName() const override { return local_cluster_name_; } @@ -462,7 +462,7 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggable ads_mux_; + GrpcMuxSharedPtr ads_mux_; LoadStatsReporterPtr load_stats_reporter_; // The name of the local cluster of this Envoy instance if defined, else the empty string. std::string local_cluster_name_; diff --git a/test/mocks/upstream/mocks.h b/test/mocks/upstream/mocks.h index 05155dbf1a5d9..67f009616c9e3 100644 --- a/test/mocks/upstream/mocks.h +++ b/test/mocks/upstream/mocks.h @@ -322,7 +322,7 @@ class MockClusterManager : public ClusterManager { MOCK_METHOD1(removeCluster, bool(const std::string& cluster)); MOCK_METHOD0(shutdown, void()); MOCK_CONST_METHOD0(bindConfig, const envoy::api::v2::core::BindConfig&()); - MOCK_METHOD0(adsMux, std::shared_ptr()); + MOCK_METHOD0(adsMux, GrpcMuxSharedPtr()); MOCK_METHOD0(grpcAsyncClientManager, Grpc::AsyncClientManager&()); MOCK_CONST_METHOD0(versionInfo, const std::string()); MOCK_CONST_METHOD0(localClusterName, const std::string&()); From 933fea53b67bbff965621df50058f3f7f2504c3b Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 30 Aug 2019 16:13:04 -0400 Subject: [PATCH 44/56] fix breakage, protect another adsMux pause call against null Signed-off-by: Fred Douglas --- include/envoy/upstream/cluster_manager.h | 6 +++--- source/common/config/delta_subscription_state.h | 3 +++ source/common/config/grpc_mux_impl.h | 2 +- source/common/config/new_grpc_mux_impl.cc | 3 ++- source/common/router/scoped_rds.cc | 12 ++++++++---- source/common/upstream/cds_api_impl.cc | 7 ++----- source/common/upstream/cluster_manager_impl.cc | 8 ++++---- source/common/upstream/cluster_manager_impl.h | 5 +++-- source/server/lds_api.cc | 6 +----- test/common/upstream/cluster_manager_impl_test.cc | 10 ---------- test/mocks/upstream/mocks.h | 2 +- 11 files changed, 28 insertions(+), 36 deletions(-) diff --git a/include/envoy/upstream/cluster_manager.h b/include/envoy/upstream/cluster_manager.h index df3dd11c2d3ce..ba56548672cf1 100644 --- a/include/envoy/upstream/cluster_manager.h +++ b/include/envoy/upstream/cluster_manager.h @@ -180,14 +180,14 @@ class ClusterManager { virtual const envoy::api::v2::core::BindConfig& bindConfig() const PURE; /** - * Return a reference to the singleton xDS-over-gRPC provider for upstream control plane muxing of - * xDS. This is treated somewhat as a special case in ClusterManager, since it does not relate + * Returns a shared_ptr to the singleton xDS-over-gRPC provider for upstream control plane muxing + * of xDS. This is treated somewhat as a special case in ClusterManager, since it does not relate * logically to the management of clusters but instead is required early in ClusterManager/server * initialization and in various sites that need ClusterManager for xDS API interfacing. * * @return GrpcMux& ADS API provider referencee. */ - virtual GrpcMuxSharedPtr adsMux() PURE; + virtual Config::GrpcMuxSharedPtr adsMux() PURE; /** * @return Grpc::AsyncClientManager& the cluster manager's gRPC client manager. diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index 3475fca2b55aa..7018ab617617c 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -37,7 +37,10 @@ class DeltaSubscriptionState : public Logger::Loggable { void handleEstablishmentFailure(); + // Returns the next gRPC request proto to be sent off to the server, based on this object's + // understanding of the current protocol state, and new resources that Envoy wants to request. envoy::api::v2::DeltaDiscoveryRequest getNextRequestAckless(); + // The WithAck version first calls the Ackless version, then adds in the passed-in ack. envoy::api::v2::DeltaDiscoveryRequest getNextRequestWithAck(const UpdateAck& ack); private: diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index 6f1c362c76424..83821a91504a6 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -135,7 +135,7 @@ class NullGrpcMuxImpl : public GrpcMux, GrpcStreamCallbacks&, SubscriptionCallbacks&, std::chrono::milliseconds) override { diff --git a/source/common/config/new_grpc_mux_impl.cc b/source/common/config/new_grpc_mux_impl.cc index 7f37cb0006481..e0840188ee5d2 100644 --- a/source/common/config/new_grpc_mux_impl.cc +++ b/source/common/config/new_grpc_mux_impl.cc @@ -10,7 +10,8 @@ namespace Envoy { namespace Config { -NewGrpcMuxImpl::NewGrpcMuxImpl(Grpc::RawAsyncClientPtr async_client, Event::Dispatcher& dispatcher, +NewGrpcMuxImpl::NewGrpcMuxImpl(Grpc::RawAsyncClientPtr&& async_client, + Event::Dispatcher& dispatcher, const Protobuf::MethodDescriptor& service_method, Runtime::RandomGenerator& random, Stats::Scope& scope, const RateLimitSettings& rate_limit_settings, diff --git a/source/common/router/scoped_rds.cc b/source/common/router/scoped_rds.cc index 64af41177addf..23e86c6e50483 100644 --- a/source/common/router/scoped_rds.cc +++ b/source/common/router/scoped_rds.cc @@ -221,8 +221,10 @@ void ScopedRdsConfigSubscription::onConfigUpdate( // Pause RDS to not send a burst of RDS requests until we start all the new subscriptions. // In the case if factory_context_.initManager() is uninitialized, RDS is already paused either // by Server init or LDS init. - factory_context_.clusterManager().adsMux()->pause( - Envoy::Config::TypeUrl::get().RouteConfiguration); + if (factory_context_.clusterManager().adsMux()) { + factory_context_.clusterManager().adsMux()->pause( + Envoy::Config::TypeUrl::get().RouteConfiguration); + } resume_rds = std::make_unique([this, &noop_init_manager, version_info] { // For new RDS subscriptions created after listener warming up, we don't wait for them to warm // up. @@ -234,8 +236,10 @@ void ScopedRdsConfigSubscription::onConfigUpdate( // New RDS subscriptions should have been created, now lift the floodgate. // Note in the case of partial acceptance, accepted RDS subscriptions should be started // despite of any error. - factory_context_.clusterManager().adsMux()->resume( - Envoy::Config::TypeUrl::get().RouteConfiguration); + if (factory_context_.clusterManager().adsMux()) { + factory_context_.clusterManager().adsMux()->resume( + Envoy::Config::TypeUrl::get().RouteConfiguration); + } }); } std::vector exception_msgs; diff --git a/source/common/upstream/cds_api_impl.cc b/source/common/upstream/cds_api_impl.cc index fa3786a6e55ce..f267bf00c65bf 100644 --- a/source/common/upstream/cds_api_impl.cc +++ b/source/common/upstream/cds_api_impl.cc @@ -61,12 +61,9 @@ void CdsApiImpl::onConfigUpdate( const std::string& system_version_info) { if (cm_.adsMux()) { cm_.adsMux()->pause(Config::TypeUrl::get().ClusterLoadAssignment); + Cleanup eds_resume( + [this] { cm_.adsMux()->resume(Config::TypeUrl::get().ClusterLoadAssignment); }); } - Cleanup eds_resume([this] { - if (cm_.adsMux()) { - cm_.adsMux()->resume(Config::TypeUrl::get().ClusterLoadAssignment); - } - }); ENVOY_LOG(info, "cds: add {} cluster(s), remove {} cluster(s)", added_resources.size(), removed_resources.size()); diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index 05ddde1b61192..45dac7d5c573a 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -138,16 +138,16 @@ void ClusterManagerInitHelper::maybeFinishInitialize() { // If the first CDS response doesn't have any primary cluster, ClusterLoadAssignment // should be already paused by CdsApiImpl::onConfigUpdate(). Need to check that to // avoid double pause ClusterLoadAssignment. - if (cm_.adsMux().paused(Config::TypeUrl::get().ClusterLoadAssignment)) { + if (cm_.adsMux() == nullptr || + cm_.adsMux()->paused(Config::TypeUrl::get().ClusterLoadAssignment)) { initializeSecondaryClusters(); } else { - cm_.adsMux().pause(Config::TypeUrl::get().ClusterLoadAssignment); + cm_.adsMux()->pause(Config::TypeUrl::get().ClusterLoadAssignment); Cleanup eds_resume( - [this] { cm_.adsMux().resume(Config::TypeUrl::get().ClusterLoadAssignment); }); + [this] { cm_.adsMux()->resume(Config::TypeUrl::get().ClusterLoadAssignment); }); initializeSecondaryClusters(); } } - return; } diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index bbd30e4a852db..0b6290eddb891 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -220,7 +220,7 @@ class ClusterManagerImpl : public ClusterManager, Logger::Loggablepause(Config::TypeUrl::get().RouteConfiguration); + Cleanup rds_resume([this] { cm_.adsMux()->resume(Config::TypeUrl::get().RouteConfiguration); }); } - Cleanup rds_resume([this] { - if (cm_.adsMux()) { - cm_.adsMux()->resume(Config::TypeUrl::get().RouteConfiguration); - } - }); bool any_applied = false; // We do all listener removals before adding the new listeners. This allows adding a new listener diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index 51238c082ba99..f6ce85256d4e6 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -2841,12 +2841,7 @@ TEST_F(ClusterManagerInitHelperTest, InitSecondaryWithoutEdsPaused) { ON_CALL(cluster1, initializePhase()).WillByDefault(Return(Cluster::InitializePhase::Secondary)); init_helper_.addCluster(cluster1); - const auto& type_url = Config::TypeUrl::get().ClusterLoadAssignment; - EXPECT_CALL(cm_.ads_mux_, paused(Eq(ByRef(type_url)))).WillRepeatedly(Return(false)); - EXPECT_CALL(cm_.ads_mux_, pause(Eq(ByRef(type_url)))); EXPECT_CALL(cluster1, initialize(_)); - EXPECT_CALL(cm_.ads_mux_, resume(Eq(ByRef(type_url)))); - init_helper_.onStaticLoadComplete(); EXPECT_CALL(*this, onClusterInit(Ref(cluster1))); @@ -2868,12 +2863,7 @@ TEST_F(ClusterManagerInitHelperTest, InitSecondaryWithEdsPaused) { ON_CALL(cluster1, initializePhase()).WillByDefault(Return(Cluster::InitializePhase::Secondary)); init_helper_.addCluster(cluster1); - const auto& type_url = Config::TypeUrl::get().ClusterLoadAssignment; - EXPECT_CALL(cm_.ads_mux_, paused(Eq(ByRef(type_url)))).WillRepeatedly(Return(true)); - EXPECT_CALL(cm_.ads_mux_, pause(Eq(ByRef(type_url)))).Times(0); EXPECT_CALL(cluster1, initialize(_)); - EXPECT_CALL(cm_.ads_mux_, resume(Eq(ByRef(type_url)))).Times(0); - init_helper_.onStaticLoadComplete(); EXPECT_CALL(*this, onClusterInit(Ref(cluster1))); diff --git a/test/mocks/upstream/mocks.h b/test/mocks/upstream/mocks.h index 67f009616c9e3..939cf72dadf21 100644 --- a/test/mocks/upstream/mocks.h +++ b/test/mocks/upstream/mocks.h @@ -322,7 +322,7 @@ class MockClusterManager : public ClusterManager { MOCK_METHOD1(removeCluster, bool(const std::string& cluster)); MOCK_METHOD0(shutdown, void()); MOCK_CONST_METHOD0(bindConfig, const envoy::api::v2::core::BindConfig&()); - MOCK_METHOD0(adsMux, GrpcMuxSharedPtr()); + MOCK_METHOD0(adsMux, Config::GrpcMuxSharedPtr()); MOCK_METHOD0(grpcAsyncClientManager, Grpc::AsyncClientManager&()); MOCK_CONST_METHOD0(versionInfo, const std::string()); MOCK_CONST_METHOD0(localClusterName, const std::string&()); From e45ae53b368e1352fdd1802f5f96411d086e61ba Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 6 Sep 2019 15:31:08 -0400 Subject: [PATCH 45/56] fix Cleanup i mistakenly brought inside an if, thanks very very much to Xin Signed-off-by: Fred Douglas --- source/common/upstream/cds_api_impl.cc | 4 +++- source/server/lds_api.cc | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/source/common/upstream/cds_api_impl.cc b/source/common/upstream/cds_api_impl.cc index f267bf00c65bf..e35a19fcf530c 100644 --- a/source/common/upstream/cds_api_impl.cc +++ b/source/common/upstream/cds_api_impl.cc @@ -59,9 +59,11 @@ void CdsApiImpl::onConfigUpdate( const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info) { + + std::unique_ptr maybe_eds_resume; if (cm_.adsMux()) { cm_.adsMux()->pause(Config::TypeUrl::get().ClusterLoadAssignment); - Cleanup eds_resume( + maybe_eds_resume = std::make_unique( [this] { cm_.adsMux()->resume(Config::TypeUrl::get().ClusterLoadAssignment); }); } diff --git a/source/server/lds_api.cc b/source/server/lds_api.cc index b4e8cda674719..6b5581c7be9b5 100644 --- a/source/server/lds_api.cc +++ b/source/server/lds_api.cc @@ -31,9 +31,11 @@ void LdsApiImpl::onConfigUpdate( const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info) { + std::unique_ptr maybe_eds_resume; if (cm_.adsMux()) { cm_.adsMux()->pause(Config::TypeUrl::get().RouteConfiguration); - Cleanup rds_resume([this] { cm_.adsMux()->resume(Config::TypeUrl::get().RouteConfiguration); }); + maybe_eds_resume = std::make_unique( + [this] { cm_.adsMux()->resume(Config::TypeUrl::get().RouteConfiguration); }); } bool any_applied = false; From 5b7489b643b97ac7d0e092417f1da665452ede33 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Mon, 9 Sep 2019 12:28:40 -0400 Subject: [PATCH 46/56] snapshot Signed-off-by: Fred Douglas --- source/common/config/new_grpc_mux_impl.cc | 5 ++++- test/common/grpc/grpc_client_integration.h | 10 +++------- test/integration/ads_integration_test.cc | 7 ++++--- test/integration/cds_integration_test.cc | 3 ++- test/integration/rtds_integration_test.cc | 3 ++- test/integration/scoped_rds_integration_test.cc | 4 +++- 6 files changed, 18 insertions(+), 14 deletions(-) diff --git a/source/common/config/new_grpc_mux_impl.cc b/source/common/config/new_grpc_mux_impl.cc index e0840188ee5d2..8faa3a86f19b1 100644 --- a/source/common/config/new_grpc_mux_impl.cc +++ b/source/common/config/new_grpc_mux_impl.cc @@ -150,6 +150,7 @@ void NewGrpcMuxImpl::addSubscription(const std::string& type_url, } void NewGrpcMuxImpl::trySendDiscoveryRequests() { + auto old_queue_size = pausable_ack_queue_.size(); while (true) { // Do any of our subscriptions even want to send a request? absl::optional maybe_request_type = whoWantsToSendDiscoveryRequest(); @@ -184,7 +185,9 @@ void NewGrpcMuxImpl::trySendDiscoveryRequests() { grpc_stream_.sendMessage(sub->second->sub_state_.getNextRequestAckless()); } } - grpc_stream_.maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); + if (old_queue_size != pausable_ack_queue_.size()) { + grpc_stream_.maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); + } } // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check diff --git a/test/common/grpc/grpc_client_integration.h b/test/common/grpc/grpc_client_integration.h index 7db8014ac1485..86017cf821d15 100644 --- a/test/common/grpc/grpc_client_integration.h +++ b/test/common/grpc/grpc_client_integration.h @@ -91,20 +91,16 @@ class DeltaSotwIntegrationParamTest #define DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS \ testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ testing::Values(Grpc::ClientType::EnvoyGrpc, Grpc::ClientType::GoogleGrpc), \ - testing::Bool()) + testing::Values(Grpc::SotwOrDelta::Sotw, Grpc::SotwOrDelta::Delta)) #else #define GRPC_CLIENT_INTEGRATION_PARAMS \ testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ testing::Values(Grpc::ClientType::EnvoyGrpc)) #define DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS \ testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ - testing::Values(Grpc::ClientType::EnvoyGrpc), testing::Bool()) -#endif // ENVOY_GOOGLE_GRPC - -#define DELTA_INTEGRATION_PARAMS \ - testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ - testing::Values(Grpc::ClientType::EnvoyGrpc, Grpc::ClientType::GoogleGrpc), \ + testing::Values(Grpc::ClientType::EnvoyGrpc), \ testing::Values(Grpc::SotwOrDelta::Sotw, Grpc::SotwOrDelta::Delta)) +#endif // ENVOY_GOOGLE_GRPC } // namespace Grpc } // namespace Envoy diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index b2b131fa99a69..91e916ed451f7 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -26,7 +26,8 @@ using testing::AssertionResult; namespace Envoy { -INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, AdsIntegrationTest, DELTA_INTEGRATION_PARAMS); +INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, AdsIntegrationTest, + DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS); // Validate basic config delivery and upgrade. TEST_P(AdsIntegrationTest, Basic) { @@ -616,7 +617,7 @@ class AdsFailIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, }; INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, AdsFailIntegrationTest, - DELTA_INTEGRATION_PARAMS); + DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS); // Validate that we don't crash on failed ADS stream. TEST_P(AdsFailIntegrationTest, ConnectDisconnect) { @@ -668,7 +669,7 @@ class AdsConfigIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, }; INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, AdsConfigIntegrationTest, - DELTA_INTEGRATION_PARAMS); + DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS); // This is s regression validating that we don't crash on EDS static Cluster that uses ADS. TEST_P(AdsConfigIntegrationTest, EdsClusterWithAdsConfigSource) { diff --git a/test/integration/cds_integration_test.cc b/test/integration/cds_integration_test.cc index 1ea5f5fe67e89..d3013870ea949 100644 --- a/test/integration/cds_integration_test.cc +++ b/test/integration/cds_integration_test.cc @@ -122,7 +122,8 @@ class CdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public Ht bool test_skipped_{true}; }; -INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, CdsIntegrationTest, DELTA_INTEGRATION_PARAMS); +INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, CdsIntegrationTest, + DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS); // 1) Envoy starts up with no static clusters (other than the CDS-over-gRPC server). // 2) Envoy is told of a cluster via CDS. diff --git a/test/integration/rtds_integration_test.cc b/test/integration/rtds_integration_test.cc index 02283b3927c22..3ffcc5cf54a5c 100644 --- a/test/integration/rtds_integration_test.cc +++ b/test/integration/rtds_integration_test.cc @@ -110,7 +110,8 @@ class RtdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public H uint32_t initial_keys_{}; }; -INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, RtdsIntegrationTest, DELTA_INTEGRATION_PARAMS); +INSTANTIATE_TEST_SUITE_P(IpVersionsClientTypeDelta, RtdsIntegrationTest, + DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS); TEST_P(RtdsIntegrationTest, RtdsReload) { initialize(); diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 4bcc2a38997bc..a2ef034a5ae63 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -13,7 +13,7 @@ namespace Envoy { namespace { class ScopedRdsIntegrationTest : public HttpIntegrationTest, - public Grpc::DeltaSotwGrpcClientIntegrationParamTest { + public Grpc::DeltaSotwIntegrationParamTest { protected: struct FakeUpstreamInfo { FakeHttpConnectionPtr connection_; @@ -217,6 +217,8 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, response); } + bool isDelta() { return sotwOrDelta() == Grpc::SotwOrDelta::Delta; } + const std::string srds_config_name_{"foo-scoped-routes"}; FakeUpstreamInfo scoped_rds_upstream_info_; FakeUpstreamInfo rds_upstream_info_; From a7a59ca360f900d7bf3050b8a7c355ad94c8615f Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Mon, 9 Sep 2019 12:53:41 -0400 Subject: [PATCH 47/56] fix typo in docs, deta to delta Signed-off-by: Fred Douglas --- .../intro/arch_overview/operations/dynamic_configuration.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/root/intro/arch_overview/operations/dynamic_configuration.rst b/docs/root/intro/arch_overview/operations/dynamic_configuration.rst index 676284d7594a9..99bb323ed7c2f 100644 --- a/docs/root/intro/arch_overview/operations/dynamic_configuration.rst +++ b/docs/root/intro/arch_overview/operations/dynamic_configuration.rst @@ -94,7 +94,7 @@ Aggregated xDS ("ADS") EDS, CDS, etc. are each separate services, with different REST/gRPC service names, e.g. StreamListeners, StreamSecrets. For users looking to enforce the order in which resources of different types reach Envoy, there is aggregated xDS, a single gRPC service that carries all resource types in a single gRPC stream. (ADS is only supported by gRPC). :ref:`More details about ADS `. -.. _arch_overview_dynamic_config_deta: +.. _arch_overview_dynamic_config_delta: Delta gRPC xDS ----------------------------- From c1de3f7339989c3f7f76e7139f6af1601fa4918d Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Mon, 9 Sep 2019 16:26:42 -0400 Subject: [PATCH 48/56] clang_tidy Signed-off-by: Fred Douglas --- source/common/config/delta_subscription_impl.h | 2 +- source/common/config/delta_subscription_state.h | 6 +++--- source/common/config/new_grpc_mux_impl.cc | 2 +- source/common/config/pausable_ack_queue.cc | 11 +++++++++-- test/common/config/delta_subscription_impl_test.cc | 2 +- .../config/filesystem_subscription_test_harness.h | 2 +- test/integration/integration.cc | 8 ++++---- 7 files changed, 20 insertions(+), 13 deletions(-) diff --git a/source/common/config/delta_subscription_impl.h b/source/common/config/delta_subscription_impl.h index 1a1ad64489f52..b45be7c4da85f 100644 --- a/source/common/config/delta_subscription_impl.h +++ b/source/common/config/delta_subscription_impl.h @@ -35,7 +35,7 @@ class DeltaSubscriptionImpl : public Subscription, public SubscriptionCallbacks DeltaSubscriptionImpl(GrpcMuxSharedPtr context, absl::string_view type_url, SubscriptionCallbacks& callbacks, SubscriptionStats stats, std::chrono::milliseconds init_fetch_timeout, bool is_aggregated); - ~DeltaSubscriptionImpl(); + ~DeltaSubscriptionImpl() override; void pause(); void resume(); diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index 7018ab617617c..b1a6e58758822 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -43,6 +43,9 @@ class DeltaSubscriptionState : public Logger::Loggable { // The WithAck version first calls the Ackless version, then adds in the passed-in ack. envoy::api::v2::DeltaDiscoveryRequest getNextRequestWithAck(const UpdateAck& ack); + DeltaSubscriptionState(const DeltaSubscriptionState&) = delete; + DeltaSubscriptionState& operator=(const DeltaSubscriptionState&) = delete; + private: void handleGoodResponse(const envoy::api::v2::DeltaDiscoveryResponse& message); void handleBadResponse(const EnvoyException& e, UpdateAck& ack); @@ -95,9 +98,6 @@ class DeltaSubscriptionState : public Logger::Loggable { // Feel free to change to unordered if you can figure out how to make it work. std::set names_added_; std::set names_removed_; - - DeltaSubscriptionState(const DeltaSubscriptionState&) = delete; - DeltaSubscriptionState& operator=(const DeltaSubscriptionState&) = delete; }; } // namespace Config diff --git a/source/common/config/new_grpc_mux_impl.cc b/source/common/config/new_grpc_mux_impl.cc index 8faa3a86f19b1..d0541f056e596 100644 --- a/source/common/config/new_grpc_mux_impl.cc +++ b/source/common/config/new_grpc_mux_impl.cc @@ -98,7 +98,7 @@ void NewGrpcMuxImpl::onEstablishmentFailure() { void NewGrpcMuxImpl::onWriteable() { trySendDiscoveryRequests(); } void NewGrpcMuxImpl::kickOffAck(UpdateAck ack) { - pausable_ack_queue_.push(ack); + pausable_ack_queue_.push(std::move(ack)); trySendDiscoveryRequests(); } diff --git a/source/common/config/pausable_ack_queue.cc b/source/common/config/pausable_ack_queue.cc index 783febb1425f0..34951e38b15c0 100644 --- a/source/common/config/pausable_ack_queue.cc +++ b/source/common/config/pausable_ack_queue.cc @@ -7,16 +7,19 @@ namespace Envoy { namespace Config { -void PausableAckQueue::push(UpdateAck x) { storage_.push_back(x); } +void PausableAckQueue::push(UpdateAck x) { storage_.push_back(std::move(x)); } + size_t PausableAckQueue::size() const { return storage_.size(); } + bool PausableAckQueue::empty() { - for (auto entry : storage_) { + for (const auto& entry : storage_) { if (!paused_[entry.type_url_]) { return false; } } return true; } + const UpdateAck& PausableAckQueue::front() { for (const auto& entry : storage_) { if (!paused_[entry.type_url_]) { @@ -26,6 +29,7 @@ const UpdateAck& PausableAckQueue::front() { RELEASE_ASSERT(!storage_.empty(), "front() on an empty queue is undefined behavior!"); NOT_REACHED_GCOVR_EXCL_LINE; } + void PausableAckQueue::pop() { for (auto it = storage_.begin(); it != storage_.end(); ++it) { if (!paused_[it->type_url_]) { @@ -36,17 +40,20 @@ void PausableAckQueue::pop() { RELEASE_ASSERT(!storage_.empty(), "pop() on an empty queue is undefined behavior!"); NOT_REACHED_GCOVR_EXCL_LINE; } + void PausableAckQueue::pause(const std::string& type_url) { // It's ok to pause a subscription that doesn't exist yet. auto& pause_entry = paused_[type_url]; ASSERT(!pause_entry); pause_entry = true; } + void PausableAckQueue::resume(const std::string& type_url) { auto& pause_entry = paused_[type_url]; ASSERT(pause_entry); pause_entry = false; } + bool PausableAckQueue::paused(const std::string& type_url) const { auto entry = paused_.find(type_url); if (entry == paused_.end()) { diff --git a/test/common/config/delta_subscription_impl_test.cc b/test/common/config/delta_subscription_impl_test.cc index 62ad4ac59f9a9..f8127f93cfdff 100644 --- a/test/common/config/delta_subscription_impl_test.cc +++ b/test/common/config/delta_subscription_impl_test.cc @@ -130,7 +130,7 @@ TEST(DeltaSubscriptionImplFixturelessTest, NoGrpcStream) { NiceMock random; Envoy::Config::RateLimitSettings rate_limit_settings; NiceMock> callbacks; - Grpc::MockAsyncClient* async_client = new Grpc::MockAsyncClient(); + auto* async_client = new Grpc::MockAsyncClient(); const Protobuf::MethodDescriptor* method_descriptor = Protobuf::DescriptorPool::generated_pool()->FindMethodByName( diff --git a/test/common/config/filesystem_subscription_test_harness.h b/test/common/config/filesystem_subscription_test_harness.h index 6f4614a4db667..1958a4270bd69 100644 --- a/test/common/config/filesystem_subscription_test_harness.h +++ b/test/common/config/filesystem_subscription_test_harness.h @@ -47,7 +47,7 @@ class FilesystemSubscriptionTestHarness : public SubscriptionTestHarness { subscription_.updateResourceInterest(cluster_names); } - void updateFile(const std::string json, bool run_dispatcher = true) { + void updateFile(const std::string& json, bool run_dispatcher = true) { // Write JSON contents to file, rename to path_ and run dispatcher to catch // inotify. const std::string temp_path = TestEnvironment::writeStringToFileForTest("eds.json.tmp", json); diff --git a/test/integration/integration.cc b/test/integration/integration.cc index c2c79f7cd10ef..cf4baa464d10c 100644 --- a/test/integration/integration.cc +++ b/test/integration/integration.cc @@ -608,17 +608,17 @@ AssertionResult BaseIntegrationTest::compareSotwDiscoveryRequest( return AssertionSuccess(); } -AssertionResult compareSets(std::set set1, std::set set2, +AssertionResult compareSets(const std::set& set1, const std::set& set2, absl::string_view name) { if (set1 == set2) { return AssertionSuccess(); } auto failure = AssertionFailure() << name << " field not as expected.\nExpected: {"; - for (auto x : set1) { + for (const auto& x : set1) { failure << x << ", "; } failure << "}\nActual: {"; - for (auto x : set2) { + for (const auto& x : set2) { failure << x << ", "; } return failure << "}"; @@ -669,7 +669,7 @@ AssertionResult BaseIntegrationTest::compareDeltaDiscoveryRequest( if (expected_error_code != Grpc::Status::GrpcStatus::Ok && request.error_detail().message().find(expected_error_substring) == std::string::npos) { return AssertionFailure() << "\"" << expected_error_substring - << "\" is not a substring of error message \"" + << "\" is not a substring of actual error message \"" << request.error_detail().message() << "\""; } return AssertionSuccess(); From f0e783810636cfd44982d6821c93b0b2cf60238e Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Wed, 11 Sep 2019 11:42:59 -0400 Subject: [PATCH 49/56] merge master Signed-off-by: Fred Douglas --- source/common/config/grpc_mux_impl.cc | 5 +---- source/common/config/new_grpc_mux_impl.cc | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index c1eda78b6dd67..b8158b9bd8252 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -232,10 +232,7 @@ void GrpcMuxImpl::queueDiscoveryRequest(const std::string& queue_item) { void GrpcMuxImpl::clearRequestQueue() { grpc_stream_.maybeUpdateQueueSizeStat(0); - // TODO(fredlas) when we have C++17: request_queue_ = {}; - while (!request_queue_.empty()) { - request_queue_.pop(); - } + request_queue_ = {}; } void GrpcMuxImpl::drainRequests() { diff --git a/source/common/config/new_grpc_mux_impl.cc b/source/common/config/new_grpc_mux_impl.cc index d0541f056e596..52cdaf36cad12 100644 --- a/source/common/config/new_grpc_mux_impl.cc +++ b/source/common/config/new_grpc_mux_impl.cc @@ -185,7 +185,7 @@ void NewGrpcMuxImpl::trySendDiscoveryRequests() { grpc_stream_.sendMessage(sub->second->sub_state_.getNextRequestAckless()); } } - if (old_queue_size != pausable_ack_queue_.size()) { + if (pausable_ack_queue_.size() != old_queue_size || pausable_ack_queue_.size() != 0) { grpc_stream_.maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); } } From 422662782108c5601993e1f9779f57b22b964a6e Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 12 Sep 2019 10:51:39 -0400 Subject: [PATCH 50/56] a little cleanup in new_grpc_mux_impl Signed-off-by: Fred Douglas --- source/common/config/new_grpc_mux_impl.cc | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/source/common/config/new_grpc_mux_impl.cc b/source/common/config/new_grpc_mux_impl.cc index 52cdaf36cad12..ed550bb960dc1 100644 --- a/source/common/config/new_grpc_mux_impl.cc +++ b/source/common/config/new_grpc_mux_impl.cc @@ -139,7 +139,9 @@ void NewGrpcMuxImpl::updateWatch(const std::string& type_url, Watch* watch, auto added_removed = sub->second->watch_map_.updateWatchInterest(watch, resources); sub->second->sub_state_.updateSubscriptionInterest(added_removed.added_, added_removed.removed_); // Tell the server about our new interests, if there are any. - trySendDiscoveryRequests(); + if (sub->second->sub_state_.subscriptionUpdatePending()) { + trySendDiscoveryRequests(); + } } void NewGrpcMuxImpl::addSubscription(const std::string& type_url, @@ -150,7 +152,6 @@ void NewGrpcMuxImpl::addSubscription(const std::string& type_url, } void NewGrpcMuxImpl::trySendDiscoveryRequests() { - auto old_queue_size = pausable_ack_queue_.size(); while (true) { // Do any of our subscriptions even want to send a request? absl::optional maybe_request_type = whoWantsToSendDiscoveryRequest(); @@ -185,9 +186,7 @@ void NewGrpcMuxImpl::trySendDiscoveryRequests() { grpc_stream_.sendMessage(sub->second->sub_state_.getNextRequestAckless()); } } - if (pausable_ack_queue_.size() != old_queue_size || pausable_ack_queue_.size() != 0) { - grpc_stream_.maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); - } + grpc_stream_.maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); } // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check @@ -227,10 +226,7 @@ absl::optional NewGrpcMuxImpl::whoWantsToSendDiscoveryRequest() { // subscriptions were initiated. for (const auto& sub_type : subscription_ordering_) { auto sub = subscriptions_.find(sub_type); - if (sub == subscriptions_.end()) { - continue; - } - if (sub->second->sub_state_.subscriptionUpdatePending() && + if (sub != subscriptions_.end() && sub->second->sub_state_.subscriptionUpdatePending() && !pausable_ack_queue_.paused(sub_type)) { return sub->first; } From 60dfeabb8b895b6ff536bdd3d3cf17358dab3dc2 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 19 Sep 2019 17:04:18 -0400 Subject: [PATCH 51/56] fix merge drift revealed by fix of CONSTRUCT_ON_FIRST_USE Signed-off-by: Fred Douglas --- source/common/config/grpc_mux_impl.cc | 5 +-- source/common/upstream/cds_api_impl.cc | 1 - test/integration/ads_integration.cc | 1 + test/integration/ads_integration.h | 12 ++++--- test/integration/ads_integration_test.cc | 41 +++++++++++++----------- test/integration/integration.h | 8 +++-- 6 files changed, 40 insertions(+), 28 deletions(-) diff --git a/source/common/config/grpc_mux_impl.cc b/source/common/config/grpc_mux_impl.cc index b8158b9bd8252..fdb3fa603758a 100644 --- a/source/common/config/grpc_mux_impl.cc +++ b/source/common/config/grpc_mux_impl.cc @@ -164,8 +164,9 @@ void GrpcMuxImpl::onDiscoveryResponse( GrpcMuxCallbacks& callbacks = api_state_[type_url].watches_.front()->callbacks_; for (const auto& resource : message->resources()) { if (type_url != resource.type_url()) { - throw EnvoyException(fmt::format("{} does not match {} type URL in DiscoveryResponse {}", - resource.type_url(), type_url, message->DebugString())); + throw EnvoyException( + fmt::format("{} does not match the message-wide type URL {} in DiscoveryResponse {}", + resource.type_url(), type_url, message->DebugString())); } const std::string resource_name = callbacks.resourceName(resource); resources.emplace(resource_name, resource); diff --git a/source/common/upstream/cds_api_impl.cc b/source/common/upstream/cds_api_impl.cc index e35a19fcf530c..a2404c0769446 100644 --- a/source/common/upstream/cds_api_impl.cc +++ b/source/common/upstream/cds_api_impl.cc @@ -59,7 +59,6 @@ void CdsApiImpl::onConfigUpdate( const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info) { - std::unique_ptr maybe_eds_resume; if (cm_.adsMux()) { cm_.adsMux()->pause(Config::TypeUrl::get().ClusterLoadAssignment); diff --git a/test/integration/ads_integration.cc b/test/integration/ads_integration.cc index a2073cac79e82..6138f963e13aa 100644 --- a/test/integration/ads_integration.cc +++ b/test/integration/ads_integration.cc @@ -25,6 +25,7 @@ AdsIntegrationTest::AdsIntegrationTest() use_lds_ = false; create_xds_upstream_ = true; tls_xds_upstream_ = true; + sotw_or_delta_ = sotwOrDelta(); } void AdsIntegrationTest::TearDown() { diff --git a/test/integration/ads_integration.h b/test/integration/ads_integration.h index 69aa472c2f9bf..31956bd6bb126 100644 --- a/test/integration/ads_integration.h +++ b/test/integration/ads_integration.h @@ -12,10 +12,12 @@ #include "test/common/grpc/grpc_client_integration.h" #include "test/integration/http_integration.h" +// TODO(fredlas) set_node_on_first_message_only was true; the delta+SotW unification +// work restores it here. namespace Envoy { -static const std::string& AdsIntegrationConfig(const std::string& api_type) { - CONSTRUCT_ON_FIRST_USE(std::string, fmt::format( - R"EOF( +static std::string AdsIntegrationConfig(const std::string& api_type) { + // Note: do not use CONSTRUCT_ON_FIRST_USE here! + return fmt::format(R"EOF( dynamic_resources: lds_config: ads: {{}} @@ -23,7 +25,7 @@ static const std::string& AdsIntegrationConfig(const std::string& api_type) { ads: {{}} ads_config: api_type: {} - set_node_on_first_message_only: true + set_node_on_first_message_only: false static_resources: clusters: name: dummy_cluster @@ -43,7 +45,7 @@ static const std::string& AdsIntegrationConfig(const std::string& api_type) { address: 127.0.0.1 port_value: 0 )EOF", - api_type)); + api_type); } class AdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public HttpIntegrationTest { diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index 91e916ed451f7..442b5dd8293b4 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -55,9 +55,8 @@ TEST_P(AdsIntegrationTest, Failure) { EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "", {}, {}, {})); EXPECT_TRUE(compareDiscoveryRequest( - Config::TypeUrl::get().Cluster, "", {}, {}, {}, false, Grpc::Status::GrpcStatus::Internal, - fmt::format("{} does not match {}", Config::TypeUrl::get().ClusterLoadAssignment, - Config::TypeUrl::get().Cluster))); + Config::TypeUrl::get().Cluster, "", {}, {}, {}, true, Grpc::Status::GrpcStatus::Internal, + fmt::format("does not match the message-wide type URL {}", Config::TypeUrl::get().Cluster))); sendDiscoveryResponse(Config::TypeUrl::get().Cluster, {buildCluster("cluster_0")}, {buildCluster("cluster_0")}, {}, "1"); @@ -69,11 +68,11 @@ TEST_P(AdsIntegrationTest, Failure) { {buildCluster("cluster_0")}, {}, "1"); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "1", {}, {}, {})); - EXPECT_TRUE( - compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "", {"cluster_0"}, {}, - {}, false, Grpc::Status::GrpcStatus::Internal, - fmt::format("{} does not match {}", Config::TypeUrl::get().Cluster, - Config::TypeUrl::get().ClusterLoadAssignment))); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "", + {"cluster_0"}, {}, {}, true, + Grpc::Status::GrpcStatus::Internal, + fmt::format("does not match the message-wide type URL {}", + Config::TypeUrl::get().ClusterLoadAssignment))); sendDiscoveryResponse( Config::TypeUrl::get().ClusterLoadAssignment, {buildClusterLoadAssignment("cluster_0")}, {buildClusterLoadAssignment("cluster_0")}, {}, "1"); @@ -85,9 +84,8 @@ TEST_P(AdsIntegrationTest, Failure) { {buildRouteConfig("listener_0", "route_config_0")}, {}, "1"); EXPECT_TRUE(compareDiscoveryRequest( - Config::TypeUrl::get().Listener, "", {}, {}, {}, false, Grpc::Status::GrpcStatus::Internal, - fmt::format("{} does not match {}", Config::TypeUrl::get().RouteConfiguration, - Config::TypeUrl::get().Listener))); + Config::TypeUrl::get().Listener, "", {}, {}, {}, true, Grpc::Status::GrpcStatus::Internal, + fmt::format("does not match the message-wide type URL {}", Config::TypeUrl::get().Listener))); sendDiscoveryResponse( Config::TypeUrl::get().Listener, {buildListener("listener_0", "route_config_0")}, {buildListener("listener_0", "route_config_0")}, {}, "1"); @@ -99,11 +97,11 @@ TEST_P(AdsIntegrationTest, Failure) { {buildListener("route_config_0", "cluster_0")}, {}, "1"); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "1", {}, {}, {})); - EXPECT_TRUE( - compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "", {"route_config_0"}, {}, - {}, false, Grpc::Status::GrpcStatus::Internal, - fmt::format("{} does not match {}", Config::TypeUrl::get().Listener, - Config::TypeUrl::get().RouteConfiguration))); + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().RouteConfiguration, "", + {"route_config_0"}, {}, {}, true, + Grpc::Status::GrpcStatus::Internal, + fmt::format("does not match the message-wide type URL {}", + Config::TypeUrl::get().RouteConfiguration))); sendDiscoveryResponse( Config::TypeUrl::get().RouteConfiguration, {buildRouteConfig("route_config_0", "cluster_0")}, {buildRouteConfig("route_config_0", "cluster_0")}, {}, "1"); @@ -211,7 +209,7 @@ TEST_P(AdsIntegrationTest, RedisClusterRemoval) { {buildRedisCluster("redis_cluster")}, {}, "1"); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "", - {"redis_cluster"}, {}, {})); + {"redis_cluster"}, {"redis_cluster"}, {})); sendDiscoveryResponse( Config::TypeUrl::get().ClusterLoadAssignment, {buildClusterLoadAssignment("redis_cluster")}, {buildClusterLoadAssignment("redis_cluster")}, {}, "1"); @@ -364,6 +362,12 @@ TEST_P(AdsIntegrationTest, CdsPausedDuringWarming) { test_server_->server().clusterManager().adsMux()->paused(Config::TypeUrl::get().Cluster)); // CDS is resumed and EDS response was acknowledged. + if (sotw_or_delta_ == Grpc::SotwOrDelta::Delta) { + // Envoy will ACK both Cluster messages. Since they arrived while CDS was paused, they aren't + // sent until CDS is unpaused. Since version 3 has already arrived by the time the version 2 + // ACK goes out, they're both acknowledging version 3. + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "3", {}, {}, {})); + } EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "3", {}, {}, {})); EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "2", {"warming_cluster_2", "warming_cluster_1"}, {}, {})); @@ -763,7 +767,8 @@ TEST_P(AdsIntegrationTest, ListenerDrainBeforeServerStart) { // Remove listener. EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Listener, "1", {}, {}, {})); - sendDiscoveryResponse(Config::TypeUrl::get().Listener, {}, {}, {}, "1"); + sendDiscoveryResponse(Config::TypeUrl::get().Listener, {}, {}, + {"listener_0"}, "2"); test_server_->waitForGaugeEq("listener_manager.total_listeners_active", 0); } diff --git a/test/integration/integration.h b/test/integration/integration.h index f4768b73feb93..7cd2109cc8593 100644 --- a/test/integration/integration.h +++ b/test/integration/integration.h @@ -220,12 +220,14 @@ class BaseIntegrationTest : Logger::Loggable { // sending/receiving to/from the (imaginary) xDS server. You should almost always use // compareDiscoveryRequest() and sendDiscoveryResponse(), but the SotW/delta-specific versions are // available if you're writing a SotW/delta-specific test. + // TODO(fredlas) expect_node was defaulting false here; the delta+SotW unification work restores + // it. AssertionResult compareDiscoveryRequest(const std::string& expected_type_url, const std::string& expected_version, const std::vector& expected_resource_names, const std::vector& expected_resource_names_added, const std::vector& expected_resource_names_removed, - bool expect_node = false, + bool expect_node = true, const Protobuf::int32 expected_error_code = Grpc::Status::GrpcStatus::Ok, const std::string& expected_error_message = ""); template @@ -257,9 +259,11 @@ class BaseIntegrationTest : Logger::Loggable { const Protobuf::int32 expected_error_code = Grpc::Status::GrpcStatus::Ok, const std::string& expected_error_message = ""); + // TODO(fredlas) expect_node was defaulting false here; the delta+SotW unification work restores + // it. AssertionResult compareSotwDiscoveryRequest( const std::string& expected_type_url, const std::string& expected_version, - const std::vector& expected_resource_names, bool expect_node = false, + const std::vector& expected_resource_names, bool expect_node = true, const Protobuf::int32 expected_error_code = Grpc::Status::GrpcStatus::Ok, const std::string& expected_error_message = ""); From dc2fc7c270a5cda3a688b5f66492f1041c691a6e Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 19 Sep 2019 17:47:42 -0400 Subject: [PATCH 52/56] the pedantic spellchecker has decided that unpaused isnt a word Signed-off-by: Fred Douglas --- tools/spelling_dictionary.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/spelling_dictionary.txt b/tools/spelling_dictionary.txt index 7b925d7b16be3..393c3b88d2854 100644 --- a/tools/spelling_dictionary.txt +++ b/tools/spelling_dictionary.txt @@ -817,6 +817,7 @@ unordered unowned unparented unpause +unpaused unpopulated unprotect unref From a12d914f3396f1e1dfb81c328c1fc3ecbf090b37 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Thu, 19 Sep 2019 18:21:06 -0400 Subject: [PATCH 53/56] disable more instances of set_node_on_first_message_only Signed-off-by: Fred Douglas --- test/config/utility.cc | 4 +++- test/integration/rtds_integration_test.cc | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/test/config/utility.cc b/test/config/utility.cc index 7a6b3885fd81f..e93bb5c333ddb 100644 --- a/test/config/utility.cc +++ b/test/config/utility.cc @@ -155,6 +155,8 @@ name: envoy.squash nanos: 0 )EOF"; +// TODO(fredlas) set_node_on_first_message_only was true; the delta+SotW unification +// work restores it here. // TODO(#6327) cleaner approach to testing with static config. std::string ConfigHelper::discoveredClustersBootstrap(const std::string& api_type) { return fmt::format( @@ -172,7 +174,7 @@ std::string ConfigHelper::discoveredClustersBootstrap(const std::string& api_typ grpc_services: envoy_grpc: cluster_name: my_cds_cluster - set_node_on_first_message_only: true + set_node_on_first_message_only: false static_resources: clusters: - name: my_cds_cluster diff --git a/test/integration/rtds_integration_test.cc b/test/integration/rtds_integration_test.cc index 3ffcc5cf54a5c..6f223f1a626dc 100644 --- a/test/integration/rtds_integration_test.cc +++ b/test/integration/rtds_integration_test.cc @@ -6,6 +6,8 @@ namespace Envoy { namespace { +// TODO(fredlas) set_node_on_first_message_only was true; the delta+SotW unification +// work restores it here. std::string tdsBootstrapConfig(absl::string_view api_type) { return fmt::format(R"EOF( static_resources: @@ -36,7 +38,7 @@ std::string tdsBootstrapConfig(absl::string_view api_type) { grpc_services: envoy_grpc: cluster_name: rtds_cluster - set_node_on_first_message_only: true + set_node_on_first_message_only: false - name: some_admin_layer admin_layer: {{}} admin: From dcf287cd00c800f2ccf74fa51dc7ad35e479fa8e Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Fri, 27 Sep 2019 18:28:59 -0400 Subject: [PATCH 54/56] rearrange update_attempt inc, assert on nonexistent sub, add nonexistent sub test Signed-off-by: Fred Douglas --- .../common/config/delta_subscription_impl.cc | 11 +- source/common/config/new_grpc_mux_impl.cc | 46 +++---- test/common/config/BUILD | 24 ++++ test/common/config/new_grpc_mux_impl_test.cc | 115 ++++++++++++++++++ 4 files changed, 165 insertions(+), 31 deletions(-) create mode 100644 test/common/config/new_grpc_mux_impl_test.cc diff --git a/source/common/config/delta_subscription_impl.cc b/source/common/config/delta_subscription_impl.cc index 651cf3aa475bb..ed34dac39bea0 100644 --- a/source/common/config/delta_subscription_impl.cc +++ b/source/common/config/delta_subscription_impl.cc @@ -43,9 +43,9 @@ void DeltaSubscriptionImpl::updateResourceInterest( void DeltaSubscriptionImpl::onConfigUpdate( const Protobuf::RepeatedPtrField& resources, const std::string& version_info) { + stats_.update_attempt_.inc(); callbacks_.onConfigUpdate(resources, version_info); stats_.update_success_.inc(); - stats_.update_attempt_.inc(); stats_.version_.set(HashUtil::xxHash64(version_info)); } @@ -53,9 +53,9 @@ void DeltaSubscriptionImpl::onConfigUpdate( const Protobuf::RepeatedPtrField& added_resources, const Protobuf::RepeatedPtrField& removed_resources, const std::string& system_version_info) { + stats_.update_attempt_.inc(); callbacks_.onConfigUpdate(added_resources, removed_resources, system_version_info); stats_.update_success_.inc(); - stats_.update_attempt_.inc(); stats_.version_.set(HashUtil::xxHash64(system_version_info)); } @@ -67,9 +67,15 @@ void DeltaSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reaso // So, don't onConfigUpdateFailed() here. Instead, allow a retry of the gRPC stream. // If init_fetch_timeout_ is non-zero, the server will continue startup after that timeout. stats_.update_failure_.inc(); + // TODO(fredlas) remove; it only makes sense to count start() and updateResourceInterest() + // as attempts. + stats_.update_attempt_.inc(); break; case Envoy::Config::ConfigUpdateFailureReason::FetchTimedout: stats_.init_fetch_timeout_.inc(); + // TODO(fredlas) remove; it only makes sense to count start() and updateResourceInterest() + // as attempts. + stats_.update_attempt_.inc(); callbacks_.onConfigUpdateFailed(reason, e); break; case Envoy::Config::ConfigUpdateFailureReason::UpdateRejected: @@ -79,7 +85,6 @@ void DeltaSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reaso callbacks_.onConfigUpdateFailed(reason, e); break; } - stats_.update_attempt_.inc(); } std::string DeltaSubscriptionImpl::resourceName(const ProtobufWkt::Any& resource) { diff --git a/source/common/config/new_grpc_mux_impl.cc b/source/common/config/new_grpc_mux_impl.cc index ed550bb960dc1..d22593d0bf997 100644 --- a/source/common/config/new_grpc_mux_impl.cc +++ b/source/common/config/new_grpc_mux_impl.cc @@ -35,10 +35,8 @@ Watch* NewGrpcMuxImpl::addOrUpdateWatch(const std::string& type_url, Watch* watc void NewGrpcMuxImpl::removeWatch(const std::string& type_url, Watch* watch) { updateWatch(type_url, watch, {}); auto entry = subscriptions_.find(type_url); - if (entry == subscriptions_.end()) { - ENVOY_LOG(error, "removeWatch() called for non-existent subscription {}.", type_url); - return; - } + RELEASE_ASSERT(entry != subscriptions_.end(), + fmt::format("removeWatch() called for non-existent subscription {}.", type_url)); entry->second->watch_map_.removeWatch(watch); } @@ -102,11 +100,12 @@ void NewGrpcMuxImpl::kickOffAck(UpdateAck ack) { trySendDiscoveryRequests(); } -// TODO TODO but yeah this should just be gone!!!!!!!!! +// TODO(fredlas) to be removed from the GrpcMux interface very soon. GrpcMuxWatchPtr NewGrpcMuxImpl::subscribe(const std::string&, const std::set&, GrpcMuxCallbacks&) { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + void NewGrpcMuxImpl::start() { grpc_stream_.establishNewStream(); } Watch* NewGrpcMuxImpl::addWatch(const std::string& type_url, const std::set& resources, @@ -132,13 +131,11 @@ void NewGrpcMuxImpl::updateWatch(const std::string& type_url, Watch* watch, const std::set& resources) { ASSERT(watch != nullptr); auto sub = subscriptions_.find(type_url); - if (sub == subscriptions_.end()) { - ENVOY_LOG(error, "Watch of {} has no subscription to update.", type_url); - return; - } + RELEASE_ASSERT(sub != subscriptions_.end(), + fmt::format("Watch of {} has no subscription to update.", type_url)); auto added_removed = sub->second->watch_map_.updateWatchInterest(watch, resources); sub->second->sub_state_.updateSubscriptionInterest(added_removed.added_, added_removed.removed_); - // Tell the server about our new interests, if there are any. + // Tell the server about our change in interest, if any. if (sub->second->sub_state_.subscriptionUpdatePending()) { trySendDiscoveryRequests(); } @@ -162,15 +159,10 @@ void NewGrpcMuxImpl::trySendDiscoveryRequests() { std::string next_request_type_url = maybe_request_type.value(); // If we don't have a subscription object for this request's type_url, drop the request. auto sub = subscriptions_.find(next_request_type_url); - if (sub == subscriptions_.end()) { - ENVOY_LOG(error, "Not sending discovery request for non-existent subscription {}.", - next_request_type_url); - // It's safe to assume the front of the ACK queue is of this type, because that's the only - // way whoWantsToSendDiscoveryRequest() could return something for a non-existent - // subscription. - pausable_ack_queue_.pop(); - continue; - } + RELEASE_ASSERT(sub != subscriptions_.end(), + fmt::format("Tried to send discovery request for non-existent subscription {}.", + next_request_type_url)); + // Try again later if paused/rate limited/stream down. if (!canSendDiscoveryRequest(next_request_type_url)) { break; @@ -192,15 +184,13 @@ void NewGrpcMuxImpl::trySendDiscoveryRequests() { // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check // whether we *want* to send a DeltaDiscoveryRequest). bool NewGrpcMuxImpl::canSendDiscoveryRequest(const std::string& type_url) { - if (pausable_ack_queue_.paused(type_url)) { - ASSERT(false, - fmt::format("canSendDiscoveryRequest() called on paused type_url {}. Pausedness is " - "supposed to be filtered out by whoWantsToSendDiscoveryRequest(). " - "Returning false, but your xDS might be about to get head-of-line blocked " - "- permanently, if the pause is never undone.", - type_url)); - return false; - } else if (!grpc_stream_.grpcStreamAvailable()) { + RELEASE_ASSERT( + !pausable_ack_queue_.paused(type_url), + fmt::format("canSendDiscoveryRequest() called on paused type_url {}. Pausedness is " + "supposed to be filtered out by whoWantsToSendDiscoveryRequest(). ", + type_url)); + + if (!grpc_stream_.grpcStreamAvailable()) { ENVOY_LOG(trace, "No stream available to send a discovery request for {}.", type_url); return false; } else if (!grpc_stream_.checkRateLimitAllowsDrain()) { diff --git a/test/common/config/BUILD b/test/common/config/BUILD index 60101fa4aa22b..ad8eff35f1543 100644 --- a/test/common/config/BUILD +++ b/test/common/config/BUILD @@ -95,6 +95,30 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "new_grpc_mux_impl_test", + srcs = ["new_grpc_mux_impl_test.cc"], + deps = [ + "//source/common/config:new_grpc_mux_lib", + "//source/common/config:protobuf_link_hacks", + "//source/common/config:resources_lib", + "//source/common/protobuf", + "//source/common/stats:isolated_store_lib", + "//test/mocks:common_lib", + "//test/mocks/config:config_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/test_common:logging_lib", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/api/v2:discovery_cc", + "@envoy_api//envoy/api/v2:eds_cc", + "@envoy_api//envoy/service/discovery/v2:ads_cc", + ], +) + envoy_cc_test( name = "grpc_stream_test", srcs = ["grpc_stream_test.cc"], diff --git a/test/common/config/new_grpc_mux_impl_test.cc b/test/common/config/new_grpc_mux_impl_test.cc new file mode 100644 index 0000000000000..3cb3ed7ae63ff --- /dev/null +++ b/test/common/config/new_grpc_mux_impl_test.cc @@ -0,0 +1,115 @@ +#include + +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/api/v2/eds.pb.h" + +#include "common/common/empty_string.h" +#include "common/config/new_grpc_mux_impl.h" +#include "common/config/protobuf_link_hacks.h" +#include "common/config/resources.h" +#include "common/config/utility.h" +#include "common/protobuf/protobuf.h" +#include "common/stats/isolated_store_impl.h" + +#include "test/mocks/common.h" +#include "test/mocks/config/mocks.h" +#include "test/mocks/event/mocks.h" +#include "test/mocks/grpc/mocks.h" +#include "test/mocks/local_info/mocks.h" +#include "test/mocks/runtime/mocks.h" +#include "test/test_common/logging.h" +#include "test/test_common/simulated_time_system.h" +#include "test/test_common/test_time.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::_; +using testing::AtLeast; +using testing::InSequence; +using testing::Invoke; +using testing::IsSubstring; +using testing::NiceMock; +using testing::Return; +using testing::ReturnRef; + +namespace Envoy { +namespace Config { +namespace { + +// We test some mux specific stuff below, other unit test coverage for singleton use of +// NewGrpcMuxImpl is provided in [grpc_]subscription_impl_test.cc. +class NewGrpcMuxImplTestBase : public testing::Test { +public: + NewGrpcMuxImplTestBase() + : async_client_(new Grpc::MockAsyncClient()), + control_plane_connected_state_( + stats_.gauge("control_plane.connected_state", Stats::Gauge::ImportMode::NeverImport)) {} + + void setup() { + grpc_mux_ = std::make_unique( + std::unique_ptr(async_client_), dispatcher_, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), + random_, stats_, rate_limit_settings_, local_info_); + } + + NiceMock dispatcher_; + NiceMock random_; + Grpc::MockAsyncClient* async_client_; + NiceMock async_stream_; + std::unique_ptr grpc_mux_; + NiceMock> callbacks_; + NiceMock local_info_; + Stats::IsolatedStoreImpl stats_; + Envoy::Config::RateLimitSettings rate_limit_settings_; + Stats::Gauge& control_plane_connected_state_; +}; + +class NewGrpcMuxImplTest : public NewGrpcMuxImplTestBase { +public: + Event::SimulatedTimeSystem time_system_; +}; + +// Test that we simply ignore a message for an unknown type_url, with no ill effects. +TEST_F(NewGrpcMuxImplTest, DiscoveryResponseNonexistentSub) { + setup(); + + const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; + grpc_mux_->addOrUpdateWatch(type_url, nullptr, {}, callbacks_, std::chrono::milliseconds(0)); + + EXPECT_CALL(*async_client_, startRaw(_, _, _)).WillOnce(Return(&async_stream_)); + grpc_mux_->start(); + + { + auto unexpected_response = std::make_unique(); + unexpected_response->set_type_url(type_url); + unexpected_response->set_system_version_info("0"); + EXPECT_CALL(callbacks_, onConfigUpdate(_, _, "0")).Times(0); + grpc_mux_->onDiscoveryResponse(std::move(unexpected_response)); + } + { + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_system_version_info("1"); + envoy::api::v2::ClusterLoadAssignment load_assignment; + load_assignment.set_cluster_name("x"); + response->add_resources()->mutable_resource()->PackFrom(load_assignment); + EXPECT_CALL(callbacks_, onConfigUpdate(_, _, "1")) + .WillOnce( + Invoke([&load_assignment]( + const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField&, const std::string&) { + EXPECT_EQ(1, added_resources.size()); + envoy::api::v2::ClusterLoadAssignment expected_assignment; + added_resources[0].resource().UnpackTo(&expected_assignment); + EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); + })); + grpc_mux_->onDiscoveryResponse(std::move(response)); + } +} + +} // namespace +} // namespace Config +} // namespace Envoy From 590e800009526f41fa0a0e95d9bbc86052d1f272 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Mon, 30 Sep 2019 14:54:01 -0400 Subject: [PATCH 55/56] a couple of clang_tidy things Signed-off-by: Fred Douglas --- source/common/config/delta_subscription_state.cc | 4 ++-- source/common/config/delta_subscription_state.h | 2 +- test/common/config/new_grpc_mux_impl_test.cc | 4 ---- 3 files changed, 3 insertions(+), 7 deletions(-) diff --git a/source/common/config/delta_subscription_state.cc b/source/common/config/delta_subscription_state.cc index ed7f11fbdfced..93ddc18164dae 100644 --- a/source/common/config/delta_subscription_state.cc +++ b/source/common/config/delta_subscription_state.cc @@ -6,12 +6,12 @@ namespace Envoy { namespace Config { -DeltaSubscriptionState::DeltaSubscriptionState(const std::string& type_url, +DeltaSubscriptionState::DeltaSubscriptionState(std::string type_url, SubscriptionCallbacks& callbacks, const LocalInfo::LocalInfo& local_info, std::chrono::milliseconds init_fetch_timeout, Event::Dispatcher& dispatcher) - : type_url_(type_url), callbacks_(callbacks), local_info_(local_info), + : type_url_(std::move(type_url)), callbacks_(callbacks), local_info_(local_info), init_fetch_timeout_(init_fetch_timeout) { if (init_fetch_timeout_.count() > 0 && !init_fetch_timeout_timer_) { init_fetch_timeout_timer_ = dispatcher.createTimer([this]() -> void { diff --git a/source/common/config/delta_subscription_state.h b/source/common/config/delta_subscription_state.h index b1a6e58758822..6442d4b7a84ff 100644 --- a/source/common/config/delta_subscription_state.h +++ b/source/common/config/delta_subscription_state.h @@ -19,7 +19,7 @@ namespace Config { // being multiplexed together by ADS. class DeltaSubscriptionState : public Logger::Loggable { public: - DeltaSubscriptionState(const std::string& type_url, SubscriptionCallbacks& callbacks, + DeltaSubscriptionState(std::string type_url, SubscriptionCallbacks& callbacks, const LocalInfo::LocalInfo& local_info, std::chrono::milliseconds init_fetch_timeout, Event::Dispatcher& dispatcher); diff --git a/test/common/config/new_grpc_mux_impl_test.cc b/test/common/config/new_grpc_mux_impl_test.cc index 3cb3ed7ae63ff..98150c9b7915a 100644 --- a/test/common/config/new_grpc_mux_impl_test.cc +++ b/test/common/config/new_grpc_mux_impl_test.cc @@ -26,13 +26,9 @@ #include "gtest/gtest.h" using testing::_; -using testing::AtLeast; -using testing::InSequence; using testing::Invoke; -using testing::IsSubstring; using testing::NiceMock; using testing::Return; -using testing::ReturnRef; namespace Envoy { namespace Config { From 6be011ea3fc69b9b214a6624fdcac0c14f434871 Mon Sep 17 00:00:00 2001 From: Fred Douglas Date: Tue, 1 Oct 2019 12:47:37 -0400 Subject: [PATCH 56/56] correct and simplify pausable_ack_queue RELEASE_ASSERTs Signed-off-by: Fred Douglas --- source/common/config/pausable_ack_queue.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/common/config/pausable_ack_queue.cc b/source/common/config/pausable_ack_queue.cc index 34951e38b15c0..0217717de0355 100644 --- a/source/common/config/pausable_ack_queue.cc +++ b/source/common/config/pausable_ack_queue.cc @@ -26,7 +26,7 @@ const UpdateAck& PausableAckQueue::front() { return entry; } } - RELEASE_ASSERT(!storage_.empty(), "front() on an empty queue is undefined behavior!"); + RELEASE_ASSERT(false, "front() on an empty queue is undefined behavior!"); NOT_REACHED_GCOVR_EXCL_LINE; } @@ -37,7 +37,7 @@ void PausableAckQueue::pop() { return; } } - RELEASE_ASSERT(!storage_.empty(), "pop() on an empty queue is undefined behavior!"); + RELEASE_ASSERT(false, "pop() on an empty queue is undefined behavior!"); NOT_REACHED_GCOVR_EXCL_LINE; }