diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst index e7ac57c6b6d1c..c72e20c05389c 100644 --- a/docs/root/version_history/current.rst +++ b/docs/root/version_history/current.rst @@ -152,6 +152,8 @@ New Features :ref:`max_upstream_rx_datagram_size ` UDP proxy configuration to allow configuration of upstream max UDP datagram size. The defaults for both remain 1500 bytes. +* xds: re-introduced unification of delta and sotw xDS multiplexers, based on work in https://github.com/envoyproxy/envoy/pull/8974. Added a new runtime config `envoy.reloadable_features.unified_mux` that when set to true, switches xDS to use the unified multiplexer implementation shared by both regular- and delta-xDS. + Deprecated ---------- diff --git a/include/envoy/config/grpc_mux.h b/include/envoy/config/grpc_mux.h index fc1c603b74ac7..6f2992eec4653 100644 --- a/include/envoy/config/grpc_mux.h +++ b/include/envoy/config/grpc_mux.h @@ -31,6 +31,8 @@ struct ControlPlaneStats { GENERATE_TEXT_READOUT_STRUCT) }; +struct Watch; + /** * Handle on a muxed gRPC subscription. The subscription is canceled on destruction. */ @@ -110,6 +112,59 @@ class GrpcMux { using TypeUrlMap = absl::flat_hash_map; static TypeUrlMap& typeUrlMap() { MUTABLE_CONSTRUCT_ON_FIRST_USE(TypeUrlMap, {}); } + + // Unified mux interface starts here + /** + * Start a configuration subscription asynchronously for some API type and resources. + * @param type_url type URL corresponding to xDS API, e.g. + * type.googleapis.com/envoy.api.v2.Cluster. + * @param resources set of resource names to watch for. If this is empty, then all + * resources for type_url will result in callbacks. + * @param callbacks the callbacks to be notified of configuration updates. These must be valid + * until GrpcMuxWatch is destroyed. + * @param resource_decoder how incoming opaque resource objects are to be decoded. + * @param use_namespace_matching if namespace watch should be created. This is used for creating + * watches on collections of resources; individual members of a collection are identified by the + * namespace in resource name. + * @return Watch* an opaque watch token added or updated, to be used in future addOrUpdateWatch + * calls. + */ + virtual Watch* addWatch(const std::string& type_url, + const absl::flat_hash_set& resources, + SubscriptionCallbacks& callbacks, OpaqueResourceDecoder& resource_decoder, + std::chrono::milliseconds init_fetch_timeout, + const bool use_namespace_matching) PURE; + + // Updates the list of resource names watched by the given watch. If an added name is new across + // the whole subscription, or if a removed name has no other watch interested in it, then the + // subscription will enqueue and attempt to send an appropriate discovery request. + virtual void updateWatch(const std::string& type_url, Watch* watch, + const absl::flat_hash_set& resources, + const bool creating_namespace_watch) PURE; + + /** + * Cleanup of a Watch* added by addOrUpdateWatch(). Receiving a Watch* from addOrUpdateWatch() + * makes you responsible for eventually invoking this cleanup. + * @param type_url type URL corresponding to xDS API e.g. type.googleapis.com/envoy.api.v2.Cluster + * @param watch the watch to be cleaned up. + */ + virtual void removeWatch(const std::string& type_url, Watch* watch) PURE; + + /** + * Retrieves the current pause state as set by pause()/resume(). + * @param type_url type URL corresponding to xDS API, e.g. + * type.googleapis.com/envoy.api.v2.Cluster + * @return bool whether the API is paused. + */ + virtual bool paused(const std::string& type_url) const PURE; + + /** + * Passes through to all multiplexed SubscriptionStates. To be called when something + * definitive happens with the initial fetch: either an update is successfully received, + * or some sort of error happened.*/ + virtual void disableInitFetchTimeoutTimer() PURE; + + virtual bool isUnified() const { return false; } }; using GrpcMuxPtr = std::unique_ptr; diff --git a/source/common/config/BUILD b/source/common/config/BUILD index de123c9f86c5b..3ba55c291e9bb 100644 --- a/source/common/config/BUILD +++ b/source/common/config/BUILD @@ -340,6 +340,7 @@ envoy_cc_library( "//include/envoy/config:subscription_interface", "//include/envoy/upstream:cluster_manager_interface", "//source/common/common:minimal_logger_lib", + "//source/common/config/unified_mux:grpc_subscription_lib", "//source/common/http:utility_lib", "//source/common/protobuf", "@envoy_api//envoy/config/core/v3:pkg_cc_proto", diff --git a/source/common/config/grpc_mux_impl.h b/source/common/config/grpc_mux_impl.h index d0c61792f562c..0ffb6f05762d7 100644 --- a/source/common/config/grpc_mux_impl.h +++ b/source/common/config/grpc_mux_impl.h @@ -74,6 +74,24 @@ class GrpcMuxImpl : public GrpcMux, return grpc_stream_; } + // unified GrpcMux interface, not implemented by legacy multiplexers + Watch* addWatch(const std::string&, const absl::flat_hash_set&, + SubscriptionCallbacks&, OpaqueResourceDecoder&, std::chrono::milliseconds, + const bool) override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } + + void updateWatch(const std::string&, Watch*, const absl::flat_hash_set&, + const bool) override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } + + void removeWatch(const std::string&, Watch*) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + + bool paused(const std::string&) const override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + + void disableInitFetchTimeoutTimer() override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + private: void drainRequests(); void setRetryTimer(); @@ -206,6 +224,24 @@ class NullGrpcMuxImpl : public GrpcMux, void onEstablishmentFailure() override {} void onDiscoveryResponse(std::unique_ptr&&, ControlPlaneStats&) override {} + + // unified GrpcMux interface, not implemented by legacy multiplexers + Watch* addWatch(const std::string&, const absl::flat_hash_set&, + SubscriptionCallbacks&, OpaqueResourceDecoder&, std::chrono::milliseconds, + const bool) override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } + + void updateWatch(const std::string&, Watch*, const absl::flat_hash_set&, + const bool) override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } + + void removeWatch(const std::string&, Watch*) override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + + bool paused(const std::string&) const override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + + void disableInitFetchTimeoutTimer() override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } }; } // namespace Config diff --git a/source/common/config/new_grpc_mux_impl.h b/source/common/config/new_grpc_mux_impl.h index 4e9d1ac869293..694c4c24117e1 100644 --- a/source/common/config/new_grpc_mux_impl.h +++ b/source/common/config/new_grpc_mux_impl.h @@ -87,6 +87,17 @@ class NewGrpcMuxImpl return subscriptions_; } + // unified GrpcMux interface, not implemented by legacy multiplexers + Watch* addWatch(const std::string&, const absl::flat_hash_set&, + SubscriptionCallbacks&, OpaqueResourceDecoder&, std::chrono::milliseconds, + const bool) override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } + + bool paused(const std::string&) const override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + + void disableInitFetchTimeoutTimer() override { NOT_IMPLEMENTED_GCOVR_EXCL_LINE; } + private: class WatchImpl : public GrpcMuxWatch { public: @@ -112,14 +123,14 @@ class NewGrpcMuxImpl NewGrpcMuxImpl& parent_; }; - void removeWatch(const std::string& type_url, Watch* watch); + void removeWatch(const std::string& type_url, Watch* watch) override; // Updates the list of resource names watched by the given watch. If an added name is new across // the whole subscription, or if a removed name has no other watch interested in it, then the // subscription will enqueue and attempt to send an appropriate discovery request. void updateWatch(const std::string& type_url, Watch* watch, const absl::flat_hash_set& resources, - bool creating_namespace_watch = false); + bool creating_namespace_watch = false) override; void addSubscription(const std::string& type_url, const bool use_namespace_matching); diff --git a/source/common/config/pausable_ack_queue.cc b/source/common/config/pausable_ack_queue.cc index dc6f01773f6a1..ed2138314922d 100644 --- a/source/common/config/pausable_ack_queue.cc +++ b/source/common/config/pausable_ack_queue.cc @@ -20,6 +20,8 @@ bool PausableAckQueue::empty() { return true; } +void PausableAckQueue::clear() { storage_.clear(); } + const UpdateAck& PausableAckQueue::front() { for (const auto& entry : storage_) { if (pauses_[entry.type_url_] == 0) { diff --git a/source/common/config/pausable_ack_queue.h b/source/common/config/pausable_ack_queue.h index 5535e262598f5..eb04358398df7 100644 --- a/source/common/config/pausable_ack_queue.h +++ b/source/common/config/pausable_ack_queue.h @@ -23,6 +23,7 @@ class PausableAckQueue { void pause(const std::string& type_url); void resume(const std::string& type_url); bool paused(const std::string& type_url) const; + void clear(); private: // It's ok for non-existent subs to be paused/resumed. The cleanest way to support that is to give diff --git a/source/common/config/subscription_factory_impl.cc b/source/common/config/subscription_factory_impl.cc index 4e125e115a2e0..884a8ad34b0b7 100644 --- a/source/common/config/subscription_factory_impl.cc +++ b/source/common/config/subscription_factory_impl.cc @@ -8,6 +8,8 @@ #include "common/config/http_subscription_impl.h" #include "common/config/new_grpc_mux_impl.h" #include "common/config/type_to_endpoint.h" +#include "common/config/unified_mux/grpc_mux_impl.h" +#include "common/config/unified_mux/grpc_subscription_impl.h" #include "common/config/utility.h" #include "common/config/xds_resource.h" #include "common/http/utility.h" @@ -57,6 +59,19 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( resource_decoder, stats, Utility::configSourceInitialFetchTimeout(config), validation_visitor_); case envoy::config::core::v3::ApiConfigSource::GRPC: + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.unified_mux")) { + return std::make_unique( + std::make_shared( + Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), + api_config_source, scope, true) + ->create(), + dispatcher_, sotwGrpcMethod(type_url, transport_api_version), transport_api_version, + api_.randomGenerator(), scope, Utility::parseRateLimitSettings(api_config_source), + local_info_, api_config_source.set_node_on_first_message_only()), + type_url, callbacks, resource_decoder, stats, dispatcher_.timeSource(), + Utility::configSourceInitialFetchTimeout(config), + /*is_aggregated*/ false, use_namespace_matching); + } return std::make_unique( std::make_shared( local_info_, @@ -70,6 +85,20 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( Utility::configSourceInitialFetchTimeout(config), /*is_aggregated*/ false, use_namespace_matching); case envoy::config::core::v3::ApiConfigSource::DELTA_GRPC: { + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.unified_mux")) { + return std::make_unique( + std::make_shared( + Config::Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), + api_config_source, scope, true) + ->create(), + dispatcher_, deltaGrpcMethod(type_url, transport_api_version), + transport_api_version, api_.randomGenerator(), scope, + Utility::parseRateLimitSettings(api_config_source), local_info_, + api_config_source.set_node_on_first_message_only()), + type_url, callbacks, resource_decoder, stats, dispatcher_.timeSource(), + Utility::configSourceInitialFetchTimeout(config), /*is_aggregated*/ false, + use_namespace_matching); + } return std::make_unique( std::make_shared( Config::Utility::factoryForGrpcApiConfigSource(cm_.grpcAsyncClientManager(), @@ -87,6 +116,11 @@ SubscriptionPtr SubscriptionFactoryImpl::subscriptionFromConfigSource( } } case envoy::config::core::v3::ConfigSource::ConfigSourceSpecifierCase::kAds: { + if (cm_.adsMux()->isUnified()) { + return std::make_unique( + cm_.adsMux(), type_url, callbacks, resource_decoder, stats, dispatcher_.timeSource(), + Utility::configSourceInitialFetchTimeout(config), true, use_namespace_matching); + } return std::make_unique( cm_.adsMux(), callbacks, resource_decoder, stats, type_url, dispatcher_, Utility::configSourceInitialFetchTimeout(config), true, use_namespace_matching); @@ -125,6 +159,11 @@ SubscriptionPtr SubscriptionFactoryImpl::collectionSubscriptionFromUrl( switch (api_config_source.api_type()) { case envoy::config::core::v3::ApiConfigSource::AGGREGATED_DELTA_GRPC: { + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.unified_mux")) { + return std::make_unique( + collection_locator, cm_.adsMux(), callbacks, resource_decoder, stats, + dispatcher_.timeSource(), Utility::configSourceInitialFetchTimeout(config), false); + } return std::make_unique( collection_locator, cm_.adsMux(), callbacks, resource_decoder, stats, dispatcher_, Utility::configSourceInitialFetchTimeout(config), false); diff --git a/source/common/config/unified_mux/BUILD b/source/common/config/unified_mux/BUILD new file mode 100644 index 0000000000000..d5f3f0c6cded7 --- /dev/null +++ b/source/common/config/unified_mux/BUILD @@ -0,0 +1,87 @@ +load( + "//bazel:envoy_build_system.bzl", + "envoy_cc_library", + "envoy_package", +) + +licenses(["notice"]) # Apache 2 + +envoy_package() + +envoy_cc_library( + name = "grpc_mux_lib", + srcs = ["grpc_mux_impl.cc"], + hdrs = ["grpc_mux_impl.h"], + deps = [ + ":delta_subscription_state_lib", + ":sotw_subscription_state_lib", + "//include/envoy/event:dispatcher_interface", + "//include/envoy/grpc:async_client_interface", + "//source/common/config:api_version_lib", + "//source/common/config:decoded_resource_lib", + "//source/common/config:grpc_stream_lib", + "//source/common/config:pausable_ack_queue_lib", + "//source/common/config:watch_map_lib", + "//source/common/config:xds_context_params_lib", + "//source/common/config:xds_resource_lib", + "//source/common/memory:utils_lib", + "@envoy_api//envoy/api/v2:pkg_cc_proto", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "grpc_subscription_lib", + srcs = ["grpc_subscription_impl.cc"], + hdrs = ["grpc_subscription_impl.h"], + deps = [ + ":grpc_mux_lib", + "//include/envoy/config:subscription_interface", + "//source/common/config:grpc_stream_lib", + "//source/common/config:utility_lib", + "//source/common/config:xds_resource_lib", + "//source/common/protobuf:utility_lib", + ], +) + +envoy_cc_library( + name = "delta_subscription_state_lib", + srcs = ["delta_subscription_state.cc"], + hdrs = ["delta_subscription_state.h"], + deps = [ + ":subscription_state_lib", + "//source/common/config:api_version_lib", + "//source/common/config:utility_lib", + "//source/common/grpc:common_lib", + "//source/common/protobuf", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "sotw_subscription_state_lib", + srcs = ["sotw_subscription_state.cc"], + hdrs = ["sotw_subscription_state.h"], + deps = [ + ":subscription_state_lib", + "//source/common/config:utility_lib", + "//source/common/grpc:common_lib", + "//source/common/protobuf", + "@envoy_api//envoy/api/v2:pkg_cc_proto", + ], +) + +envoy_cc_library( + name = "subscription_state_lib", + srcs = ["subscription_state.cc"], + hdrs = ["subscription_state.h"], + deps = [ + "//include/envoy/config:subscription_interface", + "//include/envoy/event:dispatcher_interface", + "//include/envoy/local_info:local_info_interface", + "//source/common/common:minimal_logger_lib", + "//source/common/config:ttl_lib", + "//source/common/config:update_ack_lib", + "@envoy_api//envoy/api/v2:pkg_cc_proto", + ], +) diff --git a/source/common/config/unified_mux/delta_subscription_state.cc b/source/common/config/unified_mux/delta_subscription_state.cc new file mode 100644 index 0000000000000..bd4cbb748d791 --- /dev/null +++ b/source/common/config/unified_mux/delta_subscription_state.cc @@ -0,0 +1,234 @@ +#include "common/config/unified_mux/delta_subscription_state.h" + +#include "envoy/event/dispatcher.h" +#include "envoy/service/discovery/v3/discovery.pb.h" + +#include "common/common/assert.h" +#include "common/common/hash.h" +#include "common/config/utility.h" +#include "common/runtime/runtime_features.h" + +namespace Envoy { +namespace Config { +namespace UnifiedMux { + +DeltaSubscriptionState::DeltaSubscriptionState(std::string type_url, + UntypedConfigUpdateCallbacks& watch_map, + std::chrono::milliseconds init_fetch_timeout, + Event::Dispatcher& dispatcher) + : SubscriptionState(std::move(type_url), watch_map, init_fetch_timeout, dispatcher) {} + +DeltaSubscriptionState::~DeltaSubscriptionState() = default; + +DeltaSubscriptionStateFactory::DeltaSubscriptionStateFactory(Event::Dispatcher& dispatcher) + : dispatcher_(dispatcher) {} + +DeltaSubscriptionStateFactory::~DeltaSubscriptionStateFactory() = default; + +std::unique_ptr +DeltaSubscriptionStateFactory::makeSubscriptionState(const std::string& type_url, + UntypedConfigUpdateCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { + return std::make_unique(type_url, callbacks, init_fetch_timeout, + dispatcher_); +} +void DeltaSubscriptionState::updateSubscriptionInterest( + const absl::flat_hash_set& cur_added, + const absl::flat_hash_set& cur_removed) { + for (const auto& a : cur_added) { + resource_state_[a] = ResourceState::waitingForServer(); + // If interest in a resource is removed-then-added (all before a discovery request + // can be sent), we must treat it as a "new" addition: our user may have forgotten its + // copy of the resource after instructing us to remove it, and need to be reminded of it. + names_removed_.erase(a); + names_added_.insert(a); + } + for (const auto& r : cur_removed) { + resource_state_.erase(r); + // Ideally, when interest in a resource is added-then-removed in between requests, + // we would avoid putting a superfluous "unsubscribe [resource that was never subscribed]" + // in the request. However, the removed-then-added case *does* need to go in the request, + // and due to how we accomplish that, it's difficult to distinguish remove-add-remove from + // add-remove (because "remove-add" has to be treated as equivalent to just "add"). + names_added_.erase(r); + names_removed_.insert(r); + } +} + +// Not having sent any requests yet counts as an "update pending" since you're supposed to resend +// the entirety of your interest at the start of a stream, even if nothing has changed. +bool DeltaSubscriptionState::subscriptionUpdatePending() const { + return !names_added_.empty() || !names_removed_.empty() || + !any_request_sent_yet_in_current_stream_ || dynamicContextChanged(); +} + +UpdateAck DeltaSubscriptionState::handleResponse(const void* response_proto_ptr) { + auto* response = + static_cast(response_proto_ptr); + // We *always* copy the response's nonce into the next request, even if we're going to make that + // request a NACK by setting error_detail. + UpdateAck ack(response->nonce(), type_url()); + try { + handleGoodResponse(*response); + } catch (const EnvoyException& e) { + handleBadResponse(e, ack); + } + return ack; +} + +bool DeltaSubscriptionState::isHeartbeatResource( + const envoy::service::discovery::v3::Resource& resource) const { + if (!supports_heartbeats_ && + !Runtime::runtimeFeatureEnabled("envoy.reloadable_features.vhds_heartbeats")) { + return false; + } + const auto itr = resource_state_.find(resource.name()); + if (itr == resource_state_.end()) { + return false; + } + + return !resource.has_resource() && !itr->second.isWaitingForServer() && + resource.version() == itr->second.version(); +} + +void DeltaSubscriptionState::handleGoodResponse( + const envoy::service::discovery::v3::DeltaDiscoveryResponse& message) { + absl::flat_hash_set names_added_removed; + Protobuf::RepeatedPtrField non_heartbeat_resources; + for (const auto& resource : message.resources()) { + if (!names_added_removed.insert(resource.name()).second) { + throw EnvoyException( + fmt::format("duplicate name {} found among added/updated resources", resource.name())); + } + if (isHeartbeatResource(resource)) { + continue; + } + non_heartbeat_resources.Add()->CopyFrom(resource); + // DeltaDiscoveryResponses for unresolved aliases don't contain an actual resource + if (!resource.has_resource() && resource.aliases_size() > 0) { + continue; + } + if (message.type_url() != resource.resource().type_url()) { + throw EnvoyException(fmt::format("type URL {} embedded in an individual Any does not match " + "the message-wide type URL {} in DeltaDiscoveryResponse {}", + resource.resource().type_url(), message.type_url(), + message.DebugString())); + } + } + for (const auto& name : message.removed_resources()) { + if (!names_added_removed.insert(name).second) { + throw EnvoyException( + fmt::format("duplicate name {} found in the union of added+removed resources", name)); + } + } + + { + const auto scoped_update = ttl_.scopedTtlUpdate(); + for (const auto& resource : message.resources()) { + addResourceState(resource); + } + } + + callbacks().onConfigUpdate(non_heartbeat_resources, message.removed_resources(), + message.system_version_info()); + + // If a resource is gone, there is no longer a meaningful version for it that makes sense to + // provide to the server upon stream reconnect: either it will continue to not exist, in which + // case saying nothing is fine, or the server will bring back something new, which we should + // receive regardless (which is the logic that not specifying a version will get you). + // + // So, leave the version map entry present but blank. It will be left out of + // initial_resource_versions messages, but will remind us to explicitly tell the server "I'm + // cancelling my subscription" when we lose interest. + for (const auto& resource_name : message.removed_resources()) { + if (resource_state_.find(resource_name) != resource_state_.end()) { + resource_state_[resource_name] = ResourceState::waitingForServer(); + } + } + ENVOY_LOG(debug, "Delta config for {} accepted with {} resources added, {} removed", type_url(), + message.resources().size(), message.removed_resources().size()); +} + +void DeltaSubscriptionState::handleBadResponse(const EnvoyException& e, UpdateAck& ack) { + // Note that error_detail being set is what indicates that a DeltaDiscoveryRequest is a NACK. + ack.error_detail_.set_code(Grpc::Status::WellKnownGrpcStatus::Internal); + ack.error_detail_.set_message(Config::Utility::truncateGrpcStatusMessage(e.what())); + ENVOY_LOG(warn, "delta config for {} rejected: {}", type_url(), e.what()); + callbacks().onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &e); +} + +void DeltaSubscriptionState::handleEstablishmentFailure() { + callbacks().onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, + nullptr); +} + +envoy::service::discovery::v3::DeltaDiscoveryRequest* +DeltaSubscriptionState::getNextRequestInternal() { + auto* request = new envoy::service::discovery::v3::DeltaDiscoveryRequest; + request->set_type_url(type_url()); + if (!any_request_sent_yet_in_current_stream_) { + any_request_sent_yet_in_current_stream_ = true; + // initial_resource_versions "must be populated for first request in a stream". + // Also, since this might be a new server, we must explicitly state *all* of our subscription + // interest. + for (auto const& [resource_name, resource_state] : resource_state_) { + // Populate initial_resource_versions with the resource versions we currently have. + // Resources we are interested in, but are still waiting to get any version of from the + // server, do not belong in initial_resource_versions. (But do belong in new subscriptions!) + if (!resource_state.isWaitingForServer()) { + (*request->mutable_initial_resource_versions())[resource_name] = resource_state.version(); + } + // As mentioned above, fill resource_names_subscribe with everything, including names we + // have yet to receive any resource for. + names_added_.insert(resource_name); + } + names_removed_.clear(); + } + + std::copy(names_added_.begin(), names_added_.end(), + Protobuf::RepeatedFieldBackInserter(request->mutable_resource_names_subscribe())); + std::copy(names_removed_.begin(), names_removed_.end(), + Protobuf::RepeatedFieldBackInserter(request->mutable_resource_names_unsubscribe())); + names_added_.clear(); + names_removed_.clear(); + + return request; +} + +void* DeltaSubscriptionState::getNextRequestAckless() { return getNextRequestInternal(); } + +void* DeltaSubscriptionState::getNextRequestWithAck(const UpdateAck& ack) { + envoy::service::discovery::v3::DeltaDiscoveryRequest* request = getNextRequestInternal(); + request->set_response_nonce(ack.nonce_); + ENVOY_LOG(debug, "ACK for {} will have nonce {}", type_url(), ack.nonce_); + if (ack.error_detail_.code() != Grpc::Status::WellKnownGrpcStatus::Ok) { + // Don't needlessly make the field present-but-empty if status is ok. + request->mutable_error_detail()->CopyFrom(ack.error_detail_); + } + return request; +} + +void DeltaSubscriptionState::addResourceState( + const envoy::service::discovery::v3::Resource& resource) { + if (resource.has_ttl()) { + ttl_.add(std::chrono::milliseconds(DurationUtil::durationToMilliseconds(resource.ttl())), + resource.name()); + } else { + ttl_.clear(resource.name()); + } + + resource_state_[resource.name()] = ResourceState(resource.version()); +} + +void DeltaSubscriptionState::ttlExpiryCallback(const std::vector& expired) { + Protobuf::RepeatedPtrField removed_resources; + for (const auto& resource : expired) { + resource_state_[resource] = ResourceState::waitingForServer(); + removed_resources.Add(std::string(resource)); + } + callbacks().onConfigUpdate({}, removed_resources, ""); +} + +} // namespace UnifiedMux +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/unified_mux/delta_subscription_state.h b/source/common/config/unified_mux/delta_subscription_state.h new file mode 100644 index 0000000000000..0d9a22ff47969 --- /dev/null +++ b/source/common/config/unified_mux/delta_subscription_state.h @@ -0,0 +1,111 @@ +#pragma once + +#include "envoy/grpc/status.h" + +#include "common/common/assert.h" +#include "common/common/logger.h" +#include "common/config/api_version.h" +#include "common/config/unified_mux/subscription_state.h" + +#include "absl/container/node_hash_map.h" +#include "absl/types/optional.h" + +namespace Envoy { +namespace Config { +namespace UnifiedMux { + +// Tracks the state of a delta xDS-over-gRPC protocol session. +class DeltaSubscriptionState : public SubscriptionState { +public: + DeltaSubscriptionState(std::string type_url, UntypedConfigUpdateCallbacks& watch_map, + std::chrono::milliseconds init_fetch_timeout, + Event::Dispatcher& dispatcher); + + ~DeltaSubscriptionState() override; + + // Update which resources we're interested in subscribing to. + void updateSubscriptionInterest(const absl::flat_hash_set& cur_added, + const absl::flat_hash_set& cur_removed) override; + + // Whether there was a change in our subscription interest we have yet to inform the server of. + bool subscriptionUpdatePending() const override; + + void markStreamFresh() override { any_request_sent_yet_in_current_stream_ = false; } + + UpdateAck handleResponse(const void* response_proto_ptr) override; + + void handleEstablishmentFailure() override; + + // Returns the next gRPC request proto to be sent off to the server, based on this object's + // understanding of the current protocol state, and new resources that Envoy wants to request. + // Returns a new'd pointer, meant to be owned by the caller. + void* getNextRequestAckless() override; + // The WithAck version first calls the Ack-less version, then adds in the passed-in ack. + // Returns a new'd pointer, meant to be owned by the caller. + void* getNextRequestWithAck(const UpdateAck& ack) override; + + void ttlExpiryCallback(const std::vector& expired) override; + + DeltaSubscriptionState(const DeltaSubscriptionState&) = delete; + DeltaSubscriptionState& operator=(const DeltaSubscriptionState&) = delete; + +private: + // Returns a new'd pointer, meant to be owned by the caller. + envoy::service::discovery::v3::DeltaDiscoveryRequest* getNextRequestInternal(); + bool isHeartbeatResource(const envoy::service::discovery::v3::Resource& resource) const; + void handleGoodResponse(const envoy::service::discovery::v3::DeltaDiscoveryResponse& message); + void handleBadResponse(const EnvoyException& e, UpdateAck& ack); + void addResourceState(const envoy::service::discovery::v3::Resource& resource); + + class ResourceState { + public: + explicit ResourceState(absl::string_view version) : version_(version) {} + // Builds a ResourceVersion in the waitingForServer state. + ResourceState() = default; + // Self-documenting alias of default constructor. + static ResourceState waitingForServer() { return ResourceState(); } + + // If true, we currently have no version of this resource - we are waiting for the server to + // provide us with one. + bool isWaitingForServer() const { return version_ == absl::nullopt; } + + // Must not be called if waitingForServer() == true. + std::string version() const { + ASSERT(version_.has_value()); + return version_.value_or(""); + } + + private: + absl::optional version_; + }; + + // A map from resource name to per-resource version. The keys of this map are exactly the resource + // names we are currently interested in. Those in the waitingForServer state currently don't have + // any version for that resource: we need to inform the server if we lose interest in them, but we + // also need to *not* include them in the initial_resource_versions map upon a reconnect. + absl::node_hash_map resource_state_; + + bool any_request_sent_yet_in_current_stream_{}; + + // Tracks changes in our subscription interest since the previous DeltaDiscoveryRequest we sent. + // TODO: Can't use absl::flat_hash_set due to ordering issues in gTest expectation matching. + // Feel free to change to an unordered container once we figure out how to make it work. + std::set names_added_; + std::set names_removed_; +}; + +class DeltaSubscriptionStateFactory : public SubscriptionStateFactory { +public: + DeltaSubscriptionStateFactory(Event::Dispatcher& dispatcher); + ~DeltaSubscriptionStateFactory() override; + std::unique_ptr + makeSubscriptionState(const std::string& type_url, UntypedConfigUpdateCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) override; + +private: + Event::Dispatcher& dispatcher_; +}; + +} // namespace UnifiedMux +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/unified_mux/grpc_mux_impl.cc b/source/common/config/unified_mux/grpc_mux_impl.cc new file mode 100644 index 0000000000000..95a925875aaa3 --- /dev/null +++ b/source/common/config/unified_mux/grpc_mux_impl.cc @@ -0,0 +1,432 @@ +#include "common/config/unified_mux/grpc_mux_impl.h" + +#include "envoy/service/discovery/v3/discovery.pb.h" + +#include "common/common/assert.h" +#include "common/common/backoff_strategy.h" +#include "common/config/decoded_resource_impl.h" +#include "common/config/utility.h" +#include "common/config/version_converter.h" +#include "common/config/xds_context_params.h" +#include "common/config/xds_resource.h" +#include "common/memory/utils.h" +#include "common/protobuf/protobuf.h" +#include "common/protobuf/utility.h" + +namespace Envoy { +namespace Config { +namespace UnifiedMux { + +GrpcMuxImpl::GrpcMuxImpl(std::unique_ptr subscription_state_factory, + bool skip_subsequent_node, const LocalInfo::LocalInfo& local_info, + envoy::config::core::v3::ApiVersion transport_api_version) + : subscription_state_factory_(std::move(subscription_state_factory)), + skip_subsequent_node_(skip_subsequent_node), local_info_(local_info), + dynamic_update_callback_handle_(local_info.contextProvider().addDynamicContextUpdateCallback( + [this](absl::string_view resource_type_url) { + onDynamicContextUpdate(resource_type_url); + })), + transport_api_version_(transport_api_version), + enable_type_url_downgrade_and_upgrade_(Runtime::runtimeFeatureEnabled( + "envoy.reloadable_features.enable_type_url_downgrade_and_upgrade")) { + Config::Utility::checkLocalInfo("ads", local_info); +} + +void GrpcMuxImpl::onDynamicContextUpdate(absl::string_view resource_type_url) { + ENVOY_LOG(debug, "GrpcMuxImpl::onDynamicContextUpdate for {}", resource_type_url); + auto sub = subscriptions_.find(resource_type_url); + if (sub == subscriptions_.end()) { + return; + } + sub->second->setDynamicContextChanged(); + trySendDiscoveryRequests(); +} + +Watch* GrpcMuxImpl::addWatch(const std::string& type_url, + const absl::flat_hash_set& resources, + SubscriptionCallbacks& callbacks, + OpaqueResourceDecoder& resource_decoder, + std::chrono::milliseconds init_fetch_timeout, + const bool use_namespace_matching) { + auto watch_map = watch_maps_.find(type_url); + if (watch_map == watch_maps_.end()) { + // We don't yet have a subscription for type_url! Make one! + if (enable_type_url_downgrade_and_upgrade_) { + registerVersionedTypeUrl(type_url); + } + watch_map = + watch_maps_.emplace(type_url, std::make_unique(use_namespace_matching)).first; + subscriptions_.emplace(type_url, subscription_state_factory_->makeSubscriptionState( + type_url, *watch_maps_[type_url], init_fetch_timeout)); + subscription_ordering_.emplace_back(type_url); + } + + Watch* watch = watch_map->second->addWatch(callbacks, resource_decoder); + // updateWatch() queues a discovery request if any of 'resources' are not yet subscribed. + updateWatch(type_url, watch, resources, use_namespace_matching); + return watch; +} + +// Updates the list of resource names watched by the given watch. If an added name is new across +// the whole subscription, or if a removed name has no other watch interested in it, then the +// subscription will enqueue and attempt to send an appropriate discovery request. +void GrpcMuxImpl::updateWatch(const std::string& type_url, Watch* watch, + const absl::flat_hash_set& resources, + const bool creating_namespace_watch) { + ENVOY_LOG(debug, "GrpcMuxImpl::updateWatch for {}", type_url); + ASSERT(watch != nullptr); + SubscriptionState& sub = subscriptionStateFor(type_url); + WatchMap& watch_map = watchMapFor(type_url); + + // If this is a glob collection subscription, we need to compute actual context parameters. + absl::flat_hash_set xdstp_resources; + // TODO(htuch): add support for resources beyond glob collections, the constraints below around + // resource size and ID reflect the progress of the xdstp:// implementation. + if (!resources.empty() && XdsResourceIdentifier::hasXdsTpScheme(*resources.begin())) { + // Callers must be asking for a single resource, the collection. + ASSERT(resources.size() == 1); + auto resource = XdsResourceIdentifier::decodeUrn(*resources.begin()); + // We only know how to deal with glob collections and static context parameters right now. + // TODO(htuch): add support for dynamic context params and list collections in the future. + if (absl::EndsWith(resource.id(), "/*")) { + auto encoded_context = XdsContextParams::encodeResource( + local_info_.contextProvider().nodeContext(), resource.context(), {}, {}); + resource.mutable_context()->CopyFrom(encoded_context); + XdsResourceIdentifier::EncodeOptions encode_options; + encode_options.sort_context_params_ = true; + xdstp_resources.insert(XdsResourceIdentifier::encodeUrn(resource, encode_options)); + } else { + // TODO(htuch): We will handle list collections here in future work. + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } + } + + auto added_removed = + watch_map.updateWatchInterest(watch, xdstp_resources.empty() ? resources : xdstp_resources); + if (creating_namespace_watch && xdstp_resources.empty()) { + // This is to prevent sending out of requests that contain prefixes instead of resource names + sub.updateSubscriptionInterest({}, {}); + } else { + sub.updateSubscriptionInterest(added_removed.added_, added_removed.removed_); + } + + // Tell the server about our change in interest, if any. + if (sub.subscriptionUpdatePending()) { + trySendDiscoveryRequests(); + } +} + +void GrpcMuxImpl::removeWatch(const std::string& type_url, Watch* watch) { + updateWatch(type_url, watch, {}); + watchMapFor(type_url).removeWatch(watch); +} + +ScopedResume GrpcMuxImpl::pause(const std::string& type_url) { + return pause(std::vector{type_url}); +} + +ScopedResume GrpcMuxImpl::pause(const std::vector type_urls) { + for (const auto& type_url : type_urls) { + pausable_ack_queue_.pause(type_url); + } + + return std::make_unique([this, type_urls]() { + for (const auto& type_url : type_urls) { + pausable_ack_queue_.resume(type_url); + trySendDiscoveryRequests(); + } + }); +} + +bool GrpcMuxImpl::paused(const std::string& type_url) const { + return pausable_ack_queue_.paused(type_url); +} + +void GrpcMuxImpl::registerVersionedTypeUrl(const std::string& type_url) { + TypeUrlMap& type_url_map = typeUrlMap(); + if (type_url_map.find(type_url) != type_url_map.end()) { + return; + } + // If type_url is v3, earlier_type_url will contain v2 type url. + absl::optional earlier_type_url = ApiTypeOracle::getEarlierTypeUrl(type_url); + // Register v2 to v3 and v3 to v2 type_url mapping in the hash map. + if (earlier_type_url.has_value()) { + type_url_map[earlier_type_url.value()] = type_url; + type_url_map[type_url] = earlier_type_url.value(); + } +} + +void GrpcMuxImpl::genericHandleResponse(const std::string& type_url, + const void* response_proto_ptr) { + auto sub = subscriptions_.find(type_url); + // If this type url is not watched, try another version type url. + if (enable_type_url_downgrade_and_upgrade_ && sub == subscriptions_.end()) { + registerVersionedTypeUrl(type_url); + TypeUrlMap& type_url_map = typeUrlMap(); + if (type_url_map.find(type_url) != type_url_map.end()) { + sub = subscriptions_.find(type_url_map[type_url]); + } + } + if (sub == subscriptions_.end()) { + ENVOY_LOG(warn, + "The server sent an xDS response proto with type_url {}, which we have " + "not subscribed to. Ignoring.", + type_url); + return; + } + pausable_ack_queue_.push(sub->second->handleResponse(response_proto_ptr)); + trySendDiscoveryRequests(); + Memory::Utils::tryShrinkHeap(); +} + +void GrpcMuxImpl::start() { + ENVOY_LOG(debug, "GrpcMuxImpl now trying to establish a stream"); + establishGrpcStream(); +} + +void GrpcMuxImpl::handleEstablishedStream() { + ENVOY_LOG(debug, "GrpcMuxImpl stream successfully established"); + for (auto& [type_url, subscription_state] : subscriptions_) { + subscription_state->markStreamFresh(); + } + set_any_request_sent_yet_in_current_stream(false); + maybeUpdateQueueSizeStat(0); + pausable_ack_queue_.clear(); + trySendDiscoveryRequests(); +} + +void GrpcMuxImpl::disableInitFetchTimeoutTimer() { + for (auto& [type_url, subscription_state] : subscriptions_) { + subscription_state->disableInitFetchTimeoutTimer(); + } +} + +void GrpcMuxImpl::handleStreamEstablishmentFailure() { + ENVOY_LOG(debug, "GrpcMuxImpl stream failed to establish"); + // If this happens while Envoy is still initializing, the onConfigUpdateFailed() we ultimately + // call on CDS will cause LDS to start up, which adds to subscriptions_ here. So, to avoid a + // crash, the iteration needs to dance around a little: collect pointers to all + // SubscriptionStates, call on all those pointers we haven't yet called on, repeat if there are + // now more SubscriptionStates. + absl::flat_hash_map all_subscribed; + absl::flat_hash_map already_called; + do { + for (auto& [type_url, subscription_state] : subscriptions_) { + all_subscribed[type_url] = subscription_state.get(); + } + for (auto& sub : all_subscribed) { + if (already_called.insert(sub).second) { // insert succeeded ==> not already called + sub.second->handleEstablishmentFailure(); + } + } + } while (all_subscribed.size() != subscriptions_.size()); +} + +SubscriptionState& GrpcMuxImpl::subscriptionStateFor(const std::string& type_url) { + auto sub = subscriptions_.find(type_url); + RELEASE_ASSERT(sub != subscriptions_.end(), + fmt::format("Tried to look up SubscriptionState for non-existent subscription {}.", + type_url)); + return *sub->second; +} + +WatchMap& GrpcMuxImpl::watchMapFor(const std::string& type_url) { + auto watch_map = watch_maps_.find(type_url); + RELEASE_ASSERT( + watch_map != watch_maps_.end(), + fmt::format("Tried to look up WatchMap for non-existent subscription {}.", type_url)); + return *watch_map->second; +} + +void GrpcMuxImpl::trySendDiscoveryRequests() { + while (true) { + // Do any of our subscriptions even want to send a request? + absl::optional request_type_if_any = whoWantsToSendDiscoveryRequest(); + if (!request_type_if_any.has_value()) { + break; + } + // If so, which one (by type_url)? + std::string next_request_type_url = request_type_if_any.value(); + SubscriptionState& sub = subscriptionStateFor(next_request_type_url); + ENVOY_LOG(debug, "GrpcMuxImpl wants to send discovery request for {}", next_request_type_url); + // Try again later if paused/rate limited/stream down. + if (!canSendDiscoveryRequest(next_request_type_url)) { + break; + } + void* request; + // Get our subscription state to generate the appropriate discovery request, and send. + if (!pausable_ack_queue_.empty()) { + // Because ACKs take precedence over plain requests, if there is anything in the queue, it's + // safe to assume it's of the type_url that we're wanting to send. + // + // getNextRequestWithAck() returns a raw unowned pointer, which sendGrpcMessage deletes. + request = sub.getNextRequestWithAck(pausable_ack_queue_.popFront()); + ENVOY_LOG(debug, "GrpcMuxImpl sent ACK discovery request for {}", next_request_type_url); + } else { + // Returns a raw unowned pointer, which sendGrpcMessage deletes. + request = sub.getNextRequestAckless(); + ENVOY_LOG(debug, "GrpcMuxImpl sent non-ACK discovery request for {}", next_request_type_url); + } + ENVOY_LOG(debug, "GrpcMuxImpl skip_subsequent_node: {}", skip_subsequent_node()); + sendGrpcMessage(request, sub); + } + maybeUpdateQueueSizeStat(pausable_ack_queue_.size()); +} + +// Checks whether external conditions allow sending a discovery request. (Does not check +// whether we *want* to send a discovery request). +bool GrpcMuxImpl::canSendDiscoveryRequest(const std::string& type_url) { + RELEASE_ASSERT( + !pausable_ack_queue_.paused(type_url), + fmt::format("canSendDiscoveryRequest() called on paused type_url {}. Pausedness is " + "supposed to be filtered out by whoWantsToSendDiscoveryRequest(). ", + type_url)); + + if (!grpcStreamAvailable()) { + ENVOY_LOG(trace, "No stream available to send a discovery request for {}.", type_url); + return false; + } else if (!rateLimitAllowsDrain()) { + ENVOY_LOG(trace, "{} discovery request hit rate limit; will try later.", type_url); + return false; + } + return true; +} + +// Checks whether we have something to say in a discovery request, which can be an ACK and/or +// a subscription update. (Does not check whether we *can* send that discovery request). +// Returns the type_url we should send the discovery request for (if any). +// First, prioritizes ACKs over non-ACK subscription interest updates. +// Then, prioritizes non-ACK updates in the order the various types +// of subscriptions were activated. +absl::optional GrpcMuxImpl::whoWantsToSendDiscoveryRequest() { + // All ACKs are sent before plain updates. trySendDiscoveryRequests() relies on this. So, choose + // type_url from pausable_ack_queue_ if possible, before looking at pending updates. + if (!pausable_ack_queue_.empty()) { + return pausable_ack_queue_.front().type_url_; + } + // If we're looking to send multiple non-ACK requests, send them in the order that their + // subscriptions were initiated. + for (const auto& sub_type : subscription_ordering_) { + SubscriptionState& sub = subscriptionStateFor(sub_type); + if (sub.subscriptionUpdatePending() && !pausable_ack_queue_.paused(sub_type)) { + return sub_type; + } + } + return absl::nullopt; +} + +// Delta- and SotW-specific concrete subclasses: +GrpcMuxDelta::GrpcMuxDelta(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, + envoy::config::core::v3::ApiVersion transport_api_version, + Random::RandomGenerator& random, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node) + : GrpcMuxImpl(std::make_unique(dispatcher), skip_subsequent_node, + local_info, transport_api_version), + grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, + rate_limit_settings) {} + +// GrpcStreamCallbacks for GrpcMuxDelta +void GrpcMuxDelta::onStreamEstablished() { handleEstablishedStream(); } +void GrpcMuxDelta::onEstablishmentFailure() { handleStreamEstablishmentFailure(); } +void GrpcMuxDelta::onWriteable() { trySendDiscoveryRequests(); } +void GrpcMuxDelta::onDiscoveryResponse( + std::unique_ptr&& message, + ControlPlaneStats&) { + genericHandleResponse(message->type_url(), message.get()); +} + +void GrpcMuxDelta::establishGrpcStream() { grpc_stream_.establishNewStream(); } +void GrpcMuxDelta::sendGrpcMessage(void* msg_proto_ptr, SubscriptionState& sub_state) { + std::unique_ptr typed_proto( + static_cast(msg_proto_ptr)); + if (sub_state.dynamicContextChanged() || !any_request_sent_yet_in_current_stream() || + !skip_subsequent_node()) { + typed_proto->mutable_node()->MergeFrom(local_info().node()); + } + VersionConverter::prepareMessageForGrpcWire(*typed_proto, transport_api_version()); + grpc_stream_.sendMessage(*typed_proto); + set_any_request_sent_yet_in_current_stream(true); + sub_state.clearDynamicContextChanged(); +} +void GrpcMuxDelta::maybeUpdateQueueSizeStat(uint64_t size) { + grpc_stream_.maybeUpdateQueueSizeStat(size); +} +bool GrpcMuxDelta::grpcStreamAvailable() const { return grpc_stream_.grpcStreamAvailable(); } +bool GrpcMuxDelta::rateLimitAllowsDrain() { return grpc_stream_.checkRateLimitAllowsDrain(); } + +void GrpcMuxDelta::requestOnDemandUpdate(const std::string& type_url, + const absl::flat_hash_set& for_update) { + SubscriptionState& sub = subscriptionStateFor(type_url); + sub.updateSubscriptionInterest(for_update, {}); + // Tell the server about our change in interest, if any. + if (sub.subscriptionUpdatePending()) { + trySendDiscoveryRequests(); + } +} + +GrpcMuxSotw::GrpcMuxSotw(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, + envoy::config::core::v3::ApiVersion transport_api_version, + Random::RandomGenerator& random, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, + const LocalInfo::LocalInfo& local_info, bool skip_subsequent_node) + : GrpcMuxImpl(std::make_unique(dispatcher), skip_subsequent_node, + local_info, transport_api_version), + grpc_stream_(this, std::move(async_client), service_method, random, dispatcher, scope, + rate_limit_settings) {} + +// GrpcStreamCallbacks for GrpcMuxSotw +void GrpcMuxSotw::onStreamEstablished() { handleEstablishedStream(); } +void GrpcMuxSotw::onEstablishmentFailure() { handleStreamEstablishmentFailure(); } +void GrpcMuxSotw::onWriteable() { trySendDiscoveryRequests(); } +void GrpcMuxSotw::onDiscoveryResponse( + std::unique_ptr&& message, + ControlPlaneStats& control_plane_stats) { + if (message->has_control_plane()) { + control_plane_stats.identifier_.set(message->control_plane().identifier()); + } + genericHandleResponse(message->type_url(), message.get()); +} + +void GrpcMuxSotw::establishGrpcStream() { grpc_stream_.establishNewStream(); } + +void GrpcMuxSotw::sendGrpcMessage(void* msg_proto_ptr, SubscriptionState& sub_state) { + std::unique_ptr typed_proto( + static_cast(msg_proto_ptr)); + if (sub_state.dynamicContextChanged() || !any_request_sent_yet_in_current_stream() || + !skip_subsequent_node()) { + typed_proto->mutable_node()->MergeFrom(local_info().node()); + } + VersionConverter::prepareMessageForGrpcWire(*typed_proto, transport_api_version()); + grpc_stream_.sendMessage(*typed_proto); + set_any_request_sent_yet_in_current_stream(true); + sub_state.clearDynamicContextChanged(); +} + +void GrpcMuxSotw::maybeUpdateQueueSizeStat(uint64_t size) { + grpc_stream_.maybeUpdateQueueSizeStat(size); +} + +bool GrpcMuxSotw::grpcStreamAvailable() const { return grpc_stream_.grpcStreamAvailable(); } +bool GrpcMuxSotw::rateLimitAllowsDrain() { return grpc_stream_.checkRateLimitAllowsDrain(); } + +Watch* NullGrpcMuxImpl::addWatch(const std::string&, const absl::flat_hash_set&, + SubscriptionCallbacks&, OpaqueResourceDecoder&, + std::chrono::milliseconds, const bool) { + throw EnvoyException("ADS must be configured to support an ADS config source"); +} + +void NullGrpcMuxImpl::updateWatch(const std::string&, Watch*, + const absl::flat_hash_set&, const bool) { + throw EnvoyException("ADS must be configured to support an ADS config source"); +} + +void NullGrpcMuxImpl::removeWatch(const std::string&, Watch*) { + throw EnvoyException("ADS must be configured to support an ADS config source"); +} + +} // namespace UnifiedMux +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/unified_mux/grpc_mux_impl.h b/source/common/config/unified_mux/grpc_mux_impl.h new file mode 100644 index 0000000000000..7fe666468f510 --- /dev/null +++ b/source/common/config/unified_mux/grpc_mux_impl.h @@ -0,0 +1,268 @@ +#pragma once + +#include +#include +#include + +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/common/random_generator.h" +#include "envoy/common/time.h" +#include "envoy/common/token_bucket.h" +#include "envoy/config/grpc_mux.h" +#include "envoy/config/subscription.h" +#include "envoy/event/dispatcher.h" +#include "envoy/grpc/status.h" +#include "envoy/service/discovery/v3/discovery.pb.h" +#include "envoy/upstream/cluster_manager.h" + +#include "common/common/logger.h" +#include "common/common/utility.h" +#include "common/config/api_version.h" +#include "common/config/grpc_stream.h" +#include "common/config/pausable_ack_queue.h" +#include "common/config/unified_mux/delta_subscription_state.h" +#include "common/config/unified_mux/sotw_subscription_state.h" +#include "common/config/watch_map.h" +#include "common/grpc/common.h" +#include "common/runtime/runtime_features.h" + +#include "absl/container/node_hash_map.h" + +namespace Envoy { +namespace Config { +namespace UnifiedMux { + +// Manages subscriptions to one or more type of resource. The logical protocol +// state of those subscription(s) is handled by SubscriptionState. +// This class owns the GrpcStream used to talk to the server, maintains queuing +// logic to properly order the subscription(s)' various messages, and allows +// starting/stopping/pausing of the subscriptions. +class GrpcMuxImpl : public GrpcMux, Logger::Loggable { +public: + GrpcMuxImpl(std::unique_ptr subscription_state_factory, + bool skip_subsequent_node, const LocalInfo::LocalInfo& local_info, + envoy::config::core::v3::ApiVersion transport_api_version); + + Watch* addWatch(const std::string& type_url, const absl::flat_hash_set& resources, + SubscriptionCallbacks& callbacks, OpaqueResourceDecoder& resource_decoder, + std::chrono::milliseconds init_fetch_timeout, + const bool use_namespace_matching = false) override; + void updateWatch(const std::string& type_url, Watch* watch, + const absl::flat_hash_set& resources, + const bool creating_namespace_watch = false) override; + void removeWatch(const std::string& type_url, Watch* watch) override; + + ScopedResume pause(const std::string& type_url) override; + ScopedResume pause(const std::vector type_urls) override; + bool paused(const std::string& type_url) const override; + void start() override; + void disableInitFetchTimeoutTimer() override; + void registerVersionedTypeUrl(const std::string& type_url); + const absl::flat_hash_map>& + subscriptions() const { + return subscriptions_; + } + + // legacy mux interface not implemented by unified mux. + GrpcMuxWatchPtr addWatch(const std::string&, const absl::flat_hash_set&, + SubscriptionCallbacks&, OpaqueResourceDecoder&, const bool) override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } + + void requestOnDemandUpdate(const std::string&, const absl::flat_hash_set&) override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } + + bool isUnified() const override { return true; } + +protected: + // Everything related to GrpcStream must remain abstract. GrpcStream (and the gRPC-using classes + // that underlie it) are templated on protobufs. That means that a single implementation that + // supports different types of protobufs cannot use polymorphism to share code. The workaround: + // the GrpcStream will be owned by a derived class, and all code that would touch grpc_stream_ is + // seen here in the base class as calls to abstract functions, to be provided by those derived + // classes. + virtual void establishGrpcStream() PURE; + // Deletes msg_proto_ptr. + virtual void sendGrpcMessage(void* msg_proto_ptr, SubscriptionState& sub_state) PURE; + virtual void maybeUpdateQueueSizeStat(uint64_t size) PURE; + virtual bool grpcStreamAvailable() const PURE; + virtual bool rateLimitAllowsDrain() PURE; + + SubscriptionState& subscriptionStateFor(const std::string& type_url); + WatchMap& watchMapFor(const std::string& type_url); + void handleEstablishedStream(); + void handleStreamEstablishmentFailure(); + void genericHandleResponse(const std::string& type_url, const void* response_proto_ptr); + void trySendDiscoveryRequests(); + bool skip_subsequent_node() const { return skip_subsequent_node_; } + bool any_request_sent_yet_in_current_stream() const { + return any_request_sent_yet_in_current_stream_; + } + void set_any_request_sent_yet_in_current_stream(bool value) { + any_request_sent_yet_in_current_stream_ = value; + } + const LocalInfo::LocalInfo& local_info() const { return local_info_; } + const envoy::config::core::v3::ApiVersion& transport_api_version() const { + return transport_api_version_; + } + +private: + // Checks whether external conditions allow sending a DeltaDiscoveryRequest. (Does not check + // whether we *want* to send a DeltaDiscoveryRequest). + bool canSendDiscoveryRequest(const std::string& type_url); + + // Checks whether we have something to say in a DeltaDiscoveryRequest, which can be an ACK and/or + // a subscription update. (Does not check whether we *can* send that DeltaDiscoveryRequest). + // Returns the type_url we should send the DeltaDiscoveryRequest for (if any). + // First, prioritizes ACKs over non-ACK subscription interest updates. + // Then, prioritizes non-ACK updates in the order the various types + // of subscriptions were activated (as tracked by subscription_ordering_). + absl::optional whoWantsToSendDiscoveryRequest(); + + // Invoked when dynamic context parameters change for a resource type. + void onDynamicContextUpdate(absl::string_view resource_type_url); + + // Resource (N)ACKs we're waiting to send, stored in the order that they should be sent in. All + // of our different resource types' ACKs are mixed together in this queue. See class for + // description of how it interacts with pause() and resume(). + PausableAckQueue pausable_ack_queue_; + + // Makes SubscriptionStates, to be held in the subscriptions_ map. Whether this GrpcMux is doing + // delta or state of the world xDS is determined by which concrete subclass this variable gets. + std::unique_ptr subscription_state_factory_; + + // Map key is type_url. + // Only addWatch() should insert into these maps. + absl::flat_hash_map> subscriptions_; + absl::flat_hash_map> watch_maps_; + + // Determines the order of initial discovery requests. (Assumes that subscriptions are added + // to this GrpcMux in the order of Envoy's dependency ordering). + std::list subscription_ordering_; + + // Whether to enable the optimization of only including the node field in the very first + // discovery request in an xDS gRPC stream (really just one: *not* per-type_url). + const bool skip_subsequent_node_; + + // State to help with skip_subsequent_node's logic. + bool any_request_sent_yet_in_current_stream_{}; + + // Used to populate the [Delta]DiscoveryRequest's node field. That field is the same across + // all type_urls, and moreover, the 'skip_subsequent_node' logic needs to operate across all + // the type_urls. So, while the SubscriptionStates populate every other field of these messages, + // this one is up to GrpcMux. + const LocalInfo::LocalInfo& local_info_; + Common::CallbackHandlePtr dynamic_update_callback_handle_; + + const envoy::config::core::v3::ApiVersion transport_api_version_; + const bool enable_type_url_downgrade_and_upgrade_; +}; + +class GrpcMuxDelta + : public GrpcMuxImpl, + public GrpcStreamCallbacks { +public: + GrpcMuxDelta(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, + envoy::config::core::v3::ApiVersion transport_api_version, + Random::RandomGenerator& random, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, const LocalInfo::LocalInfo& local_info, + bool skip_subsequent_node); + + // GrpcStreamCallbacks + void onStreamEstablished() override; + void onEstablishmentFailure() override; + void onWriteable() override; + void onDiscoveryResponse( + std::unique_ptr&& message, + ControlPlaneStats& control_plane_stats) override; + void requestOnDemandUpdate(const std::string& type_url, + const absl::flat_hash_set& for_update) override; + +protected: + void establishGrpcStream() override; + void sendGrpcMessage(void* msg_proto_ptr, SubscriptionState& sub_state) override; + void maybeUpdateQueueSizeStat(uint64_t size) override; + bool grpcStreamAvailable() const override; + bool rateLimitAllowsDrain() override; + +private: + GrpcStream + grpc_stream_; +}; + +class GrpcMuxSotw : public GrpcMuxImpl, + public GrpcStreamCallbacks { +public: + GrpcMuxSotw(Grpc::RawAsyncClientPtr&& async_client, Event::Dispatcher& dispatcher, + const Protobuf::MethodDescriptor& service_method, + envoy::config::core::v3::ApiVersion transport_api_version, + Random::RandomGenerator& random, Stats::Scope& scope, + const RateLimitSettings& rate_limit_settings, const LocalInfo::LocalInfo& local_info, + bool skip_subsequent_node); + + // GrpcStreamCallbacks + void onStreamEstablished() override; + void onEstablishmentFailure() override; + void onWriteable() override; + void + onDiscoveryResponse(std::unique_ptr&& message, + ControlPlaneStats& control_plane_stats) override; + void requestOnDemandUpdate(const std::string&, const absl::flat_hash_set&) override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + }; + GrpcStream& + grpcStreamForTest() { + return grpc_stream_; + } + +protected: + void establishGrpcStream() override; + void sendGrpcMessage(void* msg_proto_ptr, SubscriptionState& sub_state) override; + void maybeUpdateQueueSizeStat(uint64_t size) override; + bool grpcStreamAvailable() const override; + bool rateLimitAllowsDrain() override; + +private: + GrpcStream + grpc_stream_; +}; + +class NullGrpcMuxImpl : public GrpcMux { +public: + void start() override {} + + ScopedResume pause(const std::string&) override { + return std::make_unique([]() {}); + } + ScopedResume pause(const std::vector) override { + return std::make_unique([]() {}); + } + bool paused(const std::string&) const override { return false; } + void disableInitFetchTimeoutTimer() override {} + + Watch* addWatch(const std::string&, const absl::flat_hash_set&, + SubscriptionCallbacks&, OpaqueResourceDecoder&, std::chrono::milliseconds, + const bool) override; + void updateWatch(const std::string&, Watch*, const absl::flat_hash_set&, + const bool) override; + void removeWatch(const std::string&, Watch*) override; + + // legacy mux interface not implemented by unified mux. + GrpcMuxWatchPtr addWatch(const std::string&, const absl::flat_hash_set&, + SubscriptionCallbacks&, OpaqueResourceDecoder&, const bool) override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } + + void requestOnDemandUpdate(const std::string&, const absl::flat_hash_set&) override { + NOT_IMPLEMENTED_GCOVR_EXCL_LINE; + } +}; + +} // namespace UnifiedMux +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/unified_mux/grpc_subscription_impl.cc b/source/common/config/unified_mux/grpc_subscription_impl.cc new file mode 100644 index 0000000000000..a07fb97c07879 --- /dev/null +++ b/source/common/config/unified_mux/grpc_subscription_impl.cc @@ -0,0 +1,146 @@ +#include "common/config/unified_mux/grpc_subscription_impl.h" + +#include + +#include "common/config/xds_resource.h" +#include "common/protobuf/type_util.h" + +namespace Envoy { +namespace Config { +namespace UnifiedMux { + +constexpr std::chrono::milliseconds UpdateDurationLogThreshold = std::chrono::milliseconds(50); + +GrpcSubscriptionImpl::GrpcSubscriptionImpl(GrpcMuxSharedPtr grpc_mux, absl::string_view type_url, + SubscriptionCallbacks& callbacks, + OpaqueResourceDecoder& resource_decoder, + SubscriptionStats stats, TimeSource& time_source, + std::chrono::milliseconds init_fetch_timeout, + bool is_aggregated, bool use_namespace_matching) + : grpc_mux_(std::move(grpc_mux)), type_url_(type_url), callbacks_(callbacks), + resource_decoder_(resource_decoder), stats_(stats), time_source_(time_source), + init_fetch_timeout_(init_fetch_timeout), is_aggregated_(is_aggregated), + use_namespace_matching_(use_namespace_matching) {} + +GrpcSubscriptionImpl::~GrpcSubscriptionImpl() { + if (watch_) { + grpc_mux_->removeWatch(type_url_, watch_); + } +} + +ScopedResume GrpcSubscriptionImpl::pause() { return grpc_mux_->pause(type_url_); } + +// Config::Subscription +void GrpcSubscriptionImpl::start(const absl::flat_hash_set& resources) { + // ADS initial request batching relies on the users of the GrpcMux *not* calling start on it, + // whereas non-ADS xDS users must call it themselves. + if (!is_aggregated_) { + grpc_mux_->start(); + } + watch_ = grpc_mux_->addWatch(type_url_, resources, *this, resource_decoder_, init_fetch_timeout_, + use_namespace_matching_); + stats_.update_attempt_.inc(); + ENVOY_LOG(debug, "{} subscription started", type_url_); +} + +void GrpcSubscriptionImpl::updateResourceInterest( + const absl::flat_hash_set& update_to_these_names) { + grpc_mux_->updateWatch(type_url_, watch_, update_to_these_names, use_namespace_matching_); + stats_.update_attempt_.inc(); +} + +void GrpcSubscriptionImpl::requestOnDemandUpdate( + const absl::flat_hash_set& for_update) { + grpc_mux_->requestOnDemandUpdate(type_url_, for_update); + stats_.update_attempt_.inc(); +} + +// Config::SubscriptionCallbacks +void GrpcSubscriptionImpl::onConfigUpdate(const std::vector& resources, + const std::string& version_info) { + ENVOY_LOG(debug, "{} received SotW update", type_url_); + stats_.update_attempt_.inc(); + grpc_mux_->disableInitFetchTimeoutTimer(); + auto start = time_source_.monotonicTime(); + callbacks_.onConfigUpdate(resources, version_info); + std::chrono::milliseconds update_duration = + std::chrono::duration_cast(time_source_.monotonicTime() - start); + stats_.update_success_.inc(); + stats_.update_time_.set(DateUtil::nowToMilliseconds(time_source_)); + stats_.version_.set(HashUtil::xxHash64(version_info)); + stats_.version_text_.set(version_info); + stats_.update_duration_.recordValue(update_duration.count()); + ENVOY_LOG(debug, "SotW update for {} accepted with {} resources with version {}", type_url_, + resources.size(), version_info); + + if (update_duration > UpdateDurationLogThreshold) { + ENVOY_LOG(debug, "gRPC config update took {} ms! Resources names: {}", update_duration.count(), + absl::StrJoin(resources, ",", ResourceNameFormatter())); + } +} + +void GrpcSubscriptionImpl::onConfigUpdate( + const std::vector& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) { + stats_.update_attempt_.inc(); + grpc_mux_->disableInitFetchTimeoutTimer(); + auto start = time_source_.monotonicTime(); + callbacks_.onConfigUpdate(added_resources, removed_resources, system_version_info); + std::chrono::milliseconds update_duration = + std::chrono::duration_cast(time_source_.monotonicTime() - start); + stats_.update_success_.inc(); + stats_.update_time_.set(DateUtil::nowToMilliseconds(time_source_)); + stats_.version_.set(HashUtil::xxHash64(system_version_info)); + stats_.version_text_.set(system_version_info); + stats_.update_duration_.recordValue(update_duration.count()); +} + +void GrpcSubscriptionImpl::onConfigUpdateFailed(ConfigUpdateFailureReason reason, + const EnvoyException* e) { + switch (reason) { + case Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure: + // This is a gRPC-stream-level establishment failure, not an xDS-protocol-level failure. + // So, don't onConfigUpdateFailed() here. Instead, allow a retry of the gRPC stream. + // If init_fetch_timeout_ is non-zero, the server will continue startup after that timeout. + stats_.update_failure_.inc(); + ENVOY_LOG(debug, "{} update failed: ConnectionFailure", type_url_); + break; + case Envoy::Config::ConfigUpdateFailureReason::FetchTimedout: + stats_.init_fetch_timeout_.inc(); + grpc_mux_->disableInitFetchTimeoutTimer(); + callbacks_.onConfigUpdateFailed(reason, e); + ENVOY_LOG(debug, "{} update failed: FetchTimedout", type_url_); + break; + case Envoy::Config::ConfigUpdateFailureReason::UpdateRejected: + // We expect Envoy exception to be thrown when update is rejected. + ASSERT(e != nullptr); + grpc_mux_->disableInitFetchTimeoutTimer(); + stats_.update_rejected_.inc(); + callbacks_.onConfigUpdateFailed(reason, e); + ENVOY_LOG(debug, "{} update failed: UpdateRejected", type_url_); + break; + } + + stats_.update_attempt_.inc(); +} + +GrpcCollectionSubscriptionImpl::GrpcCollectionSubscriptionImpl( + const xds::core::v3::ResourceLocator& collection_locator, GrpcMuxSharedPtr grpc_mux, + SubscriptionCallbacks& callbacks, OpaqueResourceDecoder& resource_decoder, + SubscriptionStats stats, TimeSource& time_source, std::chrono::milliseconds init_fetch_timeout, + bool is_aggregated) + : GrpcSubscriptionImpl( + grpc_mux, TypeUtil::descriptorFullNameToTypeUrl(collection_locator.resource_type()), + callbacks, resource_decoder, stats, time_source, init_fetch_timeout, is_aggregated, + false), + collection_locator_(collection_locator) {} + +void GrpcCollectionSubscriptionImpl::start(const absl::flat_hash_set& resource_names) { + ASSERT(resource_names.empty()); + GrpcSubscriptionImpl::start({XdsResourceIdentifier::encodeUrl(collection_locator_)}); +} + +} // namespace UnifiedMux +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/unified_mux/grpc_subscription_impl.h b/source/common/config/unified_mux/grpc_subscription_impl.h new file mode 100644 index 0000000000000..79281b8dddf03 --- /dev/null +++ b/source/common/config/unified_mux/grpc_subscription_impl.h @@ -0,0 +1,109 @@ +#pragma once + +#include +#include + +#include "envoy/config/grpc_mux.h" +#include "envoy/config/subscription.h" +#include "envoy/event/dispatcher.h" + +#include "common/common/logger.h" + +#include "xds/core/v3/resource_locator.pb.h" + +namespace Envoy { +namespace Config { +namespace UnifiedMux { + +// GrpcSubscriptionImpl provides a top-level interface to the Envoy's gRPC communication with +// an xDS server, for use by the various xDS users within Envoy. It is built around a (shared) +// GrpcMuxImpl, and the further machinery underlying that. An xDS user indicates interest in +// various resources via start() and updateResourceInterest(). It receives updates to those +// resources via the SubscriptionCallbacks it provides. Multiple users can each have their own +// Subscription object for the same type_url; GrpcMuxImpl maintains a subscription to the +// union of interested resources, and delivers to the users just the resource updates that they +// are "watching" for. +// +// GrpcSubscriptionImpl and GrpcMuxImpl are both built to provide both regular xDS and ADS, +// distinguished by whether multiple GrpcSubscriptionImpls are sharing a single GrpcMuxImpl. +// (Also distinguished by the gRPC method string, but that's taken care of in SubscriptionFactory). +// +// Why does GrpcSubscriptionImpl itself implement the SubscriptionCallbacks interface? So that it +// can write to SubscriptionStats (which needs to live out here in the GrpcSubscriptionImpl) upon a +// config update. GrpcSubscriptionImpl presents itself to WatchMap as the SubscriptionCallbacks, +// and then, after incrementing stats, passes through to the real callbacks_. +class GrpcSubscriptionImpl : public Subscription, + protected SubscriptionCallbacks, + Logger::Loggable { +public: + // is_aggregated: whether our GrpcMux is also providing ADS to other Subscriptions, or whether + // it's all ours. The practical difference is that we ourselves must call start() on it only if + // we are the sole owner. + GrpcSubscriptionImpl(GrpcMuxSharedPtr grpc_mux, absl::string_view type_url, + SubscriptionCallbacks& callbacks, OpaqueResourceDecoder& resource_decoder, + SubscriptionStats stats, TimeSource& time_source, + std::chrono::milliseconds init_fetch_timeout, bool is_aggregated, + bool use_namespace_matching); + ~GrpcSubscriptionImpl() override; + + // Config::Subscription + void start(const absl::flat_hash_set& resource_names) override; + void + updateResourceInterest(const absl::flat_hash_set& update_to_these_names) override; + void requestOnDemandUpdate(const absl::flat_hash_set& add_these_names) override; + // Config::SubscriptionCallbacks (all pass through to callbacks_!) + void onConfigUpdate(const std::vector& resources, + const std::string& version_info) override; + void onConfigUpdate(const std::vector& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) override; + void onConfigUpdateFailed(ConfigUpdateFailureReason reason, const EnvoyException* e) override; + + GrpcMuxSharedPtr getGrpcMuxForTest() { return grpc_mux_; } + + ScopedResume pause(); + +private: + void disableInitFetchTimeoutTimer(); + + GrpcMuxSharedPtr grpc_mux_; + const std::string type_url_; + SubscriptionCallbacks& callbacks_; + OpaqueResourceDecoder& resource_decoder_; + SubscriptionStats stats_; + Watch* watch_{nullptr}; + TimeSource& time_source_; + // NOTE: if another subscription of the same type_url has already been started, this value will be + // ignored in favor of the other subscription's. + std::chrono::milliseconds init_fetch_timeout_; + Event::TimerPtr init_fetch_timeout_timer_; + const bool is_aggregated_; + const bool use_namespace_matching_; + + struct ResourceNameFormatter { + void operator()(std::string* out, const Config::DecodedResourceRef& resource) { + out->append(resource.get().name()); + } + }; +}; + +using GrpcSubscriptionImplPtr = std::unique_ptr; +using GrpcSubscriptionImplSharedPtr = std::shared_ptr; + +class GrpcCollectionSubscriptionImpl : public GrpcSubscriptionImpl { +public: + GrpcCollectionSubscriptionImpl(const xds::core::v3::ResourceLocator& collection_locator, + GrpcMuxSharedPtr grpc_mux, SubscriptionCallbacks& callbacks, + OpaqueResourceDecoder& resource_decoder, SubscriptionStats stats, + TimeSource& time_source, + std::chrono::milliseconds init_fetch_timeout, bool is_aggregated); + + void start(const absl::flat_hash_set& resource_names) override; + +private: + xds::core::v3::ResourceLocator collection_locator_; +}; + +} // namespace UnifiedMux +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/unified_mux/sotw_subscription_state.cc b/source/common/config/unified_mux/sotw_subscription_state.cc new file mode 100644 index 0000000000000..52eebf686e56e --- /dev/null +++ b/source/common/config/unified_mux/sotw_subscription_state.cc @@ -0,0 +1,187 @@ +#include "common/config/unified_mux/sotw_subscription_state.h" + +#include "common/common/assert.h" +#include "common/common/hash.h" +#include "common/config/utility.h" + +namespace Envoy { +namespace Config { +namespace UnifiedMux { + +SotwSubscriptionState::SotwSubscriptionState(std::string type_url, + UntypedConfigUpdateCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout, + Event::Dispatcher& dispatcher) + : SubscriptionState(std::move(type_url), callbacks, init_fetch_timeout, dispatcher) {} + +SotwSubscriptionState::~SotwSubscriptionState() = default; + +SotwSubscriptionStateFactory::SotwSubscriptionStateFactory(Event::Dispatcher& dispatcher) + : dispatcher_(dispatcher) {} + +SotwSubscriptionStateFactory::~SotwSubscriptionStateFactory() = default; + +std::unique_ptr +SotwSubscriptionStateFactory::makeSubscriptionState(const std::string& type_url, + UntypedConfigUpdateCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) { + return std::make_unique(type_url, callbacks, init_fetch_timeout, + dispatcher_); +} + +void SotwSubscriptionState::updateSubscriptionInterest( + const absl::flat_hash_set& cur_added, + const absl::flat_hash_set& cur_removed) { + for (const auto& a : cur_added) { + names_tracked_.insert(a); + } + for (const auto& r : cur_removed) { + names_tracked_.erase(r); + } + if (!cur_added.empty() || !cur_removed.empty()) { + update_pending_ = true; + } +} + +// Not having sent any requests yet counts as an "update pending" since you're supposed to resend +// the entirety of your interest at the start of a stream, even if nothing has changed. +bool SotwSubscriptionState::subscriptionUpdatePending() const { + return update_pending_ || dynamicContextChanged(); +} + +void SotwSubscriptionState::markStreamFresh() { + last_good_version_info_ = absl::nullopt; + last_good_nonce_ = absl::nullopt; + update_pending_ = true; + clearDynamicContextChanged(); +} + +UpdateAck SotwSubscriptionState::handleResponse(const void* response_proto_ptr) { + auto* response = + static_cast(response_proto_ptr); + // We *always* copy the response's nonce into the next request, even if we're going to make that + // request a NACK by setting error_detail. + UpdateAck ack(response->nonce(), type_url()); + ENVOY_LOG(debug, "Handling response for {}", type_url()); + try { + handleGoodResponse(*response); + } catch (const EnvoyException& e) { + handleBadResponse(e, ack); + } + return ack; +} + +void SotwSubscriptionState::handleGoodResponse( + const envoy::service::discovery::v3::DiscoveryResponse& message) { + Protobuf::RepeatedPtrField non_heartbeat_resources; + std::vector resources_with_ttl( + message.resources().size()); + + for (const auto& any : message.resources()) { + if (!any.Is() && + any.type_url() != message.type_url()) { + throw EnvoyException(fmt::format("type URL {} embedded in an individual Any does not match " + "the message-wide type URL {} in DiscoveryResponse {}", + any.type_url(), message.type_url(), message.DebugString())); + } + + // ttl changes (including removing of the ttl timer) are only done when an Any is wrapped in a + // Resource (which contains ttl duration). + if (any.Is()) { + resources_with_ttl.emplace(resources_with_ttl.end()); + MessageUtil::unpackTo(any, resources_with_ttl.back()); + + if (isHeartbeatResource(resources_with_ttl.back(), message.version_info())) { + continue; + } + } + non_heartbeat_resources.Add()->CopyFrom(any); + } + + { + const auto scoped_update = ttl_.scopedTtlUpdate(); + for (auto& resource : resources_with_ttl) { + setResourceTtl(resource); + } + } + + callbacks().onConfigUpdate(non_heartbeat_resources, message.version_info()); + // Now that we're passed onConfigUpdate() without an exception thrown, we know we're good. + last_good_version_info_ = message.version_info(); + last_good_nonce_ = message.nonce(); + ENVOY_LOG(debug, "Config update for {} (version {}) accepted with {} resources", type_url(), + message.version_info(), message.resources().size()); +} + +void SotwSubscriptionState::handleBadResponse(const EnvoyException& e, UpdateAck& ack) { + // Note that error_detail being set is what indicates that a DeltaDiscoveryRequest is a NACK. + ack.error_detail_.set_code(Grpc::Status::WellKnownGrpcStatus::Internal); + ack.error_detail_.set_message(Config::Utility::truncateGrpcStatusMessage(e.what())); + ENVOY_LOG(warn, "gRPC state-of-the-world config for {} rejected: {}", type_url(), e.what()); + callbacks().onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, &e); +} + +void SotwSubscriptionState::handleEstablishmentFailure() { + ENVOY_LOG(debug, "SotwSubscriptionState establishment failed for {}", type_url()); + callbacks().onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, + nullptr); +} + +envoy::service::discovery::v3::DiscoveryRequest* SotwSubscriptionState::getNextRequestInternal() { + auto* request = new envoy::service::discovery::v3::DiscoveryRequest; + request->set_type_url(type_url()); + std::copy(names_tracked_.begin(), names_tracked_.end(), + Protobuf::RepeatedFieldBackInserter(request->mutable_resource_names())); + if (last_good_version_info_.has_value()) { + request->set_version_info(last_good_version_info_.value()); + } + // Default response_nonce to the last known good one. If we are being called by + // getNextRequestWithAck(), this value will be overwritten. + if (last_good_nonce_.has_value()) { + request->set_response_nonce(last_good_nonce_.value()); + } + + update_pending_ = false; + return request; +} + +void* SotwSubscriptionState::getNextRequestAckless() { return getNextRequestInternal(); } + +void* SotwSubscriptionState::getNextRequestWithAck(const UpdateAck& ack) { + envoy::service::discovery::v3::DiscoveryRequest* request = getNextRequestInternal(); + request->set_response_nonce(ack.nonce_); + ENVOY_LOG(debug, "ACK for {} will have nonce {}", type_url(), ack.nonce_); + if (ack.error_detail_.code() != Grpc::Status::WellKnownGrpcStatus::Ok) { + // Don't needlessly make the field present-but-empty if status is ok. + request->mutable_error_detail()->CopyFrom(ack.error_detail_); + } + return request; +} + +void SotwSubscriptionState::setResourceTtl( + const envoy::service::discovery::v3::Resource& resource) { + if (resource.has_ttl()) { + ttl_.add(std::chrono::milliseconds(DurationUtil::durationToMilliseconds(resource.ttl())), + resource.name()); + } else { + ttl_.clear(resource.name()); + } +} + +void SotwSubscriptionState::ttlExpiryCallback(const std::vector& expired) { + Protobuf::RepeatedPtrField removed_resources; + for (const auto& resource : expired) { + removed_resources.Add(std::string(resource)); + } + callbacks().onConfigUpdate({}, removed_resources, ""); +} + +bool SotwSubscriptionState::isHeartbeatResource( + const envoy::service::discovery::v3::Resource& resource, const std::string& version) { + return !resource.has_resource() && last_good_version_info_.has_value() && + version == last_good_version_info_.value(); +} + +} // namespace UnifiedMux +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/unified_mux/sotw_subscription_state.h b/source/common/config/unified_mux/sotw_subscription_state.h new file mode 100644 index 0000000000000..2889241ca5af0 --- /dev/null +++ b/source/common/config/unified_mux/sotw_subscription_state.h @@ -0,0 +1,91 @@ +#pragma once + +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/grpc/status.h" + +#include "common/common/assert.h" +#include "common/common/hash.h" +#include "common/config/unified_mux/subscription_state.h" + +#include "absl/types/optional.h" + +namespace Envoy { +namespace Config { +namespace UnifiedMux { + +// Tracks the state of a "state-of-the-world" (i.e. not delta) xDS-over-gRPC protocol session. +class SotwSubscriptionState : public SubscriptionState { +public: + // Note that, outside of tests, we expect callbacks to always be a WatchMap. + SotwSubscriptionState(std::string type_url, UntypedConfigUpdateCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout, + Event::Dispatcher& dispatcher); + ~SotwSubscriptionState() override; + + // Update which resources we're interested in subscribing to. + void updateSubscriptionInterest(const absl::flat_hash_set& cur_added, + const absl::flat_hash_set& cur_removed) override; + + // Whether there was a change in our subscription interest we have yet to inform the server of. + bool subscriptionUpdatePending() const override; + + void markStreamFresh() override; + + // message is expected to be a envoy::service::discovery::v3::DiscoveryResponse. + UpdateAck handleResponse(const void* response_proto_ptr) override; + + void handleEstablishmentFailure() override; + + // Returns the next gRPC request proto to be sent off to the server, based on this object's + // understanding of the current protocol state, and new resources that Envoy wants to request. + // Returns a new'd pointer, meant to be owned by the caller. + void* getNextRequestAckless() override; + // The WithAck version first calls the ack-less version, then adds in the passed-in ack. + // Returns a new'd pointer, meant to be owned by the caller. + void* getNextRequestWithAck(const UpdateAck& ack) override; + + void ttlExpiryCallback(const std::vector& expired) override; + + SotwSubscriptionState(const SotwSubscriptionState&) = delete; + SotwSubscriptionState& operator=(const SotwSubscriptionState&) = delete; + +private: + // Returns a new'd pointer, meant to be owned by the caller. + envoy::service::discovery::v3::DiscoveryRequest* getNextRequestInternal(); + + void handleGoodResponse(const envoy::service::discovery::v3::DiscoveryResponse& message); + void handleBadResponse(const EnvoyException& e, UpdateAck& ack); + + bool isHeartbeatResource(const envoy::service::discovery::v3::Resource& resource, + const std::string& version); + void setResourceTtl(const envoy::service::discovery::v3::Resource& resource); + + // The version_info carried by the last accepted DiscoveryResponse. + // Remains empty until one is accepted. + absl::optional last_good_version_info_; + // The nonce carried by the last accepted DiscoveryResponse. + // Remains empty until one is accepted. + // Used when it's time to make a spontaneous (i.e. not primarily meant as an ACK) request. + absl::optional last_good_nonce_; + + // Starts true because we should send a request upon subscription start. + bool update_pending_{true}; + + absl::flat_hash_set names_tracked_; +}; + +class SotwSubscriptionStateFactory : public SubscriptionStateFactory { +public: + SotwSubscriptionStateFactory(Event::Dispatcher& dispatcher); + ~SotwSubscriptionStateFactory() override; + std::unique_ptr + makeSubscriptionState(const std::string& type_url, UntypedConfigUpdateCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) override; + +private: + Event::Dispatcher& dispatcher_; +}; + +} // namespace UnifiedMux +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/unified_mux/subscription_state.cc b/source/common/config/unified_mux/subscription_state.cc new file mode 100644 index 0000000000000..d466a7691073a --- /dev/null +++ b/source/common/config/unified_mux/subscription_state.cc @@ -0,0 +1,41 @@ +#include "common/config/unified_mux/subscription_state.h" + +#include +#include + +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/common/pure.h" +#include "envoy/config/subscription.h" + +namespace Envoy { +namespace Config { +namespace UnifiedMux { + +SubscriptionState::SubscriptionState(std::string type_url, UntypedConfigUpdateCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout, + Event::Dispatcher& dispatcher) + // TODO(snowp): Hard coding VHDS here is temporary until we can move it away from relying on + // empty resources as updates. + : supports_heartbeats_(type_url != "envoy.config.route.v3.VirtualHost"), + ttl_([this](const std::vector& expired) { ttlExpiryCallback(expired); }, + dispatcher, dispatcher.timeSource()), + type_url_(std::move(type_url)), callbacks_(callbacks), dispatcher_(dispatcher) { + if (init_fetch_timeout.count() > 0 && !init_fetch_timeout_timer_) { + init_fetch_timeout_timer_ = dispatcher.createTimer([this]() -> void { + ENVOY_LOG(warn, "config: initial fetch timed out for {}", type_url_); + callbacks_.onConfigUpdateFailed(ConfigUpdateFailureReason::FetchTimedout, nullptr); + }); + init_fetch_timeout_timer_->enableTimer(init_fetch_timeout); + } +} + +void SubscriptionState::disableInitFetchTimeoutTimer() { + if (init_fetch_timeout_timer_) { + init_fetch_timeout_timer_->disableTimer(); + init_fetch_timeout_timer_.reset(); + } +} + +} // namespace UnifiedMux +} // namespace Config +} // namespace Envoy diff --git a/source/common/config/unified_mux/subscription_state.h b/source/common/config/unified_mux/subscription_state.h new file mode 100644 index 0000000000000..4bc3cc6adee79 --- /dev/null +++ b/source/common/config/unified_mux/subscription_state.h @@ -0,0 +1,94 @@ +#pragma once + +#include +#include + +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/common/pure.h" +#include "envoy/config/subscription.h" +#include "envoy/event/dispatcher.h" + +#include "common/config/ttl.h" +#include "common/config/update_ack.h" +#include "common/protobuf/protobuf.h" + +#include "absl/strings/string_view.h" + +namespace Envoy { +namespace Config { +namespace UnifiedMux { + +// Tracks the protocol state of an individual ongoing xDS-over-gRPC session, for a single type_url. +// There can be multiple SubscriptionStates active, one per type_url. They will all be +// blissfully unaware of each other's existence, even when their messages are being multiplexed +// together by ADS. +// This is the abstract parent class for both the delta and state-of-the-world xDS variants. +class SubscriptionState : public Logger::Loggable { +public: + // Note that, outside of tests, we expect callbacks to always be a WatchMap. + SubscriptionState(std::string type_url, UntypedConfigUpdateCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout, Event::Dispatcher& dispatcher); + virtual ~SubscriptionState() = default; + + // Update which resources we're interested in subscribing to. + virtual void updateSubscriptionInterest(const absl::flat_hash_set& cur_added, + const absl::flat_hash_set& cur_removed) PURE; + + void setDynamicContextChanged() { dynamic_context_changed_ = true; } + void clearDynamicContextChanged() { dynamic_context_changed_ = false; } + bool dynamicContextChanged() const { return dynamic_context_changed_; } + + // Whether there was a change in our subscription interest we have yet to inform the server of. + virtual bool subscriptionUpdatePending() const PURE; + + virtual void markStreamFresh() PURE; + + // Implementations expect either a DeltaDiscoveryResponse or DiscoveryResponse. The caller is + // expected to know which it should be providing. + virtual UpdateAck handleResponse(const void* response_proto_ptr) PURE; + + virtual void handleEstablishmentFailure() PURE; + + // Returns the next gRPC request proto to be sent off to the server, based on this object's + // understanding of the current protocol state, and new resources that Envoy wants to request. + // Returns a new'd pointer, meant to be owned by the caller, who is expected to know what type the + // pointer actually is. + virtual void* getNextRequestAckless() PURE; + // The WithAck version first calls the ack-less version, then adds in the passed-in ack. + // Returns a new'd pointer, meant to be owned by the caller, who is expected to know what type the + // pointer actually is. + virtual void* getNextRequestWithAck(const UpdateAck& ack) PURE; + + void disableInitFetchTimeoutTimer(); + + virtual void ttlExpiryCallback(const std::vector& type_url) PURE; + +protected: + std::string type_url() const { return type_url_; } + UntypedConfigUpdateCallbacks& callbacks() const { return callbacks_; } + + // Not all xDS resources supports heartbeats due to there being specific information encoded in + // an empty response, which is indistinguishable from a heartbeat in some cases. For now we just + // disable heartbeats for these resources (currently only VHDS). + const bool supports_heartbeats_; + TtlManager ttl_; + const std::string type_url_; + // callbacks_ is expected (outside of tests) to be a WatchMap. + UntypedConfigUpdateCallbacks& callbacks_; + Event::Dispatcher& dispatcher_; + Event::TimerPtr init_fetch_timeout_timer_; + bool dynamic_context_changed_{}; +}; + +class SubscriptionStateFactory { +public: + virtual ~SubscriptionStateFactory() = default; + // Note that, outside of tests, we expect callbacks to always be a WatchMap. + virtual std::unique_ptr + makeSubscriptionState(const std::string& type_url, UntypedConfigUpdateCallbacks& callbacks, + std::chrono::milliseconds init_fetch_timeout) PURE; +}; + +} // namespace UnifiedMux +} // namespace Config +} // namespace Envoy diff --git a/source/common/runtime/runtime_features.cc b/source/common/runtime/runtime_features.cc index 67e1930f6806c..30a89f9633a2d 100644 --- a/source/common/runtime/runtime_features.cc +++ b/source/common/runtime/runtime_features.cc @@ -118,6 +118,8 @@ constexpr const char* disabled_runtime_features[] = { "envoy.reloadable_features.remove_legacy_json", // Sentinel and test flag. "envoy.reloadable_features.test_feature_false", + // TODO (dmitri-d) flip to true to enable unified mux by default + "envoy.reloadable_features.unified_mux", }; RuntimeFeatures::RuntimeFeatures() { diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD index d69db58fb866f..75b21b537ede1 100644 --- a/source/common/upstream/BUILD +++ b/source/common/upstream/BUILD @@ -56,6 +56,7 @@ envoy_cc_library( "//source/common/config:subscription_factory_lib", "//source/common/config:utility_lib", "//source/common/config:version_converter_lib", + "//source/common/config/unified_mux:grpc_mux_lib", "//source/common/grpc:async_client_manager_lib", "//source/common/http:async_client_lib", "//source/common/http:mixed_conn_pool", diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index dc731e6038b3b..632487eddffd2 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -21,6 +21,7 @@ #include "common/common/fmt.h" #include "common/common/utility.h" #include "common/config/new_grpc_mux_impl.h" +#include "common/config/unified_mux/grpc_mux_impl.h" #include "common/config/utility.h" #include "common/config/version_converter.h" #include "common/grpc/async_client_manager_impl.h" @@ -317,40 +318,87 @@ ClusterManagerImpl::ClusterManagerImpl( if (dyn_resources.has_ads_config()) { if (dyn_resources.ads_config().api_type() == envoy::config::core::v3::ApiConfigSource::DELTA_GRPC) { - ads_mux_ = std::make_shared( - Config::Utility::factoryForGrpcApiConfigSource(*async_client_manager_, - dyn_resources.ads_config(), stats, false) - ->create(), - main_thread_dispatcher, - *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( - Config::Utility::getAndCheckTransportVersion(dyn_resources.ads_config()) == - envoy::config::core::v3::ApiVersion::V3 - // TODO(htuch): consolidate with type_to_endpoint.cc, once we sort out the future - // direction of that module re: https://github.com/envoyproxy/envoy/issues/10650. - ? "envoy.service.discovery.v3.AggregatedDiscoveryService.DeltaAggregatedResources" - : "envoy.service.discovery.v2.AggregatedDiscoveryService." - "DeltaAggregatedResources"), - Config::Utility::getAndCheckTransportVersion(dyn_resources.ads_config()), random_, stats_, - Envoy::Config::Utility::parseRateLimitSettings(dyn_resources.ads_config()), local_info); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.unified_mux")) { + ads_mux_ = std::make_shared( + Config::Utility::factoryForGrpcApiConfigSource(*async_client_manager_, + dyn_resources.ads_config(), stats, false) + ->create(), + main_thread_dispatcher, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + Config::Utility::getAndCheckTransportVersion(dyn_resources.ads_config()) == + envoy::config::core::v3::ApiVersion::V3 + // TODO(htuch): consolidate with type_to_endpoint.cc, once we sort out the + // future direction of that module re: + // https://github.com/envoyproxy/envoy/issues/10650. + ? "envoy.service.discovery.v3.AggregatedDiscoveryService." + "DeltaAggregatedResources" + : "envoy.service.discovery.v2.AggregatedDiscoveryService." + "DeltaAggregatedResources"), + Config::Utility::getAndCheckTransportVersion(dyn_resources.ads_config()), random_, + stats_, Envoy::Config::Utility::parseRateLimitSettings(dyn_resources.ads_config()), + local_info, dyn_resources.ads_config().set_node_on_first_message_only()); + } else { + ads_mux_ = std::make_shared( + Config::Utility::factoryForGrpcApiConfigSource(*async_client_manager_, + dyn_resources.ads_config(), stats, false) + ->create(), + main_thread_dispatcher, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + Config::Utility::getAndCheckTransportVersion(dyn_resources.ads_config()) == + envoy::config::core::v3::ApiVersion::V3 + // TODO(htuch): consolidate with type_to_endpoint.cc, once we sort out the + // future direction of that module re: + // https://github.com/envoyproxy/envoy/issues/10650. + ? "envoy.service.discovery.v3.AggregatedDiscoveryService." + "DeltaAggregatedResources" + : "envoy.service.discovery.v2.AggregatedDiscoveryService." + "DeltaAggregatedResources"), + Config::Utility::getAndCheckTransportVersion(dyn_resources.ads_config()), random_, + stats_, Envoy::Config::Utility::parseRateLimitSettings(dyn_resources.ads_config()), + local_info); + } } else { - ads_mux_ = std::make_shared( - local_info, - Config::Utility::factoryForGrpcApiConfigSource(*async_client_manager_, - dyn_resources.ads_config(), stats, false) - ->create(), - main_thread_dispatcher, - *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( - Config::Utility::getAndCheckTransportVersion(dyn_resources.ads_config()) == - envoy::config::core::v3::ApiVersion::V3 - // TODO(htuch): consolidate with type_to_endpoint.cc, once we sort out the future - // direction of that module re: https://github.com/envoyproxy/envoy/issues/10650. - ? "envoy.service.discovery.v3.AggregatedDiscoveryService." - "StreamAggregatedResources" - : "envoy.service.discovery.v2.AggregatedDiscoveryService." - "StreamAggregatedResources"), - Config::Utility::getAndCheckTransportVersion(dyn_resources.ads_config()), random_, stats_, - Envoy::Config::Utility::parseRateLimitSettings(dyn_resources.ads_config()), - bootstrap.dynamic_resources().ads_config().set_node_on_first_message_only()); + if (Runtime::runtimeFeatureEnabled("envoy.reloadable_features.unified_mux")) { + ads_mux_ = std::make_shared( + Config::Utility::factoryForGrpcApiConfigSource(*async_client_manager_, + dyn_resources.ads_config(), stats, false) + ->create(), + main_thread_dispatcher, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + dyn_resources.ads_config().transport_api_version() == + envoy::config::core::v3::ApiVersion::V3 + // TODO(htuch): consolidate with type_to_endpoint.cc, once we sort out the + // future direction of that module re: + // https://github.com/envoyproxy/envoy/issues/10650. + ? "envoy.service.discovery.v3.AggregatedDiscoveryService." + "StreamAggregatedResources" + : "envoy.service.discovery.v2.AggregatedDiscoveryService." + "StreamAggregatedResources"), + Config::Utility::getAndCheckTransportVersion(dyn_resources.ads_config()), random_, + stats_, Envoy::Config::Utility::parseRateLimitSettings(dyn_resources.ads_config()), + local_info, + bootstrap.dynamic_resources().ads_config().set_node_on_first_message_only()); + } else { + ads_mux_ = std::make_shared( + local_info, + Config::Utility::factoryForGrpcApiConfigSource(*async_client_manager_, + dyn_resources.ads_config(), stats, false) + ->create(), + main_thread_dispatcher, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + Config::Utility::getAndCheckTransportVersion(dyn_resources.ads_config()) == + envoy::config::core::v3::ApiVersion::V3 + // TODO(htuch): consolidate with type_to_endpoint.cc, once we sort out the + // future direction of that module re: + // https://github.com/envoyproxy/envoy/issues/10650. + ? "envoy.service.discovery.v3.AggregatedDiscoveryService." + "StreamAggregatedResources" + : "envoy.service.discovery.v2.AggregatedDiscoveryService." + "StreamAggregatedResources"), + Config::Utility::getAndCheckTransportVersion(dyn_resources.ads_config()), random_, + stats_, Envoy::Config::Utility::parseRateLimitSettings(dyn_resources.ads_config()), + bootstrap.dynamic_resources().ads_config().set_node_on_first_message_only()); + } } } else { ads_mux_ = std::make_unique(); diff --git a/test/common/config/BUILD b/test/common/config/BUILD index 56b8e4b5ac98c..20c21fb030aba 100644 --- a/test/common/config/BUILD +++ b/test/common/config/BUILD @@ -70,6 +70,28 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "unified_delta_subscription_impl_test", + srcs = ["unified_delta_subscription_impl_test.cc"], + deps = [ + ":unified_delta_subscription_test_harness", + "//source/common/config:api_version_lib", + "//source/common/config/unified_mux:grpc_subscription_lib", + "//source/common/stats:isolated_store_lib", + "//test/mocks:common_lib", + "//test/mocks/config:config_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/test_common:logging_lib", + "@envoy_api//envoy/api/v2:pkg_cc_proto", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) + envoy_cc_test( name = "delta_subscription_state_test", srcs = ["delta_subscription_state_test.cc"], @@ -91,6 +113,25 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "unified_delta_subscription_state_test", + srcs = ["unified_delta_subscription_state_test.cc"], + deps = [ + "//source/common/config/unified_mux:delta_subscription_state_lib", + "//source/common/stats:isolated_store_lib", + "//test/mocks:common_lib", + "//test/mocks/config:config_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/test_common:logging_lib", + "//test/test_common:test_runtime_lib", + "@envoy_api//envoy/config/cluster/v3:pkg_cc_proto", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) + envoy_cc_test( name = "filesystem_subscription_impl_test", srcs = ["filesystem_subscription_impl_test.cc"], @@ -177,6 +218,35 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "unified_grpc_mux_impl_test", + srcs = ["unified_grpc_mux_impl_test.cc"], + deps = [ + "//source/common/config:api_version_lib", + "//source/common/config:protobuf_link_hacks", + "//source/common/config:version_converter_lib", + "//source/common/config/unified_mux:grpc_mux_lib", + "//source/common/config/unified_mux:grpc_subscription_lib", + "//source/common/protobuf", + "//test/common/stats:stat_test_utility_lib", + "//test/mocks:common_lib", + "//test/mocks/config:config_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/test_common:logging_lib", + "//test/test_common:resources_lib", + "//test/test_common:simulated_time_system_lib", + "//test/test_common:test_runtime_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/api/v2:pkg_cc_proto", + "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", + "@envoy_api//envoy/config/route/v3:pkg_cc_proto", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) + envoy_cc_test( name = "grpc_stream_test", srcs = ["grpc_stream_test.cc"], @@ -224,6 +294,39 @@ envoy_cc_test_library( ], ) +envoy_cc_test( + name = "unified_grpc_subscription_impl_test", + srcs = ["unified_grpc_subscription_impl_test.cc"], + deps = [ + ":unified_grpc_subscription_test_harness", + "//source/common/buffer:zero_copy_input_stream_lib", + ], +) + +envoy_cc_test_library( + name = "unified_grpc_subscription_test_harness", + hdrs = ["unified_grpc_subscription_test_harness.h"], + deps = [ + ":subscription_test_harness", + "//source/common/common:hash_lib", + "//source/common/config:api_version_lib", + "//source/common/config:version_converter_lib", + "//source/common/config/unified_mux:grpc_mux_lib", + "//source/common/config/unified_mux:grpc_subscription_lib", + "//test/mocks/config:config_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/upstream:cluster_manager_mocks", + "//test/test_common:resources_lib", + "//test/test_common:utility_lib", + "@envoy_api//envoy/api/v2:pkg_cc_proto", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) + envoy_cc_test_library( name = "delta_subscription_test_harness", hdrs = ["delta_subscription_test_harness.h"], @@ -244,6 +347,26 @@ envoy_cc_test_library( ], ) +envoy_cc_test_library( + name = "unified_delta_subscription_test_harness", + hdrs = ["unified_delta_subscription_test_harness.h"], + deps = [ + ":subscription_test_harness", + "//source/common/common:utility_lib", + "//source/common/config:version_converter_lib", + "//source/common/config/unified_mux:grpc_subscription_lib", + "//source/common/grpc:common_lib", + "//test/mocks/config:config_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/runtime:runtime_mocks", + "@envoy_api//envoy/config/core/v3:pkg_cc_proto", + "@envoy_api//envoy/config/endpoint/v3:pkg_cc_proto", + "@envoy_api//envoy/service/discovery/v3:pkg_cc_proto", + ], +) + envoy_cc_test( name = "http_subscription_impl_test", srcs = ["http_subscription_impl_test.cc"], @@ -275,6 +398,23 @@ envoy_cc_test_library( ], ) +envoy_cc_test( + name = "sotw_subscription_state_test", + srcs = ["sotw_subscription_state_test.cc"], + deps = [ + "//source/common/config:resource_name_lib", + "//source/common/config/unified_mux:sotw_subscription_state_lib", + "//source/common/stats:isolated_store_lib", + "//test/mocks:common_lib", + "//test/mocks/config:config_mocks", + "//test/mocks/event:event_mocks", + "//test/mocks/grpc:grpc_mocks", + "//test/mocks/local_info:local_info_mocks", + "//test/mocks/runtime:runtime_mocks", + "//test/test_common:logging_lib", + ], +) + envoy_cc_test( name = "opaque_resource_decoder_impl_test", srcs = ["opaque_resource_decoder_impl_test.cc"], @@ -321,6 +461,18 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "unified_subscription_impl_test", + srcs = ["unified_subscription_impl_test.cc"], + deps = [ + ":filesystem_subscription_test_harness", + ":http_subscription_test_harness", + ":subscription_test_harness", + ":unified_delta_subscription_test_harness", + ":unified_grpc_subscription_test_harness", + ], +) + envoy_cc_test_library( name = "subscription_test_harness", srcs = ["subscription_test_harness.h"], diff --git a/test/common/config/sotw_subscription_state_test.cc b/test/common/config/sotw_subscription_state_test.cc new file mode 100644 index 0000000000000..34dcd5b4b665b --- /dev/null +++ b/test/common/config/sotw_subscription_state_test.cc @@ -0,0 +1,186 @@ +#include "common/config/resource_name.h" +#include "common/config/unified_mux/sotw_subscription_state.h" +#include "common/config/utility.h" +#include "common/stats/isolated_store_impl.h" + +#include "test/mocks/config/mocks.h" +#include "test/mocks/event/mocks.h" +#include "test/mocks/local_info/mocks.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::NiceMock; +using testing::Throw; +using testing::UnorderedElementsAre; + +namespace Envoy { +namespace Config { +namespace { + +class SotwSubscriptionStateTest : public testing::Test { +protected: + SotwSubscriptionStateTest() + : state_(Config::getTypeUrl( + envoy::config::core::v3::ApiVersion::V3), + callbacks_, std::chrono::milliseconds(0U), dispatcher_) { + state_.updateSubscriptionInterest({"name1", "name2", "name3"}, {}); + auto cur_request = getNextDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names(), UnorderedElementsAre("name1", "name2", "name3")); + } + + std::unique_ptr getNextDiscoveryRequestAckless() { + auto* ptr = static_cast(state_.getNextRequestAckless()); + return std::unique_ptr(ptr); + } + + UpdateAck deliverDiscoveryResponse(const std::vector& resource_names, + const std::string& version_info, const std::string& nonce) { + envoy::service::discovery::v3::DiscoveryResponse response; + response.set_version_info(version_info); + response.set_nonce(nonce); + response.set_type_url(Config::getTypeUrl( + envoy::config::core::v3::ApiVersion::V3)); + Protobuf::RepeatedPtrField typed_resources; + for (const auto& resource_name : resource_names) { + envoy::config::endpoint::v3::ClusterLoadAssignment* load_assignment = typed_resources.Add(); + load_assignment->set_cluster_name(resource_name); + response.add_resources()->PackFrom(*load_assignment); + } + EXPECT_CALL(callbacks_, onConfigUpdate(_, version_info)); + return state_.handleResponse(&response); + } + + UpdateAck deliverBadDiscoveryResponse(const std::string& version_info, const std::string& nonce) { + envoy::service::discovery::v3::DiscoveryResponse message; + message.set_version_info(version_info); + message.set_nonce(nonce); + EXPECT_CALL(callbacks_, onConfigUpdate(_, _)).WillOnce(Throw(EnvoyException("oh no"))); + return state_.handleResponse(&message); + } + + NiceMock callbacks_; + NiceMock dispatcher_; + // We start out interested in three resources: name1, name2, and name3. + UnifiedMux::SotwSubscriptionState state_; +}; + +// Basic gaining/losing interest in resources should lead to changes in subscriptions. +TEST_F(SotwSubscriptionStateTest, SubscribeAndUnsubscribe) { + { + state_.updateSubscriptionInterest({"name4"}, {"name1"}); + auto cur_request = getNextDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names(), UnorderedElementsAre("name2", "name3", "name4")); + } + { + state_.updateSubscriptionInterest({"name1"}, {"name3", "name4"}); + auto cur_request = getNextDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names(), UnorderedElementsAre("name1", "name2")); + } +} + +// Unlike delta, if SotW gets multiple interest updates before being able to send a request, they +// all collapse to a single update. However, even if the updates all cancel each other out, there +// still will be a request generated. All of the following tests explore different such cases. +TEST_F(SotwSubscriptionStateTest, RemoveThenAdd) { + state_.updateSubscriptionInterest({}, {"name3"}); + state_.updateSubscriptionInterest({"name3"}, {}); + auto cur_request = getNextDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names(), UnorderedElementsAre("name1", "name2", "name3")); +} + +TEST_F(SotwSubscriptionStateTest, AddThenRemove) { + state_.updateSubscriptionInterest({"name4"}, {}); + state_.updateSubscriptionInterest({}, {"name4"}); + auto cur_request = getNextDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names(), UnorderedElementsAre("name1", "name2", "name3")); +} + +TEST_F(SotwSubscriptionStateTest, AddRemoveAdd) { + state_.updateSubscriptionInterest({"name4"}, {}); + state_.updateSubscriptionInterest({}, {"name4"}); + state_.updateSubscriptionInterest({"name4"}, {}); + auto cur_request = getNextDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names(), + UnorderedElementsAre("name1", "name2", "name3", "name4")); +} + +TEST_F(SotwSubscriptionStateTest, RemoveAddRemove) { + state_.updateSubscriptionInterest({}, {"name3"}); + state_.updateSubscriptionInterest({"name3"}, {}); + state_.updateSubscriptionInterest({}, {"name3"}); + auto cur_request = getNextDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names(), UnorderedElementsAre("name1", "name2")); +} + +TEST_F(SotwSubscriptionStateTest, BothAddAndRemove) { + state_.updateSubscriptionInterest({"name4"}, {"name1", "name2", "name3"}); + state_.updateSubscriptionInterest({"name1", "name2", "name3"}, {"name4"}); + state_.updateSubscriptionInterest({"name4"}, {"name1", "name2", "name3"}); + auto cur_request = getNextDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names(), UnorderedElementsAre("name4")); +} + +TEST_F(SotwSubscriptionStateTest, CumulativeUpdates) { + state_.updateSubscriptionInterest({"name4"}, {}); + state_.updateSubscriptionInterest({"name5"}, {}); + auto cur_request = getNextDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names(), + UnorderedElementsAre("name1", "name2", "name3", "name4", "name5")); +} + +// Verifies that a sequence of good and bad responses from the server all get the appropriate +// ACKs/NACKs from Envoy. +TEST_F(SotwSubscriptionStateTest, AckGenerated) { + // The xDS server's first response includes items for name1 and 2, but not 3. + { + UpdateAck ack = deliverDiscoveryResponse({"name1", "name2"}, "version1", "nonce1"); + EXPECT_EQ("nonce1", ack.nonce_); + EXPECT_EQ(Grpc::Status::WellKnownGrpcStatus::Ok, ack.error_detail_.code()); + } + // The next response updates 1 and 2, and adds 3. + { + UpdateAck ack = deliverDiscoveryResponse({"name1", "name2", "name3"}, "version2", "nonce2"); + EXPECT_EQ("nonce2", ack.nonce_); + EXPECT_EQ(Grpc::Status::WellKnownGrpcStatus::Ok, ack.error_detail_.code()); + } + // The next response tries but fails to update all 3, and so should produce a NACK. + { + UpdateAck ack = deliverBadDiscoveryResponse("version3", "nonce3"); + EXPECT_EQ("nonce3", ack.nonce_); + EXPECT_NE(Grpc::Status::WellKnownGrpcStatus::Ok, ack.error_detail_.code()); + } + // The last response successfully updates all 3. + { + UpdateAck ack = deliverDiscoveryResponse({"name1", "name2", "name3"}, "version4", "nonce4"); + EXPECT_EQ("nonce4", ack.nonce_); + EXPECT_EQ(Grpc::Status::WellKnownGrpcStatus::Ok, ack.error_detail_.code()); + } +} + +TEST_F(SotwSubscriptionStateTest, CheckUpdatePending) { + // Note that the test fixture ctor causes the first request to be "sent", so we start in the + // middle of a stream, with our initially interested resources having been requested already. + EXPECT_FALSE(state_.subscriptionUpdatePending()); + state_.updateSubscriptionInterest({}, {}); // no change + EXPECT_FALSE(state_.subscriptionUpdatePending()); + state_.markStreamFresh(); + EXPECT_TRUE(state_.subscriptionUpdatePending()); // no change, BUT fresh stream + state_.updateSubscriptionInterest({}, {"name3"}); // one removed + EXPECT_TRUE(state_.subscriptionUpdatePending()); + state_.updateSubscriptionInterest({"name3"}, {}); // one added + EXPECT_TRUE(state_.subscriptionUpdatePending()); +} + +TEST_F(SotwSubscriptionStateTest, HandleEstablishmentFailure) { + // Although establishment failure is not supposed to cause an onConfigUpdateFailed() on the + // ultimate actual subscription callbacks, the callbacks reference held is actually to + // the WatchMap, which then calls GrpcSubscriptionImpl(s). It is the GrpcSubscriptionImpl + // that will decline to pass on an onConfigUpdateFailed(ConnectionFailure). + EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, _)); + state_.handleEstablishmentFailure(); +} + +} // namespace +} // namespace Config +} // namespace Envoy diff --git a/test/common/config/subscription_factory_impl_test.cc b/test/common/config/subscription_factory_impl_test.cc index 0687e8aa50ae0..bae70b579e03e 100644 --- a/test/common/config/subscription_factory_impl_test.cc +++ b/test/common/config/subscription_factory_impl_test.cc @@ -72,6 +72,10 @@ class SubscriptionFactoryTest : public testing::Test { SubscriptionFactoryImpl subscription_factory_; }; +class SubscriptionFactoryTestUnifiedOrLegacyMux : public SubscriptionFactoryTest, + public testing::WithParamInterface { +}; + class SubscriptionFactoryTestApiConfigSource : public SubscriptionFactoryTest, public testing::WithParamInterface {}; @@ -83,6 +87,10 @@ TEST_F(SubscriptionFactoryTest, NoConfigSpecifier) { "Missing config source specifier in envoy::config::core::v3::ConfigSource"); } +INSTANTIATE_TEST_SUITE_P(SubscriptionFactoryTestUnifiedOrLegacyMux, + SubscriptionFactoryTestUnifiedOrLegacyMux, + ::testing::Values("true", "false")); + TEST_F(SubscriptionFactoryTest, RestClusterEmpty) { envoy::config::core::v3::ConfigSource config; Upstream::ClusterManager::ClusterSet primary_clusters; @@ -94,7 +102,11 @@ TEST_F(SubscriptionFactoryTest, RestClusterEmpty) { "API configs must have either a gRPC service or a cluster name defined:"); } -TEST_F(SubscriptionFactoryTest, GrpcClusterEmpty) { +TEST_P(SubscriptionFactoryTestUnifiedOrLegacyMux, GrpcClusterEmpty) { + TestScopedRuntime scoped_runtime; + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.unified_mux", GetParam()}}); + envoy::config::core::v3::ConfigSource config; Upstream::ClusterManager::ClusterSet primary_clusters; @@ -120,7 +132,11 @@ TEST_F(SubscriptionFactoryTest, RestClusterSingleton) { subscriptionFromConfigSource(config); } -TEST_F(SubscriptionFactoryTest, GrpcClusterSingleton) { +TEST_P(SubscriptionFactoryTestUnifiedOrLegacyMux, GrpcClusterSingleton) { + TestScopedRuntime scoped_runtime; + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.unified_mux", GetParam()}}); + envoy::config::core::v3::ConfigSource config; Upstream::ClusterManager::ClusterSet primary_clusters; @@ -168,7 +184,11 @@ TEST_F(SubscriptionFactoryTest, RestClusterMultiton) { config.mutable_api_config_source()->GetTypeName())); } -TEST_F(SubscriptionFactoryTest, GrpcClusterMultiton) { +TEST_P(SubscriptionFactoryTestUnifiedOrLegacyMux, GrpcClusterMultiton) { + TestScopedRuntime scoped_runtime; + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.unified_mux", GetParam()}}); + envoy::config::core::v3::ConfigSource config; Upstream::ClusterManager::ClusterSet primary_clusters; @@ -303,7 +323,11 @@ TEST_F(SubscriptionFactoryTest, HttpSubscriptionNoRefreshDelay) { "refresh_delay is required for REST API configuration sources"); } -TEST_F(SubscriptionFactoryTest, GrpcSubscription) { +TEST_P(SubscriptionFactoryTestUnifiedOrLegacyMux, GrpcSubscription) { + TestScopedRuntime scoped_runtime; + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.unified_mux", GetParam()}}); + envoy::config::core::v3::ConfigSource config; auto* api_config_source = config.mutable_api_config_source(); api_config_source->set_api_type(envoy::config::core::v3::ApiConfigSource::GRPC); @@ -338,7 +362,11 @@ TEST_F(SubscriptionFactoryTest, GrpcCollectionSubscriptionBadType) { "envoy.config.endpoint.v3.ClusterLoadAssignment in xdstp:///foo"); } -TEST_F(SubscriptionFactoryTest, GrpcCollectionSubscriptionUnsupportedApiType) { +TEST_P(SubscriptionFactoryTestUnifiedOrLegacyMux, GrpcCollectionSubscriptionUnsupportedApiType) { + TestScopedRuntime scoped_runtime; + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.unified_mux", GetParam()}}); + envoy::config::core::v3::ConfigSource config; auto* api_config_source = config.mutable_api_config_source(); api_config_source->set_api_type(envoy::config::core::v3::ApiConfigSource::DELTA_GRPC); @@ -354,7 +382,11 @@ TEST_F(SubscriptionFactoryTest, GrpcCollectionSubscriptionUnsupportedApiType) { EnvoyException, "Unknown xdstp:// transport API type in api_type: DELTA_GRPC"); } -TEST_F(SubscriptionFactoryTest, GrpcCollectionSubscription) { +TEST_P(SubscriptionFactoryTestUnifiedOrLegacyMux, GrpcCollectionSubscription) { + TestScopedRuntime scoped_runtime; + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.unified_mux", GetParam()}}); + envoy::config::core::v3::ConfigSource config; auto* api_config_source = config.mutable_api_config_source(); api_config_source->set_api_type(envoy::config::core::v3::ApiConfigSource::AGGREGATED_DELTA_GRPC); @@ -365,7 +397,9 @@ TEST_F(SubscriptionFactoryTest, GrpcCollectionSubscription) { EXPECT_CALL(cm_, primaryClusters()).WillOnce(ReturnRef(primary_clusters)); GrpcMuxSharedPtr ads_mux = std::make_shared>(); EXPECT_CALL(cm_, adsMux()).WillOnce(Return(ads_mux)); - EXPECT_CALL(dispatcher_, createTimer_(_)); + if ("false" == GetParam()) { // Legacy grpc mux + EXPECT_CALL(dispatcher_, createTimer_(_)); + } // onConfigUpdateFailed() should not be called for gRPC stream connection failure EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, _)).Times(0); collectionSubscriptionFromUrl("xdstp://foo/envoy.config.endpoint.v3.ClusterLoadAssignment/bar", @@ -374,7 +408,11 @@ TEST_F(SubscriptionFactoryTest, GrpcCollectionSubscription) { } // Use of the V2 transport fails by default. -TEST_F(SubscriptionFactoryTest, LogWarningOnDeprecatedV2Transport) { +TEST_P(SubscriptionFactoryTestUnifiedOrLegacyMux, LogWarningOnDeprecatedV2Transport) { + TestScopedRuntime scoped_runtime; + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.unified_mux", GetParam()}}); + envoy::config::core::v3::ConfigSource config; config.mutable_api_config_source()->set_api_type(envoy::config::core::v3::ApiConfigSource::GRPC); @@ -383,7 +421,6 @@ TEST_F(SubscriptionFactoryTest, LogWarningOnDeprecatedV2Transport) { config.mutable_api_config_source()->add_grpc_services()->mutable_envoy_grpc()->set_cluster_name( "static_cluster"); - TestScopedRuntime scoped_runtime; Upstream::ClusterManager::ClusterSet primary_clusters; primary_clusters.insert("static_cluster"); EXPECT_CALL(cm_, primaryClusters()).WillOnce(ReturnRef(primary_clusters)); @@ -396,7 +433,11 @@ TEST_F(SubscriptionFactoryTest, LogWarningOnDeprecatedV2Transport) { } // Use of AUTO transport fails by default. This will encourage folks to upgrade to explicit V3. -TEST_F(SubscriptionFactoryTest, LogWarningOnDeprecatedAutoTransport) { +TEST_P(SubscriptionFactoryTestUnifiedOrLegacyMux, LogWarningOnDeprecatedAutoTransport) { + TestScopedRuntime scoped_runtime; + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.unified_mux", GetParam()}}); + envoy::config::core::v3::ConfigSource config; config.mutable_api_config_source()->set_api_type(envoy::config::core::v3::ApiConfigSource::GRPC); @@ -405,7 +446,6 @@ TEST_F(SubscriptionFactoryTest, LogWarningOnDeprecatedAutoTransport) { config.mutable_api_config_source()->add_grpc_services()->mutable_envoy_grpc()->set_cluster_name( "static_cluster"); - TestScopedRuntime scoped_runtime; Upstream::ClusterManager::ClusterSet primary_clusters; primary_clusters.insert("static_cluster"); EXPECT_CALL(cm_, primaryClusters()).WillOnce(ReturnRef(primary_clusters)); diff --git a/test/common/config/unified_delta_subscription_impl_test.cc b/test/common/config/unified_delta_subscription_impl_test.cc new file mode 100644 index 0000000000000..591f30e6ae939 --- /dev/null +++ b/test/common/config/unified_delta_subscription_impl_test.cc @@ -0,0 +1,167 @@ +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/config/core/v3/base.pb.h" +#include "envoy/config/endpoint/v3/endpoint.pb.h" +#include "envoy/service/discovery/v3/discovery.pb.h" + +#include "common/buffer/zero_copy_input_stream_impl.h" +#include "common/config/api_version.h" + +#include "test/common/config/unified_delta_subscription_test_harness.h" + +namespace Envoy { +namespace Config { +namespace UnifiedMux { +namespace { + +class DeltaSubscriptionImplTest : public DeltaSubscriptionTestHarness, public testing::Test { +protected: + DeltaSubscriptionImplTest() = default; + + // We need to destroy the subscription before the test's destruction, because the subscription's + // destructor removes its watch from the GrpcMuxDelta, and that removal process involves + // some things held by the test fixture. + void TearDown() override { doSubscriptionTearDown(); } +}; + +TEST_F(DeltaSubscriptionImplTest, UpdateResourcesCausesRequest) { + startSubscription({"name1", "name2", "name3"}); + expectSendMessage({"name4"}, {"name1", "name2"}, Grpc::Status::WellKnownGrpcStatus::Ok, "", {}); + subscription_->updateResourceInterest({"name3", "name4"}); + expectSendMessage({"name1", "name2"}, {}, Grpc::Status::WellKnownGrpcStatus::Ok, "", {}); + subscription_->updateResourceInterest({"name1", "name2", "name3", "name4"}); + expectSendMessage({}, {"name1", "name2"}, Grpc::Status::WellKnownGrpcStatus::Ok, "", {}); + subscription_->updateResourceInterest({"name3", "name4"}); + expectSendMessage({"name1", "name2"}, {}, Grpc::Status::WellKnownGrpcStatus::Ok, "", {}); + subscription_->updateResourceInterest({"name1", "name2", "name3", "name4"}); + expectSendMessage({}, {"name1", "name2", "name3"}, Grpc::Status::WellKnownGrpcStatus::Ok, "", {}); + subscription_->updateResourceInterest({"name4"}); +} + +// Checks that after a pause(), no requests are sent until resume(). +// Also demonstrates the collapsing of subscription interest updates into a single +// request. (This collapsing happens any time multiple updates arrive before a request +// can be sent, not just with pausing: rate limiting or a down gRPC stream would also do it). +TEST_F(DeltaSubscriptionImplTest, PauseHoldsRequest) { + startSubscription({"name1", "name2", "name3"}); + auto resume_sub = subscription_->pause(); + // If nested pause wasn't handled correctly, the single expectedSendMessage below would be + // insufficient. + auto nested_resume_sub = subscription_->pause(); + + expectSendMessage({"name4"}, {"name1", "name2"}, Grpc::Status::WellKnownGrpcStatus::Ok, "", {}); + // If not for the pause, these updates would make the expectSendMessage fail due to too many + // messages being sent. + subscription_->updateResourceInterest({"name3", "name4"}); + subscription_->updateResourceInterest({"name1", "name2", "name3", "name4"}); + subscription_->updateResourceInterest({"name3", "name4"}); + subscription_->updateResourceInterest({"name1", "name2", "name3", "name4"}); + subscription_->updateResourceInterest({"name3", "name4"}); +} + +TEST_F(DeltaSubscriptionImplTest, ResponseCausesAck) { + startSubscription({"name1"}); + deliverConfigUpdate({"name1"}, "someversion", true); +} + +// Checks that after a pause(), no ACK requests are sent until resume(), but that after the +// resume, *all* ACKs that arrived during the pause are sent (in order). +TEST_F(DeltaSubscriptionImplTest, PauseQueuesAcks) { + startSubscription({"name1", "name2", "name3"}); + auto resume_sub = subscription_->pause(); + // The server gives us our first version of resource name1. + // subscription_ now wants to ACK name1 (but can't due to pause). + { + auto message = std::make_unique(); + auto* resource = message->mutable_resources()->Add(); + resource->set_name("name1"); + resource->set_version("version1A"); + const std::string nonce = std::to_string(HashUtil::xxHash64("version1A")); + message->set_nonce(nonce); + message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); + nonce_acks_required_.push(nonce); + auto shared_mux = subscription_->getGrpcMuxForTest(); + static_cast(shared_mux.get()) + ->onDiscoveryResponse(std::move(message), control_plane_stats_); + } + // The server gives us our first version of resource name2. + // subscription_ now wants to ACK name1 and then name2 (but can't due to pause). + { + auto message = std::make_unique(); + auto* resource = message->mutable_resources()->Add(); + resource->set_name("name2"); + resource->set_version("version2A"); + const std::string nonce = std::to_string(HashUtil::xxHash64("version2A")); + message->set_nonce(nonce); + message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); + nonce_acks_required_.push(nonce); + auto shared_mux = subscription_->getGrpcMuxForTest(); + static_cast(shared_mux.get()) + ->onDiscoveryResponse(std::move(message), control_plane_stats_); + } + // The server gives us an updated version of resource name1. + // subscription_ now wants to ACK name1A, then name2, then name1B (but can't due to pause). + { + auto message = std::make_unique(); + auto* resource = message->mutable_resources()->Add(); + resource->set_name("name1"); + resource->set_version("version1B"); + const std::string nonce = std::to_string(HashUtil::xxHash64("version1B")); + message->set_nonce(nonce); + message->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); + nonce_acks_required_.push(nonce); + auto shared_mux = subscription_->getGrpcMuxForTest(); + static_cast(shared_mux.get()) + ->onDiscoveryResponse(std::move(message), control_plane_stats_); + } + // All ACK sendMessage()s will happen upon calling resume(). + EXPECT_CALL(async_stream_, sendMessageRaw_(_, _)) + .WillRepeatedly(Invoke([this](Buffer::InstancePtr& buffer, bool) { + API_NO_BOOST(envoy::api::v2::DeltaDiscoveryRequest) message; + EXPECT_TRUE(Grpc::Common::parseBufferInstance(std::move(buffer), message)); + const std::string nonce = message.response_nonce(); + if (!nonce.empty()) { + nonce_acks_sent_.push(nonce); + } + })); + // DeltaSubscriptionTestHarness's dtor will check that all ACKs were sent with the correct nonces, + // in the correct order. +} + +TEST(DeltaSubscriptionImplFixturelessTest, NoGrpcStream) { + Stats::IsolatedStoreImpl stats_store; + SubscriptionStats stats(Utility::generateStats(stats_store)); + + envoy::config::core::v3::Node node; + node.set_id("fo0"); + NiceMock local_info; + EXPECT_CALL(local_info, node()).WillRepeatedly(testing::ReturnRef(node)); + + NiceMock dispatcher; + NiceMock random; + Envoy::Config::RateLimitSettings rate_limit_settings; + NiceMock callbacks; + NiceMock resource_decoder; + auto* async_client = new Grpc::MockAsyncClient(); + + const Protobuf::MethodDescriptor* method_descriptor = + Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.api.v2.EndpointDiscoveryService.StreamEndpoints"); + std::shared_ptr xds_context = std::make_shared( + std::unique_ptr(async_client), dispatcher, *method_descriptor, + envoy::config::core::v3::ApiVersion::AUTO, random, stats_store, rate_limit_settings, + local_info, true); + + auto subscription = std::make_unique( + xds_context, Config::TypeUrl::get().ClusterLoadAssignment, callbacks, resource_decoder, stats, + dispatcher.timeSource(), std::chrono::milliseconds(12345), false, false); + + EXPECT_CALL(*async_client, startRaw(_, _, _, _)).WillOnce(Return(nullptr)); + + subscription->start({"name1"}); + subscription->updateResourceInterest({"name1", "name2"}); +} + +} // namespace +} // namespace UnifiedMux +} // namespace Config +} // namespace Envoy diff --git a/test/common/config/unified_delta_subscription_state_test.cc b/test/common/config/unified_delta_subscription_state_test.cc new file mode 100644 index 0000000000000..efb5760ab28c4 --- /dev/null +++ b/test/common/config/unified_delta_subscription_state_test.cc @@ -0,0 +1,540 @@ +#include + +#include "envoy/config/cluster/v3/cluster.pb.h" +#include "envoy/service/discovery/v3/discovery.pb.h" + +#include "common/config/unified_mux/delta_subscription_state.h" +#include "common/config/utility.h" +#include "common/stats/isolated_store_impl.h" + +#include "test/mocks/config/mocks.h" +#include "test/mocks/event/mocks.h" +#include "test/mocks/local_info/mocks.h" +#include "test/test_common/simulated_time_system.h" +#include "test/test_common/test_runtime.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::NiceMock; +using testing::Throw; +using testing::UnorderedElementsAre; + +namespace Envoy { +namespace Config { +namespace UnifiedMux { +namespace { + +const char TypeUrl[] = "type.googleapis.com/envoy.api.v2.Cluster"; + +class DeltaSubscriptionStateTestBase : public testing::Test { +protected: + DeltaSubscriptionStateTestBase(const std::string& type_url) + : timer_(new Event::MockTimer(&dispatcher_)), + state_(type_url, callbacks_, std::chrono::milliseconds(0U), dispatcher_) { + state_.updateSubscriptionInterest({"name1", "name2", "name3"}, {}); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names_subscribe(), + UnorderedElementsAre("name1", "name2", "name3")); + } + + std::unique_ptr + getNextDeltaDiscoveryRequestAckless() { + auto* ptr = static_cast( + state_.getNextRequestAckless()); + return std::unique_ptr(ptr); + } + + UpdateAck deliverDiscoveryResponse( + const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& version_info, absl::optional nonce = absl::nullopt, + bool expect_config_update_call = true, absl::optional updated_resources = {}) { + envoy::service::discovery::v3::DeltaDiscoveryResponse message; + *message.mutable_resources() = added_resources; + *message.mutable_removed_resources() = removed_resources; + message.set_system_version_info(version_info); + if (nonce.has_value()) { + message.set_nonce(nonce.value()); + } + EXPECT_CALL(callbacks_, onConfigUpdate(_, _, _)) + .Times(expect_config_update_call ? 1 : 0) + .WillRepeatedly(Invoke([updated_resources](const auto& added, const auto&, const auto&) { + if (updated_resources) { + EXPECT_EQ(added.size(), *updated_resources); + } + })); + return state_.handleResponse(&message); + } + + UpdateAck deliverBadDiscoveryResponse( + const Protobuf::RepeatedPtrField& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& version_info, std::string nonce, std::string error_message) { + envoy::service::discovery::v3::DeltaDiscoveryResponse message; + *message.mutable_resources() = added_resources; + *message.mutable_removed_resources() = removed_resources; + message.set_system_version_info(version_info); + message.set_nonce(nonce); + EXPECT_CALL(callbacks_, onConfigUpdate(_, _, _)).WillOnce(Throw(EnvoyException(error_message))); + return state_.handleResponse(&message); + } + + NiceMock callbacks_; + NiceMock dispatcher_; + Event::MockTimer* timer_; + // We start out interested in three resources: name1, name2, and name3. + DeltaSubscriptionState state_; +}; + +Protobuf::RepeatedPtrField +populateRepeatedResource(std::vector> items) { + Protobuf::RepeatedPtrField add_to; + for (const auto& item : items) { + auto* resource = add_to.Add(); + resource->set_name(item.first); + resource->set_version(item.second); + } + return add_to; +} + +class DeltaSubscriptionStateTest : public DeltaSubscriptionStateTestBase { +public: + DeltaSubscriptionStateTest() : DeltaSubscriptionStateTestBase(TypeUrl) {} +}; + +// Basic gaining/losing interest in resources should lead to subscription updates. +TEST_F(DeltaSubscriptionStateTest, SubscribeAndUnsubscribe) { + { + state_.updateSubscriptionInterest({"name4"}, {"name1"}); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name4")); + EXPECT_THAT(cur_request->resource_names_unsubscribe(), UnorderedElementsAre("name1")); + } + { + state_.updateSubscriptionInterest({"name1"}, {"name3", "name4"}); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name1")); + EXPECT_THAT(cur_request->resource_names_unsubscribe(), UnorderedElementsAre("name3", "name4")); + } +} + +// Delta xDS reliably queues up and sends all discovery requests, even in situations where it isn't +// strictly necessary. E.g.: if you subscribe but then unsubscribe to a given resource, all before a +// request was able to be sent, two requests will be sent. The following tests demonstrate this. +// +// If Envoy decided it wasn't interested in a resource and then (before a request was sent) decided +// it was again, for all we know, it dropped that resource in between and needs to retrieve it +// again. So, we *should* send a request "re-"subscribing. This means that the server needs to +// interpret the resource_names_subscribe field as "send these resources even if you think Envoy +// already has them". +TEST_F(DeltaSubscriptionStateTest, RemoveThenAdd) { + state_.updateSubscriptionInterest({}, {"name3"}); + state_.updateSubscriptionInterest({"name3"}, {}); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name3")); + EXPECT_TRUE(cur_request->resource_names_unsubscribe().empty()); +} + +// Due to how our implementation provides the required behavior tested in RemoveThenAdd, the +// add-then-remove case *also* causes the resource to be referred to in the request (as an +// unsubscribe). +// Unlike the remove-then-add case, this one really is unnecessary, and ideally we would have +// the request simply not include any mention of the resource. Oh well. +// This test is just here to illustrate that this behavior exists, not to enforce that it +// should be like this. What *is* important: the server must happily and cleanly ignore +// "unsubscribe from [resource name I have never before referred to]" requests. +TEST_F(DeltaSubscriptionStateTest, AddThenRemove) { + state_.updateSubscriptionInterest({"name4"}, {}); + state_.updateSubscriptionInterest({}, {"name4"}); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_TRUE(cur_request->resource_names_subscribe().empty()); + EXPECT_THAT(cur_request->resource_names_unsubscribe(), UnorderedElementsAre("name4")); +} + +// add/remove/add == add. +TEST_F(DeltaSubscriptionStateTest, AddRemoveAdd) { + state_.updateSubscriptionInterest({"name4"}, {}); + state_.updateSubscriptionInterest({}, {"name4"}); + state_.updateSubscriptionInterest({"name4"}, {}); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name4")); + EXPECT_TRUE(cur_request->resource_names_unsubscribe().empty()); +} + +// remove/add/remove == remove. +TEST_F(DeltaSubscriptionStateTest, RemoveAddRemove) { + state_.updateSubscriptionInterest({}, {"name3"}); + state_.updateSubscriptionInterest({"name3"}, {}); + state_.updateSubscriptionInterest({}, {"name3"}); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_TRUE(cur_request->resource_names_subscribe().empty()); + EXPECT_THAT(cur_request->resource_names_unsubscribe(), UnorderedElementsAre("name3")); +} + +// Starts with 1,2,3. 4 is added/removed/added. In those same updates, 1,2,3 are +// removed/added/removed. End result should be 4 added and 1,2,3 removed. +TEST_F(DeltaSubscriptionStateTest, BothAddAndRemove) { + state_.updateSubscriptionInterest({"name4"}, {"name1", "name2", "name3"}); + state_.updateSubscriptionInterest({"name1", "name2", "name3"}, {"name4"}); + state_.updateSubscriptionInterest({"name4"}, {"name1", "name2", "name3"}); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name4")); + EXPECT_THAT(cur_request->resource_names_unsubscribe(), + UnorderedElementsAre("name1", "name2", "name3")); +} + +TEST_F(DeltaSubscriptionStateTest, CumulativeUpdates) { + state_.updateSubscriptionInterest({"name4"}, {}); + state_.updateSubscriptionInterest({"name5"}, {}); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name4", "name5")); + EXPECT_TRUE(cur_request->resource_names_unsubscribe().empty()); +} + +// Verifies that a sequence of good and bad responses from the server all get the appropriate +// ACKs/NACKs from Envoy. +TEST_F(DeltaSubscriptionStateTest, AckGenerated) { + // The xDS server's first response includes items for name1 and 2, but not 3. + { + Protobuf::RepeatedPtrField added_resources = + populateRepeatedResource({{"name1", "version1A"}, {"name2", "version2A"}}); + EXPECT_CALL(*timer_, disableTimer()); + UpdateAck ack = deliverDiscoveryResponse(added_resources, {}, "debug1", "nonce1"); + EXPECT_EQ("nonce1", ack.nonce_); + EXPECT_EQ(Grpc::Status::WellKnownGrpcStatus::Ok, ack.error_detail_.code()); + } + // The next response updates 1 and 2, and adds 3. + { + Protobuf::RepeatedPtrField added_resources = + populateRepeatedResource( + {{"name1", "version1B"}, {"name2", "version2B"}, {"name3", "version3A"}}); + EXPECT_CALL(*timer_, disableTimer()); + UpdateAck ack = deliverDiscoveryResponse(added_resources, {}, "debug2", "nonce2"); + EXPECT_EQ("nonce2", ack.nonce_); + EXPECT_EQ(Grpc::Status::WellKnownGrpcStatus::Ok, ack.error_detail_.code()); + } + // The next response tries but fails to update all 3, and so should produce a NACK. + { + Protobuf::RepeatedPtrField added_resources = + populateRepeatedResource( + {{"name1", "version1C"}, {"name2", "version2C"}, {"name3", "version3B"}}); + EXPECT_CALL(*timer_, disableTimer()); + UpdateAck ack = deliverBadDiscoveryResponse(added_resources, {}, "debug3", "nonce3", "oh no"); + EXPECT_EQ("nonce3", ack.nonce_); + EXPECT_NE(Grpc::Status::WellKnownGrpcStatus::Ok, ack.error_detail_.code()); + } + // The last response successfully updates all 3. + { + Protobuf::RepeatedPtrField added_resources = + populateRepeatedResource( + {{"name1", "version1D"}, {"name2", "version2D"}, {"name3", "version3C"}}); + EXPECT_CALL(*timer_, disableTimer()); + UpdateAck ack = deliverDiscoveryResponse(added_resources, {}, "debug4", "nonce4"); + EXPECT_EQ("nonce4", ack.nonce_); + EXPECT_EQ(Grpc::Status::WellKnownGrpcStatus::Ok, ack.error_detail_.code()); + } + // Bad response error detail is truncated if it's too large. + { + const std::string very_large_error_message(1 << 20, 'A'); + Protobuf::RepeatedPtrField added_resources = + populateRepeatedResource( + {{"name1", "version1D"}, {"name2", "version2D"}, {"name3", "version3D"}}); + EXPECT_CALL(*timer_, disableTimer()); + UpdateAck ack = deliverBadDiscoveryResponse(added_resources, {}, "debug5", "nonce5", + very_large_error_message); + EXPECT_EQ("nonce5", ack.nonce_); + EXPECT_NE(Grpc::Status::WellKnownGrpcStatus::Ok, ack.error_detail_.code()); + EXPECT_TRUE(absl::EndsWith(ack.error_detail_.message(), "AAAAAAA...(truncated)")); + EXPECT_LT(ack.error_detail_.message().length(), very_large_error_message.length()); + } +} + +// Tests population of the initial_resource_versions map in the first request of a new stream. +// Tests that +// 1) resources we have a version of are present in the map, +// 2) resources we are interested in but don't have are not present, and +// 3) resources we have lost interest in are not present. +TEST_F(DeltaSubscriptionStateTest, ResourceGoneLeadsToBlankInitialVersion) { + { + // The xDS server's first update includes items for name1 and 2, but not 3. + Protobuf::RepeatedPtrField add1_2 = + populateRepeatedResource({{"name1", "version1A"}, {"name2", "version2A"}}); + EXPECT_CALL(*timer_, disableTimer()); + deliverDiscoveryResponse(add1_2, {}, "debugversion1"); + state_.markStreamFresh(); // simulate a stream reconnection + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_EQ("version1A", cur_request->initial_resource_versions().at("name1")); + EXPECT_EQ("version2A", cur_request->initial_resource_versions().at("name2")); + EXPECT_EQ(cur_request->initial_resource_versions().end(), + cur_request->initial_resource_versions().find("name3")); + } + + { + // The next update updates 1, removes 2, and adds 3. The map should then have 1 and 3. + Protobuf::RepeatedPtrField add1_3 = + populateRepeatedResource({{"name1", "version1B"}, {"name3", "version3A"}}); + Protobuf::RepeatedPtrField remove2; + *remove2.Add() = "name2"; + EXPECT_CALL(*timer_, disableTimer()).Times(2); + deliverDiscoveryResponse(add1_3, remove2, "debugversion2"); + state_.markStreamFresh(); // simulate a stream reconnection + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_EQ("version1B", cur_request->initial_resource_versions().at("name1")); + EXPECT_EQ(cur_request->initial_resource_versions().end(), + cur_request->initial_resource_versions().find("name2")); + EXPECT_EQ("version3A", cur_request->initial_resource_versions().at("name3")); + } + + { + // The next update removes 1 and 3. The map we send the server should be empty... + Protobuf::RepeatedPtrField remove1_3; + *remove1_3.Add() = "name1"; + *remove1_3.Add() = "name3"; + deliverDiscoveryResponse({}, remove1_3, "debugversion3"); + state_.markStreamFresh(); // simulate a stream reconnection + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_TRUE(cur_request->initial_resource_versions().empty()); + } + + { + // ...but our own map should remember our interest. In particular, losing interest in a + // resource should cause its name to appear in the next request's resource_names_unsubscribe. + state_.updateSubscriptionInterest({"name4"}, {"name1", "name2"}); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_THAT(cur_request->resource_names_subscribe(), UnorderedElementsAre("name4")); + EXPECT_THAT(cur_request->resource_names_unsubscribe(), UnorderedElementsAre("name1", "name2")); + } +} + +// Upon a reconnection, the server is supposed to assume a blank slate for the Envoy's state +// (hence the need for initial_resource_versions). The resource_names_subscribe of the first +// message must therefore be every resource the Envoy is interested in. +// +// resource_names_unsubscribe, on the other hand, is always blank in the first request - even if, +// in between the last request of the last stream and the first request of the new stream, Envoy +// lost interest in a resource. The unsubscription implicitly takes effect by simply saying +// nothing about the resource in the newly reconnected stream. +TEST_F(DeltaSubscriptionStateTest, SubscribeAndUnsubscribeAfterReconnect) { + Protobuf::RepeatedPtrField add1_2 = + populateRepeatedResource({{"name1", "version1A"}, {"name2", "version2A"}}); + EXPECT_CALL(*timer_, disableTimer()); + deliverDiscoveryResponse(add1_2, {}, "debugversion1"); + + state_.updateSubscriptionInterest({"name4"}, {"name1"}); + state_.markStreamFresh(); // simulate a stream reconnection + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + // Regarding the resource_names_subscribe field: + // name1: do not include: we lost interest. + // name2: yes do include: we're interested and we have a version of it. + // name3: yes do include: even though we don't have a version of it, we are interested. + // name4: yes do include: we are newly interested. (If this wasn't a stream reconnect, only + // name4 + // would belong in this subscribe field). + EXPECT_THAT(cur_request->resource_names_subscribe(), + UnorderedElementsAre("name2", "name3", "name4")); + EXPECT_TRUE(cur_request->resource_names_unsubscribe().empty()); +} + +// initial_resource_versions should not be present on messages after the first in a stream. +TEST_F(DeltaSubscriptionStateTest, InitialVersionMapFirstMessageOnly) { + // First, verify that the first message of a new stream sends initial versions. + { + // The xDS server's first update gives us all three resources. + Protobuf::RepeatedPtrField add_all = + populateRepeatedResource( + {{"name1", "version1A"}, {"name2", "version2A"}, {"name3", "version3A"}}); + EXPECT_CALL(*timer_, disableTimer()); + deliverDiscoveryResponse(add_all, {}, "debugversion1"); + state_.markStreamFresh(); // simulate a stream reconnection + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_EQ("version1A", cur_request->initial_resource_versions().at("name1")); + EXPECT_EQ("version2A", cur_request->initial_resource_versions().at("name2")); + EXPECT_EQ("version3A", cur_request->initial_resource_versions().at("name3")); + } + // Then, after updating the resources but not reconnecting the stream, verify that initial + // versions are not sent. + { + state_.updateSubscriptionInterest({"name4"}, {}); + // The xDS server updates our resources, and gives us our newly requested one too. + Protobuf::RepeatedPtrField add_all = + populateRepeatedResource({{"name1", "version1B"}, + {"name2", "version2B"}, + {"name3", "version3B"}, + {"name4", "version4A"}}); + EXPECT_CALL(*timer_, disableTimer()); + deliverDiscoveryResponse(add_all, {}, "debugversion2"); + auto cur_request = getNextDeltaDiscoveryRequestAckless(); + EXPECT_TRUE(cur_request->initial_resource_versions().empty()); + } +} + +TEST_F(DeltaSubscriptionStateTest, CheckUpdatePending) { + // Note that the test fixture ctor causes the first request to be "sent", so we start in the + // middle of a stream, with our initially interested resources having been requested already. + EXPECT_FALSE(state_.subscriptionUpdatePending()); + state_.updateSubscriptionInterest({}, {}); // no change + EXPECT_FALSE(state_.subscriptionUpdatePending()); + state_.markStreamFresh(); + EXPECT_TRUE(state_.subscriptionUpdatePending()); // no change, BUT fresh stream + state_.updateSubscriptionInterest({}, {"name3"}); // one removed + EXPECT_TRUE(state_.subscriptionUpdatePending()); + state_.updateSubscriptionInterest({"name3"}, {}); // one added + EXPECT_TRUE(state_.subscriptionUpdatePending()); +} + +// The next three tests test that duplicate resource names (whether additions or removals) cause +// DeltaSubscriptionState to reject the update without even trying to hand it to the consuming +// API's onConfigUpdate(). +TEST_F(DeltaSubscriptionStateTest, DuplicatedAdd) { + Protobuf::RepeatedPtrField additions = + populateRepeatedResource({{"name1", "version1A"}, {"name1", "sdfsdfsdfds"}}); + UpdateAck ack = deliverDiscoveryResponse(additions, {}, "debugversion1", absl::nullopt, false); + EXPECT_EQ("duplicate name name1 found among added/updated resources", + ack.error_detail_.message()); +} + +TEST_F(DeltaSubscriptionStateTest, DuplicatedRemove) { + Protobuf::RepeatedPtrField removals; + *removals.Add() = "name1"; + *removals.Add() = "name1"; + UpdateAck ack = deliverDiscoveryResponse({}, removals, "debugversion1", absl::nullopt, false); + EXPECT_EQ("duplicate name name1 found in the union of added+removed resources", + ack.error_detail_.message()); +} + +TEST_F(DeltaSubscriptionStateTest, AddedAndRemoved) { + Protobuf::RepeatedPtrField additions = + populateRepeatedResource({{"name1", "version1A"}}); + Protobuf::RepeatedPtrField removals; + *removals.Add() = "name1"; + UpdateAck ack = + deliverDiscoveryResponse(additions, removals, "debugversion1", absl::nullopt, false); + EXPECT_EQ("duplicate name name1 found in the union of added+removed resources", + ack.error_detail_.message()); +} + +TEST_F(DeltaSubscriptionStateTest, HandleEstablishmentFailure) { + // Although establishment failure is not supposed to cause an onConfigUpdateFailed() on the + // ultimate actual subscription callbacks, DeltaSubscriptionState's callbacks are actually + // the WatchMap, which then calls GrpcSubscriptionImpl(s). It is the GrpcSubscriptionImpl + // that will decline to pass on an onConfigUpdateFailed(ConnectionFailure). + EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, _)); + state_.handleEstablishmentFailure(); +} + +TEST_F(DeltaSubscriptionStateTest, ResourceTTL) { + Event::SimulatedTimeSystem time_system; + time_system.setSystemTime(std::chrono::milliseconds(0)); + + auto create_resource_with_ttl = [](absl::optional ttl_s, + bool include_resource) { + Protobuf::RepeatedPtrField added_resources; + auto* resource = added_resources.Add(); + resource->set_name("name1"); + resource->set_version("version1A"); + + if (include_resource) { + resource->mutable_resource(); + } + + if (ttl_s) { + ProtobufWkt::Duration ttl; + ttl.set_seconds(ttl_s->count()); + resource->mutable_ttl()->CopyFrom(ttl); + } + + return added_resources; + }; + + { + EXPECT_CALL(*timer_, enabled()); + EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(1000), _)); + deliverDiscoveryResponse(create_resource_with_ttl(std::chrono::seconds(1), true), {}, "debug1", + "nonce1"); + } + + { + // Increase the TTL. + EXPECT_CALL(*timer_, enabled()); + EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(2000), _)); + deliverDiscoveryResponse(create_resource_with_ttl(std::chrono::seconds(2), true), {}, "debug1", + "nonce1", true, 1); + } + + { + // Refresh the TTL with a heartbeat. The resource should not be passed to the update callbacks. + EXPECT_CALL(*timer_, enabled()); + deliverDiscoveryResponse(create_resource_with_ttl(std::chrono::seconds(2), false), {}, "debug1", + "nonce1", true, 0); + } + + // Remove the TTL. + EXPECT_CALL(*timer_, disableTimer()); + deliverDiscoveryResponse(create_resource_with_ttl(absl::nullopt, true), {}, "debug1", "nonce1", + true, 1); + + // Add back the TTL. + EXPECT_CALL(*timer_, enabled()); + EXPECT_CALL(*timer_, enableTimer(_, _)); + deliverDiscoveryResponse(create_resource_with_ttl(std::chrono::seconds(2), true), {}, "debug1", + "nonce1"); + + EXPECT_CALL(callbacks_, onConfigUpdate(_, _, _)); + EXPECT_CALL(*timer_, disableTimer()); + time_system.setSystemTime(std::chrono::seconds(2)); + + // Invoke the TTL. + timer_->invokeCallback(); +} + +class VhdsDeltaSubscriptionStateTest : public DeltaSubscriptionStateTestBase { +public: + VhdsDeltaSubscriptionStateTest() + : DeltaSubscriptionStateTestBase("envoy.config.route.v3.VirtualHost") {} +}; + +TEST_F(VhdsDeltaSubscriptionStateTest, ResourceTTL) { + Event::SimulatedTimeSystem time_system; + time_system.setSystemTime(std::chrono::milliseconds(0)); + + TestScopedRuntime scoped_runtime; + + auto create_resource_with_ttl = [](bool include_resource) { + Protobuf::RepeatedPtrField added_resources; + auto* resource = added_resources.Add(); + resource->set_name("name1"); + resource->set_version("version1A"); + + if (include_resource) { + resource->mutable_resource(); + } + + ProtobufWkt::Duration ttl; + ttl.set_seconds(1); + resource->mutable_ttl()->CopyFrom(ttl); + + return added_resources; + }; + + EXPECT_CALL(*timer_, enabled()); + EXPECT_CALL(*timer_, enableTimer(std::chrono::milliseconds(1000), _)); + deliverDiscoveryResponse(create_resource_with_ttl(true), {}, "debug1", "nonce1", true, 1); + + // Heartbeat update should not be propagated to the subscription callback. + EXPECT_CALL(*timer_, enabled()); + deliverDiscoveryResponse(create_resource_with_ttl(false), {}, "debug1", "nonce1", true, 0); + + // When runtime flag is disabled, maintain old behavior where we do propagate + // the update to the subscription callback. + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.vhds_heartbeats", "false"}}); + + EXPECT_CALL(*timer_, enabled()); + deliverDiscoveryResponse(create_resource_with_ttl(false), {}, "debug1", "nonce1", true, 1); +} + +} // namespace +} // namespace UnifiedMux +} // namespace Config +} // namespace Envoy diff --git a/test/common/config/unified_delta_subscription_test_harness.h b/test/common/config/unified_delta_subscription_test_harness.h new file mode 100644 index 0000000000000..3f6e58f426a41 --- /dev/null +++ b/test/common/config/unified_delta_subscription_test_harness.h @@ -0,0 +1,223 @@ +#pragma once + +#include + +#include "envoy/config/core/v3/base.pb.h" +#include "envoy/config/endpoint/v3/endpoint.pb.h" +#include "envoy/config/endpoint/v3/endpoint.pb.validate.h" +#include "envoy/service/discovery/v3/discovery.pb.h" + +#include "common/config/unified_mux/grpc_mux_impl.h" +#include "common/config/unified_mux/grpc_subscription_impl.h" +#include "common/config/version_converter.h" +#include "common/grpc/common.h" + +#include "test/common/config/subscription_test_harness.h" +#include "test/mocks/common.h" +#include "test/mocks/config/mocks.h" +#include "test/mocks/event/mocks.h" +#include "test/mocks/grpc/mocks.h" +#include "test/mocks/local_info/mocks.h" +#include "test/mocks/stats/mocks.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::Mock; +using testing::NiceMock; +using testing::Return; + +namespace Envoy { +namespace Config { +namespace UnifiedMux { +namespace { + +class DeltaSubscriptionTestHarness : public SubscriptionTestHarness { +public: + DeltaSubscriptionTestHarness() : DeltaSubscriptionTestHarness(std::chrono::milliseconds(0)) {} + DeltaSubscriptionTestHarness(std::chrono::milliseconds init_fetch_timeout) + : method_descriptor_(Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.api.v2.EndpointDiscoveryService.StreamEndpoints")), + async_client_(new Grpc::MockAsyncClient()) { + node_.set_id("fo0"); + EXPECT_CALL(local_info_, node()).WillRepeatedly(testing::ReturnRef(node_)); + EXPECT_CALL(dispatcher_, createTimer_(_)); + grpc_mux_ = std::make_shared( + std::unique_ptr(async_client_), dispatcher_, *method_descriptor_, + envoy::config::core::v3::ApiVersion::AUTO, random_, stats_store_, rate_limit_settings_, + local_info_, false); + subscription_ = std::make_unique( + grpc_mux_, Config::TypeUrl::get().ClusterLoadAssignment, callbacks_, resource_decoder_, + stats_, dispatcher_.timeSource(), init_fetch_timeout, false, false); + + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + } + + void doSubscriptionTearDown() override { + if (subscription_started_) { + EXPECT_CALL(async_stream_, sendMessageRaw_(_, _)); + subscription_.reset(); + } + } + + ~DeltaSubscriptionTestHarness() override { + while (!nonce_acks_required_.empty()) { + if (nonce_acks_sent_.empty()) { + // It's not enough to EXPECT_FALSE(nonce_acks_sent_.empty()), we need to skip the following + // EXPECT_EQ, otherwise the undefined .front() can get pretty bad. + EXPECT_FALSE(nonce_acks_sent_.empty()); + break; + } + EXPECT_EQ(nonce_acks_required_.front(), nonce_acks_sent_.front()); + nonce_acks_required_.pop(); + nonce_acks_sent_.pop(); + } + EXPECT_TRUE(nonce_acks_sent_.empty()); + } + + void startSubscription(const std::set& cluster_names) override { + subscription_started_ = true; + last_cluster_names_ = cluster_names; + expectSendMessage(last_cluster_names_, ""); + ttl_timer_ = new NiceMock(&dispatcher_); + subscription_->start(flattenResources(cluster_names)); + } + + void expectSendMessage(const std::set& cluster_names, const std::string& version, + bool expect_node = false) override { + UNREFERENCED_PARAMETER(version); + UNREFERENCED_PARAMETER(expect_node); + expectSendMessage(cluster_names, {}, Grpc::Status::WellKnownGrpcStatus::Ok, "", {}); + } + + void expectSendMessage(const std::set& subscribe, + const std::set& unsubscribe, const Protobuf::int32 error_code, + const std::string& error_message, + std::map initial_resource_versions) { + API_NO_BOOST(envoy::api::v2::DeltaDiscoveryRequest) expected_request; + expected_request.mutable_node()->CopyFrom(API_DOWNGRADE(node_)); + std::copy( + subscribe.begin(), subscribe.end(), + Protobuf::RepeatedFieldBackInserter(expected_request.mutable_resource_names_subscribe())); + std::copy( + unsubscribe.begin(), unsubscribe.end(), + Protobuf::RepeatedFieldBackInserter(expected_request.mutable_resource_names_unsubscribe())); + if (!last_response_nonce_.empty()) { + nonce_acks_required_.push(last_response_nonce_); + last_response_nonce_ = ""; + } + expected_request.set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); + + for (auto const& resource : initial_resource_versions) { + (*expected_request.mutable_initial_resource_versions())[resource.first] = resource.second; + } + + if (error_code != Grpc::Status::WellKnownGrpcStatus::Ok) { + ::google::rpc::Status* error_detail = expected_request.mutable_error_detail(); + error_detail->set_code(error_code); + error_detail->set_message(error_message); + } + EXPECT_CALL(async_stream_, + sendMessageRaw_( + Grpc::ProtoBufferEqIgnoringField(expected_request, "response_nonce"), false)) + .WillOnce([this](Buffer::InstancePtr& buffer, bool) { + API_NO_BOOST(envoy::api::v2::DeltaDiscoveryRequest) message; + EXPECT_TRUE(Grpc::Common::parseBufferInstance(std::move(buffer), message)); + const std::string nonce = message.response_nonce(); + if (!nonce.empty()) { + nonce_acks_sent_.push(nonce); + } + }); + } + + void deliverConfigUpdate(const std::vector& cluster_names, + const std::string& version, bool accept) override { + auto response = std::make_unique(); + last_response_nonce_ = std::to_string(HashUtil::xxHash64(version)); + response->set_nonce(last_response_nonce_); + response->set_system_version_info(version); + response->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); + + Protobuf::RepeatedPtrField typed_resources; + for (const auto& cluster : cluster_names) { + if (std::find(last_cluster_names_.begin(), last_cluster_names_.end(), cluster) != + last_cluster_names_.end()) { + envoy::config::endpoint::v3::ClusterLoadAssignment* load_assignment = typed_resources.Add(); + load_assignment->set_cluster_name(cluster); + auto* resource = response->add_resources(); + resource->set_name(cluster); + resource->set_version(version); + resource->mutable_resource()->PackFrom(*load_assignment); + } + } + Protobuf::RepeatedPtrField removed_resources; + EXPECT_CALL(callbacks_, onConfigUpdate(_, _, version)).WillOnce(ThrowOnRejectedConfig(accept)); + if (accept) { + expectSendMessage({}, version); + } else { + EXPECT_CALL(callbacks_, onConfigUpdateFailed( + Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, _)); + expectSendMessage({}, {}, Grpc::Status::WellKnownGrpcStatus::Internal, "bad config", {}); + } + auto shared_mux = subscription_->getGrpcMuxForTest(); + static_cast(shared_mux.get()) + ->onDiscoveryResponse(std::move(response), control_plane_stats_); + Mock::VerifyAndClearExpectations(&async_stream_); + } + + void updateResourceInterest(const std::set& cluster_names) override { + std::set sub; + std::set unsub; + + std::set_difference(cluster_names.begin(), cluster_names.end(), last_cluster_names_.begin(), + last_cluster_names_.end(), std::inserter(sub, sub.begin())); + std::set_difference(last_cluster_names_.begin(), last_cluster_names_.end(), + cluster_names.begin(), cluster_names.end(), + std::inserter(unsub, unsub.begin())); + + expectSendMessage(sub, unsub, Grpc::Status::WellKnownGrpcStatus::Ok, "", {}); + subscription_->updateResourceInterest(flattenResources(cluster_names)); + last_cluster_names_ = cluster_names; + } + + void expectConfigUpdateFailed() override { + EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, nullptr)); + } + + void expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds timeout) override { + init_timeout_timer_ = new Event::MockTimer(&dispatcher_); + EXPECT_CALL(*init_timeout_timer_, enableTimer(timeout, _)); + } + + void expectDisableInitFetchTimeoutTimer() override { + EXPECT_CALL(*init_timeout_timer_, disableTimer()); + } + + void callInitFetchTimeoutCb() override { init_timeout_timer_->invokeCallback(); } + + const Protobuf::MethodDescriptor* method_descriptor_; + Grpc::MockAsyncClient* async_client_; + Event::MockDispatcher dispatcher_; + NiceMock random_; + NiceMock local_info_; + Grpc::MockAsyncStream async_stream_; + std::shared_ptr grpc_mux_; + std::unique_ptr subscription_; + std::string last_response_nonce_; + std::set last_cluster_names_; + Envoy::Config::RateLimitSettings rate_limit_settings_; + Event::MockTimer* ttl_timer_; + Event::MockTimer* init_timeout_timer_; + envoy::config::core::v3::Node node_; + NiceMock callbacks_; + TestUtility::TestOpaqueResourceDecoderImpl + resource_decoder_{"cluster_name"}; + std::queue nonce_acks_required_; + std::queue nonce_acks_sent_; + bool subscription_started_{}; +}; + +} // namespace +} // namespace UnifiedMux +} // namespace Config +} // namespace Envoy diff --git a/test/common/config/unified_grpc_mux_impl_test.cc b/test/common/config/unified_grpc_mux_impl_test.cc new file mode 100644 index 0000000000000..19e3c7996ecc1 --- /dev/null +++ b/test/common/config/unified_grpc_mux_impl_test.cc @@ -0,0 +1,1122 @@ +#include + +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/config/endpoint/v3/endpoint.pb.h" +#include "envoy/config/endpoint/v3/endpoint.pb.validate.h" +#include "envoy/config/route/v3/route_components.pb.h" +#include "envoy/config/route/v3/route_components.pb.validate.h" +#include "envoy/service/discovery/v3/discovery.pb.h" + +#include "common/common/empty_string.h" +#include "common/config/api_version.h" +#include "common/config/protobuf_link_hacks.h" +#include "common/config/unified_mux/grpc_mux_impl.h" +#include "common/config/utility.h" +#include "common/config/version_converter.h" +#include "common/protobuf/protobuf.h" +#include "common/stats/isolated_store_impl.h" + +#include "test/common/stats/stat_test_utility.h" +#include "test/mocks/common.h" +#include "test/mocks/config/mocks.h" +#include "test/mocks/event/mocks.h" +#include "test/mocks/grpc/mocks.h" +#include "test/mocks/local_info/mocks.h" +#include "test/mocks/runtime/mocks.h" +#include "test/test_common/logging.h" +#include "test/test_common/resources.h" +#include "test/test_common/simulated_time_system.h" +#include "test/test_common/test_runtime.h" +#include "test/test_common/test_time.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::_; +using testing::AtLeast; +using testing::InSequence; +using testing::Invoke; +using testing::IsSubstring; +using testing::NiceMock; +using testing::Return; +using testing::ReturnRef; + +namespace Envoy { +namespace Config { +namespace UnifiedMux { +namespace { + +// We test some mux specific stuff below, other unit test coverage for singleton use of GrpcMuxImpl +// is provided in [grpc_]subscription_impl_test.cc. +class GrpcMuxImplTestBase : public testing::Test { +public: + GrpcMuxImplTestBase() + : async_client_(new Grpc::MockAsyncClient()), + control_plane_stats_(Utility::generateControlPlaneStats(stats_)), + control_plane_connected_state_( + stats_.gauge("control_plane.connected_state", Stats::Gauge::ImportMode::NeverImport)), + control_plane_pending_requests_( + stats_.gauge("control_plane.pending_requests", Stats::Gauge::ImportMode::NeverImport)), + resource_decoder_(TestUtility::TestOpaqueResourceDecoderImpl< + envoy::config::endpoint::v3::ClusterLoadAssignment>("cluster_name")) {} + + void setup() { + grpc_mux_ = std::make_unique( + std::unique_ptr(async_client_), dispatcher_, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), + envoy::config::core::v3::ApiVersion::AUTO, random_, stats_, rate_limit_settings_, + local_info_, true); + } + + void setup(const RateLimitSettings& custom_rate_limit_settings) { + grpc_mux_ = std::make_unique( + std::unique_ptr(async_client_), dispatcher_, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), + envoy::config::core::v3::ApiVersion::AUTO, random_, stats_, custom_rate_limit_settings, + local_info_, true); + } + + void expectSendMessage(const std::string& type_url, + const std::vector& resource_names, const std::string& version, + bool first = false, const std::string& nonce = "", + const Protobuf::int32 error_code = Grpc::Status::WellKnownGrpcStatus::Ok, + const std::string& error_message = "") { + API_NO_BOOST(envoy::api::v2::DiscoveryRequest) expected_request; + if (first) { + expected_request.mutable_node()->CopyFrom(API_DOWNGRADE(local_info_.node())); + } + for (const auto& resource : resource_names) { + expected_request.add_resource_names(resource); + } + if (!version.empty()) { + expected_request.set_version_info(version); + } + expected_request.set_response_nonce(nonce); + expected_request.set_type_url(type_url); + if (error_code != Grpc::Status::WellKnownGrpcStatus::Ok) { + ::google::rpc::Status* error_detail = expected_request.mutable_error_detail(); + error_detail->set_code(error_code); + error_detail->set_message(error_message); + } + EXPECT_CALL( + async_stream_, + sendMessageRaw_(Grpc::ProtoBufferEqIgnoreRepeatedFieldOrdering(expected_request), false)); + } + + // These tests were written around GrpcMuxWatch, an RAII type returned by the old subscribe(). + // To preserve these tests for the new code, we need an RAII watch handler. That is + // GrpcSubscriptionImpl, but to keep things simple, we'll fake it. (What we really care about + // is the destructor, which is identical to the real one). + class FakeGrpcSubscription { + public: + FakeGrpcSubscription(GrpcMux* grpc_mux, std::string type_url, Watch* watch) + : grpc_mux_(grpc_mux), type_url_(std::move(type_url)), watch_(watch) {} + ~FakeGrpcSubscription() { grpc_mux_->removeWatch(type_url_, watch_); } + + private: + GrpcMux* const grpc_mux_; + std::string type_url_; + Watch* const watch_; + }; + + FakeGrpcSubscription makeWatch(const std::string& type_url, + const absl::flat_hash_set& resources) { + return FakeGrpcSubscription(grpc_mux_.get(), type_url, + grpc_mux_->addWatch(type_url, resources, callbacks_, + resource_decoder_, + std::chrono::milliseconds(0))); + } + + FakeGrpcSubscription makeWatch(const std::string& type_url, + const absl::flat_hash_set& resources, + NiceMock& callbacks, + Config::OpaqueResourceDecoder& resource_decoder) { + return FakeGrpcSubscription(grpc_mux_.get(), type_url, + grpc_mux_->addWatch(type_url, resources, callbacks, + resource_decoder, + std::chrono::milliseconds(0))); + } + + NiceMock dispatcher_; + NiceMock random_; + Grpc::MockAsyncClient* async_client_; + Grpc::MockAsyncStream async_stream_; + NiceMock local_info_; + std::unique_ptr grpc_mux_; + NiceMock callbacks_; + Stats::TestUtil::TestStore stats_; + ControlPlaneStats control_plane_stats_; + Envoy::Config::RateLimitSettings rate_limit_settings_; + Stats::Gauge& control_plane_connected_state_; + Stats::Gauge& control_plane_pending_requests_; + TestUtility::TestOpaqueResourceDecoderImpl + resource_decoder_; +}; + +class GrpcMuxImplTest : public GrpcMuxImplTestBase { +public: + Event::SimulatedTimeSystem time_system_; +}; + +// Validate behavior when multiple type URL watches are maintained, watches are created/destroyed. +TEST_F(GrpcMuxImplTest, MultipleTypeUrlStreams) { + setup(); + InSequence s; + + FakeGrpcSubscription foo_sub = makeWatch("type_url_foo", {"x", "y"}); + FakeGrpcSubscription bar_sub = makeWatch("type_url_bar", {}); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage("type_url_foo", {"x", "y"}, "", true); + expectSendMessage("type_url_bar", {}, ""); + grpc_mux_->start(); + EXPECT_EQ(1, control_plane_connected_state_.value()); + expectSendMessage("type_url_bar", {"z"}, ""); + FakeGrpcSubscription bar_z_sub = makeWatch("type_url_bar", {"z"}); + expectSendMessage("type_url_bar", {"zz", "z"}, ""); + FakeGrpcSubscription bar_zz_sub = makeWatch("type_url_bar", {"zz"}); + expectSendMessage("type_url_bar", {"z"}, ""); + expectSendMessage("type_url_bar", {}, ""); + expectSendMessage("type_url_foo", {}, ""); +} + +// Validate behavior when multiple type URL watches are maintained and the stream is reset. +TEST_F(GrpcMuxImplTest, ResetStream) { + InSequence s; + + auto* timer = new Event::MockTimer(&dispatcher_); + // TTL timers. + new Event::MockTimer(&dispatcher_); + new Event::MockTimer(&dispatcher_); + new Event::MockTimer(&dispatcher_); + + setup(); + FakeGrpcSubscription foo_sub = makeWatch("type_url_foo", {"x", "y"}); + FakeGrpcSubscription bar_sub = makeWatch("type_url_bar", {}); + FakeGrpcSubscription baz_sub = makeWatch("type_url_baz", {"z"}); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage("type_url_foo", {"x", "y"}, "", true); + expectSendMessage("type_url_bar", {}, ""); + expectSendMessage("type_url_baz", {"z"}, ""); + grpc_mux_->start(); + + // Send another message for foo so that the node is cleared in the cached request. + // This is to test that the the node is set again in the first message below. + expectSendMessage("type_url_foo", {"z", "x", "y"}, ""); + FakeGrpcSubscription foo_z_sub = makeWatch("type_url_foo", {"z"}); + + EXPECT_CALL(callbacks_, + onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)) + .Times(4); + EXPECT_CALL(random_, random()); + EXPECT_CALL(*timer, enableTimer(_, _)); + grpc_mux_->grpcStreamForTest().onRemoteClose(Grpc::Status::WellKnownGrpcStatus::Canceled, ""); + EXPECT_EQ(0, control_plane_connected_state_.value()); + EXPECT_EQ(0, control_plane_pending_requests_.value()); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage("type_url_foo", {"z", "x", "y"}, "", true); + expectSendMessage("type_url_bar", {}, ""); + expectSendMessage("type_url_baz", {"z"}, ""); + expectSendMessage("type_url_foo", {"x", "y"}, ""); + timer->invokeCallback(); + + expectSendMessage("type_url_baz", {}, ""); + expectSendMessage("type_url_foo", {}, ""); +} + +// Validate pause-resume behavior. +TEST_F(GrpcMuxImplTest, PauseResume) { + setup(); + InSequence s; + grpc_mux_->addWatch("type_url_foo", {"x", "y"}, callbacks_, resource_decoder_, + std::chrono::milliseconds(0)); + { + ScopedResume a = grpc_mux_->pause("type_url_foo"); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + grpc_mux_->start(); + expectSendMessage("type_url_foo", {"x", "y"}, "", true); + } + { + ScopedResume a = grpc_mux_->pause("type_url_bar"); + expectSendMessage("type_url_foo", {"z", "x", "y"}, ""); + grpc_mux_->addWatch("type_url_foo", {"z"}, callbacks_, resource_decoder_, + std::chrono::milliseconds(0)); + } + { + ScopedResume a = grpc_mux_->pause("type_url_foo"); + grpc_mux_->addWatch("type_url_foo", {"zz"}, callbacks_, resource_decoder_, + std::chrono::milliseconds(0)); + expectSendMessage("type_url_foo", {"zz", "z", "x", "y"}, ""); + } + // When nesting, we only have a single resumption. + { + ScopedResume a = grpc_mux_->pause("type_url_foo"); + ScopedResume b = grpc_mux_->pause("type_url_foo"); + grpc_mux_->addWatch("type_url_foo", {"zzz"}, callbacks_, resource_decoder_, + std::chrono::milliseconds(0)); + expectSendMessage("type_url_foo", {"zzz", "zz", "z", "x", "y"}, ""); + } + grpc_mux_->pause("type_url_foo")->cancel(); +} + +// Validate behavior when type URL mismatches occur. +TEST_F(GrpcMuxImplTest, TypeUrlMismatch) { + setup(); + + auto invalid_response = std::make_unique(); + InSequence s; + FakeGrpcSubscription foo_sub = makeWatch("type_url_foo", {"x", "y"}); + + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage("type_url_foo", {"x", "y"}, "", true); + grpc_mux_->start(); + + { + auto response = std::make_unique(); + response->set_type_url("type_url_bar"); + response->set_version_info("bar-version"); + grpc_mux_->onDiscoveryResponse(std::move(response), control_plane_stats_); + } + + { + invalid_response->set_type_url("type_url_foo"); + invalid_response->set_version_info("foo-version"); + invalid_response->mutable_resources()->Add()->set_type_url("type_url_bar"); + EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, _)) + .WillOnce(Invoke([](Envoy::Config::ConfigUpdateFailureReason, const EnvoyException* e) { + EXPECT_TRUE( + IsSubstring("", "", + "type URL type_url_bar embedded in an individual Any does not match the " + "message-wide type URL type_url_foo in DiscoveryResponse", + e->what())); + })); + + expectSendMessage( + "type_url_foo", {"x", "y"}, "", false, "", Grpc::Status::WellKnownGrpcStatus::Internal, + fmt::format("type URL type_url_bar embedded in an individual Any does not match the " + "message-wide type URL type_url_foo in DiscoveryResponse {}", + invalid_response->DebugString())); + grpc_mux_->onDiscoveryResponse(std::move(invalid_response), control_plane_stats_); + } + expectSendMessage("type_url_foo", {}, ""); +} + +TEST_F(GrpcMuxImplTest, RpcErrorMessageTruncated) { + setup(); + auto invalid_response = std::make_unique(); + InSequence s; + FakeGrpcSubscription foo_sub = makeWatch("type_url_foo", {"x", "y"}); + + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage("type_url_foo", {"x", "y"}, "", true); + grpc_mux_->start(); + + { // Large error message sent back to management server is truncated. + const std::string very_large_type_url(1 << 20, 'A'); + invalid_response->set_type_url("type_url_foo"); + invalid_response->set_version_info("invalid"); + invalid_response->mutable_resources()->Add()->set_type_url(very_large_type_url); + EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, _)) + .WillOnce(Invoke([&very_large_type_url](Envoy::Config::ConfigUpdateFailureReason, + const EnvoyException* e) { + EXPECT_TRUE( + IsSubstring("", "", + fmt::format("type URL {} embedded in an individual Any does not match " + "the message-wide type URL type_url_foo in DiscoveryResponse", + very_large_type_url), // Local error message is not truncated. + e->what())); + })); + expectSendMessage("type_url_foo", {"x", "y"}, "", false, "", + Grpc::Status::WellKnownGrpcStatus::Internal, + fmt::format("type URL {}...(truncated)", std::string(4087, 'A'))); + grpc_mux_->onDiscoveryResponse(std::move(invalid_response), control_plane_stats_); + } + expectSendMessage("type_url_foo", {}, ""); +} + +envoy::service::discovery::v3::Resource heartbeatResource(std::chrono::milliseconds ttl, + const std::string& name) { + envoy::service::discovery::v3::Resource resource; + + resource.mutable_ttl()->CopyFrom(Protobuf::util::TimeUtil::MillisecondsToDuration(ttl.count())); + resource.set_name(name); + + return resource; +} + +envoy::service::discovery::v3::Resource +resourceWithTtl(std::chrono::milliseconds ttl, + envoy::config::endpoint::v3::ClusterLoadAssignment& cla) { + envoy::service::discovery::v3::Resource resource; + resource.mutable_resource()->PackFrom(cla); + resource.mutable_ttl()->CopyFrom(Protobuf::util::TimeUtil::MillisecondsToDuration(ttl.count())); + + resource.set_name(cla.cluster_name()); + + return resource; +} +envoy::service::discovery::v3::Resource +resourceWithEmptyTtl(envoy::config::endpoint::v3::ClusterLoadAssignment& cla) { + envoy::service::discovery::v3::Resource resource; + resource.mutable_resource()->PackFrom(cla); + resource.set_name(cla.cluster_name()); + return resource; +} +// Validates the behavior when the TTL timer expires. +TEST_F(GrpcMuxImplTest, ResourceTTL) { + setup(); + + time_system_.setSystemTime(std::chrono::seconds(0)); + + TestUtility::TestOpaqueResourceDecoderImpl + resource_decoder("cluster_name"); + const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; + InSequence s; + auto* ttl_timer = new Event::MockTimer(&dispatcher_); + FakeGrpcSubscription eds_sub = makeWatch(type_url, {"x"}, callbacks_, resource_decoder); + + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage(type_url, {"x"}, "", true); + grpc_mux_->start(); + + { + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_version_info("1"); + envoy::config::endpoint::v3::ClusterLoadAssignment load_assignment; + load_assignment.set_cluster_name("x"); + + auto wrapped_resource = resourceWithTtl(std::chrono::milliseconds(1000), load_assignment); + response->add_resources()->PackFrom(wrapped_resource); + + EXPECT_CALL(*ttl_timer, enabled()); + EXPECT_CALL(*ttl_timer, enableTimer(std::chrono::milliseconds(1000), _)); + EXPECT_CALL(callbacks_, onConfigUpdate(_, "1")) + .WillOnce(Invoke([](const std::vector& resources, const std::string&) { + EXPECT_EQ(1, resources.size()); + })); + expectSendMessage(type_url, {"x"}, "1"); + grpc_mux_->onDiscoveryResponse(std::move(response), control_plane_stats_); + } + + // Increase the TTL. + { + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_version_info("1"); + envoy::config::endpoint::v3::ClusterLoadAssignment load_assignment; + load_assignment.set_cluster_name("x"); + auto wrapped_resource = resourceWithTtl(std::chrono::milliseconds(10000), load_assignment); + response->add_resources()->PackFrom(wrapped_resource); + + EXPECT_CALL(*ttl_timer, enabled()); + EXPECT_CALL(*ttl_timer, enableTimer(std::chrono::milliseconds(10000), _)); + EXPECT_CALL(callbacks_, onConfigUpdate(_, "1")) + .WillOnce(Invoke([](const std::vector& resources, const std::string&) { + EXPECT_EQ(1, resources.size()); + })); + // No update, just a change in TTL. + expectSendMessage(type_url, {"x"}, "1"); + grpc_mux_->onDiscoveryResponse(std::move(response), control_plane_stats_); + } + + // Refresh the TTL with a heartbeat response. + { + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_version_info("1"); + auto wrapped_resource = heartbeatResource(std::chrono::milliseconds(10000), "x"); + response->add_resources()->PackFrom(wrapped_resource); + + EXPECT_CALL(*ttl_timer, enabled()); + EXPECT_CALL(callbacks_, onConfigUpdate(_, "1")) + .WillOnce(Invoke([](const std::vector& resources, const std::string&) { + EXPECT_TRUE(resources.empty()); + })); + + // No update, just a change in TTL. + expectSendMessage(type_url, {"x"}, "1"); + grpc_mux_->onDiscoveryResponse(std::move(response), control_plane_stats_); + } + + // Remove the TTL. + { + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_version_info("1"); + envoy::config::endpoint::v3::ClusterLoadAssignment load_assignment; + load_assignment.set_cluster_name("x"); + response->add_resources()->PackFrom(resourceWithEmptyTtl(load_assignment)); + + EXPECT_CALL(*ttl_timer, disableTimer()); + EXPECT_CALL(callbacks_, onConfigUpdate(_, "1")) + .WillOnce(Invoke([](const std::vector& resources, const std::string&) { + EXPECT_EQ(1, resources.size()); + })); + expectSendMessage(type_url, {"x"}, "1"); + grpc_mux_->onDiscoveryResponse(std::move(response), control_plane_stats_); + } + + // Put the TTL back. + { + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_version_info("1"); + envoy::config::endpoint::v3::ClusterLoadAssignment load_assignment; + load_assignment.set_cluster_name("x"); + auto wrapped_resource = resourceWithTtl(std::chrono::milliseconds(10000), load_assignment); + response->add_resources()->PackFrom(wrapped_resource); + + EXPECT_CALL(*ttl_timer, enabled()); + EXPECT_CALL(*ttl_timer, enableTimer(std::chrono::milliseconds(10000), _)); + EXPECT_CALL(callbacks_, onConfigUpdate(_, "1")) + .WillOnce(Invoke([](const std::vector& resources, const std::string&) { + EXPECT_EQ(1, resources.size()); + })); + // No update, just a change in TTL. + expectSendMessage(type_url, {"x"}, "1"); + grpc_mux_->onDiscoveryResponse(std::move(response), control_plane_stats_); + } + + time_system_.setSystemTime(std::chrono::seconds(11)); + EXPECT_CALL(callbacks_, onConfigUpdate(_, _, "")) + .WillOnce(Invoke([](auto, const auto& removed, auto) { + EXPECT_EQ(1, removed.size()); + EXPECT_EQ("x", removed.Get(0)); + })); + // Fire the TTL timer. + EXPECT_CALL(*ttl_timer, disableTimer()); + ttl_timer->invokeCallback(); + + expectSendMessage(type_url, {}, "1"); +} + +// Validate behavior when watches has an unknown resource name. +TEST_F(GrpcMuxImplTest, WildcardWatch) { + setup(); + + const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; + FakeGrpcSubscription foo_sub = makeWatch(type_url, {}, callbacks_, resource_decoder_); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage(type_url, {}, "", true); + grpc_mux_->start(); + + { + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_version_info("1"); + envoy::config::endpoint::v3::ClusterLoadAssignment load_assignment; + load_assignment.set_cluster_name("x"); + response->add_resources()->PackFrom(load_assignment); + EXPECT_CALL(callbacks_, onConfigUpdate(_, "1")) + .WillOnce(Invoke([&load_assignment](const std::vector& resources, + const std::string&) { + EXPECT_EQ(1, resources.size()); + const auto& expected_assignment = + dynamic_cast( + resources[0].get().resource()); + EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); + })); + expectSendMessage(type_url, {}, "1"); + grpc_mux_->onDiscoveryResponse(std::move(response), control_plane_stats_); + } +} + +// Validate behavior when watches specify resources (potentially overlapping). +TEST_F(GrpcMuxImplTest, WatchDemux) { + setup(); + // We will not require InSequence here: an update that causes multiple onConfigUpdates + // causes them in an indeterminate order, based on the whims of the hash map. + const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; + + NiceMock foo_callbacks; + FakeGrpcSubscription foo_sub = makeWatch(type_url, {"x", "y"}, foo_callbacks, resource_decoder_); + NiceMock bar_callbacks; + FakeGrpcSubscription bar_sub = makeWatch(type_url, {"y", "z"}, bar_callbacks, resource_decoder_); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + // Should dedupe the "x" resource. + expectSendMessage(type_url, {"y", "z", "x"}, "", true); + grpc_mux_->start(); + + // Send just x; only foo_callbacks should receive an onConfigUpdate(). + { + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_version_info("1"); + envoy::config::endpoint::v3::ClusterLoadAssignment load_assignment; + load_assignment.set_cluster_name("x"); + response->add_resources()->PackFrom(load_assignment); + EXPECT_CALL(bar_callbacks, onConfigUpdate(_, "1")).Times(0); + EXPECT_CALL(foo_callbacks, onConfigUpdate(_, "1")) + .WillOnce(Invoke([&load_assignment](const std::vector& resources, + const std::string&) { + EXPECT_EQ(1, resources.size()); + const auto& expected_assignment = + dynamic_cast( + resources[0].get().resource()); + EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); + })); + expectSendMessage(type_url, {"y", "z", "x"}, "1"); + grpc_mux_->onDiscoveryResponse(std::move(response), control_plane_stats_); + } + + // Send x y and z; foo_ and bar_callbacks should both receive onConfigUpdate()s, carrying {x,y} + // and {y,z} respectively. + { + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_version_info("2"); + envoy::config::endpoint::v3::ClusterLoadAssignment load_assignment_x; + load_assignment_x.set_cluster_name("x"); + response->add_resources()->PackFrom(load_assignment_x); + envoy::config::endpoint::v3::ClusterLoadAssignment load_assignment_y; + load_assignment_y.set_cluster_name("y"); + response->add_resources()->PackFrom(load_assignment_y); + envoy::config::endpoint::v3::ClusterLoadAssignment load_assignment_z; + load_assignment_z.set_cluster_name("z"); + response->add_resources()->PackFrom(load_assignment_z); + EXPECT_CALL(bar_callbacks, onConfigUpdate(_, "2")) + .WillOnce(Invoke([&load_assignment_y, &load_assignment_z]( + const std::vector& resources, const std::string&) { + EXPECT_EQ(2, resources.size()); + const auto& expected_assignment = + dynamic_cast( + resources[0].get().resource()); + EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment_y)); + const auto& expected_assignment_1 = + dynamic_cast( + resources[1].get().resource()); + EXPECT_TRUE(TestUtility::protoEqual(expected_assignment_1, load_assignment_z)); + })); + EXPECT_CALL(foo_callbacks, onConfigUpdate(_, "2")) + .WillOnce(Invoke([&load_assignment_x, &load_assignment_y]( + const std::vector& resources, const std::string&) { + EXPECT_EQ(2, resources.size()); + const auto& expected_assignment = + dynamic_cast( + resources[0].get().resource()); + EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment_x)); + const auto& expected_assignment_1 = + dynamic_cast( + resources[1].get().resource()); + EXPECT_TRUE(TestUtility::protoEqual(expected_assignment_1, load_assignment_y)); + })); + expectSendMessage(type_url, {"y", "z", "x"}, "2"); + grpc_mux_->onDiscoveryResponse(std::move(response), control_plane_stats_); + } + + expectSendMessage(type_url, {"x", "y"}, "2"); + expectSendMessage(type_url, {}, "2"); +} + +// Validate behavior when we have multiple watchers that send empty updates. +TEST_F(GrpcMuxImplTest, MultipleWatcherWithEmptyUpdates) { + setup(); + InSequence s; + const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; + NiceMock foo_callbacks; + FakeGrpcSubscription foo_sub = makeWatch(type_url, {"x", "y"}, foo_callbacks, resource_decoder_); + + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage(type_url, {"x", "y"}, "", true); + grpc_mux_->start(); + + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_version_info("1"); + + EXPECT_CALL(foo_callbacks, onConfigUpdate(_, "1")).Times(0); + expectSendMessage(type_url, {"x", "y"}, "1"); + grpc_mux_->onDiscoveryResponse(std::move(response), control_plane_stats_); + + expectSendMessage(type_url, {}, "1"); +} + +// Validate behavior when we have Single Watcher that sends Empty updates. +TEST_F(GrpcMuxImplTest, SingleWatcherWithEmptyUpdates) { + setup(); + const std::string& type_url = Config::TypeUrl::get().Cluster; + NiceMock foo_callbacks; + FakeGrpcSubscription foo_sub = makeWatch(type_url, {}, foo_callbacks, resource_decoder_); + + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage(type_url, {}, "", true); + grpc_mux_->start(); + + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_version_info("1"); + // Validate that onConfigUpdate is called with empty resources. + EXPECT_CALL(foo_callbacks, onConfigUpdate(_, "1")) + .WillOnce(Invoke([](const std::vector& resources, const std::string&) { + EXPECT_TRUE(resources.empty()); + })); + expectSendMessage(type_url, {}, "1"); + grpc_mux_->onDiscoveryResponse(std::move(response), control_plane_stats_); +} + +// Exactly one test requires a mock time system to provoke behavior that cannot +// easily be achieved with a SimulatedTimeSystem. +class GrpcMuxImplTestWithMockTimeSystem : public GrpcMuxImplTestBase { +public: + Event::DelegatingTestTimeSystem mock_time_system_; +}; + +// Verifies that rate limiting is not enforced with defaults. +TEST_F(GrpcMuxImplTestWithMockTimeSystem, TooManyRequestsWithDefaultSettings) { + + auto ttl_timer = new Event::MockTimer(&dispatcher_); + // Retry timer, + new Event::MockTimer(&dispatcher_); + + // Validate that rate limiter is not created. + EXPECT_CALL(*mock_time_system_, monotonicTime()).Times(0); + + setup(); + + EXPECT_CALL(async_stream_, sendMessageRaw_(_, false)).Times(AtLeast(99)); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + + const auto onReceiveMessage = [&](uint64_t burst) { + for (uint64_t i = 0; i < burst; i++) { + auto response = std::make_unique(); + response->set_version_info("type_url_baz"); + response->set_nonce("type_url_bar"); + response->set_type_url("type_url_foo"); + EXPECT_CALL(*ttl_timer, disableTimer()); + grpc_mux_->onDiscoveryResponse(std::move(response), control_plane_stats_); + } + }; + + FakeGrpcSubscription foo_sub = makeWatch("type_url_foo", {"x"}); + expectSendMessage("type_url_foo", {"x"}, "", true); + grpc_mux_->start(); + + // Exhausts the limit. + onReceiveMessage(99); + + // API calls go over the limit but we do not see the stat incremented. + onReceiveMessage(1); + EXPECT_EQ(0, stats_.counter("control_plane.rate_limit_enforced").value()); +} + +// Verifies that default rate limiting is enforced with empty RateLimitSettings. +TEST_F(GrpcMuxImplTest, TooManyRequestsWithEmptyRateLimitSettings) { + // Validate that request drain timer is created. + + auto ttl_timer = new Event::MockTimer(&dispatcher_); + Event::MockTimer* drain_request_timer = new Event::MockTimer(&dispatcher_); + Event::MockTimer* retry_timer = new Event::MockTimer(&dispatcher_); + + RateLimitSettings custom_rate_limit_settings; + custom_rate_limit_settings.enabled_ = true; + setup(custom_rate_limit_settings); + + // Attempt to send 99 messages. One of them is rate limited (and we never drain). + EXPECT_CALL(async_stream_, sendMessageRaw_(_, false)).Times(99); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + + const auto onReceiveMessage = [&](uint64_t burst) { + for (uint64_t i = 0; i < burst; i++) { + auto response = std::make_unique(); + response->set_version_info("type_url_baz"); + response->set_nonce("type_url_bar"); + response->set_type_url("type_url_foo"); + EXPECT_CALL(*ttl_timer, disableTimer()); + grpc_mux_->onDiscoveryResponse(std::move(response), control_plane_stats_); + } + }; + + FakeGrpcSubscription foo_sub = makeWatch("type_url_foo", {"x"}); + expectSendMessage("type_url_foo", {"x"}, "", true); + grpc_mux_->start(); + + // Validate that drain_request_timer is enabled when there are no tokens. + EXPECT_CALL(*drain_request_timer, enableTimer(std::chrono::milliseconds(100), _)); + // The drain timer enable is checked twice, once when we limit, again when the watch is destroyed. + EXPECT_CALL(*drain_request_timer, enabled()).Times(11); + onReceiveMessage(110); + EXPECT_EQ(11, stats_.counter("control_plane.rate_limit_enforced").value()); + EXPECT_EQ(11, control_plane_pending_requests_.value()); + + // Validate that when we reset a stream with pending requests, it reverts back to the initial + // query (i.e. the queue is discarded). + EXPECT_CALL(callbacks_, + onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)); + EXPECT_CALL(random_, random()); + EXPECT_CALL(*retry_timer, enableTimer(_, _)); + grpc_mux_->grpcStreamForTest().onRemoteClose(Grpc::Status::WellKnownGrpcStatus::Canceled, ""); + EXPECT_EQ(11, control_plane_pending_requests_.value()); + EXPECT_EQ(0, control_plane_connected_state_.value()); + EXPECT_CALL(async_stream_, sendMessageRaw_(_, false)); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + time_system_.setMonotonicTime(std::chrono::seconds(30)); + retry_timer->invokeCallback(); + EXPECT_EQ(0, control_plane_pending_requests_.value()); + // One more message on the way out when the watch is destroyed. + EXPECT_CALL(async_stream_, sendMessageRaw_(_, false)); +} + +// Verifies that rate limiting is enforced with custom RateLimitSettings. +TEST_F(GrpcMuxImplTest, TooManyRequestsWithCustomRateLimitSettings) { + // Validate that request drain timer is created. + + // TTL timer. + auto ttl_timer = new Event::MockTimer(&dispatcher_); + Event::MockTimer* drain_request_timer = new Event::MockTimer(&dispatcher_); + // Retry timer. + new Event::MockTimer(&dispatcher_); + + RateLimitSettings custom_rate_limit_settings; + custom_rate_limit_settings.enabled_ = true; + custom_rate_limit_settings.max_tokens_ = 250; + custom_rate_limit_settings.fill_rate_ = 2; + setup(custom_rate_limit_settings); + + EXPECT_CALL(async_stream_, sendMessageRaw_(_, false)).Times(AtLeast(260)); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + + const auto onReceiveMessage = [&](uint64_t burst) { + for (uint64_t i = 0; i < burst; i++) { + auto response = std::make_unique(); + response->set_version_info("type_url_baz"); + response->set_nonce("type_url_bar"); + response->set_type_url("type_url_foo"); + EXPECT_CALL(*ttl_timer, disableTimer()); + grpc_mux_->onDiscoveryResponse(std::move(response), control_plane_stats_); + } + }; + + FakeGrpcSubscription foo_sub = makeWatch("type_url_foo", {"x"}); + expectSendMessage("type_url_foo", {"x"}, "", true); + grpc_mux_->start(); + + // Validate that rate limit is not enforced for 100 requests. + onReceiveMessage(100); + EXPECT_EQ(0, stats_.counter("control_plane.rate_limit_enforced").value()); + + // Validate that drain_request_timer is enabled when there are no tokens. + EXPECT_CALL(*drain_request_timer, enableTimer(std::chrono::milliseconds(500), _)); + EXPECT_CALL(*drain_request_timer, enabled()).Times(11); + onReceiveMessage(160); + EXPECT_EQ(11, stats_.counter("control_plane.rate_limit_enforced").value()); + EXPECT_EQ(11, control_plane_pending_requests_.value()); + + // Validate that drain requests call when there are multiple requests in queue. + time_system_.setMonotonicTime(std::chrono::seconds(10)); + drain_request_timer->invokeCallback(); + + // Check that the pending_requests stat is updated with the queue drain. + EXPECT_EQ(0, control_plane_pending_requests_.value()); +} + +// Verifies that a message with no resources is accepted. +TEST_F(GrpcMuxImplTest, UnwatchedTypeAcceptsEmptyResources) { + setup(); + + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + + const std::string& type_url = Config::TypeUrl::get().ClusterLoadAssignment; + + grpc_mux_->start(); + { + // subscribe and unsubscribe to simulate a cluster added and removed + expectSendMessage(type_url, {"y"}, "", true); + FakeGrpcSubscription temp_sub = makeWatch(type_url, {"y"}); + expectSendMessage(type_url, {}, ""); + } + + // simulate the server sending empty CLA message to notify envoy that the CLA was removed. + auto response = std::make_unique(); + response->set_nonce("bar"); + response->set_version_info("1"); + response->set_type_url(type_url); + + // Although the update will change nothing for us, we will "accept" it, and so according + // to the spec we should ACK it. + expectSendMessage(type_url, {}, "1", false, "bar"); + grpc_mux_->onDiscoveryResponse(std::move(response), control_plane_stats_); + + // When we become interested in "x", we should send a request indicating that interest. + expectSendMessage(type_url, {"x"}, "1", false, "bar"); + FakeGrpcSubscription sub = makeWatch(type_url, {"x"}); + + // Watch destroyed -> interest gone -> unsubscribe request. + expectSendMessage(type_url, {}, "1", false, "bar"); +} + +// Verifies that a message with some resources is accepted even when there are no watches. +// Rationale: SotW gRPC xDS has always been willing to accept updates that include +// uninteresting resources. It should not matter whether those uninteresting resources +// are accompanied by interesting ones. +// Note: this was previously "rejects", not "accepts". See +// https://github.com/envoyproxy/envoy/pull/8350#discussion_r328218220 for discussion. +TEST_F(GrpcMuxImplTest, UnwatchedTypeAcceptsResources) { + setup(); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + const std::string& type_url = + Config::getTypeUrl( + envoy::config::core::v3::ApiVersion::V3); + grpc_mux_->start(); + + // subscribe and unsubscribe so that the type is known to envoy + { + expectSendMessage(type_url, {"y"}, "", true); + expectSendMessage(type_url, {}, ""); + FakeGrpcSubscription delete_immediately = makeWatch(type_url, {"y"}); + } + auto response = std::make_unique(); + response->set_type_url(type_url); + envoy::config::endpoint::v3::ClusterLoadAssignment load_assignment; + load_assignment.set_cluster_name("x"); + response->add_resources()->PackFrom(load_assignment); + response->set_version_info("1"); + + expectSendMessage(type_url, {}, "1"); + grpc_mux_->onDiscoveryResponse(std::move(response), control_plane_stats_); +} + +TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyClusterName) { + EXPECT_CALL(local_info_, clusterName()).WillOnce(ReturnRef(EMPTY_STRING)); + EXPECT_THROW_WITH_MESSAGE( + UnifiedMux::GrpcMuxSotw( + std::unique_ptr(async_client_), dispatcher_, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), + envoy::config::core::v3::ApiVersion::AUTO, random_, stats_, rate_limit_settings_, + local_info_, true), + EnvoyException, + "ads: node 'id' and 'cluster' are required. Set it either in 'node' config or via " + "--service-node and --service-cluster options."); +} + +TEST_F(GrpcMuxImplTest, BadLocalInfoEmptyNodeName) { + EXPECT_CALL(local_info_, nodeName()).WillOnce(ReturnRef(EMPTY_STRING)); + EXPECT_THROW_WITH_MESSAGE( + UnifiedMux::GrpcMuxSotw( + std::unique_ptr(async_client_), dispatcher_, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), + envoy::config::core::v3::ApiVersion::AUTO, random_, stats_, rate_limit_settings_, + local_info_, true), + EnvoyException, + "ads: node 'id' and 'cluster' are required. Set it either in 'node' config or via " + "--service-node and --service-cluster options."); +} + +// DeltaDiscoveryResponse that comes in response to an on-demand request updates the watch with +// resource's name. The watch is initially created with an alias used in the on-demand request. +TEST_F(GrpcMuxImplTest, ConfigUpdateWithAliases) { + std::unique_ptr grpc_mux = std::make_unique( + std::unique_ptr(async_client_), dispatcher_, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), + envoy::config::core::v3::ApiVersion::AUTO, random_, stats_, rate_limit_settings_, local_info_, + true); + const std::string& type_url = Config::getTypeUrl( + envoy::config::core::v3::ApiVersion::V3); + NiceMock async_stream; + MockSubscriptionCallbacks callbacks; + TestUtility::TestOpaqueResourceDecoderImpl + resource_decoder("prefix"); + grpc_mux->addWatch(type_url, {"prefix"}, callbacks, resource_decoder, + std::chrono::milliseconds(0), true); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream)); + grpc_mux->start(); + + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_system_version_info("1"); + + envoy::config::route::v3::VirtualHost vhost; + vhost.set_name("vhost_1"); + vhost.add_domains("domain1.test"); + vhost.add_domains("domain2.test"); + + response->add_resources()->mutable_resource()->PackFrom(vhost); + response->mutable_resources()->at(0).set_name("prefix/vhost_1"); + response->mutable_resources()->at(0).add_aliases("prefix/domain1.test"); + response->mutable_resources()->at(0).add_aliases("prefix/domain2.test"); + + EXPECT_CALL(callbacks, onConfigUpdate(_, _, "1")); + + grpc_mux->onDiscoveryResponse(std::move(response), control_plane_stats_); +} + +// DeltaDiscoveryResponse that comes in response to an on-demand request that couldn't be resolved +// will contain an empty Resource. The Resource's aliases field will be populated with the alias +// originally used in the request. +TEST_F(GrpcMuxImplTest, ConfigUpdateWithNotFoundResponse) { + std::unique_ptr grpc_mux = std::make_unique( + std::unique_ptr(async_client_), dispatcher_, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.discovery.v2.AggregatedDiscoveryService.StreamAggregatedResources"), + envoy::config::core::v3::ApiVersion::AUTO, random_, stats_, rate_limit_settings_, local_info_, + true); + const std::string& type_url = Config::getTypeUrl( + envoy::config::core::v3::ApiVersion::V3); + NiceMock async_stream; + MockSubscriptionCallbacks callbacks; + TestUtility::TestOpaqueResourceDecoderImpl + resource_decoder("prefix"); + + grpc_mux->addWatch(type_url, {"prefix"}, callbacks, resource_decoder, + std::chrono::milliseconds(0), true); + + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream)); + grpc_mux->start(); + + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_system_version_info("1"); + + response->add_resources(); + response->mutable_resources()->at(0).set_name("prefix/not-found"); + response->mutable_resources()->at(0).add_aliases("prefix/domain1.test"); + + EXPECT_CALL(callbacks, onConfigUpdate(_, _, "1")); + + grpc_mux->onDiscoveryResponse(std::move(response), control_plane_stats_); +} + +// Send discovery request with v2 resource type_url, receive discovery response with v3 resource +// type_url. +TEST_F(GrpcMuxImplTest, WatchV2ResourceV3) { + TestScopedRuntime scoped_runtime; + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.enable_type_url_downgrade_and_upgrade", "true"}}); + setup(); + + InSequence s; + const std::string& v2_type_url = Config::TypeUrl::get().ClusterLoadAssignment; + const std::string& v3_type_url = + Config::getTypeUrl( + envoy::config::core::v3::ApiVersion::V3); + auto foo_sub = makeWatch(v2_type_url, {}, callbacks_, resource_decoder_); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage(v2_type_url, {}, "", true); + grpc_mux_->start(); + { + auto response = std::make_unique(); + response->set_type_url(v3_type_url); + response->set_version_info("1"); + envoy::config::endpoint::v3::ClusterLoadAssignment load_assignment; + load_assignment.set_cluster_name("x"); + response->add_resources()->PackFrom(load_assignment); + EXPECT_CALL(callbacks_, onConfigUpdate(_, "1")) + .WillOnce(Invoke([&load_assignment](const std::vector& resources, + const std::string&) { + EXPECT_EQ(1, resources.size()); + const auto& expected_assignment = + dynamic_cast( + resources[0].get().resource()); + EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); + })); + expectSendMessage(v2_type_url, {}, "1"); + grpc_mux_->onDiscoveryResponse(std::move(response), control_plane_stats_); + } +} + +// Send discovery request with v3 resource type_url, receive discovery response with v2 resource +// type_url. +TEST_F(GrpcMuxImplTest, WatchV3ResourceV2) { + TestScopedRuntime scoped_runtime; + Runtime::LoaderSingleton::getExisting()->mergeValues( + {{"envoy.reloadable_features.enable_type_url_downgrade_and_upgrade", "true"}}); + setup(); + + InSequence s; + const std::string& v2_type_url = Config::TypeUrl::get().ClusterLoadAssignment; + const std::string& v3_type_url = + Config::getTypeUrl( + envoy::config::core::v3::ApiVersion::V3); + auto foo_sub = makeWatch(v3_type_url, {}, callbacks_, resource_decoder_); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage(v3_type_url, {}, "", true); + grpc_mux_->start(); + + { + + auto response = std::make_unique(); + response->set_type_url(v2_type_url); + response->set_version_info("1"); + envoy::config::endpoint::v3::ClusterLoadAssignment load_assignment; + load_assignment.set_cluster_name("x"); + response->add_resources()->PackFrom(load_assignment); + EXPECT_CALL(callbacks_, onConfigUpdate(_, "1")) + .WillOnce(Invoke([&load_assignment](const std::vector& resources, + const std::string&) { + EXPECT_EQ(1, resources.size()); + const auto& expected_assignment = + dynamic_cast( + resources[0].get().resource()); + EXPECT_TRUE(TestUtility::protoEqual(expected_assignment, load_assignment)); + })); + expectSendMessage(v3_type_url, {}, "1"); + grpc_mux_->onDiscoveryResponse(std::move(response), control_plane_stats_); + } +} + +// Validate behavior when dynamic context parameters are updated. +TEST_F(GrpcMuxImplTest, DynamicContextParameters) { + setup(); + InSequence s; + grpc_mux_->addWatch("foo", {"x", "y"}, callbacks_, resource_decoder_, + std::chrono::milliseconds(0)); + grpc_mux_->addWatch("bar", {}, callbacks_, resource_decoder_, std::chrono::milliseconds(0)); + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage("foo", {"x", "y"}, "", true); + expectSendMessage("bar", {}, ""); + grpc_mux_->start(); + // Unknown type, shouldn't do anything. + local_info_.context_provider_.update_cb_handler_.runCallbacks("baz"); + // Update to foo type should resend Node. + expectSendMessage("foo", {"x", "y"}, "", true); + local_info_.context_provider_.update_cb_handler_.runCallbacks("foo"); + // Update to bar type should resend Node. + expectSendMessage("bar", {}, "", true); + local_info_.context_provider_.update_cb_handler_.runCallbacks("bar"); +} + +// Test that we simply ignore a message for an unknown type_url, with no ill effects. +TEST_F(GrpcMuxImplTest, DiscoveryResponseNonexistentSub) { + setup(); + + const std::string& type_url = + Config::getTypeUrl( + envoy::config::core::v3::ApiVersion::V3); + grpc_mux_->addWatch(type_url, {}, callbacks_, resource_decoder_, std::chrono::milliseconds(0), + false); + + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage(type_url, {}, "", true); + grpc_mux_->start(); + { + auto unexpected_response = std::make_unique(); + unexpected_response->set_type_url("unexpected_type_url"); + unexpected_response->set_version_info("0"); + EXPECT_CALL(callbacks_, onConfigUpdate(_, _, "0")).Times(0); + grpc_mux_->onDiscoveryResponse(std::move(unexpected_response), control_plane_stats_); + } + auto response = std::make_unique(); + response->set_type_url(type_url); + response->set_version_info("1"); + envoy::config::endpoint::v3::ClusterLoadAssignment load_assignment; + load_assignment.set_cluster_name("x"); + response->add_resources()->PackFrom(load_assignment); + EXPECT_CALL(callbacks_, onConfigUpdate(_, "1")) + .WillOnce(Invoke([&load_assignment](const std::vector& resources, + const std::string&) -> void { + EXPECT_EQ(1, resources.size()); + EXPECT_TRUE(TestUtility::protoEqual(resources[0].get().resource(), load_assignment)); + })); + expectSendMessage(type_url, {}, "1"); + grpc_mux_->onDiscoveryResponse(std::move(response), control_plane_stats_); +} + +} // namespace +} // namespace UnifiedMux +} // namespace Config +} // namespace Envoy diff --git a/test/common/config/unified_grpc_subscription_impl_test.cc b/test/common/config/unified_grpc_subscription_impl_test.cc new file mode 100644 index 0000000000000..7802ab1a14bac --- /dev/null +++ b/test/common/config/unified_grpc_subscription_impl_test.cc @@ -0,0 +1,109 @@ +#include "test/common/config/unified_grpc_subscription_test_harness.h" + +#include "gtest/gtest.h" + +using testing::InSequence; + +namespace Envoy { +namespace Config { +namespace UnifiedMux { +namespace { + +class GrpcSubscriptionImplTest : public testing::Test, public GrpcSubscriptionTestHarness {}; + +// Validate that stream creation results in a timer based retry and can recover. +TEST_F(GrpcSubscriptionImplTest, StreamCreationFailure) { + InSequence s; + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(nullptr)); + + // onConfigUpdateFailed() should not be called for gRPC stream connection failure + EXPECT_CALL(callbacks_, + onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)) + .Times(0); + EXPECT_CALL(random_, random()); + EXPECT_CALL(*timer_, enableTimer(_, _)); + ttl_timer_ = new NiceMock(&dispatcher_); + subscription_->start({"cluster0", "cluster1"}); + EXPECT_TRUE(statsAre(2, 0, 0, 0, 0, 0, 0, "")); + // Ensure this doesn't cause an issue by sending a request, since we don't + // have a gRPC stream. + subscription_->updateResourceInterest({"cluster2"}); + + // Retry and succeed. + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + + expectSendMessage({"cluster2"}, "", true); + timer_->invokeCallback(); + EXPECT_TRUE(statsAre(3, 0, 0, 0, 0, 0, 0, "")); + verifyControlPlaneStats(1); +} + +// Validate that the client can recover from a remote stream closure via retry. +TEST_F(GrpcSubscriptionImplTest, RemoteStreamClose) { + startSubscription({"cluster0", "cluster1"}); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0, 0, "")); + // onConfigUpdateFailed() should not be called for gRPC stream connection failure + EXPECT_CALL(callbacks_, + onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)) + .Times(0); + EXPECT_CALL(*timer_, enableTimer(_, _)); + EXPECT_CALL(random_, random()); + auto shared_mux = subscription_->getGrpcMuxForTest(); + static_cast(shared_mux.get()) + ->grpcStreamForTest() + .onRemoteClose(Grpc::Status::WellKnownGrpcStatus::Canceled, ""); + EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0, 0, "")); + verifyControlPlaneStats(0); + + // Retry and succeed. + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + expectSendMessage({"cluster0", "cluster1"}, "", true); + timer_->invokeCallback(); + EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0, 0, "")); +} + +// Validate that when the management server gets multiple requests for the same version, it can +// ignore later ones. This allows the nonce to be used. +TEST_F(GrpcSubscriptionImplTest, RepeatedNonce) { + startSubscription({"cluster0", "cluster1"}); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0, 0, "")); + // First with the initial, empty version update to "0". + updateResourceInterest({"cluster2"}); + EXPECT_TRUE(statsAre(2, 0, 0, 0, 0, 0, 0, "")); + deliverConfigUpdate({"cluster0", "cluster2"}, "0", false); + EXPECT_TRUE(statsAre(3, 0, 1, 0, 0, 0, 0, "")); + deliverConfigUpdate({"cluster0", "cluster2"}, "0", true); + EXPECT_TRUE(statsAre(4, 1, 1, 0, 0, TEST_TIME_MILLIS, 7148434200721666028, "0")); + // Now with version "0" update to "1". + updateResourceInterest({"cluster3"}); + EXPECT_TRUE(statsAre(5, 1, 1, 0, 0, TEST_TIME_MILLIS, 7148434200721666028, "0")); + deliverConfigUpdate({"cluster3"}, "42", false); + EXPECT_TRUE(statsAre(6, 1, 2, 0, 0, TEST_TIME_MILLIS, 7148434200721666028, "0")); + deliverConfigUpdate({"cluster3"}, "42", true); + EXPECT_TRUE(statsAre(7, 2, 2, 0, 0, TEST_TIME_MILLIS, 7919287270473417401, "42")); +} + +TEST_F(GrpcSubscriptionImplTest, UpdateTimeNotChangedOnUpdateReject) { + startSubscription({"cluster0", "cluster1"}); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0, 0, "")); + deliverConfigUpdate({"cluster0", "cluster2"}, "0", false); + EXPECT_TRUE(statsAre(2, 0, 1, 0, 0, 0, 0, "")); +} + +TEST_F(GrpcSubscriptionImplTest, UpdateTimeChangedOnUpdateSuccess) { + startSubscription({"cluster0", "cluster1"}); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0, 0, "")); + deliverConfigUpdate({"cluster0", "cluster2"}, "0", true); + EXPECT_TRUE(statsAre(2, 1, 0, 0, 0, TEST_TIME_MILLIS, 7148434200721666028, "0")); + + // Advance the simulated time and verify that a trivial update (no change) also changes the update + // time. + simTime().setSystemTime(SystemTime(std::chrono::milliseconds(TEST_TIME_MILLIS + 1))); + deliverConfigUpdate({"cluster0", "cluster2"}, "0", true); + EXPECT_TRUE(statsAre(2, 2, 0, 0, 0, TEST_TIME_MILLIS + 1, 7148434200721666028, "0")); +} + +} // namespace +} // namespace UnifiedMux +} // namespace Config +} // namespace Envoy diff --git a/test/common/config/unified_grpc_subscription_test_harness.h b/test/common/config/unified_grpc_subscription_test_harness.h new file mode 100644 index 0000000000000..cf18d1527f677 --- /dev/null +++ b/test/common/config/unified_grpc_subscription_test_harness.h @@ -0,0 +1,198 @@ +#pragma once + +#include + +#include "envoy/api/v2/discovery.pb.h" +#include "envoy/config/core/v3/base.pb.h" +#include "envoy/config/endpoint/v3/endpoint.pb.h" +#include "envoy/config/endpoint/v3/endpoint.pb.validate.h" +#include "envoy/service/discovery/v3/discovery.pb.h" + +#include "common/common/hash.h" +#include "common/config/api_version.h" +#include "common/config/unified_mux/grpc_mux_impl.h" +#include "common/config/unified_mux/grpc_subscription_impl.h" +#include "common/config/version_converter.h" + +#include "test/common/config/subscription_test_harness.h" +#include "test/mocks/config/mocks.h" +#include "test/mocks/event/mocks.h" +#include "test/mocks/grpc/mocks.h" +#include "test/mocks/local_info/mocks.h" +#include "test/mocks/upstream/cluster_manager.h" +#include "test/test_common/resources.h" +#include "test/test_common/utility.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" + +using testing::_; +using testing::Invoke; +using testing::Mock; +using testing::NiceMock; +using testing::Return; + +namespace Envoy { +namespace Config { +namespace UnifiedMux { + +class GrpcSubscriptionTestHarness : public SubscriptionTestHarness { +public: + GrpcSubscriptionTestHarness() : GrpcSubscriptionTestHarness(std::chrono::milliseconds(0)) {} + + GrpcSubscriptionTestHarness(std::chrono::milliseconds init_fetch_timeout) + : method_descriptor_(Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.api.v2.EndpointDiscoveryService.StreamEndpoints")), + async_client_(new NiceMock()) { + node_.set_id("fo0"); + node_.set_cluster("cluster_name"); + node_.mutable_locality()->set_zone("zone_name"); + EXPECT_CALL(local_info_, node()).WillRepeatedly(testing::ReturnRef(node_)); + + timer_ = new Event::MockTimer(&dispatcher_); + + subscription_ = std::make_unique( + std::make_shared(std::unique_ptr(async_client_), + dispatcher_, *method_descriptor_, + envoy::config::core::v3::ApiVersion::AUTO, random_, + stats_store_, rate_limit_settings_, local_info_, + /*skip_subsequent_node=*/true), + Config::TypeUrl::get().ClusterLoadAssignment, callbacks_, resource_decoder_, stats_, + dispatcher_.timeSource(), init_fetch_timeout, + /*is_aggregated=*/false, false); + } + + ~GrpcSubscriptionTestHarness() override { + EXPECT_CALL(async_stream_, sendMessageRaw_(_, false)); + EXPECT_CALL(dispatcher_, clearDeferredDeleteList()); + dispatcher_.clearDeferredDeleteList(); + } + + void expectSendMessage(const std::set& cluster_names, const std::string& version, + bool expect_node = false) override { + expectSendMessage(cluster_names, version, expect_node, Grpc::Status::WellKnownGrpcStatus::Ok, + ""); + } + + void expectSendMessage(const std::set& cluster_names, const std::string& version, + bool expect_node, const Protobuf::int32 error_code, + const std::string& error_message) { + UNREFERENCED_PARAMETER(expect_node); + API_NO_BOOST(envoy::api::v2::DiscoveryRequest) expected_request; + if (expect_node) { + expected_request.mutable_node()->CopyFrom(API_DOWNGRADE(node_)); + } + for (const auto& cluster : cluster_names) { + expected_request.add_resource_names(cluster); + } + if (!version.empty()) { + expected_request.set_version_info(version); + } + expected_request.set_response_nonce(last_response_nonce_); + expected_request.set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); + if (error_code != Grpc::Status::WellKnownGrpcStatus::Ok) { + ::google::rpc::Status* error_detail = expected_request.mutable_error_detail(); + error_detail->set_code(error_code); + error_detail->set_message(error_message); + } + EXPECT_CALL( + async_stream_, + sendMessageRaw_(Grpc::ProtoBufferEqIgnoreRepeatedFieldOrdering(expected_request), false)); + } + + void startSubscription(const std::set& cluster_names) override { + EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); + last_cluster_names_ = cluster_names; + expectSendMessage(last_cluster_names_, "", true); + ttl_timer_ = new NiceMock(&dispatcher_); + subscription_->start(flattenResources(cluster_names)); + } + + void deliverConfigUpdate(const std::vector& cluster_names, + const std::string& version, bool accept) override { + auto response = std::make_unique(); + response->set_version_info(version); + last_response_nonce_ = std::to_string(HashUtil::xxHash64(version)); + response->set_nonce(last_response_nonce_); + response->set_type_url(Config::TypeUrl::get().ClusterLoadAssignment); + response->mutable_control_plane()->set_identifier("ground_control_foo123"); + Protobuf::RepeatedPtrField typed_resources; + for (const auto& cluster : cluster_names) { + if (std::find(last_cluster_names_.begin(), last_cluster_names_.end(), cluster) != + last_cluster_names_.end()) { + envoy::config::endpoint::v3::ClusterLoadAssignment* load_assignment = typed_resources.Add(); + load_assignment->set_cluster_name(cluster); + response->add_resources()->PackFrom(*load_assignment); + } + } + const auto decoded_resources = + TestUtility::decodeResources( + *response, "cluster_name"); + + EXPECT_CALL(callbacks_, onConfigUpdate(DecodedResourcesEq(decoded_resources.refvec_), version)) + .WillOnce(ThrowOnRejectedConfig(accept)); + + if (accept) { + expectSendMessage(last_cluster_names_, version, false); + version_ = version; + } else { + EXPECT_CALL(callbacks_, onConfigUpdateFailed( + Envoy::Config::ConfigUpdateFailureReason::UpdateRejected, _)); + expectSendMessage(last_cluster_names_, version_, false, + Grpc::Status::WellKnownGrpcStatus::Internal, "bad config"); + } + auto shared_mux = subscription_->getGrpcMuxForTest(); + static_cast(shared_mux.get()) + ->onDiscoveryResponse(std::move(response), control_plane_stats_); + EXPECT_EQ(control_plane_stats_.identifier_.value(), "ground_control_foo123"); + Mock::VerifyAndClearExpectations(&async_stream_); + } + + void updateResourceInterest(const std::set& cluster_names) override { + expectSendMessage(cluster_names, version_); + subscription_->updateResourceInterest(flattenResources(cluster_names)); + last_cluster_names_ = cluster_names; + } + + void expectConfigUpdateFailed() override { + EXPECT_CALL(callbacks_, onConfigUpdateFailed(_, nullptr)); + } + + void expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds timeout) override { + init_timeout_timer_ = new Event::MockTimer(&dispatcher_); + EXPECT_CALL(*init_timeout_timer_, enableTimer(timeout, _)); + } + + void expectDisableInitFetchTimeoutTimer() override { + EXPECT_CALL(*init_timeout_timer_, disableTimer()); + } + + void callInitFetchTimeoutCb() override { init_timeout_timer_->invokeCallback(); } + + std::string version_; + const Protobuf::MethodDescriptor* method_descriptor_; + Grpc::MockAsyncClient* async_client_; + NiceMock cm_; + Event::MockDispatcher dispatcher_; + Random::MockRandomGenerator random_; + Event::MockTimer* timer_; + Event::MockTimer* ttl_timer_; + envoy::config::core::v3::Node node_; + NiceMock callbacks_; + TestUtility::TestOpaqueResourceDecoderImpl + resource_decoder_{"cluster_name"}; + NiceMock async_stream_; + std::shared_ptr mux_; + GrpcSubscriptionImplPtr subscription_; + std::string last_response_nonce_; + std::set last_cluster_names_; + NiceMock local_info_; + Envoy::Config::RateLimitSettings rate_limit_settings_; + Event::MockTimer* init_timeout_timer_; +}; + +// TODO(danielhochman): test with RDS and ensure version_info is same as what API returned + +} // namespace UnifiedMux +} // namespace Config +} // namespace Envoy diff --git a/test/common/config/unified_subscription_impl_test.cc b/test/common/config/unified_subscription_impl_test.cc new file mode 100644 index 0000000000000..f1f48ff4a15f9 --- /dev/null +++ b/test/common/config/unified_subscription_impl_test.cc @@ -0,0 +1,209 @@ +#include +#include + +#include "test/common/config/filesystem_subscription_test_harness.h" +#include "test/common/config/http_subscription_test_harness.h" +#include "test/common/config/subscription_test_harness.h" +#include "test/common/config/unified_delta_subscription_test_harness.h" +#include "test/common/config/unified_grpc_subscription_test_harness.h" + +using testing::InSequence; + +namespace Envoy { +namespace Config { +namespace { + +enum class SubscriptionType { + Grpc, + DeltaGrpc, + Http, + Filesystem, +}; + +// NOLINTNEXTLINE(readability-identifier-naming) +void PrintTo(const SubscriptionType sub, std::ostream* os) { + (*os) << ([sub]() -> absl::string_view { + switch (sub) { + case SubscriptionType::Grpc: + return "Grpc"; + case SubscriptionType::DeltaGrpc: + return "DeltaGrpc"; + case SubscriptionType::Http: + return "Http"; + case SubscriptionType::Filesystem: + return "Filesystem"; + default: + return "unknown"; + } + })(); +} + +class SubscriptionImplTest : public testing::TestWithParam { +public: + SubscriptionImplTest() : SubscriptionImplTest(std::chrono::milliseconds(0)) {} + SubscriptionImplTest(std::chrono::milliseconds init_fetch_timeout) { + initialize(init_fetch_timeout); + } + + void initialize(std::chrono::milliseconds init_fetch_timeout = std::chrono::milliseconds(0)) { + switch (GetParam()) { + case SubscriptionType::Grpc: + test_harness_ = std::make_unique(init_fetch_timeout); + break; + case SubscriptionType::DeltaGrpc: + test_harness_ = + std::make_unique(init_fetch_timeout); + break; + case SubscriptionType::Http: + test_harness_ = std::make_unique(init_fetch_timeout); + break; + case SubscriptionType::Filesystem: + test_harness_ = std::make_unique(); + break; + } + } + + void TearDown() override { test_harness_->doSubscriptionTearDown(); } + + void startSubscription(const std::set& cluster_names) { + test_harness_->startSubscription(cluster_names); + } + + void updateResourceInterest(const std::set& cluster_names) { + test_harness_->updateResourceInterest(cluster_names); + } + + void expectSendMessage(const std::set& cluster_names, const std::string& version, + bool expect_node) { + test_harness_->expectSendMessage(cluster_names, version, expect_node); + } + + AssertionResult statsAre(uint32_t attempt, uint32_t success, uint32_t rejected, uint32_t failure, + uint32_t init_fetch_timeout, uint64_t update_time, uint64_t version, + std::string version_text) { + return test_harness_->statsAre(attempt, success, rejected, failure, init_fetch_timeout, + update_time, version, version_text); + } + + void deliverConfigUpdate(const std::vector cluster_names, const std::string& version, + bool accept) { + test_harness_->deliverConfigUpdate(cluster_names, version, accept); + } + + void expectConfigUpdateFailed() { test_harness_->expectConfigUpdateFailed(); } + + void expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds timeout) { + test_harness_->expectEnableInitFetchTimeoutTimer(timeout); + } + + void expectDisableInitFetchTimeoutTimer() { test_harness_->expectDisableInitFetchTimeoutTimer(); } + + void callInitFetchTimeoutCb() { test_harness_->callInitFetchTimeoutCb(); } + + std::unique_ptr test_harness_; +}; + +class SubscriptionImplInitFetchTimeoutTest : public SubscriptionImplTest { +public: + SubscriptionImplInitFetchTimeoutTest() : SubscriptionImplTest(std::chrono::milliseconds(1000)) {} +}; + +SubscriptionType types[] = {SubscriptionType::Grpc, SubscriptionType::DeltaGrpc, + SubscriptionType::Http, SubscriptionType::Filesystem}; +INSTANTIATE_TEST_SUITE_P(SubscriptionImplTest, SubscriptionImplTest, testing::ValuesIn(types)); +INSTANTIATE_TEST_SUITE_P(SubscriptionImplTest, SubscriptionImplInitFetchTimeoutTest, + testing::ValuesIn(types)); + +// Validate basic request-response succeeds. +TEST_P(SubscriptionImplTest, InitialRequestResponse) { + startSubscription({"cluster0", "cluster1"}); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0, 0, "")); + deliverConfigUpdate({"cluster0", "cluster1"}, "v25-ubuntu18-beta", true); + EXPECT_TRUE( + statsAre(2, 1, 0, 0, 0, TEST_TIME_MILLIS, 18202868392629624077U, "v25-ubuntu18-beta")); +} + +// Validate that multiple streamed updates succeed. +TEST_P(SubscriptionImplTest, ResponseStream) { + startSubscription({"cluster0", "cluster1"}); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0, 0, "")); + deliverConfigUpdate({"cluster0", "cluster1"}, "1.2.3.4", true); + EXPECT_TRUE(statsAre(2, 1, 0, 0, 0, TEST_TIME_MILLIS, 14026795738668939420U, "1.2.3.4")); + deliverConfigUpdate({"cluster0", "cluster1"}, "5_6_7", true); + EXPECT_TRUE(statsAre(3, 2, 0, 0, 0, TEST_TIME_MILLIS, 7612520132475921171U, "5_6_7")); +} + +// Validate that the client can reject a config. +TEST_P(SubscriptionImplTest, RejectConfig) { + startSubscription({"cluster0", "cluster1"}); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0, 0, "")); + deliverConfigUpdate({"cluster0", "cluster1"}, "0", false); + EXPECT_TRUE(statsAre(2, 0, 1, 0, 0, 0, 0, "")); +} + +// Validate that the client can reject a config and accept the same config later. +TEST_P(SubscriptionImplTest, RejectAcceptConfig) { + startSubscription({"cluster0", "cluster1"}); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0, 0, "")); + deliverConfigUpdate({"cluster0", "cluster1"}, "0", false); + EXPECT_TRUE(statsAre(2, 0, 1, 0, 0, 0, 0, "")); + deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); + EXPECT_TRUE(statsAre(3, 1, 1, 0, 0, TEST_TIME_MILLIS, 7148434200721666028, "0")); +} + +// Validate that the client can reject a config and accept another config later. +TEST_P(SubscriptionImplTest, RejectAcceptNextConfig) { + startSubscription({"cluster0", "cluster1"}); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0, 0, "")); + deliverConfigUpdate({"cluster0", "cluster1"}, "0", false); + EXPECT_TRUE(statsAre(2, 0, 1, 0, 0, 0, 0, "")); + deliverConfigUpdate({"cluster0", "cluster1"}, "1", true); + EXPECT_TRUE(statsAre(3, 1, 1, 0, 0, TEST_TIME_MILLIS, 13237225503670494420U, "1")); +} + +// Validate that stream updates send a message with the updated resources. +TEST_P(SubscriptionImplTest, UpdateResources) { + startSubscription({"cluster0", "cluster1"}); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0, 0, "")); + deliverConfigUpdate({"cluster0", "cluster1"}, "42", true); + EXPECT_TRUE(statsAre(2, 1, 0, 0, 0, TEST_TIME_MILLIS, 7919287270473417401, "42")); + updateResourceInterest({"cluster2"}); + EXPECT_TRUE(statsAre(3, 1, 0, 0, 0, TEST_TIME_MILLIS, 7919287270473417401, "42")); +} + +// Validate that initial fetch timer is created and calls callback on timeout +TEST_P(SubscriptionImplInitFetchTimeoutTest, InitialFetchTimeout) { + if (GetParam() == SubscriptionType::Filesystem) { + return; // initial_fetch_timeout not implemented for filesystem. + } + + expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); + startSubscription({"cluster0", "cluster1"}); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0, 0, "")); + expectConfigUpdateFailed(); + expectDisableInitFetchTimeoutTimer(); + callInitFetchTimeoutCb(); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 1, 0, 0, "")); +} + +// Validate that initial fetch timer is disabled on config update +TEST_P(SubscriptionImplInitFetchTimeoutTest, DisableInitTimeoutOnSuccess) { + expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); + startSubscription({"cluster0", "cluster1"}); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0, 0, "")); + expectDisableInitFetchTimeoutTimer(); + deliverConfigUpdate({"cluster0", "cluster1"}, "0", true); +} + +// Validate that initial fetch timer is disabled on config update failed +TEST_P(SubscriptionImplInitFetchTimeoutTest, DisableInitTimeoutOnFail) { + expectEnableInitFetchTimeoutTimer(std::chrono::milliseconds(1000)); + startSubscription({"cluster0", "cluster1"}); + EXPECT_TRUE(statsAre(1, 0, 0, 0, 0, 0, 0, "")); + expectDisableInitFetchTimeoutTimer(); + deliverConfigUpdate({"cluster0", "cluster1"}, "0", false); +} + +} // namespace +} // namespace Config +} // namespace Envoy diff --git a/test/common/grpc/grpc_client_integration.h b/test/common/grpc/grpc_client_integration.h index 80321fc835925..a8a5e889ab32a 100644 --- a/test/common/grpc/grpc_client_integration.h +++ b/test/common/grpc/grpc_client_integration.h @@ -14,7 +14,7 @@ namespace Grpc { // Support parameterizing over gRPC client type. enum class ClientType { EnvoyGrpc, GoogleGrpc }; // Support parameterizing over state-of-the-world xDS vs delta xDS. -enum class SotwOrDelta { Sotw, Delta }; +enum class SotwOrDelta { Sotw, Delta, LegacySotw, LegacyDelta }; class BaseGrpcClientIntegrationParamTest { public: @@ -131,7 +131,8 @@ class DeltaSotwIntegrationParamTest #define DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS \ testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ testing::Values(Grpc::ClientType::EnvoyGrpc, Grpc::ClientType::GoogleGrpc), \ - testing::Values(Grpc::SotwOrDelta::Sotw, Grpc::SotwOrDelta::Delta)) + testing::Values(Grpc::SotwOrDelta::Sotw, Grpc::SotwOrDelta::Delta, \ + Grpc::SotwOrDelta::LegacySotw, Grpc::SotwOrDelta::LegacyDelta)) #else #define GRPC_CLIENT_INTEGRATION_PARAMS \ testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ @@ -145,7 +146,8 @@ class DeltaSotwIntegrationParamTest #define DELTA_SOTW_GRPC_CLIENT_INTEGRATION_PARAMS \ testing::Combine(testing::ValuesIn(TestEnvironment::getIpVersionsForTest()), \ testing::Values(Grpc::ClientType::EnvoyGrpc), \ - testing::Values(Grpc::SotwOrDelta::Sotw, Grpc::SotwOrDelta::Delta)) + testing::Values(Grpc::SotwOrDelta::Sotw, Grpc::SotwOrDelta::Delta, \ + Grpc::SotwOrDelta::LegacySotw, Grpc::SotwOrDelta::LegacyDelta)) #endif // ENVOY_GOOGLE_GRPC } // namespace Grpc diff --git a/test/common/upstream/BUILD b/test/common/upstream/BUILD index 14da0caf50de2..7963f9c18c2d7 100644 --- a/test/common/upstream/BUILD +++ b/test/common/upstream/BUILD @@ -125,6 +125,8 @@ envoy_cc_benchmark_binary( "//source/common/config:grpc_subscription_lib", "//source/common/config:protobuf_link_hacks", "//source/common/config:utility_lib", + "//source/common/config/unified_mux:grpc_mux_lib", + "//source/common/config/unified_mux:grpc_subscription_lib", "//source/common/upstream:eds_lib", "//source/extensions/transport_sockets/raw_buffer:config", "//source/server:transport_socket_config_lib", diff --git a/test/common/upstream/eds_speed_test.cc b/test/common/upstream/eds_speed_test.cc index f2c0c5729e0f0..b97d8c26eede4 100644 --- a/test/common/upstream/eds_speed_test.cc +++ b/test/common/upstream/eds_speed_test.cc @@ -11,6 +11,8 @@ #include "common/config/grpc_mux_impl.h" #include "common/config/grpc_subscription_impl.h" #include "common/config/protobuf_link_hacks.h" +#include "common/config/unified_mux/grpc_mux_impl.h" +#include "common/config/unified_mux/grpc_subscription_impl.h" #include "common/config/utility.h" #include "common/singleton/manager_impl.h" #include "common/upstream/eds.h" @@ -40,18 +42,27 @@ namespace Upstream { class EdsSpeedTest { public: - EdsSpeedTest(State& state, bool v2_config) - : state_(state), v2_config_(v2_config), + EdsSpeedTest(State& state, bool v2_config, bool use_unified_mux) + : state_(state), v2_config_(v2_config), use_unified_mux_(use_unified_mux), type_url_(v2_config_ ? "type.googleapis.com/envoy.api.v2.ClusterLoadAssignment" : "type.googleapis.com/envoy.config.endpoint.v3.ClusterLoadAssignment"), subscription_stats_(Config::Utility::generateStats(stats_)), - api_(Api::createApiForTest(stats_)), async_client_(new Grpc::MockAsyncClient()), - grpc_mux_(new Config::GrpcMuxImpl( - local_info_, std::unique_ptr(async_client_), dispatcher_, - *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( - "envoy.service.endpoint.v3.EndpointDiscoveryService.StreamEndpoints"), - envoy::config::core::v3::ApiVersion::AUTO, random_, stats_, {}, true)) { + api_(Api::createApiForTest(stats_)), async_client_(new Grpc::MockAsyncClient()) { + if (use_unified_mux_) { + grpc_mux_.reset(new Config::UnifiedMux::GrpcMuxSotw( + std::unique_ptr(async_client_), dispatcher_, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.endpoint.v3.EndpointDiscoveryService.StreamEndpoints"), + envoy::config::core::v3::ApiVersion::AUTO, random_, stats_, {}, local_info_, true)); + } else { + grpc_mux_.reset(new Config::GrpcMuxImpl( + local_info_, std::unique_ptr(async_client_), dispatcher_, + *Protobuf::DescriptorPool::generated_pool()->FindMethodByName( + "envoy.service.endpoint.v3.EndpointDiscoveryService.StreamEndpoints"), + envoy::config::core::v3::ApiVersion::AUTO, random_, stats_, {}, true)); + } + resetCluster(R"EOF( name: name connect_timeout: 0.25s @@ -85,9 +96,15 @@ class EdsSpeedTest { std::move(scope), false); EXPECT_EQ(initialize_phase, cluster_->initializePhase()); eds_callbacks_ = cm_.subscription_factory_.callbacks_; - subscription_ = std::make_unique( - grpc_mux_, *eds_callbacks_, resource_decoder_, subscription_stats_, type_url_, dispatcher_, - std::chrono::milliseconds(), false, false); + if (use_unified_mux_) { + subscription_ = std::make_unique( + grpc_mux_, type_url_, *eds_callbacks_, resource_decoder_, subscription_stats_, + dispatcher_.timeSource(), std::chrono::milliseconds(), false, false); + } else { + subscription_ = std::make_unique( + grpc_mux_, *eds_callbacks_, resource_decoder_, subscription_stats_, type_url_, + dispatcher_, std::chrono::milliseconds(), false, false); + } } // Set up an EDS config with multiple priorities, localities, weights and make sure @@ -137,7 +154,15 @@ class EdsSpeedTest { resource->set_type_url("type.googleapis.com/envoy.api.v2.ClusterLoadAssignment"); } state_.ResumeTiming(); - grpc_mux_->grpcStreamForTest().onReceiveMessage(std::move(response)); + if (use_unified_mux_) { + static_cast(*grpc_mux_) + .grpcStreamForTest() + .onReceiveMessage(std::move(response)); + } else { + static_cast(*grpc_mux_) + .grpcStreamForTest() + .onReceiveMessage(std::move(response)); + } ASSERT(cluster_->prioritySet().hostSetsPerPriority()[1]->hostsPerLocality().get()[0].size() == num_hosts); } @@ -145,6 +170,7 @@ class EdsSpeedTest { TestDeprecatedV2Api _deprecated_v2_api_; State& state_; const bool v2_config_; + bool use_unified_mux_; const std::string type_url_; uint64_t version_{}; bool initialized_{}; @@ -169,8 +195,10 @@ class EdsSpeedTest { Server::MockOptions options_; Grpc::MockAsyncClient* async_client_; NiceMock async_stream_; - Config::GrpcMuxImplSharedPtr grpc_mux_; - Config::GrpcSubscriptionImplPtr subscription_; + // Config::GrpcMuxImplSharedPtr grpc_mux_; + std::shared_ptr grpc_mux_; + std::unique_ptr subscription_; + // Config::GrpcSubscriptionImplPtr subscription_; }; } // namespace Upstream @@ -181,7 +209,7 @@ static void priorityAndLocalityWeighted(State& state) { Envoy::Logger::Context logging_state(spdlog::level::warn, Envoy::Logger::Logger::DEFAULT_LOG_FORMAT, lock, false); for (auto _ : state) { - Envoy::Upstream::EdsSpeedTest speed_test(state, state.range(0)); + Envoy::Upstream::EdsSpeedTest speed_test(state, state.range(0), state.range(3)); // if we've been instructed to skip tests, only run once no matter the argument: uint32_t endpoints = skipExpensiveBenchmarks() ? 1 : state.range(2); @@ -190,7 +218,7 @@ static void priorityAndLocalityWeighted(State& state) { } BENCHMARK(priorityAndLocalityWeighted) - ->Ranges({{false, true}, {false, true}, {1, 100000}}) + ->Ranges({{false, true}, {false, true}, {1, 100000}, {false, true}}) ->Unit(benchmark::kMillisecond); static void duplicateUpdate(State& state) { @@ -199,7 +227,7 @@ static void duplicateUpdate(State& state) { Envoy::Logger::Logger::DEFAULT_LOG_FORMAT, lock, false); for (auto _ : state) { - Envoy::Upstream::EdsSpeedTest speed_test(state, false); + Envoy::Upstream::EdsSpeedTest speed_test(state, false, state.range(1)); uint32_t endpoints = skipExpensiveBenchmarks() ? 1 : state.range(0); speed_test.priorityAndLocalityWeightedHelper(true, endpoints, true); @@ -207,14 +235,14 @@ static void duplicateUpdate(State& state) { } } -BENCHMARK(duplicateUpdate)->Range(1, 100000)->Unit(benchmark::kMillisecond); +BENCHMARK(duplicateUpdate)->Ranges({{1, 100000}, {false, true}})->Unit(benchmark::kMillisecond); static void healthOnlyUpdate(State& state) { Envoy::Thread::MutexBasicLockable lock; Envoy::Logger::Context logging_state(spdlog::level::warn, Envoy::Logger::Logger::DEFAULT_LOG_FORMAT, lock, false); for (auto _ : state) { - Envoy::Upstream::EdsSpeedTest speed_test(state, false); + Envoy::Upstream::EdsSpeedTest speed_test(state, false, state.range(1)); uint32_t endpoints = skipExpensiveBenchmarks() ? 1 : state.range(0); speed_test.priorityAndLocalityWeightedHelper(true, endpoints, true); @@ -222,4 +250,4 @@ static void healthOnlyUpdate(State& state) { } } -BENCHMARK(healthOnlyUpdate)->Range(1, 100000)->Unit(benchmark::kMillisecond); +BENCHMARK(healthOnlyUpdate)->Ranges({{1, 100000}, {false, true}})->Unit(benchmark::kMillisecond); diff --git a/test/integration/ads_integration.cc b/test/integration/ads_integration.cc index cf0fbeb710467..0fc71434d96cc 100644 --- a/test/integration/ads_integration.cc +++ b/test/integration/ads_integration.cc @@ -22,10 +22,16 @@ namespace Envoy { AdsIntegrationTest::AdsIntegrationTest(envoy::config::core::v3::ApiVersion resource_api_version, envoy::config::core::v3::ApiVersion transport_api_version) - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), - ConfigHelper::adsBootstrap( - sotwOrDelta() == Grpc::SotwOrDelta::Sotw ? "GRPC" : "DELTA_GRPC", - resource_api_version, transport_api_version)) { + : HttpIntegrationTest( + Http::CodecClient::Type::HTTP2, ipVersion(), + ConfigHelper::adsBootstrap(sotwOrDelta() == Grpc::SotwOrDelta::Sotw || + sotwOrDelta() == Grpc::SotwOrDelta::LegacySotw + ? "GRPC" + : "DELTA_GRPC", + resource_api_version, transport_api_version)) { + if (sotwOrDelta() == Grpc::SotwOrDelta::Sotw || sotwOrDelta() == Grpc::SotwOrDelta::Delta) { + config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", "true"); + } use_lds_ = false; create_xds_upstream_ = true; tls_xds_upstream_ = true; diff --git a/test/integration/ads_integration_test.cc b/test/integration/ads_integration_test.cc index 215d7ab943e58..3eedbdc3c21b3 100644 --- a/test/integration/ads_integration_test.cc +++ b/test/integration/ads_integration_test.cc @@ -329,8 +329,13 @@ TEST_P(AdsIntegrationTest, ResendNodeOnStreamReset) { RELEASE_ASSERT(result, result.message()); xds_stream_->startGrpcStream(); - EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "1", {"cluster_0"}, - {"cluster_0"}, {}, true)); + if (sotwOrDelta() == Grpc::SotwOrDelta::LegacySotw) { + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "1", {"cluster_0"}, + {"cluster_0"}, {}, true)); + } else { + EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().Cluster, "", {"cluster_0"}, + {"cluster_0"}, {}, true)); + } } // Validate that xds can support a mix of v2 and v3 type url. @@ -491,7 +496,7 @@ TEST_P(AdsIntegrationTest, CdsEdsReplacementWarming) { {buildTlsCluster("cluster_0")}, {}, "2"); // Inconsistent SotW and delta behaviors for warming, see // https://github.com/envoyproxy/envoy/issues/11477#issuecomment-657855029. - if (sotw_or_delta_ != Grpc::SotwOrDelta::Delta) { + if (sotw_or_delta_ == Grpc::SotwOrDelta::LegacySotw) { EXPECT_TRUE(compareDiscoveryRequest(Config::TypeUrl::get().ClusterLoadAssignment, "1", {"cluster_0"}, {}, {})); } @@ -678,7 +683,7 @@ TEST_P(AdsIntegrationTest, CdsPausedDuringWarming) { test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0); // CDS is resumed and EDS response was acknowledged. - if (sotw_or_delta_ == Grpc::SotwOrDelta::Delta) { + if (sotw_or_delta_ != Grpc::SotwOrDelta::LegacySotw) { // Envoy will ACK both Cluster messages. Since they arrived while CDS was paused, they aren't // sent until CDS is unpaused. Since version 3 has already arrived by the time the version 2 // ACK goes out, they're both acknowledging version 3. @@ -760,7 +765,7 @@ TEST_P(AdsIntegrationTest, RemoveWarmingCluster) { test_server_->waitForGaugeEq("cluster_manager.active_clusters", 3); // CDS is resumed and EDS response was acknowledged. - if (sotw_or_delta_ == Grpc::SotwOrDelta::Delta) { + if (sotw_or_delta_ != Grpc::SotwOrDelta::LegacySotw) { // Envoy will ACK both Cluster messages. Since they arrived while CDS was paused, they aren't // sent until CDS is unpaused. Since version 3 has already arrived by the time the version 2 // ACK goes out, they're both acknowledging version 3. @@ -1052,10 +1057,16 @@ class AdsFailIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public HttpIntegrationTest { public: AdsFailIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), - ConfigHelper::adsBootstrap( - sotwOrDelta() == Grpc::SotwOrDelta::Sotw ? "GRPC" : "DELTA_GRPC", - envoy::config::core::v3::ApiVersion::V3)) { + : HttpIntegrationTest( + Http::CodecClient::Type::HTTP2, ipVersion(), + ConfigHelper::adsBootstrap(sotwOrDelta() == Grpc::SotwOrDelta::Sotw || + sotwOrDelta() == Grpc::SotwOrDelta::LegacySotw + ? "GRPC" + : "DELTA_GRPC", + envoy::config::core::v3::ApiVersion::V3)) { + if (sotwOrDelta() == Grpc::SotwOrDelta::Sotw || sotwOrDelta() == Grpc::SotwOrDelta::Delta) { + config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", "true"); + } create_xds_upstream_ = true; use_lds_ = false; sotw_or_delta_ = sotwOrDelta(); @@ -1093,10 +1104,16 @@ class AdsConfigIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public HttpIntegrationTest { public: AdsConfigIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), - ConfigHelper::adsBootstrap( - sotwOrDelta() == Grpc::SotwOrDelta::Sotw ? "GRPC" : "DELTA_GRPC", - envoy::config::core::v3::ApiVersion::V3)) { + : HttpIntegrationTest( + Http::CodecClient::Type::HTTP2, ipVersion(), + ConfigHelper::adsBootstrap(sotwOrDelta() == Grpc::SotwOrDelta::Sotw || + sotwOrDelta() == Grpc::SotwOrDelta::LegacySotw + ? "GRPC" + : "DELTA_GRPC", + envoy::config::core::v3::ApiVersion::V3)) { + if (sotwOrDelta() == Grpc::SotwOrDelta::Sotw || sotwOrDelta() == Grpc::SotwOrDelta::Delta) { + config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", "true"); + } create_xds_upstream_ = true; use_lds_ = false; sotw_or_delta_ = sotwOrDelta(); @@ -1234,7 +1251,8 @@ TEST_P(AdsIntegrationTest, NodeMessage) { envoy::service::discovery::v3::DiscoveryRequest sotw_request; envoy::service::discovery::v3::DeltaDiscoveryRequest delta_request; const envoy::config::core::v3::Node* node = nullptr; - if (sotw_or_delta_ == Grpc::SotwOrDelta::Sotw) { + if (sotw_or_delta_ == Grpc::SotwOrDelta::Sotw || + sotw_or_delta_ == Grpc::SotwOrDelta::LegacySotw) { EXPECT_TRUE(xds_stream_->waitForGrpcMessage(*dispatcher_, sotw_request)); EXPECT_TRUE(sotw_request.has_node()); node = &sotw_request.node(); @@ -1279,10 +1297,16 @@ class AdsClusterFromFileIntegrationTest : public Grpc::DeltaSotwIntegrationParam public HttpIntegrationTest { public: AdsClusterFromFileIntegrationTest() - : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), - ConfigHelper::adsBootstrap( - sotwOrDelta() == Grpc::SotwOrDelta::Sotw ? "GRPC" : "DELTA_GRPC", - envoy::config::core::v3::ApiVersion::V3)) { + : HttpIntegrationTest( + Http::CodecClient::Type::HTTP2, ipVersion(), + ConfigHelper::adsBootstrap(sotwOrDelta() == Grpc::SotwOrDelta::Sotw || + sotwOrDelta() == Grpc::SotwOrDelta::LegacySotw + ? "GRPC" + : "DELTA_GRPC", + envoy::config::core::v3::ApiVersion::V3)) { + if (sotwOrDelta() == Grpc::SotwOrDelta::Sotw || sotwOrDelta() == Grpc::SotwOrDelta::Delta) { + config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", "true"); + } create_xds_upstream_ = true; use_lds_ = false; sotw_or_delta_ = sotwOrDelta(); @@ -1546,7 +1570,7 @@ INSTANTIATE_TEST_SUITE_P( // There should be no variation across clients. testing::Values(Grpc::ClientType::EnvoyGrpc), // Only delta xDS is supported for XdsTp - testing::Values(Grpc::SotwOrDelta::Delta))); + testing::Values(Grpc::SotwOrDelta::Delta, Grpc::SotwOrDelta::LegacyDelta))); TEST_P(XdsTpAdsIntegrationTest, Basic) { initialize(); @@ -1753,7 +1777,7 @@ TEST_P(AdsClusterV2Test, DEPRECATED_FEATURE_TEST(CdsPausedDuringWarming)) { test_server_->waitForGaugeEq("cluster_manager.warming_clusters", 0); // CDS is resumed and EDS response was acknowledged. - if (sotw_or_delta_ == Grpc::SotwOrDelta::Delta) { + if (sotw_or_delta_ != Grpc::SotwOrDelta::LegacySotw) { // Envoy will ACK both Cluster messages. Since they arrived while CDS was paused, they aren't // sent until CDS is unpaused. Since version 3 has already arrived by the time the version 2 // ACK goes out, they're both acknowledging version 3. diff --git a/test/integration/base_integration_test.cc b/test/integration/base_integration_test.cc index 4b6bc813adbb0..3f6c6a5c5bcde 100644 --- a/test/integration/base_integration_test.cc +++ b/test/integration/base_integration_test.cc @@ -480,13 +480,14 @@ AssertionResult BaseIntegrationTest::compareDiscoveryRequest( const std::vector& expected_resource_names_added, const std::vector& expected_resource_names_removed, bool expect_node, const Protobuf::int32 expected_error_code, const std::string& expected_error_substring) { - if (sotw_or_delta_ == Grpc::SotwOrDelta::Sotw) { + if (sotw_or_delta_ == Grpc::SotwOrDelta::Sotw || + sotw_or_delta_ == Grpc::SotwOrDelta::LegacySotw) { return compareSotwDiscoveryRequest(expected_type_url, expected_version, expected_resource_names, expect_node, expected_error_code, expected_error_substring); } else { return compareDeltaDiscoveryRequest(expected_type_url, expected_resource_names_added, - expected_resource_names_removed, expected_error_code, - expected_error_substring); + expected_resource_names_removed, expect_node, + expected_error_code, expected_error_substring); } } @@ -570,13 +571,15 @@ AssertionResult BaseIntegrationTest::compareDeltaDiscoveryRequest( const std::string& expected_type_url, const std::vector& expected_resource_subscriptions, const std::vector& expected_resource_unsubscriptions, FakeStreamPtr& xds_stream, - const Protobuf::int32 expected_error_code, const std::string& expected_error_substring) { + const bool expect_node, const Protobuf::int32 expected_error_code, + const std::string& expected_error_substring) { envoy::service::discovery::v3::DeltaDiscoveryRequest request; VERIFY_ASSERTION(xds_stream->waitForGrpcMessage(*dispatcher_, request)); - // Verify all we care about node. - if (!request.has_node() || request.node().id().empty() || request.node().cluster().empty()) { - return AssertionFailure() << "Weird node field"; + if (expect_node) { + EXPECT_TRUE(request.has_node()); + EXPECT_FALSE(request.node().id().empty()); + EXPECT_FALSE(request.node().cluster().empty()); } last_node_.CopyFrom(request.node()); if (request.type_url() != expected_type_url) { diff --git a/test/integration/base_integration_test.h b/test/integration/base_integration_test.h index 2bdd0d8297b2b..874f5606b9a77 100644 --- a/test/integration/base_integration_test.h +++ b/test/integration/base_integration_test.h @@ -154,7 +154,8 @@ class BaseIntegrationTest : protected Logger::Loggable { const std::vector& added_or_updated, const std::vector& removed, const std::string& version, const bool api_downgrade = false) { - if (sotw_or_delta_ == Grpc::SotwOrDelta::Sotw) { + if (sotw_or_delta_ == Grpc::SotwOrDelta::Sotw || + sotw_or_delta_ == Grpc::SotwOrDelta::LegacySotw) { sendSotwDiscoveryResponse(type_url, state_of_the_world, version, api_downgrade); } else { sendDeltaDiscoveryResponse(type_url, added_or_updated, removed, version, api_downgrade); @@ -165,10 +166,11 @@ class BaseIntegrationTest : protected Logger::Loggable { const std::string& expected_type_url, const std::vector& expected_resource_subscriptions, const std::vector& expected_resource_unsubscriptions, + const bool expect_node = false, const Protobuf::int32 expected_error_code = Grpc::Status::WellKnownGrpcStatus::Ok, const std::string& expected_error_message = "") { return compareDeltaDiscoveryRequest(expected_type_url, expected_resource_subscriptions, - expected_resource_unsubscriptions, xds_stream_, + expected_resource_unsubscriptions, xds_stream_, expect_node, expected_error_code, expected_error_message); } @@ -176,6 +178,7 @@ class BaseIntegrationTest : protected Logger::Loggable { const std::string& expected_type_url, const std::vector& expected_resource_subscriptions, const std::vector& expected_resource_unsubscriptions, FakeStreamPtr& stream, + const bool expect_node = false, const Protobuf::int32 expected_error_code = Grpc::Status::WellKnownGrpcStatus::Ok, const std::string& expected_error_message = ""); diff --git a/test/integration/cds_integration_test.cc b/test/integration/cds_integration_test.cc index aecabc64bee69..181b25f6ec291 100644 --- a/test/integration/cds_integration_test.cc +++ b/test/integration/cds_integration_test.cc @@ -33,7 +33,10 @@ class CdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public Ht CdsIntegrationTest() : HttpIntegrationTest(Http::CodecClient::Type::HTTP2, ipVersion(), ConfigHelper::discoveredClustersBootstrap( - sotwOrDelta() == Grpc::SotwOrDelta::Sotw ? "GRPC" : "DELTA_GRPC")) { + sotwOrDelta() == Grpc::SotwOrDelta::Sotw || + sotwOrDelta() == Grpc::SotwOrDelta::LegacySotw + ? "GRPC" + : "DELTA_GRPC")) { use_lds_ = false; sotw_or_delta_ = sotwOrDelta(); } @@ -104,8 +107,9 @@ class CdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public Ht void verifyGrpcServiceMethod() { EXPECT_TRUE(xds_stream_->waitForHeadersComplete()); Envoy::Http::LowerCaseString path_string(":path"); + std::string expected_method( - sotwOrDelta() == Grpc::SotwOrDelta::Sotw + sotwOrDelta() == Grpc::SotwOrDelta::Sotw || sotwOrDelta() == Grpc::SotwOrDelta::LegacySotw ? "/envoy.service.cluster.v3.ClusterDiscoveryService/StreamClusters" : "/envoy.service.cluster.v3.ClusterDiscoveryService/DeltaClusters"); EXPECT_EQ(xds_stream_->headers().get(path_string)[0]->value(), expected_method); @@ -226,6 +230,7 @@ TEST_P(CdsIntegrationTest, TwoClusters) { // resources it already has: the reconnected stream need not start with a state-of-the-world update. TEST_P(CdsIntegrationTest, VersionsRememberedAfterReconnect) { SKIP_IF_XDS_IS(Grpc::SotwOrDelta::Sotw); + SKIP_IF_XDS_IS(Grpc::SotwOrDelta::LegacySotw); // Calls our initialize(), which includes establishing a listener, route, and cluster. testRouterHeaderOnlyRequestAndResponse(nullptr, UpstreamIndex1, "/cluster1"); diff --git a/test/integration/rtds_integration_test.cc b/test/integration/rtds_integration_test.cc index d3a3b06558561..eab7cb0be9226 100644 --- a/test/integration/rtds_integration_test.cc +++ b/test/integration/rtds_integration_test.cc @@ -80,7 +80,10 @@ class RtdsIntegrationTest : public Grpc::DeltaSotwIntegrationParamTest, public H RtdsIntegrationTest() : HttpIntegrationTest( Http::CodecClient::Type::HTTP2, ipVersion(), - tdsBootstrapConfig(sotwOrDelta() == Grpc::SotwOrDelta::Sotw ? "GRPC" : "DELTA_GRPC")) { + tdsBootstrapConfig(sotwOrDelta() == Grpc::SotwOrDelta::Sotw || + sotwOrDelta() == Grpc::SotwOrDelta::LegacySotw + ? "GRPC" + : "DELTA_GRPC")) { use_lds_ = false; create_xds_upstream_ = true; sotw_or_delta_ = sotwOrDelta(); diff --git a/test/integration/scoped_rds_integration_test.cc b/test/integration/scoped_rds_integration_test.cc index 0c6e3adf57431..7892f8fafe301 100644 --- a/test/integration/scoped_rds_integration_test.cc +++ b/test/integration/scoped_rds_integration_test.cc @@ -244,7 +244,10 @@ class ScopedRdsIntegrationTest : public HttpIntegrationTest, response); } - bool isDelta() { return sotwOrDelta() == Grpc::SotwOrDelta::Delta; } + bool isDelta() { + return sotwOrDelta() == Grpc::SotwOrDelta::Delta || + sotwOrDelta() == Grpc::SotwOrDelta::LegacyDelta; + } const std::string srds_config_name_{"foo-scoped-routes"}; FakeUpstreamInfo scoped_rds_upstream_info_; diff --git a/test/integration/vhds_integration_test.cc b/test/integration/vhds_integration_test.cc index 50601ebe8c2bb..f60ab386f3819 100644 --- a/test/integration/vhds_integration_test.cc +++ b/test/integration/vhds_integration_test.cc @@ -809,7 +809,8 @@ TEST_P(VhdsIntegrationTest, AttemptAddingDuplicateDomainNames) { virtualHostYaml("my_route/vhost_2", "vhost.duplicate"))}, {}, "2", vhds_stream_); EXPECT_TRUE(compareDeltaDiscoveryRequest(Config::TypeUrl::get().VirtualHost, {}, {}, vhds_stream_, - 13, "Only unique values for domains are permitted")); + false, 13, + "Only unique values for domains are permitted")); // Another update, this time valid, should result in no errors sendDeltaDiscoveryResponse( diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h index ae3364638092f..8f9eddf8f4183 100644 --- a/test/mocks/config/mocks.h +++ b/test/mocks/config/mocks.h @@ -122,6 +122,21 @@ class MockGrpcMux : public GrpcMux { MOCK_METHOD(void, requestOnDemandUpdate, (const std::string& type_url, const absl::flat_hash_set& add_these_names)); + + // unified mux interface + MOCK_METHOD(Watch*, addWatch, + (const std::string& type_url, const absl::flat_hash_set& resources, + SubscriptionCallbacks& callbacks, OpaqueResourceDecoder& resource_decoder, + std::chrono::milliseconds init_fetch_timeout, const bool use_namespace_matching), + (override)); + MOCK_METHOD(void, updateWatch, + (const std::string& type_url, Watch* watch, + const absl::flat_hash_set& resources, + const bool creating_namespace_watch), + (override)); + MOCK_METHOD(void, removeWatch, (const std::string& type_url, Watch* watch), (override)); + MOCK_METHOD(bool, paused, (const std::string& type_url), (const override)); + MOCK_METHOD(void, disableInitFetchTimeoutTimer, (), (override)); }; class MockGrpcStreamCallbacks @@ -188,4 +203,4 @@ class MockContextProvider : public ContextProvider { }; } // namespace Config -} // namespace Envoy +} // namespace Envoy \ No newline at end of file diff --git a/test/server/config_validation/xds_fuzz.cc b/test/server/config_validation/xds_fuzz.cc index d1543c238ad0c..37ea632ac0b8f 100644 --- a/test/server/config_validation/xds_fuzz.cc +++ b/test/server/config_validation/xds_fuzz.cc @@ -54,7 +54,7 @@ void XdsFuzzTest::updateRoute( } XdsFuzzTest::XdsFuzzTest(const test::server::config_validation::XdsTestCase& input, - envoy::config::core::v3::ApiVersion api_version) + envoy::config::core::v3::ApiVersion api_version, bool use_unified_mux) : HttpIntegrationTest( Http::CodecClient::Type::HTTP2, TestEnvironment::getIpVersionsForTest()[0], ConfigHelper::adsBootstrap(input.config().sotw_or_delta() == @@ -64,6 +64,9 @@ XdsFuzzTest::XdsFuzzTest(const test::server::config_validation::XdsTestCase& inp api_version)), verifier_(input.config().sotw_or_delta()), actions_(input.actions()), version_(1), api_version_(api_version), ip_version_(TestEnvironment::getIpVersionsForTest()[0]) { + if (use_unified_mux) { + config_helper_.addRuntimeOverride("envoy.reloadable_features.unified_mux", "true"); + } use_lds_ = false; create_xds_upstream_ = true; tls_xds_upstream_ = false; diff --git a/test/server/config_validation/xds_fuzz.h b/test/server/config_validation/xds_fuzz.h index 602c312ee8a03..82a82aad015f7 100644 --- a/test/server/config_validation/xds_fuzz.h +++ b/test/server/config_validation/xds_fuzz.h @@ -23,7 +23,7 @@ namespace Envoy { class XdsFuzzTest : public HttpIntegrationTest { public: XdsFuzzTest(const test::server::config_validation::XdsTestCase& input, - envoy::config::core::v3::ApiVersion api_version); + envoy::config::core::v3::ApiVersion api_version, bool use_unified_mux = false); envoy::config::cluster::v3::Cluster buildCluster(const std::string& name); diff --git a/test/server/config_validation/xds_fuzz_test.cc b/test/server/config_validation/xds_fuzz_test.cc index 5b41d4a01bbbd..1d9e969b46346 100644 --- a/test/server/config_validation/xds_fuzz_test.cc +++ b/test/server/config_validation/xds_fuzz_test.cc @@ -14,6 +14,8 @@ DEFINE_PROTO_FUZZER(const test::server::config_validation::XdsTestCase& input) { } XdsFuzzTest test(input, envoy::config::core::v3::ApiVersion::V3); test.replay(); + XdsFuzzTest test_with_unified_mux(input, envoy::config::core::v3::ApiVersion::V3, true); + test_with_unified_mux.replay(); } } // namespace Envoy diff --git a/tools/spelling/spelling_dictionary.txt b/tools/spelling/spelling_dictionary.txt index 52b0f16228c8e..b1bdb71e3b4f4 100644 --- a/tools/spelling/spelling_dictionary.txt +++ b/tools/spelling/spelling_dictionary.txt @@ -38,6 +38,7 @@ STATNAME SkyWalking TIDs WASI +Writeable ceil CCM CHACHA @@ -901,6 +902,7 @@ plaintext pluggable pointee poller +polymorphism popen pos posix