diff --git a/envoy/upstream/cluster_manager.h b/envoy/upstream/cluster_manager.h index f7e025b4fb91b..2c62af79305ff 100644 --- a/envoy/upstream/cluster_manager.h +++ b/envoy/upstream/cluster_manager.h @@ -38,6 +38,14 @@ #include "absl/container/node_hash_map.h" namespace Envoy { + +namespace Quic { + +class EnvoyQuicNetworkObserverRegistryFactory; +class EnvoyQuicNetworkObserverRegistry; + +} // namespace Quic + namespace Upstream { /** @@ -467,6 +475,13 @@ class ClusterManager { * Returns an EdsResourcesCache that is unique for the cluster manager. */ virtual Config::EdsResourcesCacheOptRef edsResourcesCache() PURE; + + /** + * Create a QUIC network observer registry for each worker thread using the given factory. + * @param factory used to create a registry object. + */ + virtual void createNetworkObserverRegistries( + Envoy::Quic::EnvoyQuicNetworkObserverRegistryFactory& factory) PURE; }; using ClusterManagerPtr = std::unique_ptr; @@ -515,6 +530,8 @@ class ClusterManagerFactory { /** * Allocate an HTTP connection pool for the host. Pools are separated by 'priority', * 'protocol', and 'options->hashKey()', if any. + * @param network_observer_registry if not null all the QUIC connections created by this pool + * should register to it for network events. */ virtual Http::ConnectionPool::InstancePtr allocateConnPool(Event::Dispatcher& dispatcher, HostConstSharedPtr host, @@ -524,7 +541,8 @@ class ClusterManagerFactory { const Network::ConnectionSocket::OptionsSharedPtr& options, const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, TimeSource& time_source, ClusterConnectivityState& state, - Http::PersistentQuicInfoPtr& quic_info) PURE; + Http::PersistentQuicInfoPtr& quic_info, + OptRef network_observer_registry) PURE; /** * Allocate a TCP connection pool for the host. Pools are separated by 'priority' and diff --git a/source/common/http/conn_pool_grid.cc b/source/common/http/conn_pool_grid.cc index 21dfc6a0927b3..cc59cd0ebc515 100644 --- a/source/common/http/conn_pool_grid.cc +++ b/source/common/http/conn_pool_grid.cc @@ -297,7 +297,8 @@ ConnectivityGrid::ConnectivityGrid( Upstream::ClusterConnectivityState& state, TimeSource& time_source, HttpServerPropertiesCacheSharedPtr alternate_protocols, ConnectivityOptions connectivity_options, Quic::QuicStatNames& quic_stat_names, - Stats::Scope& scope, Http::PersistentQuicInfo& quic_info) + Stats::Scope& scope, Http::PersistentQuicInfo& quic_info, + OptRef network_observer_registry) : dispatcher_(dispatcher), random_generator_(random_generator), host_(host), options_(options), transport_socket_options_(transport_socket_options), state_(state), next_attempt_duration_(std::chrono::milliseconds(kDefaultTimeoutMs)), @@ -306,7 +307,8 @@ ConnectivityGrid::ConnectivityGrid( // TODO(RyanTheOptimist): Figure out how scheme gets plumbed in here. origin_("https", getTargetHostname(transport_socket_options, host_), host_->address()->ip()->port()), - quic_info_(quic_info), priority_(priority) { + quic_info_(quic_info), priority_(priority), + network_observer_registry_(network_observer_registry) { // ProdClusterManagerFactory::allocateConnPool verifies the protocols are HTTP/1, HTTP/2 and // HTTP/3. ASSERT(connectivity_options.protocols_.size() == 3); @@ -386,7 +388,7 @@ ConnectionPool::InstancePtr ConnectivityGrid::createHttp3Pool(bool attempt_alter transport_socket_options_, state_, quic_stat_names_, *alternate_protocols_, scope_, makeOptRefFromPtr(this), - quic_info_, attempt_alternate_address); + quic_info_, network_observer_registry_, attempt_alternate_address); } void ConnectivityGrid::setupPool(ConnectionPool::Instance& pool) { diff --git a/source/common/http/conn_pool_grid.h b/source/common/http/conn_pool_grid.h index ba0e613d1d9a1..fee8a5802844b 100644 --- a/source/common/http/conn_pool_grid.h +++ b/source/common/http/conn_pool_grid.h @@ -184,7 +184,8 @@ class ConnectivityGrid : public ConnectionPool::Instance, Upstream::ClusterConnectivityState& state, TimeSource& time_source, HttpServerPropertiesCacheSharedPtr alternate_protocols, ConnectivityOptions connectivity_options, Quic::QuicStatNames& quic_stat_names, - Stats::Scope& scope, Http::PersistentQuicInfo& quic_info); + Stats::Scope& scope, Http::PersistentQuicInfo& quic_info, + OptRef network_observer_registry); ~ConnectivityGrid() override; // Event::DeferredDeletable @@ -298,6 +299,8 @@ class ConnectivityGrid : public ConnectionPool::Instance, // True iff this pool is being deferred deleted. bool deferred_deleting_{}; + + OptRef network_observer_registry_; }; } // namespace Http diff --git a/source/common/http/http3/conn_pool.cc b/source/common/http/http3/conn_pool.cc index f18fe88bd8068..761a5048db9e1 100644 --- a/source/common/http/http3/conn_pool.cc +++ b/source/common/http/http3/conn_pool.cc @@ -109,13 +109,15 @@ Http3ConnPoolImpl::Http3ConnPoolImpl( Random::RandomGenerator& random_generator, Upstream::ClusterConnectivityState& state, CreateClientFn client_fn, CreateCodecFn codec_fn, std::vector protocol, OptRef connect_callback, Http::PersistentQuicInfo& quic_info, + OptRef network_observer_registry, bool attempt_happy_eyeballs) : FixedHttpConnPoolImpl(host, priority, dispatcher, options, transport_socket_options, random_generator, state, client_fn, codec_fn, protocol, {}, nullptr), quic_info_(dynamic_cast(quic_info)), server_id_(sni(transport_socket_options, host), static_cast(host_->address()->ip()->port()), false), - connect_callback_(connect_callback), attempt_happy_eyeballs_(attempt_happy_eyeballs) {} + connect_callback_(connect_callback), attempt_happy_eyeballs_(attempt_happy_eyeballs), + network_observer_registry_(network_observer_registry) {} void Http3ConnPoolImpl::onConnected(Envoy::ConnectionPool::ActiveClient&) { if (connect_callback_ != absl::nullopt) { @@ -153,7 +155,7 @@ Http3ConnPoolImpl::createClientConnection(Quic::QuicStatNames& quic_stat_names, quic_info_, std::move(crypto_config), server_id_, dispatcher(), address, upstream_local_address.address_, quic_stat_names, rtt_cache, scope, upstream_local_address.socket_options_, transportSocketOptions(), connection_id_generator_, - host_->transportSocketFactory()); + host_->transportSocketFactory(), network_observer_registry_.ptr()); } std::unique_ptr @@ -164,7 +166,9 @@ allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_ Upstream::ClusterConnectivityState& state, Quic::QuicStatNames& quic_stat_names, OptRef rtt_cache, Stats::Scope& scope, OptRef connect_callback, - Http::PersistentQuicInfo& quic_info, bool attempt_happy_eyeballs) { + Http::PersistentQuicInfo& quic_info, + OptRef network_observer_registry, + bool attempt_happy_eyeballs) { return std::make_unique( host, priority, dispatcher, options, transport_socket_options, random_generator, state, [&quic_stat_names, rtt_cache, @@ -204,7 +208,8 @@ allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_ auto_connect); return codec; }, - std::vector{Protocol::Http3}, connect_callback, quic_info, attempt_happy_eyeballs); + std::vector{Protocol::Http3}, connect_callback, quic_info, + network_observer_registry, attempt_happy_eyeballs); } } // namespace Http3 diff --git a/source/common/http/http3/conn_pool.h b/source/common/http/http3/conn_pool.h index 932537a8a0ee8..d200a66bbe851 100644 --- a/source/common/http/http3/conn_pool.h +++ b/source/common/http/http3/conn_pool.h @@ -14,7 +14,9 @@ #include "source/common/quic/client_connection_factory_impl.h" #include "source/common/quic/envoy_quic_utils.h" #include "source/common/quic/quic_transport_socket_factory.h" +#include "source/common/quic/envoy_quic_network_observer_registry_factory.h" #include "quiche/quic/core/deterministic_connection_id_generator.h" + #else #error "http3 conn pool should not be built with QUIC disabled" #endif @@ -146,7 +148,9 @@ class Http3ConnPoolImpl : public FixedHttpConnPoolImpl { Upstream::ClusterConnectivityState& state, CreateClientFn client_fn, CreateCodecFn codec_fn, std::vector protocol, OptRef connect_callback, - Http::PersistentQuicInfo& quic_info, bool attempt_happy_eyeballs = false); + Http::PersistentQuicInfo& quic_info, + OptRef network_observer_registry, + bool attempt_happy_eyeballs = false); ~Http3ConnPoolImpl() override; ConnectionPool::Cancellable* newStream(Http::ResponseDecoder& response_decoder, @@ -183,6 +187,7 @@ class Http3ConnPoolImpl : public FixedHttpConnPoolImpl { // address. This fails over to using the primary address if the second address // in the list isn't of a different address family. bool attempt_happy_eyeballs_; + OptRef network_observer_registry_; }; std::unique_ptr @@ -193,7 +198,9 @@ allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_ Upstream::ClusterConnectivityState& state, Quic::QuicStatNames& quic_stat_names, OptRef rtt_cache, Stats::Scope& scope, OptRef connect_callback, - Http::PersistentQuicInfo& quic_info, bool attempt_happy_eyeballs = false); + Http::PersistentQuicInfo& quic_info, + OptRef network_observer_registry, + bool attempt_happy_eyeballs = false); } // namespace Http3 } // namespace Http diff --git a/source/common/quic/BUILD b/source/common/quic/BUILD index ecb713acc45d6..884ce2714010c 100644 --- a/source/common/quic/BUILD +++ b/source/common/quic/BUILD @@ -6,6 +6,7 @@ load( "//bazel:envoy_build_system.bzl", "envoy_cc_library", "envoy_package", + "envoy_select_enable_http3", "envoy_select_enable_http_datagrams", ) @@ -311,10 +312,13 @@ envoy_cc_library( srcs = [ "envoy_quic_client_session.cc", "envoy_quic_client_stream.cc", + "quic_network_connectivity_observer.cc", ], hdrs = [ "envoy_quic_client_session.h", "envoy_quic_client_stream.h", + "envoy_quic_network_observer_registry_factory.h", + "quic_network_connectivity_observer.h", ], tags = ["nofips"], deps = [ @@ -338,6 +342,17 @@ envoy_cc_library( ]), ) +envoy_cc_library( + name = "envoy_quic_network_observer_registry_factory_lib", + hdrs = [ + "envoy_quic_network_observer_registry_factory.h", + ], + tags = ["nofips"], + deps = envoy_select_enable_http3([ + ":envoy_quic_client_session_lib", + ]), +) + envoy_cc_library( name = "quic_io_handle_wrapper_lib", hdrs = ["quic_io_handle_wrapper.h"], diff --git a/source/common/quic/client_connection_factory_impl.cc b/source/common/quic/client_connection_factory_impl.cc index 08d7ce0fa05c4..17802078a16f7 100644 --- a/source/common/quic/client_connection_factory_impl.cc +++ b/source/common/quic/client_connection_factory_impl.cc @@ -37,7 +37,8 @@ std::unique_ptr createQuicNetworkConnection( const Network::ConnectionSocket::OptionsSharedPtr& options, const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, quic::ConnectionIdGeneratorInterface& generator, - Network::UpstreamTransportSocketFactory& transport_socket_factory) { + Network::UpstreamTransportSocketFactory& transport_socket_factory, + EnvoyQuicNetworkObserverRegistry* network_observer_registry) { // TODO: Quic should take into account the set_local_interface_name_on_upstream_connections config // and call maybeSetInterfaceName based on that upon acquiring a local socket. // Similar to what is done in ClientConnectionImpl::onConnected(). @@ -62,10 +63,14 @@ std::unique_ptr createQuicNetworkConnection( } // QUICHE client session always use the 1st version to start handshake. - return std::make_unique( + auto session = std::make_unique( config, quic_versions, std::move(connection), server_id, std::move(crypto_config), dispatcher, info_impl->buffer_limit_, info_impl->crypto_stream_factory_, quic_stat_names, rtt_cache, scope, transport_socket_options, transport_socket_factory); + if (network_observer_registry != nullptr) { + session->registerNetworkObserver(*network_observer_registry); + } + return session; } } // namespace Quic diff --git a/source/common/quic/client_connection_factory_impl.h b/source/common/quic/client_connection_factory_impl.h index 70170d0034c82..9e47687ac14b4 100644 --- a/source/common/quic/client_connection_factory_impl.h +++ b/source/common/quic/client_connection_factory_impl.h @@ -45,7 +45,8 @@ std::unique_ptr createQuicNetworkConnection( const Network::ConnectionSocket::OptionsSharedPtr& options, const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, quic::ConnectionIdGeneratorInterface& generator, - Network::UpstreamTransportSocketFactory& transport_socket_factory); + Network::UpstreamTransportSocketFactory& transport_socket_factory, + EnvoyQuicNetworkObserverRegistry* network_observer_registry = nullptr); } // namespace Quic } // namespace Envoy diff --git a/source/common/quic/envoy_quic_client_session.cc b/source/common/quic/envoy_quic_client_session.cc index aed2ac6115d8e..68cfb792c63d5 100644 --- a/source/common/quic/envoy_quic_client_session.cc +++ b/source/common/quic/envoy_quic_client_session.cc @@ -111,6 +111,9 @@ EnvoyQuicClientSession::EnvoyQuicClientSession( EnvoyQuicClientSession::~EnvoyQuicClientSession() { ASSERT(!connection()->connected()); network_connection_ = nullptr; + if (registry_.has_value()) { + registry_->unregisterObserver(*network_connectivity_observer_); + } } absl::string_view EnvoyQuicClientSession::requestedServerName() const { return server_id().host(); } @@ -301,5 +304,13 @@ std::vector EnvoyQuicClientSession::GetAlpnsToOffer() const { : configured_alpns_; } +void EnvoyQuicClientSession::registerNetworkObserver(EnvoyQuicNetworkObserverRegistry& registry) { + if (network_connectivity_observer_ == nullptr) { + network_connectivity_observer_ = std::make_unique(*this); + } + registry.registerObserver(*network_connectivity_observer_); + registry_ = makeOptRef(registry); +} + } // namespace Quic } // namespace Envoy diff --git a/source/common/quic/envoy_quic_client_session.h b/source/common/quic/envoy_quic_client_session.h index 5ef804e3e8ea5..65e69027bc58d 100644 --- a/source/common/quic/envoy_quic_client_session.h +++ b/source/common/quic/envoy_quic_client_session.h @@ -5,7 +5,9 @@ #include "source/common/quic/envoy_quic_client_connection.h" #include "source/common/quic/envoy_quic_client_crypto_stream_factory.h" #include "source/common/quic/envoy_quic_client_stream.h" +#include "source/common/quic/envoy_quic_network_observer_registry_factory.h" #include "source/common/quic/quic_filter_manager_connection_impl.h" +#include "source/common/quic/quic_network_connectivity_observer.h" #include "source/common/quic/quic_stat_names.h" #include "source/common/quic/quic_transport_socket_factory.h" @@ -87,6 +89,9 @@ class EnvoyQuicClientSession : public QuicFilterManagerConnectionImpl, void OnServerPreferredAddressAvailable( const quic::QuicSocketAddress& server_preferred_address) override; + // Register this session to the given registry for receiving network change events. + void registerNetworkObserver(EnvoyQuicNetworkObserverRegistry& registry); + using quic::QuicSpdyClientSession::PerformActionOnActiveStreams; protected: @@ -122,6 +127,8 @@ class EnvoyQuicClientSession : public QuicFilterManagerConnectionImpl, OptRef transport_socket_factory_; std::vector configured_alpns_; quic::HttpDatagramSupport http_datagram_support_ = quic::HttpDatagramSupport::kNone; + QuicNetworkConnectivityObserverPtr network_connectivity_observer_; + OptRef registry_; }; } // namespace Quic diff --git a/source/common/quic/envoy_quic_network_observer_registry_factory.h b/source/common/quic/envoy_quic_network_observer_registry_factory.h new file mode 100644 index 0000000000000..8556cb1d48486 --- /dev/null +++ b/source/common/quic/envoy_quic_network_observer_registry_factory.h @@ -0,0 +1,60 @@ +#pragma once + +#include + +#ifdef ENVOY_ENABLE_QUIC +#include "envoy/event/dispatcher.h" + +#include "source/common/quic/quic_network_connectivity_observer.h" +#endif + +namespace Envoy { +namespace Quic { + +#ifdef ENVOY_ENABLE_QUIC +// A registry of network connectivity observers. +class EnvoyQuicNetworkObserverRegistry { +public: + virtual ~EnvoyQuicNetworkObserverRegistry() = default; + + void registerObserver(QuicNetworkConnectivityObserver& observer) { + quic_observers_.insert(&observer); + } + + void unregisterObserver(QuicNetworkConnectivityObserver& observer) { + quic_observers_.erase(&observer); + } + +protected: + const absl::flat_hash_set& registeredQuicObservers() const { + return quic_observers_; + } + +private: + absl::flat_hash_set quic_observers_; +}; + +class EnvoyQuicNetworkObserverRegistryFactory { +public: + virtual ~EnvoyQuicNetworkObserverRegistryFactory() = default; + + virtual std::unique_ptr + createQuicNetworkObserverRegistry(Event::Dispatcher& /*dispatcher*/) { + return std::make_unique(); + } +}; + +#else + +// Dumb definitions of QUIC classes if QUIC is compiled out. +class EnvoyQuicNetworkObserverRegistry {}; +class EnvoyQuicNetworkObserverRegistryFactory {}; + +#endif + +using EnvoyQuicNetworkObserverRegistryPtr = std::unique_ptr; +using EnvoyQuicNetworkObserverRegistryFactoryPtr = + std::unique_ptr; + +} // namespace Quic +} // namespace Envoy diff --git a/source/common/quic/quic_network_connectivity_observer.cc b/source/common/quic/quic_network_connectivity_observer.cc new file mode 100644 index 0000000000000..ffc2bc44cff08 --- /dev/null +++ b/source/common/quic/quic_network_connectivity_observer.cc @@ -0,0 +1,12 @@ +#include "source/common/quic/quic_network_connectivity_observer.h" + +#include "source/common/quic/envoy_quic_client_session.h" + +namespace Envoy { +namespace Quic { + +QuicNetworkConnectivityObserver::QuicNetworkConnectivityObserver(EnvoyQuicClientSession& session) + : session_(session) {} + +} // namespace Quic +} // namespace Envoy diff --git a/source/common/quic/quic_network_connectivity_observer.h b/source/common/quic/quic_network_connectivity_observer.h new file mode 100644 index 0000000000000..0fd0f9825992e --- /dev/null +++ b/source/common/quic/quic_network_connectivity_observer.h @@ -0,0 +1,33 @@ +#pragma once + +#include + +#include "source/common/common/logger.h" + +namespace Envoy { +namespace Quic { + +class EnvoyQuicClientSession; + +// TODO(danzh) deprecate this class once QUICHE has its own more detailed network observer. +class QuicNetworkConnectivityObserver : protected Logger::Loggable { +public: + // session must outlive this object. + explicit QuicNetworkConnectivityObserver(EnvoyQuicClientSession& session); + QuicNetworkConnectivityObserver(const QuicNetworkConnectivityObserver&) = delete; + QuicNetworkConnectivityObserver& operator=(const QuicNetworkConnectivityObserver&) = delete; + + // Called when the device switches to a different network. + void onNetworkChanged() { + // TODO(danzh) close the connection if it's idle, otherwise mark it as go away. + (void)session_; + } + +private: + EnvoyQuicClientSession& session_; +}; + +using QuicNetworkConnectivityObserverPtr = std::unique_ptr; + +} // namespace Quic +} // namespace Envoy diff --git a/source/common/upstream/BUILD b/source/common/upstream/BUILD index 011eec2f43a12..a483b745f58de 100644 --- a/source/common/upstream/BUILD +++ b/source/common/upstream/BUILD @@ -113,6 +113,7 @@ envoy_cc_library( "//source/common/network:resolver_lib", "//source/common/network:utility_lib", "//source/common/protobuf:utility_lib", + "//source/common/quic:envoy_quic_network_observer_registry_factory_lib", "//source/common/quic:quic_stat_names_lib", "//source/common/router:context_lib", "//source/common/router:shadow_writer_lib", diff --git a/source/common/upstream/cluster_manager_impl.cc b/source/common/upstream/cluster_manager_impl.cc index c0e8f515788bf..6c16ac21db73b 100644 --- a/source/common/upstream/cluster_manager_impl.cc +++ b/source/common/upstream/cluster_manager_impl.cc @@ -1710,6 +1710,19 @@ Config::EdsResourcesCacheOptRef ClusterManagerImpl::edsResourcesCache() { return {}; } +void ClusterManagerImpl::createNetworkObserverRegistries( + Quic::EnvoyQuicNetworkObserverRegistryFactory& factory) { +#ifdef ENVOY_ENABLE_QUIC + tls_.runOnAllThreads([&factory](OptRef cluster_manager) { + ENVOY_LOG(trace, "cm: create network observer registry in {}", + cluster_manager->thread_local_dispatcher_.name()); + cluster_manager->createThreadLocalNetworkObserverRegistry(factory); + }); +#else + (void)factory; +#endif +} + ClusterDiscoveryManager ClusterManagerImpl::createAndSwapClusterDiscoveryManager(std::string thread_name) { ThreadLocalClusterManagerImpl& cluster_manager = *tls_; @@ -2032,7 +2045,8 @@ ClusterManagerImpl::ThreadLocalClusterManagerImpl::ClusterEntry::httpConnPoolImp parent_.thread_local_dispatcher_, host, priority, upstream_protocols, alternate_protocol_options, !upstream_options->empty() ? upstream_options : nullptr, have_transport_socket_options ? context->upstreamTransportSocketOptions() : nullptr, - parent_.parent_.time_source_, parent_.cluster_manager_state_, quic_info_); + parent_.parent_.time_source_, parent_.cluster_manager_state_, quic_info_, + parent_.getNetworkObserverRegistry()); pool->addIdleCallback([&parent = parent_, host, priority, hash_key]() { parent.httpConnPoolIsIdle(host, priority, hash_key); @@ -2210,7 +2224,8 @@ Http::ConnectionPool::InstancePtr ProdClusterManagerFactory::allocateConnPool( alternate_protocol_options, const Network::ConnectionSocket::OptionsSharedPtr& options, const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, - TimeSource& source, ClusterConnectivityState& state, Http::PersistentQuicInfoPtr& quic_info) { + TimeSource& source, ClusterConnectivityState& state, Http::PersistentQuicInfoPtr& quic_info, + OptRef network_observer_registry) { Http::HttpServerPropertiesCacheSharedPtr alternate_protocols_cache; if (alternate_protocol_options.has_value()) { @@ -2247,9 +2262,10 @@ Http::ConnectionPool::InstancePtr ProdClusterManagerFactory::allocateConnPool( return std::make_unique( dispatcher, context_.api().randomGenerator(), host, priority, options, transport_socket_options, state, source, alternate_protocols_cache, coptions, - quic_stat_names_, *stats_.rootScope(), *quic_info); + quic_stat_names_, *stats_.rootScope(), *quic_info, network_observer_registry); #else (void)quic_info; + (void)network_observer_registry; // Should be blocked by configuration checking at an earlier point. PANIC("unexpected"); #endif @@ -2281,7 +2297,8 @@ Http::ConnectionPool::InstancePtr ProdClusterManagerFactory::allocateConnPool( } return Http::Http3::allocateConnPool(dispatcher, context_.api().randomGenerator(), host, priority, options, transport_socket_options, state, - quic_stat_names_, {}, *stats_.rootScope(), {}, *quic_info); + quic_stat_names_, {}, *stats_.rootScope(), {}, *quic_info, + network_observer_registry); #else UNREFERENCED_PARAMETER(source); // Should be blocked by configuration checking at an earlier point. diff --git a/source/common/upstream/cluster_manager_impl.h b/source/common/upstream/cluster_manager_impl.h index 62f2efd2c62a1..da8e2eda753ae 100644 --- a/source/common/upstream/cluster_manager_impl.h +++ b/source/common/upstream/cluster_manager_impl.h @@ -33,6 +33,7 @@ #include "source/common/http/async_client_impl.h" #include "source/common/http/http_server_properties_cache_impl.h" #include "source/common/http/http_server_properties_cache_manager_impl.h" +#include "source/common/quic/envoy_quic_network_observer_registry_factory.h" #include "source/common/quic/quic_stat_names.h" #include "source/common/tcp/async_tcp_client_impl.h" #include "source/common/upstream/cluster_discovery_manager.h" @@ -67,15 +68,16 @@ class ProdClusterManagerFactory : public ClusterManagerFactory { // Upstream::ClusterManagerFactory ClusterManagerPtr clusterManagerFromProto(const envoy::config::bootstrap::v3::Bootstrap& bootstrap) override; - Http::ConnectionPool::InstancePtr - allocateConnPool(Event::Dispatcher& dispatcher, HostConstSharedPtr host, - ResourcePriority priority, std::vector& protocol, - const absl::optional& - alternate_protocol_options, - const Network::ConnectionSocket::OptionsSharedPtr& options, - const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, - TimeSource& time_source, ClusterConnectivityState& state, - Http::PersistentQuicInfoPtr& quic_info) override; + Http::ConnectionPool::InstancePtr allocateConnPool( + Event::Dispatcher& dispatcher, HostConstSharedPtr host, ResourcePriority priority, + std::vector& protocol, + const absl::optional& + alternate_protocol_options, + const Network::ConnectionSocket::OptionsSharedPtr& options, + const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, + TimeSource& time_source, ClusterConnectivityState& state, + Http::PersistentQuicInfoPtr& quic_info, + OptRef network_observer_registry) override; Tcp::ConnectionPool::InstancePtr allocateTcpConnPool(Event::Dispatcher& dispatcher, HostConstSharedPtr host, ResourcePriority priority, @@ -371,6 +373,9 @@ class ClusterManagerImpl : public ClusterManager, Config::EdsResourcesCacheOptRef edsResourcesCache() override; + void + createNetworkObserverRegistries(Quic::EnvoyQuicNetworkObserverRegistryFactory& factory) override; + protected: // ClusterManagerImpl's constructor should not be invoked directly; create instances from the // clusterManagerFromProto() static method. The init() method must be called after construction. @@ -695,6 +700,18 @@ class ClusterManagerImpl : public ClusterManager, */ ClusterEntry* initializeClusterInlineIfExists(absl::string_view cluster); + OptRef getNetworkObserverRegistry() { + return makeOptRefFromPtr(network_observer_registry_.get()); + } + +#ifdef ENVOY_ENABLE_QUIC + void createThreadLocalNetworkObserverRegistry( + Quic::EnvoyQuicNetworkObserverRegistryFactory& factory) { + network_observer_registry_ = + factory.createQuicNetworkObserverRegistry(thread_local_dispatcher_); + } +#endif + ClusterManagerImpl& parent_; Event::Dispatcher& thread_local_dispatcher_; // Known clusters will exclusively exist in either `thread_local_clusters_` @@ -720,6 +737,8 @@ class ClusterManagerImpl : public ClusterManager, private: static ThreadLocalClusterManagerStats generateStats(Stats::Scope& scope, const std::string& thread_name); + + Quic::EnvoyQuicNetworkObserverRegistryPtr network_observer_registry_; }; struct ClusterData : public ClusterManagerCluster { diff --git a/test/common/http/conn_pool_grid_test.cc b/test/common/http/conn_pool_grid_test.cc index 3a5606ffc3167..4ca040f8d41b9 100644 --- a/test/common/http/conn_pool_grid_test.cc +++ b/test/common/http/conn_pool_grid_test.cc @@ -188,7 +188,7 @@ class ConnectivityGridTest : public Event::TestUsingSimulatedTime, public testin grid_ = std::make_unique( dispatcher_, random_, host_, Upstream::ResourcePriority::Default, socket_options_, transport_socket_options_, state_, simTime(), alternate_protocols_, options_, - quic_stat_names_, *store_.rootScope(), *quic_connection_persistent_info_); + quic_stat_names_, *store_.rootScope(), *quic_connection_persistent_info_, registry_); grid_->cancel_ = &cancel_; grid_->info_ = &info_; grid_->encoder_ = &encoder_; @@ -234,10 +234,11 @@ class ConnectivityGridTest : public Event::TestUsingSimulatedTime, public testin StreamInfo::MockStreamInfo info_; NiceMock encoder_; - NiceMock factory_context_; testing::NiceMock thread_local_; NiceMock dispatcher_; + + Quic::EnvoyQuicNetworkObserverRegistry registry_; std::unique_ptr grid_; std::string host_impl_hostname_ = "hostname"; }; @@ -1366,7 +1367,7 @@ TEST_F(ConnectivityGridTest, RealGrid) { dispatcher_, random_, Upstream::makeTestHost(cluster_, "tcp://127.0.0.1:9000", simTime()), Upstream::ResourcePriority::Default, socket_options_, transport_socket_options_, state_, simTime(), alternate_protocols_, options_, quic_stat_names_, *store_.rootScope(), - *quic_connection_persistent_info_); + *quic_connection_persistent_info_, {}); EXPECT_EQ("connection grid", grid.protocolDescription()); EXPECT_FALSE(grid.hasActiveConnections()); @@ -1406,7 +1407,7 @@ TEST_F(ConnectivityGridTest, ConnectionCloseDuringAysnConnect) { dispatcher_, random_, Upstream::makeTestHost(cluster_, "tcp://127.0.0.1:9000", simTime()), Upstream::ResourcePriority::Default, socket_options_, transport_socket_options_, state_, simTime(), alternate_protocols_, options_, quic_stat_names_, *store_.rootScope(), - *quic_connection_persistent_info_); + *quic_connection_persistent_info_, {}); // Create the HTTP/3 pool. auto pool = ConnectivityGridForTest::forceGetOrCreateHttp3Pool(grid); diff --git a/test/common/http/http3/BUILD b/test/common/http/http3/BUILD index c187e91f91e2d..2a20dc642e1e1 100644 --- a/test/common/http/http3/BUILD +++ b/test/common/http/http3/BUILD @@ -20,6 +20,7 @@ envoy_cc_test( "//source/common/upstream:upstream_includes", "//source/common/upstream:upstream_lib", "//test/common/http:common_lib", + "//test/common/quic:test_utils_lib", "//test/common/upstream:utility_lib", "//test/mocks/event:event_mocks", "//test/mocks/http:http_mocks", diff --git a/test/common/http/http3/conn_pool_test.cc b/test/common/http/http3/conn_pool_test.cc index f45016d4c7ac7..48e2548942e2c 100644 --- a/test/common/http/http3/conn_pool_test.cc +++ b/test/common/http/http3/conn_pool_test.cc @@ -5,6 +5,7 @@ #include "source/common/quic/quic_client_transport_socket_factory.h" #include "test/common/http/common.h" +#include "test/common/quic/test_utils.h" #include "test/common/upstream/utility.h" #include "test/mocks/common.h" #include "test/mocks/event/mocks.h" @@ -69,7 +70,7 @@ class Http3ConnPoolImplTest : public Event::TestUsingSimulatedTime, public testi allocateConnPool(dispatcher_, random_, host_, Upstream::ResourcePriority::Default, options, transport_options, state_, quic_stat_names_, {}, *store_.rootScope(), makeOptRef(connect_result_callback_), - quic_info_, happy_eyeballs_); + quic_info_, {observers_}, happy_eyeballs_); EXPECT_EQ(3000, Http3ConnPoolImplPeer::getServerId(*pool_).port()); } @@ -93,6 +94,8 @@ class Http3ConnPoolImplTest : public Event::TestUsingSimulatedTime, public testi Ssl::ClientContextSharedPtr ssl_context_{new Ssl::MockClientContext()}; Stats::IsolatedStoreImpl store_; Quic::QuicStatNames quic_stat_names_{store_.symbolTable()}; + // Needs to out-live pool_; + Quic::TestNetworkObserverRegistry observers_; std::unique_ptr pool_; MockPoolConnectResultCallback connect_result_callback_; std::shared_ptr socket_option_{new Network::MockSocketOption()}; @@ -127,10 +130,10 @@ TEST_F(Http3ConnPoolImplTest, FastFailWithoutSecretsLoaded) { new Event::MockSchedulableCallback(&dispatcher_); Network::ConnectionSocket::OptionsSharedPtr options; Network::TransportSocketOptionsConstSharedPtr transport_options; - ConnectionPool::InstancePtr pool = - allocateConnPool(dispatcher_, random_, host_, Upstream::ResourcePriority::Default, options, - transport_options, state_, quic_stat_names_, {}, *store_.rootScope(), - makeOptRef(connect_result_callback_), quic_info_); + ConnectionPool::InstancePtr pool = allocateConnPool( + dispatcher_, random_, host_, Upstream::ResourcePriority::Default, options, transport_options, + state_, quic_stat_names_, {}, *store_.rootScope(), + makeOptRef(connect_result_callback_), quic_info_, {observers_}); EXPECT_EQ(static_cast(pool.get())->instantiateActiveClient(), nullptr); } @@ -154,10 +157,10 @@ TEST_F(Http3ConnPoolImplTest, FailWithSecretsBecomeEmpty) { new Event::MockSchedulableCallback(&dispatcher_); Network::ConnectionSocket::OptionsSharedPtr options; Network::TransportSocketOptionsConstSharedPtr transport_options; - ConnectionPool::InstancePtr pool = - allocateConnPool(dispatcher_, random_, host_, Upstream::ResourcePriority::Default, options, - transport_options, state_, quic_stat_names_, {}, *store_.rootScope(), - makeOptRef(connect_result_callback_), quic_info_); + ConnectionPool::InstancePtr pool = allocateConnPool( + dispatcher_, random_, host_, Upstream::ResourcePriority::Default, options, transport_options, + state_, quic_stat_names_, {}, *store_.rootScope(), + makeOptRef(connect_result_callback_), quic_info_, {observers_}); MockResponseDecoder decoder; ConnPoolCallbacks callbacks; @@ -205,6 +208,7 @@ void Http3ConnPoolImplTest::createNewStream() { std::list& clients = Http3ConnPoolImplPeer::connectingClients(*pool_); EXPECT_EQ(1u, clients.size()); + EXPECT_EQ(1u, observers_.registeredQuicObservers().size()); EXPECT_CALL(connect_result_callback_, onHandshakeComplete()).WillOnce(Invoke([cancellable]() { cancellable->cancel(Envoy::ConnectionPool::CancelPolicy::Default); })); diff --git a/test/common/quic/test_utils.h b/test/common/quic/test_utils.h index cc94aff40c5a0..b19ede56a8bd2 100644 --- a/test/common/quic/test_utils.h +++ b/test/common/quic/test_utils.h @@ -6,6 +6,7 @@ #include "source/common/quic/envoy_quic_client_connection.h" #include "source/common/quic/envoy_quic_client_session.h" #include "source/common/quic/envoy_quic_connection_debug_visitor_factory_interface.h" +#include "source/common/quic/envoy_quic_network_observer_registry_factory.h" #include "source/common/quic/envoy_quic_proof_verifier.h" #include "source/common/quic/envoy_quic_server_connection.h" #include "source/common/quic/envoy_quic_utils.h" @@ -367,5 +368,19 @@ DECLARE_FACTORY(TestEnvoyQuicConnectionDebugVisitorFactory); REGISTER_FACTORY(TestEnvoyQuicConnectionDebugVisitorFactory, Envoy::Quic::EnvoyQuicConnectionDebugVisitorFactoryInterface); +class TestNetworkObserverRegistry : public Quic::EnvoyQuicNetworkObserverRegistry { +public: + void onNetworkChanged() { + std::list existing_observers; + for (Quic::QuicNetworkConnectivityObserver* observer : registeredQuicObservers()) { + existing_observers.push_back(observer); + } + for (auto* observer : existing_observers) { + observer->onNetworkChanged(); + } + } + using Quic::EnvoyQuicNetworkObserverRegistry::registeredQuicObservers; +}; + } // namespace Quic } // namespace Envoy diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index d0640298972fa..ad2c80b5f45b1 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -2227,7 +2227,7 @@ TEST_P(ClusterManagerLifecycleTest, DynamicAddRemove) { EXPECT_EQ(1UL, cluster_manager_->clusters().active_clusters_.size()); Http::ConnectionPool::MockInstance* cp = new Http::ConnectionPool::MockInstance(); Http::ConnectionPool::Instance::IdleCb idle_cb; - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)).WillOnce(Return(cp)); + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)).WillOnce(Return(cp)); EXPECT_CALL(*cp, addIdleCallback(_)).WillOnce(SaveArg<0>(&idle_cb)); EXPECT_EQ(cp, HttpPoolDataPeer::getPool(cluster_manager_->getThreadLocalCluster("fake_cluster") ->httpConnPool(ResourcePriority::Default, @@ -2447,7 +2447,7 @@ TEST_P(ClusterManagerLifecycleTest, CloseHttpConnectionsOnHealthFailure) { })); create(parseBootstrapFromV3Json(json)); - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)).WillOnce(Return(cp1)); + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)).WillOnce(Return(cp1)); cluster_manager_->getThreadLocalCluster("some_cluster") ->httpConnPool(ResourcePriority::Default, Http::Protocol::Http11, nullptr); @@ -2459,7 +2459,7 @@ TEST_P(ClusterManagerLifecycleTest, CloseHttpConnectionsOnHealthFailure) { test_host->healthFlagSet(Host::HealthFlag::FAILED_OUTLIER_CHECK); outlier_detector.runCallbacks(test_host); - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)).WillOnce(Return(cp2)); + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)).WillOnce(Return(cp2)); cluster_manager_->getThreadLocalCluster("some_cluster") ->httpConnPool(ResourcePriority::High, Http::Protocol::Http11, nullptr); } @@ -2516,7 +2516,7 @@ TEST_P(ClusterManagerLifecycleTest, })); create(parseBootstrapFromV3Json(json)); - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)).WillOnce(Return(cp1)); + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)).WillOnce(Return(cp1)); cluster_manager_->getThreadLocalCluster("some_cluster") ->httpConnPool(ResourcePriority::Default, Http::Protocol::Http11, nullptr); @@ -2568,7 +2568,7 @@ TEST_P(ClusterManagerLifecycleTest, CloseHttpConnectionsAndDeletePoolOnHealthFai })); create(parseBootstrapFromV3Json(json)); - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)).WillOnce(Return(cp1)); + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)).WillOnce(Return(cp1)); cluster_manager_->getThreadLocalCluster("some_cluster") ->httpConnPool(ResourcePriority::Default, Http::Protocol::Http11, nullptr); @@ -2832,7 +2832,7 @@ TEST_P(ClusterManagerLifecycleTest, DynamicHostRemove) { EXPECT_CALL(initialized, ready()); cluster_manager_->setInitializedCb([&]() -> void { initialized.ready(); }); - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)) + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)) .Times(4) .WillRepeatedly(ReturnNew>()); @@ -2988,7 +2988,7 @@ TEST_P(ClusterManagerLifecycleTest, DynamicHostRemoveWithTls) { EXPECT_CALL(initialized, ready()); cluster_manager_->setInitializedCb([&]() -> void { initialized.ready(); }); - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)) + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)) .Times(4) .WillRepeatedly(ReturnNew>()); @@ -3927,7 +3927,7 @@ TEST_P(ClusterManagerLifecycleTest, DynamicHostRemoveDefaultPriority) { dns_callback(Network::DnsResolver::ResolutionStatus::Success, "", TestUtility::makeDnsResponse({"127.0.0.2"})); - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)) + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)) .WillOnce(ReturnNew>()); EXPECT_CALL(factory_, allocateTcpConnPool_) @@ -4023,7 +4023,7 @@ TEST_P(ClusterManagerLifecycleTest, ConnPoolDestroyWithDraining) { MockConnPoolWithDestroy* mock_cp = new MockConnPoolWithDestroy(); Http::ConnectionPool::Instance::IdleCb drained_cb; - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)).WillOnce(Return(mock_cp)); + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)).WillOnce(Return(mock_cp)); EXPECT_CALL(*mock_cp, addIdleCallback(_)).WillOnce(SaveArg<0>(&drained_cb)); EXPECT_CALL(*mock_cp, drainConnections(Envoy::ConnectionPool::DrainBehavior::DrainAndDelete)); @@ -4622,7 +4622,7 @@ TEST_F(ClusterManagerImplTest, UpstreamSocketOptionsPassedToConnPool) { Network::SocketOptionFactory::buildIpTransparentOptions(); EXPECT_CALL(context, upstreamSocketOptions()).WillOnce(Return(options_to_return)); - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)).WillOnce(Return(to_create)); + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)).WillOnce(Return(to_create)); auto opt_cp = cluster_manager_->getThreadLocalCluster("cluster_1") ->httpConnPool(ResourcePriority::Default, Http::Protocol::Http11, &context); @@ -4651,14 +4651,14 @@ TEST_F(ClusterManagerImplTest, UpstreamSocketOptionsUsedInConnPoolHash) { new NiceMock(); EXPECT_CALL(context1, upstreamSocketOptions()).WillOnce(Return(options1)); - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)).WillOnce(Return(to_create1)); + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)).WillOnce(Return(to_create1)); Http::ConnectionPool::Instance* cp1 = HttpPoolDataPeer::getPool( cluster_manager_->getThreadLocalCluster("cluster_1") ->httpConnPool(ResourcePriority::Default, Http::Protocol::Http11, &context1)); EXPECT_NE(nullptr, cp1); EXPECT_CALL(context2, upstreamSocketOptions()).WillOnce(Return(options2)); - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)).WillOnce(Return(to_create2)); + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)).WillOnce(Return(to_create2)); Http::ConnectionPool::Instance* cp2 = HttpPoolDataPeer::getPool( cluster_manager_->getThreadLocalCluster("cluster_1") ->httpConnPool(ResourcePriority::Default, Http::Protocol::Http11, &context2)); @@ -4691,7 +4691,7 @@ TEST_F(ClusterManagerImplTest, UpstreamSocketOptionsNullIsOkay) { Network::Socket::OptionsSharedPtr options_to_return = nullptr; EXPECT_CALL(context, upstreamSocketOptions()).WillOnce(Return(options_to_return)); - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)).WillOnce(Return(to_create)); + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)).WillOnce(Return(to_create)); auto opt_cp = cluster_manager_->getThreadLocalCluster("cluster_1") ->httpConnPool(ResourcePriority::Default, Http::Protocol::Http11, &context); @@ -4705,7 +4705,7 @@ TEST_F(ClusterManagerImplTest, HttpPoolDataForwardsCallsToConnectionPool) { Http::ConnectionPool::MockInstance* pool_mock = new Http::ConnectionPool::MockInstance(); Network::Socket::OptionsSharedPtr options_to_return = nullptr; - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)).WillOnce(Return(pool_mock)); + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)).WillOnce(Return(pool_mock)); EXPECT_CALL(*pool_mock, addIdleCallback(_)); auto opt_cp = cluster_manager_->getThreadLocalCluster("cluster_1") @@ -5882,7 +5882,7 @@ TEST_P(ClusterManagerLifecycleTest, DrainConnectionsPredicate) { 123, absl::nullopt, 100); // Using RR LB get a pool for each host. - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)) + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)) .Times(2) .WillRepeatedly(ReturnNew>()); Http::ConnectionPool::MockInstance* cp1 = HttpPoolDataPeer::getPool( @@ -5957,7 +5957,7 @@ TEST_P(ClusterManagerLifecycleTest, ConnPoolsDrainedOnHostSetChange) { EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.cluster_updated_via_merge").value()); EXPECT_EQ(0, factory_.stats_.counter("cluster_manager.update_merge_cancelled").value()); - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)) + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)) .Times(3) .WillRepeatedly(ReturnNew>()); @@ -6078,7 +6078,7 @@ TEST_P(ClusterManagerLifecycleTest, ConnPoolsNotDrainedOnHostSetChange) { 0, HostSetImpl::partitionHosts(hosts_ptr, HostsPerLocalityImpl::empty()), nullptr, hosts, {}, 123, absl::nullopt, 100); - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)) + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)) .Times(1) .WillRepeatedly(ReturnNew>()); @@ -6151,7 +6151,7 @@ TEST_P(ClusterManagerLifecycleTest, ConnPoolsIdleDeleted) { { auto* cp1 = new NiceMock(); - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)).WillOnce(Return(cp1)); + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)).WillOnce(Return(cp1)); std::function idle_callback; EXPECT_CALL(*cp1, addIdleCallback(_)).WillOnce(SaveArg<0>(&idle_callback)); @@ -6167,7 +6167,7 @@ TEST_P(ClusterManagerLifecycleTest, ConnPoolsIdleDeleted) { idle_callback(); auto* cp2 = new NiceMock(); - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)).WillOnce(Return(cp2)); + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)).WillOnce(Return(cp2)); EXPECT_CALL(*cp2, addIdleCallback(_)); // This time we expect cp2 since cp1 will have been destroyed @@ -6310,7 +6310,7 @@ TEST_F(ClusterManagerImplTest, ConnectionPoolPerDownstreamConnection) { for (size_t i = 0; i < 3; ++i) { conn_pool_vector.push_back(new Http::ConnectionPool::MockInstance()); EXPECT_CALL(*conn_pool_vector.back(), addIdleCallback(_)); - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)) + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)) .WillOnce(Return(conn_pool_vector.back())); EXPECT_CALL(downstream_connection, hashKey) .WillOnce(Invoke([i](std::vector& hash_key) { hash_key.push_back(i); })); @@ -6330,6 +6330,90 @@ TEST_F(ClusterManagerImplTest, ConnectionPoolPerDownstreamConnection) { Http::Protocol::Http11, &lb_context))); } +#ifdef ENVOY_ENABLE_QUIC +TEST_F(ClusterManagerImplTest, PassDownNetworkObserverRegistryToConnectionPool) { + const std::string yaml = R"EOF( + static_resources: + clusters: + - name: cluster_1 + connect_timeout: 0.250s + lb_policy: ROUND_ROBIN + type: STATIC + connection_pool_per_downstream_connection: true + load_assignment: + cluster_name: cluster_1 + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 11001 + )EOF"; + create(parseBootstrapFromV3Yaml(yaml)); + auto cluster1 = cluster_manager_->getThreadLocalCluster("cluster_1"); + + const std::string cluster_api = R"EOF( + name: added_via_api + connect_timeout: 0.250s + type: STATIC + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: added_via_api + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: 127.0.0.1 + port_value: 11001 + )EOF"; + // Add static cluster via api and check that addresses list is empty. + EXPECT_TRUE(cluster_manager_->addOrUpdateCluster(parseClusterFromV3Yaml(cluster_api), "v1")); + auto cluster_added_via_api = cluster_manager_->getThreadLocalCluster("added_via_api"); + + Quic::EnvoyQuicNetworkObserverRegistryFactory registry_factory; + cluster_manager_->createNetworkObserverRegistries(registry_factory); + + NiceMock lb_context; + NiceMock downstream_connection; + Network::Socket::OptionsSharedPtr options_to_return = nullptr; + ON_CALL(lb_context, downstreamConnection()).WillByDefault(Return(&downstream_connection)); + ON_CALL(downstream_connection, socketOptions()).WillByDefault(ReturnRef(options_to_return)); + + auto* pool = new Http::ConnectionPool::MockInstance(); + Quic::EnvoyQuicNetworkObserverRegistry* created_registry = nullptr; + EXPECT_CALL(*pool, addIdleCallback(_)); + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)) + .WillOnce(testing::WithArg<5>( + Invoke([pool, created_registry_ptr = &created_registry]( + OptRef network_observer_registry) { + EXPECT_TRUE(network_observer_registry.has_value()); + *created_registry_ptr = network_observer_registry.ptr(); + return pool; + }))); + EXPECT_CALL(downstream_connection, hashKey).WillOnce(Invoke([](std::vector& hash_key) { + hash_key.push_back(0); + })); + EXPECT_EQ(pool, HttpPoolDataPeer::getPool(cluster1->httpConnPool( + ResourcePriority::Default, Http::Protocol::Http11, &lb_context))); + + pool = new Http::ConnectionPool::MockInstance(); + EXPECT_CALL(*pool, addIdleCallback(_)); + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)) + .WillOnce(testing::WithArg<5>( + Invoke([pool, created_registry]( + OptRef network_observer_registry) { + EXPECT_TRUE(network_observer_registry.has_value()); + EXPECT_EQ(created_registry, network_observer_registry.ptr()); + return pool; + }))); + EXPECT_EQ(pool, HttpPoolDataPeer::getPool(cluster_added_via_api->httpConnPool( + ResourcePriority::Default, Http::Protocol::Http11, &lb_context))); +} + +#endif + TEST_F(ClusterManagerImplTest, ConnectionPoolPerDownstreamConnection_tcp) { const std::string yaml = R"EOF( static_resources: @@ -6592,7 +6676,7 @@ TEST_F(PreconnectTest, PreconnectOff) { // With preconnect set to 0, each request for a connection pool will only // allocate that conn pool. initialize(0); - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)) + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)) .Times(1) .WillRepeatedly(ReturnNew>()); auto http_handle = cluster_manager_->getThreadLocalCluster("cluster_1") @@ -6613,7 +6697,7 @@ TEST_F(PreconnectTest, PreconnectOn) { // preconnecting, so create the pool for both the current connection and the // anticipated one. initialize(1.1); - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)) + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)) .Times(2) .WillRepeatedly(ReturnNew>()); auto http_handle = cluster_manager_->getThreadLocalCluster("cluster_1") @@ -6653,7 +6737,7 @@ TEST_F(PreconnectTest, PreconnectHighHttp) { // With preconnect set to 3, the first request will kick off 3 preconnect attempts. initialize(3); int http_preconnect = 0; - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)) + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)) .Times(4) .WillRepeatedly(InvokeWithoutArgs([&]() -> Http::ConnectionPool::Instance* { auto* ret = new NiceMock(); @@ -6695,7 +6779,7 @@ TEST_F(PreconnectTest, PreconnectCappedAt3) { // With preconnect set to 20, no more than 3 connections will be preconnected. initialize(20); int http_preconnect = 0; - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)) + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)) .Times(4) .WillRepeatedly(InvokeWithoutArgs([&]() -> Http::ConnectionPool::Instance* { auto* ret = new NiceMock(); @@ -6726,7 +6810,7 @@ TEST_F(PreconnectTest, PreconnectCappedByMaybePreconnect) { // Set preconnect high, and verify preconnecting stops when maybePreconnect returns false. initialize(20); int http_preconnect_calls = 0; - EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _)) + EXPECT_CALL(factory_, allocateConnPool_(_, _, _, _, _, _)) .Times(2) .WillRepeatedly(InvokeWithoutArgs([&]() -> Http::ConnectionPool::Instance* { auto* ret = new NiceMock(); diff --git a/test/common/upstream/test_cluster_manager.h b/test/common/upstream/test_cluster_manager.h index e3c6662135f59..b532a27aee59c 100644 --- a/test/common/upstream/test_cluster_manager.h +++ b/test/common/upstream/test_cluster_manager.h @@ -90,9 +90,11 @@ class TestClusterManagerFactory : public ClusterManagerFactory { alternate_protocol_options, const Network::ConnectionSocket::OptionsSharedPtr& options, const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, TimeSource&, - ClusterConnectivityState& state, Http::PersistentQuicInfoPtr& /*quic_info*/) override { - return Http::ConnectionPool::InstancePtr{allocateConnPool_( - host, alternate_protocol_options, options, transport_socket_options, state)}; + ClusterConnectivityState& state, Http::PersistentQuicInfoPtr& /*quic_info*/, + OptRef network_observer_registry) override { + return Http::ConnectionPool::InstancePtr{allocateConnPool_(host, alternate_protocol_options, + options, transport_socket_options, + state, network_observer_registry)}; } Tcp::ConnectionPool::InstancePtr @@ -132,7 +134,8 @@ class TestClusterManagerFactory : public ClusterManagerFactory { const absl::optional& alternate_protocol_options, Network::ConnectionSocket::OptionsSharedPtr, - Network::TransportSocketOptionsConstSharedPtr, ClusterConnectivityState&)); + Network::TransportSocketOptionsConstSharedPtr, ClusterConnectivityState&, + OptRef network_observer_registry)); MOCK_METHOD(Tcp::ConnectionPool::Instance*, allocateTcpConnPool_, (HostConstSharedPtr host)); MOCK_METHOD((std::pair), clusterFromProto_, (const envoy::config::cluster::v3::Cluster& cluster, ClusterManager& cm, diff --git a/test/mocks/upstream/BUILD b/test/mocks/upstream/BUILD index edc0cf36ba12b..2b92de078cdf8 100644 --- a/test/mocks/upstream/BUILD +++ b/test/mocks/upstream/BUILD @@ -242,6 +242,7 @@ envoy_cc_mock( hdrs = ["cluster_manager_factory.h"], deps = [ "//envoy/upstream:cluster_manager_interface", + "//source/common/quic:envoy_quic_network_observer_registry_factory_lib", "//source/common/singleton:manager_impl_lib", "//test/mocks/secret:secret_mocks", "//test/test_common:thread_factory_for_test_lib", diff --git a/test/mocks/upstream/cluster_manager.h b/test/mocks/upstream/cluster_manager.h index ab2e04bbec6f2..05c58a99fb6e8 100644 --- a/test/mocks/upstream/cluster_manager.h +++ b/test/mocks/upstream/cluster_manager.h @@ -94,6 +94,8 @@ class MockClusterManager : public ClusterManager { } MOCK_METHOD(Config::EdsResourcesCacheOptRef, edsResourcesCache, ()); + MOCK_METHOD(void, createNetworkObserverRegistries, + (Quic::EnvoyQuicNetworkObserverRegistryFactory&)); envoy::config::core::v3::BindConfig& mutableBindConfig(); diff --git a/test/mocks/upstream/cluster_manager_factory.h b/test/mocks/upstream/cluster_manager_factory.h index c5ce18aaec66c..300d401add5e6 100644 --- a/test/mocks/upstream/cluster_manager_factory.h +++ b/test/mocks/upstream/cluster_manager_factory.h @@ -2,6 +2,7 @@ #include "envoy/upstream/cluster_manager.h" +#include "source/common/quic/envoy_quic_network_observer_registry_factory.h" #include "source/common/singleton/manager_impl.h" #include "test/mocks/secret/mocks.h" @@ -32,7 +33,8 @@ class MockClusterManagerFactory : public ClusterManagerFactory { const Network::ConnectionSocket::OptionsSharedPtr& options, const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options, TimeSource& source, ClusterConnectivityState& state, - Http::PersistentQuicInfoPtr& quic_info)); + Http::PersistentQuicInfoPtr& quic_info, + OptRef network_observer_registry)); MOCK_METHOD(Tcp::ConnectionPool::InstancePtr, allocateTcpConnPool, (Event::Dispatcher & dispatcher, HostConstSharedPtr host, ResourcePriority priority,