quic: add QuicNetworkConnectivityObserver and EnvoyQuicNetworkObserverRegistry interfaces

envoy/upstream/cluster_manager.h

@@ -38,6 +38,14 @@
 #include "absl/container/node_hash_map.h"
 
 namespace Envoy {
+
+namespace Quic {
+
+class EnvoyQuicNetworkObserverRegistryFactory;
+class EnvoyQuicNetworkObserverRegistry;
+
+} // namespace Quic
+
 namespace Upstream {
 
 /**
@@ -467,6 +475,13 @@ class ClusterManager {
    * Returns an EdsResourcesCache that is unique for the cluster manager.
    */
   virtual Config::EdsResourcesCacheOptRef edsResourcesCache() PURE;
+
+  /**
+   * Create a QUIC network observer registry for each worker thread using the given factory.
+   * @param factory used to create a registry object.
+   */
+  virtual void createNetworkObserverRegistries(
+      Envoy::Quic::EnvoyQuicNetworkObserverRegistryFactory& factory) PURE;
 };
 
 using ClusterManagerPtr = std::unique_ptr<ClusterManager>;
@@ -515,6 +530,8 @@ class ClusterManagerFactory {
   /**
    * Allocate an HTTP connection pool for the host. Pools are separated by 'priority',
    * 'protocol', and 'options->hashKey()', if any.
+   * @param network_observer_registry if not null all the QUIC connections created by this pool
+   * should register to it for network events.
    */
   virtual Http::ConnectionPool::InstancePtr
   allocateConnPool(Event::Dispatcher& dispatcher, HostConstSharedPtr host,
@@ -524,7 +541,8 @@ class ClusterManagerFactory {
                    const Network::ConnectionSocket::OptionsSharedPtr& options,
                    const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options,
                    TimeSource& time_source, ClusterConnectivityState& state,
-                   Http::PersistentQuicInfoPtr& quic_info) PURE;
+                   Http::PersistentQuicInfoPtr& quic_info,
+                   OptRef<Quic::EnvoyQuicNetworkObserverRegistry> network_observer_registry) PURE;
 
   /**
    * Allocate a TCP connection pool for the host. Pools are separated by 'priority' and

source/common/http/conn_pool_grid.cc

@@ -297,7 +297,8 @@ ConnectivityGrid::ConnectivityGrid(
     Upstream::ClusterConnectivityState& state, TimeSource& time_source,
     HttpServerPropertiesCacheSharedPtr alternate_protocols,
     ConnectivityOptions connectivity_options, Quic::QuicStatNames& quic_stat_names,
-    Stats::Scope& scope, Http::PersistentQuicInfo& quic_info)
+    Stats::Scope& scope, Http::PersistentQuicInfo& quic_info,
+    OptRef<Quic::EnvoyQuicNetworkObserverRegistry> network_observer_registry)
     : dispatcher_(dispatcher), random_generator_(random_generator), host_(host), options_(options),
       transport_socket_options_(transport_socket_options), state_(state),
       next_attempt_duration_(std::chrono::milliseconds(kDefaultTimeoutMs)),
@@ -306,7 +307,8 @@ ConnectivityGrid::ConnectivityGrid(
       // TODO(RyanTheOptimist): Figure out how scheme gets plumbed in here.
       origin_("https", getTargetHostname(transport_socket_options, host_),
               host_->address()->ip()->port()),
-      quic_info_(quic_info), priority_(priority) {
+      quic_info_(quic_info), priority_(priority),
+      network_observer_registry_(network_observer_registry) {
   // ProdClusterManagerFactory::allocateConnPool verifies the protocols are HTTP/1, HTTP/2 and
   // HTTP/3.
   ASSERT(connectivity_options.protocols_.size() == 3);
@@ -386,7 +388,7 @@ ConnectionPool::InstancePtr ConnectivityGrid::createHttp3Pool(bool attempt_alter
                                  transport_socket_options_, state_, quic_stat_names_,
                                  *alternate_protocols_, scope_,
                                  makeOptRefFromPtr<Http3::PoolConnectResultCallback>(this),
-                                 quic_info_, attempt_alternate_address);
+                                 quic_info_, network_observer_registry_, attempt_alternate_address);
 }
 
 void ConnectivityGrid::setupPool(ConnectionPool::Instance& pool) {

source/common/http/conn_pool_grid.h

@@ -184,7 +184,8 @@ class ConnectivityGrid : public ConnectionPool::Instance,
                    Upstream::ClusterConnectivityState& state, TimeSource& time_source,
                    HttpServerPropertiesCacheSharedPtr alternate_protocols,
                    ConnectivityOptions connectivity_options, Quic::QuicStatNames& quic_stat_names,
-                   Stats::Scope& scope, Http::PersistentQuicInfo& quic_info);
+                   Stats::Scope& scope, Http::PersistentQuicInfo& quic_info,
+                   OptRef<Quic::EnvoyQuicNetworkObserverRegistry> network_observer_registry);
   ~ConnectivityGrid() override;
 
   // Event::DeferredDeletable
@@ -298,6 +299,8 @@ class ConnectivityGrid : public ConnectionPool::Instance,
 
   // True iff this pool is being deferred deleted.
   bool deferred_deleting_{};
+
+  OptRef<Quic::EnvoyQuicNetworkObserverRegistry> network_observer_registry_;
 };
 
 } // namespace Http

source/common/http/http3/conn_pool.cc

@@ -109,13 +109,15 @@ Http3ConnPoolImpl::Http3ConnPoolImpl(
     Random::RandomGenerator& random_generator, Upstream::ClusterConnectivityState& state,
     CreateClientFn client_fn, CreateCodecFn codec_fn, std::vector<Http::Protocol> protocol,
     OptRef<PoolConnectResultCallback> connect_callback, Http::PersistentQuicInfo& quic_info,
+    OptRef<Quic::EnvoyQuicNetworkObserverRegistry> network_observer_registry,
     bool attempt_happy_eyeballs)
     : FixedHttpConnPoolImpl(host, priority, dispatcher, options, transport_socket_options,
                             random_generator, state, client_fn, codec_fn, protocol, {}, nullptr),
       quic_info_(dynamic_cast<Quic::PersistentQuicInfoImpl&>(quic_info)),
       server_id_(sni(transport_socket_options, host),
                  static_cast<uint16_t>(host_->address()->ip()->port()), false),
-      connect_callback_(connect_callback), attempt_happy_eyeballs_(attempt_happy_eyeballs) {}
+      connect_callback_(connect_callback), attempt_happy_eyeballs_(attempt_happy_eyeballs),
+      network_observer_registry_(network_observer_registry) {}
 
 void Http3ConnPoolImpl::onConnected(Envoy::ConnectionPool::ActiveClient&) {
   if (connect_callback_ != absl::nullopt) {
@@ -153,7 +155,7 @@ Http3ConnPoolImpl::createClientConnection(Quic::QuicStatNames& quic_stat_names,
       quic_info_, std::move(crypto_config), server_id_, dispatcher(), address,
       upstream_local_address.address_, quic_stat_names, rtt_cache, scope,
       upstream_local_address.socket_options_, transportSocketOptions(), connection_id_generator_,
-      host_->transportSocketFactory());
+      host_->transportSocketFactory(), network_observer_registry_.ptr());
 }
 
 std::unique_ptr<Http3ConnPoolImpl>
@@ -164,7 +166,9 @@ allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_
                  Upstream::ClusterConnectivityState& state, Quic::QuicStatNames& quic_stat_names,
                  OptRef<Http::HttpServerPropertiesCache> rtt_cache, Stats::Scope& scope,
                  OptRef<PoolConnectResultCallback> connect_callback,
-                 Http::PersistentQuicInfo& quic_info, bool attempt_happy_eyeballs) {
+                 Http::PersistentQuicInfo& quic_info,
+                 OptRef<Quic::EnvoyQuicNetworkObserverRegistry> network_observer_registry,
+                 bool attempt_happy_eyeballs) {
   return std::make_unique<Http3ConnPoolImpl>(
       host, priority, dispatcher, options, transport_socket_options, random_generator, state,
       [&quic_stat_names, rtt_cache,
@@ -204,7 +208,8 @@ allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_
             auto_connect);
         return codec;
       },
-      std::vector<Protocol>{Protocol::Http3}, connect_callback, quic_info, attempt_happy_eyeballs);
+      std::vector<Protocol>{Protocol::Http3}, connect_callback, quic_info,
+      network_observer_registry, attempt_happy_eyeballs);
 }
 
 } // namespace Http3

source/common/http/http3/conn_pool.h

@@ -14,7 +14,9 @@
 #include "source/common/quic/client_connection_factory_impl.h"
 #include "source/common/quic/envoy_quic_utils.h"
 #include "source/common/quic/quic_transport_socket_factory.h"
+#include "source/common/quic/envoy_quic_network_observer_registry_factory.h"
 #include "quiche/quic/core/deterministic_connection_id_generator.h"
+
 #else
 #error "http3 conn pool should not be built with QUIC disabled"
 #endif
@@ -146,7 +148,9 @@ class Http3ConnPoolImpl : public FixedHttpConnPoolImpl {
                     Upstream::ClusterConnectivityState& state, CreateClientFn client_fn,
                     CreateCodecFn codec_fn, std::vector<Http::Protocol> protocol,
                     OptRef<PoolConnectResultCallback> connect_callback,
-                    Http::PersistentQuicInfo& quic_info, bool attempt_happy_eyeballs = false);
+                    Http::PersistentQuicInfo& quic_info,
+                    OptRef<Quic::EnvoyQuicNetworkObserverRegistry> network_observer_registry,
+                    bool attempt_happy_eyeballs = false);
 
   ~Http3ConnPoolImpl() override;
   ConnectionPool::Cancellable* newStream(Http::ResponseDecoder& response_decoder,
@@ -183,6 +187,7 @@ class Http3ConnPoolImpl : public FixedHttpConnPoolImpl {
   // address. This fails over to using the primary address if the second address
   // in the list isn't of a different address family.
   bool attempt_happy_eyeballs_;
+  OptRef<Quic::EnvoyQuicNetworkObserverRegistry> network_observer_registry_;
 };
 
 std::unique_ptr<Http3ConnPoolImpl>
@@ -193,7 +198,9 @@ allocateConnPool(Event::Dispatcher& dispatcher, Random::RandomGenerator& random_
                  Upstream::ClusterConnectivityState& state, Quic::QuicStatNames& quic_stat_names,
                  OptRef<Http::HttpServerPropertiesCache> rtt_cache, Stats::Scope& scope,
                  OptRef<PoolConnectResultCallback> connect_callback,
-                 Http::PersistentQuicInfo& quic_info, bool attempt_happy_eyeballs = false);
+                 Http::PersistentQuicInfo& quic_info,
+                 OptRef<Quic::EnvoyQuicNetworkObserverRegistry> network_observer_registry,
+                 bool attempt_happy_eyeballs = false);
 
 } // namespace Http3
 } // namespace Http

source/common/quic/BUILD

@@ -6,6 +6,7 @@ load(
     "//bazel:envoy_build_system.bzl",
     "envoy_cc_library",
     "envoy_package",
+    "envoy_select_enable_http3",
     "envoy_select_enable_http_datagrams",
 )
 
@@ -311,10 +312,13 @@ envoy_cc_library(
     srcs = [
         "envoy_quic_client_session.cc",
         "envoy_quic_client_stream.cc",
+        "quic_network_connectivity_observer.cc",
     ],
     hdrs = [
         "envoy_quic_client_session.h",
         "envoy_quic_client_stream.h",
+        "envoy_quic_network_observer_registry_factory.h",
+        "quic_network_connectivity_observer.h",
     ],
     tags = ["nofips"],
     deps = [
@@ -338,6 +342,17 @@ envoy_cc_library(
     ]),
 )
 
+envoy_cc_library(
+    name = "envoy_quic_network_observer_registry_factory_lib",
+    hdrs = [
+        "envoy_quic_network_observer_registry_factory.h",
+    ],
+    tags = ["nofips"],
+    deps = envoy_select_enable_http3([
+        ":envoy_quic_client_session_lib",
+    ]),
+)
+
 envoy_cc_library(
     name = "quic_io_handle_wrapper_lib",
     hdrs = ["quic_io_handle_wrapper.h"],

source/common/quic/client_connection_factory_impl.cc

@@ -37,7 +37,8 @@ std::unique_ptr<Network::ClientConnection> createQuicNetworkConnection(
     const Network::ConnectionSocket::OptionsSharedPtr& options,
     const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options,
     quic::ConnectionIdGeneratorInterface& generator,
-    Network::UpstreamTransportSocketFactory& transport_socket_factory) {
+    Network::UpstreamTransportSocketFactory& transport_socket_factory,
+    EnvoyQuicNetworkObserverRegistry* network_observer_registry) {
   // TODO: Quic should take into account the set_local_interface_name_on_upstream_connections config
   // and call maybeSetInterfaceName based on that upon acquiring a local socket.
   // Similar to what is done in ClientConnectionImpl::onConnected().
@@ -62,10 +63,14 @@ std::unique_ptr<Network::ClientConnection> createQuicNetworkConnection(
   }
 
   // QUICHE client session always use the 1st version to start handshake.
-  return std::make_unique<EnvoyQuicClientSession>(
+  auto session = std::make_unique<EnvoyQuicClientSession>(
       config, quic_versions, std::move(connection), server_id, std::move(crypto_config), dispatcher,
       info_impl->buffer_limit_, info_impl->crypto_stream_factory_, quic_stat_names, rtt_cache,
       scope, transport_socket_options, transport_socket_factory);
+  if (network_observer_registry != nullptr) {
+    session->registerNetworkObserver(*network_observer_registry);
+  }
+  return session;
 }
 
 } // namespace Quic

source/common/quic/client_connection_factory_impl.h

@@ -45,7 +45,8 @@ std::unique_ptr<Network::ClientConnection> createQuicNetworkConnection(
     const Network::ConnectionSocket::OptionsSharedPtr& options,
     const Network::TransportSocketOptionsConstSharedPtr& transport_socket_options,
     quic::ConnectionIdGeneratorInterface& generator,
-    Network::UpstreamTransportSocketFactory& transport_socket_factory);
+    Network::UpstreamTransportSocketFactory& transport_socket_factory,
+    EnvoyQuicNetworkObserverRegistry* network_observer_registry = nullptr);
 
 } // namespace Quic
 } // namespace Envoy

source/common/quic/envoy_quic_client_session.cc

@@ -111,6 +111,9 @@ EnvoyQuicClientSession::EnvoyQuicClientSession(
 EnvoyQuicClientSession::~EnvoyQuicClientSession() {
   ASSERT(!connection()->connected());
   network_connection_ = nullptr;
+  if (registry_.has_value()) {
+    registry_->unregisterObserver(*network_connectivity_observer_);
+  }
 }
 
 absl::string_view EnvoyQuicClientSession::requestedServerName() const { return server_id().host(); }
@@ -301,5 +304,13 @@ std::vector<std::string> EnvoyQuicClientSession::GetAlpnsToOffer() const {
                                    : configured_alpns_;
 }
 
+void EnvoyQuicClientSession::registerNetworkObserver(EnvoyQuicNetworkObserverRegistry& registry) {
+  if (network_connectivity_observer_ == nullptr) {
+    network_connectivity_observer_ = std::make_unique<QuicNetworkConnectivityObserver>(*this);
+  }
+  registry.registerObserver(*network_connectivity_observer_);
+  registry_ = makeOptRef(registry);
+}
+
 } // namespace Quic
 } // namespace Envoy

source/common/quic/envoy_quic_client_session.h

@@ -5,7 +5,9 @@
 #include "source/common/quic/envoy_quic_client_connection.h"
 #include "source/common/quic/envoy_quic_client_crypto_stream_factory.h"
 #include "source/common/quic/envoy_quic_client_stream.h"
+#include "source/common/quic/envoy_quic_network_observer_registry_factory.h"
 #include "source/common/quic/quic_filter_manager_connection_impl.h"
+#include "source/common/quic/quic_network_connectivity_observer.h"
 #include "source/common/quic/quic_stat_names.h"
 #include "source/common/quic/quic_transport_socket_factory.h"
 
@@ -87,6 +89,9 @@ class EnvoyQuicClientSession : public QuicFilterManagerConnectionImpl,
   void OnServerPreferredAddressAvailable(
       const quic::QuicSocketAddress& server_preferred_address) override;
 
+  // Register this session to the given registry for receiving network change events.
+  void registerNetworkObserver(EnvoyQuicNetworkObserverRegistry& registry);
+
   using quic::QuicSpdyClientSession::PerformActionOnActiveStreams;
 
 protected:
@@ -122,6 +127,8 @@ class EnvoyQuicClientSession : public QuicFilterManagerConnectionImpl,
   OptRef<QuicTransportSocketFactoryBase> transport_socket_factory_;
   std::vector<std::string> configured_alpns_;
   quic::HttpDatagramSupport http_datagram_support_ = quic::HttpDatagramSupport::kNone;
+  QuicNetworkConnectivityObserverPtr network_connectivity_observer_;
+  OptRef<EnvoyQuicNetworkObserverRegistry> registry_;
 };
 
 } // namespace Quic

source/common/quic/envoy_quic_network_observer_registry_factory.h

@@ -0,0 +1,60 @@
+#pragma once
+
+#include <memory>
+
+#ifdef ENVOY_ENABLE_QUIC
+#include "envoy/event/dispatcher.h"
+
+#include "source/common/quic/quic_network_connectivity_observer.h"
+#endif
+
+namespace Envoy {
+namespace Quic {
+
+#ifdef ENVOY_ENABLE_QUIC
+// A registry of network connectivity observers.
+class EnvoyQuicNetworkObserverRegistry {
+public:
+  virtual ~EnvoyQuicNetworkObserverRegistry() = default;
+
+  void registerObserver(QuicNetworkConnectivityObserver& observer) {
+    quic_observers_.insert(&observer);
+  }
+
+  void unregisterObserver(QuicNetworkConnectivityObserver& observer) {
+    quic_observers_.erase(&observer);
+  }
+
+protected:
+  const absl::flat_hash_set<QuicNetworkConnectivityObserver*>& registeredQuicObservers() const {
+    return quic_observers_;
+  }
+
+private:
+  absl::flat_hash_set<QuicNetworkConnectivityObserver*> quic_observers_;
+};
+
+class EnvoyQuicNetworkObserverRegistryFactory {
+public:
+  virtual ~EnvoyQuicNetworkObserverRegistryFactory() = default;
+
+  virtual std::unique_ptr<EnvoyQuicNetworkObserverRegistry>
+  createQuicNetworkObserverRegistry(Event::Dispatcher& /*dispatcher*/) {
+    return std::make_unique<EnvoyQuicNetworkObserverRegistry>();
+  }
+};
+
+#else
+
+// Dumb definitions of QUIC classes if QUIC is compiled out.
+class EnvoyQuicNetworkObserverRegistry {};
+class EnvoyQuicNetworkObserverRegistryFactory {};
+
+#endif
+
+using EnvoyQuicNetworkObserverRegistryPtr = std::unique_ptr<EnvoyQuicNetworkObserverRegistry>;
+using EnvoyQuicNetworkObserverRegistryFactoryPtr =
+    std::unique_ptr<EnvoyQuicNetworkObserverRegistryFactory>;
+
+} // namespace Quic
+} // namespace Envoy

source/common/quic/quic_network_connectivity_observer.cc

@@ -0,0 +1,12 @@
+#include "source/common/quic/quic_network_connectivity_observer.h"
+
+#include "source/common/quic/envoy_quic_client_session.h"
+
+namespace Envoy {
+namespace Quic {
+
+QuicNetworkConnectivityObserver::QuicNetworkConnectivityObserver(EnvoyQuicClientSession& session)
+    : session_(session) {}
+
+} // namespace Quic
+} // namespace Envoy