quiche: support inplace filter chain update

docs/root/version_history/current.rst

@@ -114,6 +114,8 @@ New Features
 * route config: added :ref:`dynamic_metadata <envoy_v3_api_field_config.route.v3.RouteMatch.dynamic_metadata>` for routing based on dynamic metadata.
 * sxg_filter: added filter to transform response to SXG package to :ref:`contrib images <install_contrib>`. This can be enabled by setting :ref:`SXG <envoy_v3_api_msg_extensions.filters.http.sxg.v3alpha.SXG>` configuration.
 * thrift_proxy: added support for :ref:`mirroring requests <envoy_v3_api_field_extensions.filters.network.thrift_proxy.v3.RouteAction.request_mirror_policies>`.
+* udp: allows updating filter chain in-place through LDS, which is supported by Quic listener. Such listener config will be rejected in other connection-less UDP listener implementations. It can be reverted by ``envoy.reloadable_features.udp_listener_updates_filter_chain_in_place``.
+* udp: disallow L4 filter chain in config which configures connection-less UDP listener. It can be reverted by ``envoy.reloadable_features.udp_listener_updates_filter_chain_in_place``.
 
 Deprecated
 ----------

envoy/network/connection_handler.h

@@ -132,6 +132,17 @@ class ConnectionHandler {
      * Stop listening according to implementation's own definition.
      */
     virtual void shutdownListener() PURE;
+
+    /**
+     * Update the listener config.
+     */
+    virtual void updateListenerConfig(Network::ListenerConfig& config) PURE;
+
+    /**
+     * Called when the given filter chains are about to be removed.
+     */
+    virtual void onFilterChainDraining(
+        const std::list<const Network::FilterChain*>& draining_filter_chains) PURE;
   };
 
   using ActiveListenerPtr = std::unique_ptr<ActiveListener>;

source/common/quic/BUILD

@@ -265,6 +265,7 @@ envoy_cc_library(
         ":envoy_quic_utils_lib",
         ":quic_filter_manager_connection_lib",
         ":quic_stat_names_lib",
+        ":quic_transport_socket_factory_lib",
         "//source/common/buffer:buffer_lib",
         "//source/common/common:assert_lib",
         "//source/common/http:codes_lib",

source/common/quic/active_quic_listener.cc

@@ -221,6 +221,24 @@ size_t ActiveQuicListener::numPacketsExpectedPerEventLoop() const {
   return quic_dispatcher_->NumSessions() * packets_to_read_to_connection_count_ratio_;
 }
 
+void ActiveQuicListener::updateListenerConfig(Network::ListenerConfig& config) {
+  config_ = &config;
+  dynamic_cast<EnvoyQuicProofSource*>(crypto_config_->proof_source())
+      ->updateFilterChainManager(config.filterChainManager());
+  quic_dispatcher_->updateListenerConfig(config);
+}
+
+void ActiveQuicListener::onFilterChainDraining(
+    const std::list<const Network::FilterChain*>& draining_filter_chains) {
+  for (auto* filter_chain : draining_filter_chains) {
+    closeConnectionsWithFilterChain(filter_chain);
+  }
+}
+
+void ActiveQuicListener::closeConnectionsWithFilterChain(const Network::FilterChain* filter_chain) {
+  quic_dispatcher_->closeConnectionsWithFilterChain(filter_chain);
+}
+
 ActiveQuicListenerFactory::ActiveQuicListenerFactory(
     const envoy::config::listener::v3::QuicProtocolOptions& config, uint32_t concurrency,
     QuicStatNames& quic_stat_names)

source/common/quic/active_quic_listener.h

@@ -70,10 +70,15 @@ class ActiveQuicListener : public Envoy::Server::ActiveUdpListenerBase,
   void pauseListening() override;
   void resumeListening() override;
   void shutdownListener() override;
+  void updateListenerConfig(Network::ListenerConfig& config) override;
+  void onFilterChainDraining(
+      const std::list<const Network::FilterChain*>& draining_filter_chains) override;
 
 private:
   friend class ActiveQuicListenerPeer;
 
+  void closeConnectionsWithFilterChain(const Network::FilterChain* filter_chain);
+
   uint8_t random_seed_[16];
   std::unique_ptr<quic::QuicCryptoServerConfig> crypto_config_;
   Event::Dispatcher& dispatcher_;

source/common/quic/envoy_quic_client_session.h

@@ -99,7 +99,8 @@ class EnvoyQuicClientSession : public QuicFilterManagerConnectionImpl,
   // These callbacks are owned by network filters and quic session should outlive
   // them.
   Http::ConnectionCallbacks* http_connection_callbacks_{nullptr};
-  const absl::string_view host_name_;
+  // TODO(danzh) deprecate this field once server_id() is made const.
+  const std::string host_name_;
   std::shared_ptr<quic::QuicCryptoClientConfig> crypto_config_;
   EnvoyQuicCryptoClientStreamFactoryInterface& crypto_stream_factory_;
   QuicStatNames& quic_stat_names_;

source/common/quic/envoy_quic_dispatcher.cc

@@ -2,12 +2,14 @@
 
 #include <openssl/crypto.h>
 
+#include <functional>
+#include <list>
+
 #include "envoy/common/optref.h"
 
 #include "source/common/common/safe_memcpy.h"
 #include "source/common/http/utility.h"
 #include "source/common/quic/envoy_quic_server_connection.h"
-#include "source/common/quic/envoy_quic_server_session.h"
 #include "source/common/quic/envoy_quic_utils.h"
 
 namespace Envoy {
@@ -26,7 +28,7 @@ EnvoyQuicDispatcher::EnvoyQuicDispatcher(
     : quic::QuicDispatcher(&quic_config, crypto_config, version_manager, std::move(helper),
                            std::make_unique<EnvoyQuicCryptoServerStreamHelper>(),
                            std::move(alarm_factory), expected_server_connection_id_length),
-      connection_handler_(connection_handler), listener_config_(listener_config),
+      connection_handler_(connection_handler), listener_config_(&listener_config),
       listener_stats_(listener_stats), per_worker_stats_(per_worker_stats), dispatcher_(dispatcher),
       listen_socket_(listen_socket), quic_stat_names_(quic_stat_names),
       crypto_server_stream_factory_(crypto_server_stream_factory) {
@@ -52,44 +54,43 @@ void EnvoyQuicDispatcher::OnConnectionClosed(quic::QuicConnectionId connection_i
   listener_stats_.downstream_cx_active_.dec();
   per_worker_stats_.downstream_cx_active_.dec();
   connection_handler_.decNumConnections();
-  quic_stat_names_.chargeQuicConnectionCloseStats(listener_config_.listenerScope(), error, source,
+  quic_stat_names_.chargeQuicConnectionCloseStats(listener_config_->listenerScope(), error, source,
                                                   /*is_upstream*/ false);
 }
 
 std::unique_ptr<quic::QuicSession> EnvoyQuicDispatcher::CreateQuicSession(
     quic::QuicConnectionId server_connection_id, const quic::QuicSocketAddress& self_address,
-    const quic::QuicSocketAddress& peer_address, absl::string_view alpn,
+    const quic::QuicSocketAddress& peer_address, absl::string_view /*alpn*/,
     const quic::ParsedQuicVersion& version, absl::string_view sni) {
   quic::QuicConfig quic_config = config();
+  // TODO(danzh) use passed-in ALPN instead of hard-coded h3 after proof source interfaces takes in
+  // ALPN.
   Network::ConnectionSocketPtr connection_socket = createServerConnectionSocket(
-      listen_socket_.ioHandle(), self_address, peer_address, std::string(sni), alpn);
+      listen_socket_.ioHandle(), self_address, peer_address, std::string(sni), "h3");
   const Network::FilterChain* filter_chain =
-      listener_config_.filterChainManager().findFilterChain(*connection_socket);
+      listener_config_->filterChainManager().findFilterChain(*connection_socket);
 
   auto quic_connection = std::make_unique<EnvoyQuicServerConnection>(
       server_connection_id, self_address, peer_address, *helper(), *alarm_factory(), writer(),
       /*owns_writer=*/false, quic::ParsedQuicVersionVector{version}, std::move(connection_socket));
   auto quic_session = std::make_unique<EnvoyQuicServerSession>(
       quic_config, quic::ParsedQuicVersionVector{version}, std::move(quic_connection), this,
       session_helper(), crypto_config(), compressed_certs_cache(), dispatcher_,
-      listener_config_.perConnectionBufferLimitBytes(), quic_stat_names_,
-      listener_config_.listenerScope(), crypto_server_stream_factory_,
-      makeOptRefFromPtr(filter_chain == nullptr ? nullptr
-                                                : &filter_chain->transportSocketFactory()));
+      listener_config_->perConnectionBufferLimitBytes(), quic_stat_names_,
+      listener_config_->listenerScope(), crypto_server_stream_factory_);
   if (filter_chain != nullptr) {
+    // Setup filter chain before Initialize().
     const bool has_filter_initialized =
-        listener_config_.filterChainFactory().createNetworkFilterChain(
+        listener_config_->filterChainFactory().createNetworkFilterChain(
             *quic_session, filter_chain->networkFilterFactories());
     // QUIC listener must have HCM filter configured. Otherwise, stream creation later will fail.
     ASSERT(has_filter_initialized);
+    connections_by_filter_chain_[filter_chain].push_front(
+        std::reference_wrapper<Network::Connection>(*quic_session));
+    quic_session->storeConnectionMapPosition(connections_by_filter_chain_, *filter_chain,
+                                             connections_by_filter_chain_[filter_chain].begin());
   }
   quic_session->Initialize();
-  // Filter chain can't be retrieved here as self address is unknown at this
-  // point.
-  // TODO(danzh): change QUIC interface to pass in self address as it is already
-  // known. In this way, filter chain can be retrieved at this point. But one
-  // thing to pay attention is that if the retrieval fails, connection needs to
-  // be closed, and it should be added to time wait list instead of session map.
   connection_handler_.incNumConnections();
   listener_stats_.downstream_cx_active_.inc();
   listener_stats_.downstream_cx_total_.inc();
@@ -107,5 +108,27 @@ quic::QuicConnectionId EnvoyQuicDispatcher::ReplaceLongServerConnectionId(
   return new_connection_id;
 }
 
+void EnvoyQuicDispatcher::closeConnectionsWithFilterChain(
+    const Network::FilterChain* filter_chain) {
+  auto iter = connections_by_filter_chain_.find(filter_chain);
+  if (iter != connections_by_filter_chain_.end()) {
+    std::list<std::reference_wrapper<Network::Connection>>& connections = iter->second;
+    // Retain the number of connections in the list early because closing the connection will change
+    // the size.
+    const size_t num_connections = connections.size();
+    for (size_t i = 0; i < num_connections; ++i) {
+      Network::Connection& connection = connections.front().get();
+      // This will remove the connection from the list. And the last removal will remove connections
+      // from the map as well.
+      connection.close(Network::ConnectionCloseType::NoFlush);
+    }
+    ASSERT(connections_by_filter_chain_.find(filter_chain) == connections_by_filter_chain_.end());
+  }
+}
+
+void EnvoyQuicDispatcher::updateListenerConfig(Network::ListenerConfig& new_listener_config) {
+  listener_config_ = &new_listener_config;
+}
+
 } // namespace Quic
 } // namespace Envoy

source/common/quic/envoy_quic_dispatcher.h

@@ -20,6 +20,7 @@
 #include "source/server/connection_handler_impl.h"
 #include "source/server/active_listener_base.h"
 #include "source/common/quic/envoy_quic_crypto_stream_factory.h"
+#include "source/common/quic/envoy_quic_server_session.h"
 #include "source/common/quic/quic_stat_names.h"
 
 namespace Envoy {
@@ -54,6 +55,9 @@ class EnvoyQuicDispatcher : public quic::QuicDispatcher {
   void OnConnectionClosed(quic::QuicConnectionId connection_id, quic::QuicErrorCode error,
                           const std::string& error_details,
                           quic::ConnectionCloseSource source) override;
+  void closeConnectionsWithFilterChain(const Network::FilterChain* filter_chain);
+
+  void updateListenerConfig(Network::ListenerConfig& new_listener_config);
 
 protected:
   // quic::QuicDispatcher
@@ -72,13 +76,14 @@ class EnvoyQuicDispatcher : public quic::QuicDispatcher {
 
 private:
   Network::ConnectionHandler& connection_handler_;
-  Network::ListenerConfig& listener_config_;
+  Network::ListenerConfig* listener_config_{nullptr};
   Server::ListenerStats& listener_stats_;
   Server::PerHandlerListenerStats& per_worker_stats_;
   Event::Dispatcher& dispatcher_;
   Network::Socket& listen_socket_;
   QuicStatNames& quic_stat_names_;
   EnvoyQuicCryptoServerStreamFactoryInterface& crypto_server_stream_factory_;
+  FilterChainToConnectionMap connections_by_filter_chain_;
 };
 
 } // namespace Quic

source/common/quic/envoy_quic_proof_source.cc

@@ -100,13 +100,15 @@ EnvoyQuicProofSource::getTlsCertConfigAndFilterChain(const quic::QuicSocketAddre
   Network::ConnectionSocketPtr connection_socket = createServerConnectionSocket(
       listen_socket_.ioHandle(), server_address, client_address, hostname, "h3");
   const Network::FilterChain* filter_chain =
-      filter_chain_manager_.findFilterChain(*connection_socket);
+      filter_chain_manager_->findFilterChain(*connection_socket);
 
   if (filter_chain == nullptr) {
     listener_stats_.no_filter_chain_match_.inc();
     ENVOY_LOG(warn, "No matching filter chain found for handshake.");
     return {absl::nullopt, absl::nullopt};
   }
+  ENVOY_LOG(trace, "Got a matching cert chain {}", filter_chain->name());
+
   auto& transport_socket_factory =
       dynamic_cast<const QuicServerTransportSocketFactory&>(filter_chain->transportSocketFactory());
 
@@ -122,5 +124,10 @@ EnvoyQuicProofSource::getTlsCertConfigAndFilterChain(const quic::QuicSocketAddre
   return {tls_cert_configs[0].get(), *filter_chain};
 }
 
+void EnvoyQuicProofSource::updateFilterChainManager(
+    Network::FilterChainManager& filter_chain_manager) {
+  filter_chain_manager_ = &filter_chain_manager;
+}
+
 } // namespace Quic
 } // namespace Envoy

source/common/quic/envoy_quic_proof_source.h

@@ -14,7 +14,7 @@ class EnvoyQuicProofSource : public EnvoyQuicProofSourceBase {
   EnvoyQuicProofSource(Network::Socket& listen_socket,
                        Network::FilterChainManager& filter_chain_manager,
                        Server::ListenerStats& listener_stats)
-      : listen_socket_(listen_socket), filter_chain_manager_(filter_chain_manager),
+      : listen_socket_(listen_socket), filter_chain_manager_(&filter_chain_manager),
         listener_stats_(listener_stats) {}
 
   ~EnvoyQuicProofSource() override = default;
@@ -24,6 +24,8 @@ class EnvoyQuicProofSource : public EnvoyQuicProofSourceBase {
   GetCertChain(const quic::QuicSocketAddress& server_address,
                const quic::QuicSocketAddress& client_address, const std::string& hostname) override;
 
+  void updateFilterChainManager(Network::FilterChainManager& filter_chain_manager);
+
 protected:
   // quic::ProofSource
   void signPayload(const quic::QuicSocketAddress& server_address,
@@ -43,7 +45,7 @@ class EnvoyQuicProofSource : public EnvoyQuicProofSourceBase {
                                  const std::string& hostname);
 
   Network::Socket& listen_socket_;
-  Network::FilterChainManager& filter_chain_manager_;
+  Network::FilterChainManager* filter_chain_manager_{nullptr};
   Server::ListenerStats& listener_stats_;
 };
 

source/common/quic/envoy_quic_server_session.cc

@@ -1,5 +1,6 @@
 #include "source/common/quic/envoy_quic_server_session.h"
 
+#include <iterator>
 #include <memory>
 
 #include "source/common/common/assert.h"
@@ -15,15 +16,14 @@ EnvoyQuicServerSession::EnvoyQuicServerSession(
     quic::QuicCryptoServerStream::Helper* helper, const quic::QuicCryptoServerConfig* crypto_config,
     quic::QuicCompressedCertsCache* compressed_certs_cache, Event::Dispatcher& dispatcher,
     uint32_t send_buffer_limit, QuicStatNames& quic_stat_names, Stats::Scope& listener_scope,
-    EnvoyQuicCryptoServerStreamFactoryInterface& crypto_server_stream_factory,
-    OptRef<const Network::TransportSocketFactory> transport_socket_factory)
+    EnvoyQuicCryptoServerStreamFactoryInterface& crypto_server_stream_factory)
     : quic::QuicServerSessionBase(config, supported_versions, connection.get(), visitor, helper,
                                   crypto_config, compressed_certs_cache),
       QuicFilterManagerConnectionImpl(*connection, connection->connection_id(), dispatcher,
                                       send_buffer_limit),
       quic_connection_(std::move(connection)), quic_stat_names_(quic_stat_names),
-      listener_scope_(listener_scope), crypto_server_stream_factory_(crypto_server_stream_factory),
-      transport_socket_factory_(transport_socket_factory) {}
+      listener_scope_(listener_scope), crypto_server_stream_factory_(crypto_server_stream_factory) {
+}
 
 EnvoyQuicServerSession::~EnvoyQuicServerSession() {
   ASSERT(!quic_connection_->connected());
@@ -39,7 +39,9 @@ EnvoyQuicServerSession::CreateQuicCryptoServerStream(
     const quic::QuicCryptoServerConfig* crypto_config,
     quic::QuicCompressedCertsCache* compressed_certs_cache) {
   return crypto_server_stream_factory_.createEnvoyQuicCryptoServerStream(
-      crypto_config, compressed_certs_cache, this, stream_helper(), transport_socket_factory_,
+      crypto_config, compressed_certs_cache, this, stream_helper(),
+      makeOptRefFromPtr(position_.has_value() ? &position_->filter_chain_.transportSocketFactory()
+                                              : nullptr),
       dispatcher());
 }
 
@@ -89,6 +91,17 @@ void EnvoyQuicServerSession::OnConnectionClosed(const quic::QuicConnectionCloseF
                                                 quic::ConnectionCloseSource source) {
   quic::QuicServerSessionBase::OnConnectionClosed(frame, source);
   onConnectionCloseEvent(frame, source, version());
+  if (position_.has_value()) {
+    // Remove this connection from the map.
+    std::list<std::reference_wrapper<Network::Connection>>& connections =
+        position_->connection_map_[&position_->filter_chain_];
+    connections.erase(position_->iterator_);
+    if (connections.empty()) {
+      // Remove the whole entry if this is the last connection using this filter chain.
+      position_->connection_map_.erase(&position_->filter_chain_);
+    }
+    position_.reset();
+  }
 }
 
 void EnvoyQuicServerSession::Initialize() {
@@ -133,5 +146,11 @@ void EnvoyQuicServerSession::OnRstStream(const quic::QuicRstStreamFrame& frame)
                                                    /*from_self*/ false, /*is_upstream*/ false);
 }
 
+void EnvoyQuicServerSession::storeConnectionMapPosition(FilterChainToConnectionMap& connection_map,
+                                                        const Network::FilterChain& filter_chain,
+                                                        ConnectionMapIter position) {
+  position_.emplace(connection_map, filter_chain, position);
+}
+
 } // namespace Quic
 } // namespace Envoy