quic: add QUIC downstream connection close error stats.

docs/root/configuration/http/http_conn_man/stats.rst

@@ -87,7 +87,12 @@ the following statistics:
 Per listener statistics
 -----------------------
 
-Additional per listener statistics are rooted at *listener.<address>.http.<stat_prefix>.* with the
+Per listener statistics are rooted at *listener.<address>.
+
+Http per listener statistics
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Additional HTTP statistics are of the form http.<stat_prefix>.* with the
 following statistics:
 
 .. csv-table::
@@ -101,12 +106,24 @@ following statistics:
    downstream_rq_4xx, Counter, Total 4xx responses
    downstream_rq_5xx, Counter, Total 5xx responses
 
+Http3 per listener statistics
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Http3 statistics with the form of http3.<stat_prefix>.*:
+
+.. csv-table::
+   :header: Name, Type, Description
+   :widths: 1, 1, 2
+
+   [direction].[source].quic_connection_close_error_code_[error_code], Counter, A collection of counters that are lazily initialized to record each quic connection close error code that's present. direction could be *upstream* or *downstream*. source could be *self* or *peer*.
+
+
 .. _config_http_conn_man_stats_per_codec:
 
 Per codec statistics
 -----------------------
 
-Each codec has the option of adding per-codec statistics. Both http1 and http2 have codec stats.
+Each codec has the option of adding per-codec statistics. http1, http2, and http3 all have codec stats.
 
 Http1 codec statistics
 ~~~~~~~~~~~~~~~~~~~~~~

source/common/quic/BUILD

@@ -51,6 +51,19 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "quic_stat_names_lib",
+    srcs = ["quic_stat_names.cc"],
+    hdrs = ["quic_stat_names.h"],
+    tags = ["nofips"],
+    deps = [
+        "//include/envoy/stats:stats_interface",
+        "//source/common/stats:symbol_table_lib",
+        "@com_googlesource_quiche//:quic_core_error_codes_lib",
+        "@com_googlesource_quiche//:quic_core_types_lib",
+    ],
+)
+
 envoy_cc_library(
     name = "envoy_quic_proof_source_base_lib",
     srcs = ["envoy_quic_proof_source_base.cc"],
@@ -314,6 +327,7 @@ envoy_cc_library(
     hdrs = ["envoy_quic_dispatcher.h"],
     tags = ["nofips"],
     deps = [
+        "quic_stat_names_lib",
         ":envoy_quic_proof_source_lib",
         ":envoy_quic_server_connection_lib",
         ":envoy_quic_server_session_lib",

source/common/quic/active_quic_listener.cc

@@ -28,18 +28,18 @@ ActiveQuicListener::ActiveQuicListener(
     Network::UdpConnectionHandler& parent, Network::ListenerConfig& listener_config,
     const quic::QuicConfig& quic_config, Network::Socket::OptionsSharedPtr options,
     bool kernel_worker_routing, const envoy::config::core::v3::RuntimeFeatureFlag& enabled,
-    uint32_t packets_received_to_connection_count_ratio)
+    QuicStatNames& quic_stat_names, uint32_t packets_received_to_connection_count_ratio)
     : ActiveQuicListener(worker_index, concurrency, dispatcher, parent,
                          listener_config.listenSocketFactory().getListenSocket(), listener_config,
                          quic_config, std::move(options), kernel_worker_routing, enabled,
-                         packets_received_to_connection_count_ratio) {}
+                         quic_stat_names, packets_received_to_connection_count_ratio) {}
 
 ActiveQuicListener::ActiveQuicListener(
     uint32_t worker_index, uint32_t concurrency, Event::Dispatcher& dispatcher,
     Network::UdpConnectionHandler& parent, Network::SocketSharedPtr listen_socket,
     Network::ListenerConfig& listener_config, const quic::QuicConfig& quic_config,
     Network::Socket::OptionsSharedPtr options, bool kernel_worker_routing,
-    const envoy::config::core::v3::RuntimeFeatureFlag& enabled,
+    const envoy::config::core::v3::RuntimeFeatureFlag& enabled, QuicStatNames& quic_stat_names,
     uint32_t packets_to_read_to_connection_count_ratio)
     : Server::ActiveUdpListenerBase(
           worker_index, concurrency, parent, *listen_socket,
@@ -87,7 +87,7 @@ ActiveQuicListener::ActiveQuicListener(
   quic_dispatcher_ = std::make_unique<EnvoyQuicDispatcher>(
       crypto_config_.get(), quic_config, &version_manager_, std::move(connection_helper),
       std::move(alarm_factory), quic::kQuicDefaultConnectionIdLength, parent, *config_, stats_,
-      per_worker_stats_, dispatcher, listen_socket_);
+      per_worker_stats_, dispatcher, listen_socket_, quic_stat_names);
 
   // Create udp_packet_writer
   Network::UdpPacketWriterPtr udp_packet_writer =
@@ -227,8 +227,9 @@ size_t ActiveQuicListener::numPacketsExpectedPerEventLoop() const {
 }
 
 ActiveQuicListenerFactory::ActiveQuicListenerFactory(
-    const envoy::config::listener::v3::QuicProtocolOptions& config, uint32_t concurrency)
-    : concurrency_(concurrency), enabled_(config.enabled()),
+    const envoy::config::listener::v3::QuicProtocolOptions& config, uint32_t concurrency,
+    QuicStatNames& quic_stat_names)
+    : concurrency_(concurrency), enabled_(config.enabled()), quic_stat_names_(quic_stat_names),
       packets_to_read_to_connection_count_ratio_(
           PROTOBUF_GET_WRAPPED_OR_DEFAULT(config, packets_to_read_to_connection_count_ratio,
                                           DEFAULT_PACKETS_TO_READ_PER_CONNECTION)) {
@@ -314,9 +315,10 @@ Network::ConnectionHandler::ActiveUdpListenerPtr ActiveQuicListenerFactory::crea
   }
 #endif
 
-  return std::make_unique<ActiveQuicListener>(
-      worker_index, concurrency_, disptacher, parent, config, quic_config_, std::move(options),
-      kernel_worker_routing, enabled_, packets_to_read_to_connection_count_ratio_);
+  return std::make_unique<ActiveQuicListener>(worker_index, concurrency_, disptacher, parent,
+                                              config, quic_config_, std::move(options),
+                                              kernel_worker_routing, enabled_, quic_stat_names_,
+                                              packets_to_read_to_connection_count_ratio_);
 } // namespace Quic
 
 } // namespace Quic

source/common/quic/active_quic_listener.h

@@ -29,13 +29,15 @@ class ActiveQuicListener : public Envoy::Server::ActiveUdpListenerBase,
                      Network::ListenerConfig& listener_config, const quic::QuicConfig& quic_config,
                      Network::Socket::OptionsSharedPtr options, bool kernel_worker_routing,
                      const envoy::config::core::v3::RuntimeFeatureFlag& enabled,
+                     QuicStatNames& quic_stat_names,
                      uint32_t packets_to_read_to_connection_count_ratio);
 
   ActiveQuicListener(uint32_t worker_index, uint32_t concurrency, Event::Dispatcher& dispatcher,
                      Network::UdpConnectionHandler& parent, Network::SocketSharedPtr listen_socket,
                      Network::ListenerConfig& listener_config, const quic::QuicConfig& quic_config,
                      Network::Socket::OptionsSharedPtr options, bool kernel_worker_routing,
                      const envoy::config::core::v3::RuntimeFeatureFlag& enabled,
+                     QuicStatNames& quic_stat_names,
                      uint32_t packets_to_read_to_connection_count_ratio);
 
   ~ActiveQuicListener() override;
@@ -87,7 +89,7 @@ class ActiveQuicListenerFactory : public Network::ActiveUdpListenerFactory,
                                   Logger::Loggable<Logger::Id::quic> {
 public:
   ActiveQuicListenerFactory(const envoy::config::listener::v3::QuicProtocolOptions& config,
-                            uint32_t concurrency);
+                            uint32_t concurrency, QuicStatNames& quic_stat_names);
 
   // Network::ActiveUdpListenerFactory.
   Network::ConnectionHandler::ActiveUdpListenerPtr
@@ -102,6 +104,7 @@ class ActiveQuicListenerFactory : public Network::ActiveUdpListenerFactory,
   const uint32_t concurrency_;
   absl::once_flag install_bpf_once_;
   envoy::config::core::v3::RuntimeFeatureFlag enabled_;
+  QuicStatNames& quic_stat_names_;
   const uint32_t packets_to_read_to_connection_count_ratio_;
 };
 

source/common/quic/envoy_quic_dispatcher.cc

@@ -17,13 +17,13 @@ EnvoyQuicDispatcher::EnvoyQuicDispatcher(
     uint8_t expected_server_connection_id_length, Network::ConnectionHandler& connection_handler,
     Network::ListenerConfig& listener_config, Server::ListenerStats& listener_stats,
     Server::PerHandlerListenerStats& per_worker_stats, Event::Dispatcher& dispatcher,
-    Network::Socket& listen_socket)
+    Network::Socket& listen_socket, QuicStatNames& quic_stat_names)
     : quic::QuicDispatcher(&quic_config, crypto_config, version_manager, std::move(helper),
                            std::make_unique<EnvoyQuicCryptoServerStreamHelper>(),
                            std::move(alarm_factory), expected_server_connection_id_length),
       connection_handler_(connection_handler), listener_config_(listener_config),
       listener_stats_(listener_stats), per_worker_stats_(per_worker_stats), dispatcher_(dispatcher),
-      listen_socket_(listen_socket) {
+      listen_socket_(listen_socket), quic_stat_names_(quic_stat_names) {
   // Set send buffer twice of max flow control window to ensure that stream send
   // buffer always takes all the data.
   // The max amount of data buffered is the per-stream high watermark + the max
@@ -46,6 +46,8 @@ void EnvoyQuicDispatcher::OnConnectionClosed(quic::QuicConnectionId connection_i
   listener_stats_.downstream_cx_active_.dec();
   per_worker_stats_.downstream_cx_active_.dec();
   connection_handler_.decNumConnections();
+  quic_stat_names_.chargeQuicConnectionCloseStats(listener_config_.listenerScope(), error, source,
+                                                  /*is_upstream*/ false);
 }
 
 std::unique_ptr<quic::QuicSession> EnvoyQuicDispatcher::CreateQuicSession(

source/common/quic/envoy_quic_dispatcher.h

@@ -19,6 +19,7 @@
 #include "envoy/network/listener.h"
 #include "server/connection_handler_impl.h"
 #include "server/active_listener_base.h"
+#include "common/quic/quic_stat_names.h"
 
 namespace Envoy {
 namespace Quic {
@@ -43,17 +44,15 @@ class EnvoyQuicCryptoServerStreamHelper : public quic::QuicCryptoServerStreamBas
 
 class EnvoyQuicDispatcher : public quic::QuicDispatcher {
 public:
-  EnvoyQuicDispatcher(const quic::QuicCryptoServerConfig* crypto_config,
-                      const quic::QuicConfig& quic_config,
-                      quic::QuicVersionManager* version_manager,
-                      std::unique_ptr<quic::QuicConnectionHelperInterface> helper,
-                      std::unique_ptr<quic::QuicAlarmFactory> alarm_factory,
-                      uint8_t expected_server_connection_id_length,
-                      Network::ConnectionHandler& connection_handler,
-                      Network::ListenerConfig& listener_config,
-                      Server::ListenerStats& listener_stats,
-                      Server::PerHandlerListenerStats& per_worker_stats,
-                      Event::Dispatcher& dispatcher, Network::Socket& listen_socket);
+  EnvoyQuicDispatcher(
+      const quic::QuicCryptoServerConfig* crypto_config, const quic::QuicConfig& quic_config,
+      quic::QuicVersionManager* version_manager,
+      std::unique_ptr<quic::QuicConnectionHelperInterface> helper,
+      std::unique_ptr<quic::QuicAlarmFactory> alarm_factory,
+      uint8_t expected_server_connection_id_length, Network::ConnectionHandler& connection_handler,
+      Network::ListenerConfig& listener_config, Server::ListenerStats& listener_stats,
+      Server::PerHandlerListenerStats& per_worker_stats, Event::Dispatcher& dispatcher,
+      Network::Socket& listen_socket, QuicStatNames& quic_stat_names);
 
   void OnConnectionClosed(quic::QuicConnectionId connection_id, quic::QuicErrorCode error,
                           const std::string& error_details,
@@ -81,6 +80,7 @@ class EnvoyQuicDispatcher : public quic::QuicDispatcher {
   Server::PerHandlerListenerStats& per_worker_stats_;
   Event::Dispatcher& dispatcher_;
   Network::Socket& listen_socket_;
+  QuicStatNames& quic_stat_names_;
 };
 
 } // namespace Quic

source/common/quic/quic_stat_names.cc

@@ -0,0 +1,48 @@
+#include "common/quic/quic_stat_names.h"
+
+namespace Envoy {
+namespace Quic {
+
+// TODO(renjietang): Currently these stats are only available in downstream. Wire it up to upstream
+// QUIC also.
+QuicStatNames::QuicStatNames(Stats::SymbolTable& symbol_table)
+    : stat_name_pool_(symbol_table), symbol_table_(symbol_table),
+      http3_prefix_(stat_name_pool_.add("http3")), downstream_(stat_name_pool_.add("downstream")),
+      upstream_(stat_name_pool_.add("upstream")), from_self_(stat_name_pool_.add("self")),
+      from_peer_(stat_name_pool_.add("peer")) {
+  // Preallocate most used counters
+  // Most popular in client initiated connection close.
+  connectionCloseStatName(quic::QUIC_NETWORK_IDLE_TIMEOUT);
+  // Most popular in server initiated connection close.
+  connectionCloseStatName(quic::QUIC_SILENT_IDLE_TIMEOUT);
+}
+
+void QuicStatNames::incCounter(Stats::Scope& scope, const Stats::StatNameVec& names) {
+  Stats::SymbolTable::StoragePtr stat_name_storage = symbol_table_.join(names);
+  scope.counterFromStatName(Stats::StatName(stat_name_storage.get())).inc();
+}
+
+void QuicStatNames::chargeQuicConnectionCloseStats(Stats::Scope& scope,
+                                                   quic::QuicErrorCode error_code,
+                                                   quic::ConnectionCloseSource source,
+                                                   bool is_upstream) {
+  ASSERT(&symbol_table_ == &scope.symbolTable());
+
+  const Stats::StatName connection_close = connectionCloseStatName(error_code);
+  incCounter(scope, {http3_prefix_, is_upstream ? upstream_ : downstream_,
+                     source == quic::ConnectionCloseSource::FROM_SELF ? from_self_ : from_peer_,
+                     connection_close});
+}
+
+Stats::StatName QuicStatNames::connectionCloseStatName(quic::QuicErrorCode error_code) {
+  ASSERT(error_code < quic::QUIC_LAST_ERROR);
+
+  return Stats::StatName(
+      connection_error_stat_names_.get(error_code, [this, error_code]() -> const uint8_t* {
+        return stat_name_pool_.addReturningStorage(
+            absl::StrCat("quic_connection_close_error_code_", QuicErrorCodeToString(error_code)));
+      }));
+}
+
+} // namespace Quic
+} // namespace Envoy

source/common/quic/quic_stat_names.h

@@ -0,0 +1,42 @@
+#pragma once
+
+#include "envoy/stats/scope.h"
+
+#include "common/common/thread.h"
+#include "common/stats/symbol_table_impl.h"
+
+#include "quiche/quic/core/quic_error_codes.h"
+#include "quiche/quic/core/quic_types.h"
+
+namespace Envoy {
+namespace Quic {
+
+class QuicStatNames {
+public:
+  // This class holds lazily symbolized stat names and is responsible for charging them.
+  explicit QuicStatNames(Stats::SymbolTable& symbol_table);
+
+  void chargeQuicConnectionCloseStats(Stats::Scope& scope, quic::QuicErrorCode error_code,
+                                      quic::ConnectionCloseSource source, bool is_upstream);
+
+private:
+  // Find the actual counter in |scope| and increment it.
+  // An example counter name: "downstream.self.quic_connection_close_error_code_QUIC_NO_ERROR".
+  void incCounter(Stats::Scope& scope, const Stats::StatNameVec& names);
+
+  Stats::StatName connectionCloseStatName(quic::QuicErrorCode error_code);
+
+  Stats::StatNamePool stat_name_pool_;
+  Stats::SymbolTable& symbol_table_;
+  const Stats::StatName http3_prefix_;
+  const Stats::StatName downstream_;
+  const Stats::StatName upstream_;
+  const Stats::StatName from_self_;
+  const Stats::StatName from_peer_;
+  Thread::AtomicPtrArray<const uint8_t, quic::QUIC_LAST_ERROR,
+                         Thread::AtomicPtrAllocMode::DoNotDelete>
+      connection_error_stat_names_;
+};
+
+} // namespace Quic
+} // namespace Envoy

source/server/BUILD

@@ -421,6 +421,7 @@ envoy_cc_library(
         "//source/common/quic:quic_factory_lib",
         "//source/common/quic:quic_transport_socket_factory_lib",
         "//source/common/quic:udp_gso_batch_writer_lib",
+        "//source/common/quic:quic_stat_names_lib",
     ]),
 )
 

source/server/listener_impl.cc

@@ -33,7 +33,7 @@
 
 #include "extensions/filters/listener/well_known_names.h"
 
-#if defined(ENVOY_ENABLE_QUIC)
+#ifdef ENVOY_ENABLE_QUIC
 #include "common/quic/active_quic_listener.h"
 #include "common/quic/udp_gso_batch_writer.h"
 #endif
@@ -289,15 +289,21 @@ ListenerImpl::ListenerImpl(const envoy::config::listener::v3::Listener& config,
       open_connections_(std::make_shared<BasicResourceLimitImpl>(
           std::numeric_limits<uint64_t>::max(), listener_factory_context_->runtime(),
           cx_limit_runtime_key_)),
-      local_init_watcher_(fmt::format("Listener-local-init-watcher {}", name), [this] {
-        if (workers_started_) {
-          parent_.onListenerWarmed(*this);
-        } else {
-          // Notify Server that this listener is
-          // ready.
-          listener_init_target_.ready();
-        }
-      }) {
+      local_init_watcher_(fmt::format("Listener-local-init-watcher {}", name),
+                          [this] {
+                            if (workers_started_) {
+                              parent_.onListenerWarmed(*this);
+                            } else {
+                              // Notify Server that this listener is
+                              // ready.
+                              listener_init_target_.ready();
+                            }
+                          })
+#ifdef ENVOY_ENABLE_QUIC
+      ,
+      quic_stat_names_(parent_.quicStatNames())
+#endif
+{
 
   const absl::optional<std::string> runtime_val =
       listener_factory_context_->runtime().snapshot().get(cx_limit_runtime_key_);
@@ -360,10 +366,16 @@ ListenerImpl::ListenerImpl(ListenerImpl& origin,
           origin.listener_factory_context_->listener_factory_context_base_, this, *this)),
       filter_chain_manager_(address_, origin.listener_factory_context_->parentFactoryContext(),
                             initManager(), origin.filter_chain_manager_),
-      local_init_watcher_(fmt::format("Listener-local-init-watcher {}", name), [this] {
-        ASSERT(workers_started_);
-        parent_.inPlaceFilterChainUpdate(*this);
-      }) {
+      local_init_watcher_(fmt::format("Listener-local-init-watcher {}", name),
+                          [this] {
+                            ASSERT(workers_started_);
+                            parent_.inPlaceFilterChainUpdate(*this);
+                          })
+#ifdef ENVOY_ENABLE_QUIC
+      ,
+      quic_stat_names_(parent_.quicStatNames())
+#endif
+{
   buildAccessLog();
   auto socket_type = Network::Utility::protobufAddressSocketType(config.address());
   buildListenSocketOptions(socket_type);
@@ -401,9 +413,9 @@ void ListenerImpl::buildUdpListenerFactory(Network::Socket::Type socket_type,
 
   udp_listener_config_ = std::make_unique<UdpListenerConfigImpl>(config_.udp_listener_config());
   if (config_.udp_listener_config().has_quic_options()) {
-#if defined(ENVOY_ENABLE_QUIC)
+#ifdef ENVOY_ENABLE_QUIC
     udp_listener_config_->listener_factory_ = std::make_unique<Quic::ActiveQuicListenerFactory>(
-        config_.udp_listener_config().quic_options(), concurrency);
+        config_.udp_listener_config().quic_options(), concurrency, quic_stat_names_);
 #if UDP_GSO_BATCH_WRITER_COMPILETIME_SUPPORT
     // TODO(mattklein123): We should be able to use GSO without QUICHE/QUIC. Right now this causes
     // non-QUIC integration tests to fail, which I haven't investigated yet. Additionally, from