add deferred_creation util into stats

api/envoy/config/bootstrap/v3/bootstrap.proto

@@ -41,7 +41,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 // <config_overview_bootstrap>` for more detail.
 
 // Bootstrap :ref:`configuration overview <config_overview_bootstrap>`.
-// [#next-free-field: 39]
+// [#next-free-field: 40]
 message Bootstrap {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.bootstrap.v2.Bootstrap";
@@ -124,6 +124,18 @@ message Bootstrap {
     LogFormat log_format = 1;
   }
 
+  message DeferredStatOptions {
+    // When the flag is enabled, Envoy will lazily initialize a subset of the stats (see below).
+    // This will save memory and CPU cycles when creating the objects that own these stats, if those
+    // stats are never referenced throughout the lifetime of the process. However, it will incur additional
+    // memory overhead for these objects, and a small increase of CPU usage when a at least one of the stats
+    // is updated for the first time.
+    // Groups of stats that will be lazily initialized:
+    // - Cluster traffic stats: a subgroup of the :ref:`cluster statistics <config_cluster_manager_cluster_stats>`
+    // that are used when requests are routed to the cluster.
+    bool enable_deferred_creation_stats = 1;
+  }
+
   reserved 10, 11;
 
   reserved "runtime";
@@ -186,6 +198,10 @@ message Bootstrap {
   // Optional set of stats sinks.
   repeated metrics.v3.StatsSink stats_sinks = 6;
 
+  // Options to control behaviors of deferred creation compatible stats.
+  // [#not-implemented-hide:]
+  DeferredStatOptions deferred_stat_options = 39;
+
   // Configuration for internal processing of stats.
   metrics.v3.StatsConfig stats_config = 13;
 

changelogs/current.yaml

@@ -239,6 +239,11 @@ new_features:
     added new configuration field :ref:`key_formatter
     <envoy_v3_api_field_extensions.filters.network.redis_proxy.v3.RedisProxy.PrefixRoutes.Route.key_formatter>` to format redis key.
     The field supports using %KEY% as a formatter command for substituting the redis key as part of the substitution formatter expression.
+- area: stats
+  change: |
+    added config :ref:`enable_deferred_creation_stats
+    <envoy_v3_api_field_config.bootstrap.v3.Bootstrap.DeferredStatOptions.enable_deferred_creation_stats>`.
+    When set to true, enables deferred instantiation on supported stats structures.
 - area: ratelimit
   change: |
     added new configuration field :ref:`domain

envoy/server/configuration.h

@@ -84,6 +84,11 @@ class StatsConfig {
    * @return bool indicator to flush stats on-demand via the admin interface instead of on a timer.
    */
   virtual bool flushOnAdmin() const PURE;
+
+  /**
+   * @return true if deferred creation of stats is enabled.
+   */
+  virtual bool enableDeferredCreationStats() const PURE;
 };
 
 /**

envoy/stats/stats.h

@@ -217,5 +217,40 @@ using SizeFn = std::function<void(std::size_t)>;
  */
 template <typename Stat> using StatFn = std::function<void(Stat&)>;
 
+/**
+ * Interface for stats lazy initialization.
+ * To save memory and CPU consumption on blocks of stats that are never referenced throughout the
+ * process lifetime, they can be encapsulated in a DeferredCreationCompatibleInterface. Then the
+Envoy
+ * bootstrap configuration can be set to defer the instantiation of those block. Note that when the
+ * blocks of stats are created, they carry an extra 60~100 byte overhead (depending on worker thread
+ * count) due to internal bookkeeping data structures. The overhead when deferred stats are disabled
+ * is just 8 bytes.
+* See more context: https://github.com/envoyproxy/envoy/issues/23575
+ */
+template <typename StatsStructType> class DeferredCreationCompatibleInterface {
+public:
+  // Helper function to get-or-create and return the StatsStructType object.
+  virtual StatsStructType& getOrCreate() PURE;
+
+  virtual ~DeferredCreationCompatibleInterface() = default;
+};
+
+// A helper class for a lazy compatible stats struct type.
+template <typename StatsStructType> class DeferredCreationCompatibleStats {
+public:
+  explicit DeferredCreationCompatibleStats(
+      std::unique_ptr<DeferredCreationCompatibleInterface<StatsStructType>> d)
+      : data_(std::move(d)) {}
+  // Allows move construct and assign.
+  DeferredCreationCompatibleStats& operator=(DeferredCreationCompatibleStats&&) noexcept = default;
+  DeferredCreationCompatibleStats(DeferredCreationCompatibleStats&&) noexcept = default;
+
+  inline StatsStructType* operator->() { return &data_->getOrCreate(); };
+  inline StatsStructType& operator*() { return data_->getOrCreate(); };
+
+private:
+  std::unique_ptr<DeferredCreationCompatibleInterface<StatsStructType>> data_;
+};
 } // namespace Stats
 } // namespace Envoy

envoy/stats/stats_macros.h

@@ -158,16 +158,18 @@ static inline std::string statPrefixJoin(absl::string_view prefix, absl::string_
  */
 #define MAKE_STATS_STRUCT(StatsStruct, StatNamesStruct, ALL_STATS)                                 \
   struct StatsStruct {                                                                             \
+    /* Also referenced in Stats::createDeferredCompatibleStats. */                                 \
+    using StatNameType = StatNamesStruct;                                                          \
+    static const absl::string_view typeName() { return #StatsStruct; }                             \
     StatsStruct(const StatNamesStruct& stat_names, Envoy::Stats::Scope& scope,                     \
                 Envoy::Stats::StatName prefix = Envoy::Stats::StatName())                          \
         : stat_names_(stat_names)                                                                  \
               ALL_STATS(MAKE_STATS_STRUCT_COUNTER_HELPER_, MAKE_STATS_STRUCT_GAUGE_HELPER_,        \
                         MAKE_STATS_STRUCT_HISTOGRAM_HELPER_,                                       \
                         MAKE_STATS_STRUCT_TEXT_READOUT_HELPER_,                                    \
                         MAKE_STATS_STRUCT_STATNAME_HELPER_) {}                                     \
-    const StatNamesStruct& stat_names_;                                                            \
+    const StatNameType& stat_names_;                                                               \
     ALL_STATS(GENERATE_COUNTER_STRUCT, GENERATE_GAUGE_STRUCT, GENERATE_HISTOGRAM_STRUCT,           \
               GENERATE_TEXT_READOUT_STRUCT, GENERATE_STATNAME_STRUCT)                              \
   }
-
 } // namespace Envoy

source/common/stats/BUILD

@@ -36,6 +36,18 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "deferred_creation",
+    hdrs = ["deferred_creation.h"],
+    deps = [
+        "//envoy/stats:stats_interface",
+        "//source/common/common:cleanup_lib",
+        "//source/common/common:thread_lib",
+        "//source/common/stats:symbol_table_lib",
+        "//source/common/stats:utility_lib",
+    ],
+)
+
 envoy_cc_library(
     name = "histogram_lib",
     srcs = ["histogram_impl.cc"],

source/common/stats/deferred_creation.h

@@ -0,0 +1,119 @@
+#pragma once
+
+#include "envoy/common/pure.h"
+#include "envoy/stats/scope.h"
+#include "envoy/stats/stats.h"
+
+#include "source/common/common/cleanup.h"
+#include "source/common/common/thread.h"
+#include "source/common/stats/symbol_table.h"
+#include "source/common/stats/utility.h"
+
+namespace Envoy {
+namespace Stats {
+
+/**
+ * Lazy-initialization wrapper for StatsStructType, intended for deferred instantiation of a block
+ * of stats that might not be needed in a given Envoy process.
+ *
+ * This class is thread-safe -- instantiations can occur on multiple concurrent threads.
+ * This is used when
+ * :ref:`enable_deferred_creation_stats
+ * <envoy_v3_api_field_config.bootstrap.v3.Bootstrap.deferred_stat_options>` is enabled.
+ */
+template <typename StatsStructType>
+class DeferredStats : public DeferredCreationCompatibleInterface<StatsStructType> {
+public:
+  // Capture the stat names object and the scope with a ctor, that can be used to instantiate a
+  // StatsStructType object later.
+  // Caller should make sure scope and stat_names outlive this object.
+  DeferredStats(const typename StatsStructType::StatNameType& stat_names,
+                Stats::ScopeSharedPtr scope)
+      : initialized_(
+            // A lambda is used as we need to register the name into the symbol table.
+            // Note: there is no issue to capture a reference of the scope here as this lambda is
+            // only used to initialize the 'initialized_' Gauge.
+            [&scope]() -> Gauge& {
+              Stats::StatNamePool pool(scope->symbolTable());
+              return Stats::Utility::gaugeFromElements(
+                  *scope, {pool.add(StatsStructType::typeName()), pool.add("initialized")},
+                  Stats::Gauge::ImportMode::HiddenAccumulate);
+            }()),
+        ctor_([this, &stat_names, stats_scope = std::move(scope)]() -> StatsStructType* {
+          initialized_.inc();
+          // Reset ctor_ to save some RAM.
+          Cleanup reset_ctor([this] { ctor_ = nullptr; });
+          return new StatsStructType(stat_names, *stats_scope);
+        }) {
+    if (initialized_.value() > 0) {
+      getOrCreateHelper();
+    }
+  }
+  ~DeferredStats() {
+    if (ctor_ == nullptr) {
+      initialized_.dec();
+    }
+  }
+  inline StatsStructType& getOrCreate() override { return getOrCreateHelper(); }
+
+private:
+  // We can't call getOrCreate directly from constructor, otherwise the compiler complains about
+  // bypassing virtual dispatch even though it's fine.
+  inline StatsStructType& getOrCreateHelper() { return *internal_stats_.get(ctor_); }
+
+  // In order to preserve stat value continuity across a config reload, we need to automatically
+  // re-instantiate lazy stats when they are constructed, if there is already a live instantiation
+  // to the same stats. Consider the following alternate scenarios:
+
+  // Scenario 1: a cluster is instantiated but receives no requests, so its traffic-related stats
+  // are never instantiated. When this cluster gets reloaded on a config update, a new lazy-init
+  // block is created, but the stats should again not be instantiated.
+
+  // Scenario 2: a cluster is instantiated and receives traffic, so its traffic-related stats are
+  // instantiated. We must ensure that a new instance for the same cluster gets its lazy-stats
+  // instantiated before the previous cluster of the same name is destructed.
+
+  // To do that we keep an "initialized" gauge in the cluster's scope, which will be associated by
+  // name to the previous generation's cluster's lazy-init block. We use the value in this shared
+  // gauge to determine whether to instantiate the lazy block on construction.
+  Gauge& initialized_;
+  // TODO(#26957): Clean up this ctor_ by moving its ownership to AtomicPtr, and drop
+  // the setter lambda when the nested object is created.
+  std::function<StatsStructType*()> ctor_;
+  Thread::AtomicPtr<StatsStructType, Thread::AtomicPtrAllocMode::DeleteOnDestruct> internal_stats_;
+};
+
+// Non-deferred wrapper over StatsStructType. This is used when
+// :ref:`enable_deferred_creation_stats
+// <envoy_v3_api_field_config.bootstrap.v3.Bootstrap.deferred_stat_options>` is not enabled.
+template <typename StatsStructType>
+class DirectStats : public DeferredCreationCompatibleInterface<StatsStructType> {
+public:
+  DirectStats(const typename StatsStructType::StatNameType& stat_names, Stats::Scope& scope)
+      : stats_(stat_names, scope) {}
+  inline StatsStructType& getOrCreate() override { return stats_; }
+
+private:
+  StatsStructType stats_;
+};
+
+// Template that lazily initializes a StatsStruct.
+// The bootstrap config :ref:`enable_deferred_creation_stats
+// <envoy_v3_api_field_config.bootstrap.v3.Bootstrap.deferred_stat_options>` decides if
+// stats lazy initialization is enabled or not.
+template <typename StatsStructType>
+DeferredCreationCompatibleStats<StatsStructType>
+createDeferredCompatibleStats(Stats::ScopeSharedPtr scope,
+                              const typename StatsStructType::StatNameType& stat_names,
+                              bool deferred_creation) {
+  if (deferred_creation) {
+    return DeferredCreationCompatibleStats<StatsStructType>(
+        std::make_unique<DeferredStats<StatsStructType>>(stat_names, scope));
+  } else {
+    return DeferredCreationCompatibleStats<StatsStructType>(
+        std::make_unique<DirectStats<StatsStructType>>(stat_names, *scope));
+  }
+}
+
+} // namespace Stats
+} // namespace Envoy

source/docs/stats.md

@@ -295,3 +295,11 @@ from the same symbol table. To facilitate this, a test-only global singleton can
 be instantiated, via either `Stats::TestUtil::TestSymbolTable` or
 `Stats::TestUtil::TestStore`. All such structures use a singleton symbol-table
 whose lifetime is a single test method. This should resolve the assertion.
+
+
+Deferred Initialization of Stats
+================================
+
+When :ref:`enable_deferred_creation_stats <envoy_v3_api_field_config.bootstrap.v3.Bootstrap.deferred_stat_options>`
+is enabled in Bootstrap, for stats that are deferred creation compatible, the actual stats struct creation
+is deferred to first access of any member of that stats, i.e. instantiation only happens when an invocation on operator "*" or "->" happens.

source/server/configuration_impl.cc

@@ -60,7 +60,8 @@ void FilterChainUtility::buildUdpFilterChain(
   }
 }
 
-StatsConfigImpl::StatsConfigImpl(const envoy::config::bootstrap::v3::Bootstrap& bootstrap) {
+StatsConfigImpl::StatsConfigImpl(const envoy::config::bootstrap::v3::Bootstrap& bootstrap)
+    : deferred_stat_options_(bootstrap.deferred_stat_options()) {
   if (bootstrap.has_stats_flush_interval() &&
       bootstrap.stats_flush_case() !=
           envoy::config::bootstrap::v3::Bootstrap::STATS_FLUSH_NOT_SET) {
@@ -87,6 +88,11 @@ void MainImpl::initialize(const envoy::config::bootstrap::v3::Bootstrap& bootstr
   // tracing configuration is missing from the bootstrap config.
   initializeTracers(bootstrap.tracing(), server);
 
+  // stats_config_ should be set before creating the ClusterManagers so that it is available
+  // from the ServerFactoryContext when creating the static clusters and stats sinks, where
+  // stats deferred instantiation setting is read.
+  stats_config_ = std::make_unique<StatsConfigImpl>(bootstrap);
+
   const auto& secrets = bootstrap.static_resources().secrets();
   ENVOY_LOG(info, "loading {} static secret(s)", secrets.size());
   for (ssize_t i = 0; i < secrets.size(); i++) {
@@ -103,19 +109,16 @@ void MainImpl::initialize(const envoy::config::bootstrap::v3::Bootstrap& bootstr
     ENVOY_LOG(debug, "listener #{}:", i);
     server.listenerManager().addOrUpdateListener(listeners[i], "", false);
   }
-
   initializeWatchdogs(bootstrap, server);
+  // This has to happen after ClusterManager initialization, as it depends on config from
+  // ClusterManager.
   initializeStatsConfig(bootstrap, server);
 }
 
 void MainImpl::initializeStatsConfig(const envoy::config::bootstrap::v3::Bootstrap& bootstrap,
                                      Instance& server) {
   ENVOY_LOG(info, "loading stats configuration");
 
-  // stats_config_ should be set before populating the sinks so that it is available
-  // from the ServerFactoryContext when creating the stats sinks.
-  stats_config_ = std::make_unique<StatsConfigImpl>(bootstrap);
-
   for (const envoy::config::metrics::v3::StatsSink& sink_object : bootstrap.stats_sinks()) {
     // Generate factory and translate stats sink custom config.
     auto& factory = Config::Utility::getAndCheckFactory<StatsSinkFactory>(sink_object);

source/server/configuration_impl.h

@@ -56,11 +56,15 @@ class StatsConfigImpl : public StatsConfig {
   bool flushOnAdmin() const override { return flush_on_admin_; }
 
   void addSink(Stats::SinkPtr sink) { sinks_.emplace_back(std::move(sink)); }
+  bool enableDeferredCreationStats() const override {
+    return deferred_stat_options_.enable_deferred_creation_stats();
+  }
 
 private:
   std::list<Stats::SinkPtr> sinks_;
   std::chrono::milliseconds flush_interval_;
   bool flush_on_admin_{false};
+  const envoy::config::bootstrap::v3::Bootstrap::DeferredStatOptions deferred_stat_options_;
 };
 
 /**

test/common/stats/BUILD

@@ -226,6 +226,42 @@ envoy_cc_benchmark_binary(
     ],
 )
 
+envoy_cc_test(
+    name = "deferred_creation_stats_test",
+    srcs = ["deferred_creation_stats_test.cc"],
+    deps = [
+        "//envoy/stats:stats_interface",
+        "//source/common/memory:stats_lib",
+        "//source/common/stats:deferred_creation",
+        "//source/common/stats:thread_local_store_lib",
+        "//test/test_common:utility_lib",
+    ],
+)
+
+envoy_cc_benchmark_binary(
+    name = "deferred_creation_stats_benchmark",
+    srcs = ["deferred_creation_stats_speed_test.cc"],
+    external_deps = [
+        "benchmark",
+    ],
+    deps = [
+        ":real_thread_test_base",
+        "//source/common/common:random_generator_lib",
+        "//source/common/common:utility_lib",
+        "//source/common/runtime:runtime_lib",
+        "//source/common/stats:deferred_creation",
+        "//source/common/stats:isolated_store_lib",
+        "//source/common/stats:symbol_table_lib",
+        "//source/exe:process_wide_lib",
+    ],
+)
+
+envoy_benchmark_test(
+    name = "deferred_creation_stats_benchmark_test",
+    size = "large",
+    benchmark_binary = "deferred_creation_stats_benchmark",
+)
+
 envoy_benchmark_test(
     name = "symbol_table_benchmark_test",
     benchmark_binary = "symbol_table_benchmark",