Support Gauge in Stats AccessLogger

api/envoy/extensions/access_loggers/stats/v3/BUILD

@@ -6,6 +6,7 @@ licenses(["notice"])  # Apache 2
 
 api_proto_package(
     deps = [
+        "//envoy/data/accesslog/v3:pkg",
         "@xds//udpa/annotations:pkg",
         "@xds//xds/annotations/v3:pkg",
     ],

api/envoy/extensions/access_loggers/stats/v3/stats.proto

@@ -2,6 +2,8 @@ syntax = "proto3";
 
 package envoy.extensions.access_loggers.stats.v3;
 
+import "envoy/data/accesslog/v3/accesslog.proto";
+
 import "google/protobuf/wrappers.proto";
 
 import "xds/annotations/v3/status.proto";
@@ -21,12 +23,22 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 // :ref:`command operators <config_access_log_command_operators>`.
 // [#extension: envoy.access_loggers.stats]
 //
+//
 // .. warning::
 //   It is easy to configure and use this extension in ways that create very
 //   large numbers of stats in Envoy, which can cause excessive memory or CPU use
 //   leading to a denial of service in Envoy, or can overwhelm any configured
 //   stat sinks by sending too many unique metrics.
-
+//
+//   It is required to use a delta stat sink (e.g. OpenTelemetry) because this
+//   extension sets the ``Counter`` and the ``Histogram`` as evictable in the
+//   stats scope. They will be evicted periodically and only the delta values
+//   are effective. Gauges will be kept in memory for their lifetime because
+//   PAIRED_ADD/PAIRED_SUBTRACT/SET operations depend on the previous status.
+//   Please have an estimate on the cardinality of dynamic gauges introduced
+//   and keep in mind on the memory overhead.
+
+// [#next-free-field: 6]
 message Config {
   option (xds.annotations.v3.message_status).work_in_progress = true;
 
@@ -91,6 +103,92 @@ message Config {
     google.protobuf.UInt64Value value_fixed = 3 [(validate.rules).uint64 = {gt: 0}];
   }
 
+  // Configuration for a gauge stat. Gauges can be used to add, subtract, or set
+  // values, and are useful for tracking concurrency or other mutable values
+  // over time.
+  //
+  // The ``operations`` field determines how the gauge is modified based on the
+  // access log type.
+  // Using the ``SET`` operation is straightforward and self-explanatory.
+  // The following example demonstrates how to use an ``ADD/SUBTRACT`` pair to
+  // track a gauge by configuring a single stats access logger:
+  //
+  // .. code-block:: yaml
+  //
+  //   http_connection_manager:
+  //     access_log:
+  //     - name: envoy.access_loggers.stats
+  //       typed_config:
+  //         "@type": type.googleapis.com/envoy.extensions.access_loggers.stats.v3.Config
+  //         stat_prefix: "my_prefix"
+  //         gauges:
+  //         - stat:
+  //             name: "http_active_requests"
+  //           operations:
+  //           - log_type: DownstreamStart
+  //             operation_type: PAIRED_ADD
+  //           - log_type: DownstreamEnd
+  //             operation_type: PAIRED_SUBTRACT
+  //           value_fixed: 1
+  message Gauge {
+    // An operation to perform on the gauge.
+    message Operation {
+      // The operation to perform on the gauge.
+      //
+      // Usage restrictions:
+      // i). Only one PAIRED_ADD/PAIRED_SUBTRACT pair at most can be defined in
+      // the
+      // ``operations``. PAIRED_ADD/PAIRED_SUBTRACT not matched with the other
+      // isn't allowed. ii). We only support add first then subtract logic and
+      // we rely on the symmetrical log types (e.g.,
+      // DownstreamStart/DownstreamEnd) to increment and decrement the gauge.
+      // iii). During runtime, PAIRED_SUBTRACT will execute if and only if
+      // PAIRED_ADD operation has been done, tracked by inflight counter in
+      // filter state.
+      // iv). If the PAIRED_ADD operation was executed, the
+      // PAIRED_SUBTRACT will happen when the stream/connection is closed, even
+      // if the configured log type didn't happen.
+      enum OperationType {
+        // Not a valid operation. It will fail validation if used.
+        UNSPECIFIED = 0;
+
+        // Set the gauge value to the configured value.
+        SET = 1;
+
+        // Add the configured value to the gauge.
+        PAIRED_ADD = 2;
+
+        // Subtract the value from the gauge.
+        // The value subtracted will be the same as the value added by the
+        // paired PAIRED_ADD operation, regardless of the value configured
+        // for this operation.
+        PAIRED_SUBTRACT = 3;
+      }
+
+      // The access log type to trigger the operation.
+      data.accesslog.v3.AccessLogType log_type = 1 [(validate.rules).enum = {defined_only: true}];
+
+      // The operation to perform.
+      OperationType operation_type = 2 [(validate.rules).enum = {defined_only: true}];
+    }
+
+    // The name and tags of this gauge.
+    Stat stat = 1 [(validate.rules).message = {required: true}];
+
+    // The format string for the value of this gauge, using :ref:`command
+    // operators <config_access_log_command_operators>`. This must evaluate to a
+    // positive number.
+    string value_format = 2
+        [(validate.rules).string = {prefix: "%" suffix: "%" ignore_empty: true}];
+
+    // A fixed value to add/subtract/set to this gauge.
+    // One of ``value_format`` or ``value_fixed`` must be configured.
+    google.protobuf.UInt64Value value_fixed = 3 [(validate.rules).uint64 = {gt: 0}];
+
+    // The collection of operations that define how the gauge is modified.
+    repeated Operation operations = 4;
+  }
+
   // The stat prefix for the generated stats.
   string stat_prefix = 1 [(validate.rules).string = {min_len: 1}];
 
@@ -99,4 +197,7 @@ message Config {
 
   // The counters this logger will emit.
   repeated Counter counters = 4;
+
+  // The gauges this logger will emit.
+  repeated Gauge gauges = 5;
 }

changelogs/current.yaml

@@ -141,6 +141,12 @@ new_features:
 - area: mcp_router
   change: |
     Added support for MCP logging method ``logging/setLevel``.
+- area: access_log
+  change: |
+    Supported gauge in the :ref:`stats access logger <envoy_v3_api_msg_extensions.access_loggers.stats.v3.Config>`.
+- area: stats
+  change: |
+    Supported per-stat-type eviction in scope.
 - area: network
   change: |
     Added support access logging in network filters like http filters, by allowing network filters to register as access logger

contrib/istio/filters/http/istio_stats/source/istio_stats.cc

@@ -216,7 +216,10 @@ struct Context : public Singleton::Instance {
         waypoint_(pool_.add("waypoint")), istio_build_(pool_.add("istio_build")),
         component_(pool_.add("component")), proxy_(pool_.add("proxy")), tag_(pool_.add("tag")),
         istio_version_(pool_.add(extractString(local_info.node().metadata(), "ISTIO_VERSION"))),
-        scope_(scope.createScope("", true)) {
+        scope_(scope.createScope("", {/*evict_counters=*/true,
+                                      /*evict_gauges=*/true,
+                                      /*evict_histograms=*/true,
+                                      /*evict_text_readouts=*/true})) {
     all_metrics_ = {
         {"requests_total", requests_total_},
         {"request_duration_milliseconds", request_duration_milliseconds_},

envoy/stats/scope.h

@@ -21,6 +21,28 @@ class Scope;
 class Store;
 class TextReadout;
 
+/**
+ * Settings for deleting unused metrics from the scope caches.
+ */
+struct EvictionSettings {
+  /**
+   * If true, counters will be evicted if unused.
+   */
+  bool evict_counters = false;
+  /**
+   * If true, gauges will be evicted if unused.
+   */
+  bool evict_gauges = false;
+  /**
+   * If true, histograms will be evicted if unused.
+   */
+  bool evict_histograms = false;
+  /**
+   * If true, text readouts will be evicted if unused.
+   */
+  bool evict_text_readouts = false;
+};
+
 using CounterOptConstRef = absl::optional<std::reference_wrapper<const Counter>>;
 using GaugeOptConstRef = absl::optional<std::reference_wrapper<const Gauge>>;
 using HistogramOptConstRef = absl::optional<std::reference_wrapper<const Histogram>>;
@@ -82,11 +104,11 @@ class Scope : public std::enable_shared_from_this<Scope> {
    * See also scopeFromStatName, which is preferred.
    *
    * @param name supplies the scope's namespace prefix.
-   * @param evictable whether unused metrics can be deleted from the scope caches. This requires
+   * @param settings settings for deleting unused metrics from the scope caches. This requires
    * that the metrics are not stored by reference.
    * @param limits metric limits for counters, gauges and histograms allowed in this scope.
    */
-  virtual ScopeSharedPtr createScope(const std::string& name, bool evictable = false,
+  virtual ScopeSharedPtr createScope(const std::string& name, const EvictionSettings& settings = {},
                                      const ScopeStatsLimitSettings& limits = {}) PURE;
 
   /**
@@ -95,11 +117,11 @@ class Scope : public std::enable_shared_from_this<Scope> {
    * gracefully swapped in while an old scope with the same name is being destroyed.
    *
    * @param name supplies the scope's namespace prefix.
-   * @param evictable whether unused metrics can be deleted from the scope caches. This requires
+   * @param settings settings for deleting unused metrics from the scope caches. This requires
    * that the metrics are not stored by reference.
    * @param limits metric limits for counters, gauges and histograms allowed in this scope.
    */
-  virtual ScopeSharedPtr scopeFromStatName(StatName name, bool evictable = false,
+  virtual ScopeSharedPtr scopeFromStatName(StatName name, const EvictionSettings& settings = {},
                                            const ScopeStatsLimitSettings& limits = {}) PURE;
 
   /**

envoy/stats/store.h

@@ -177,9 +177,9 @@ class Store {
   /**
    * @return a scope of the given name.
    */
-  ScopeSharedPtr createScope(const std::string& name, bool evictable = false,
+  ScopeSharedPtr createScope(const std::string& name, const EvictionSettings& settings = {},
                              const ScopeStatsLimitSettings& limits = {}) {
-    return rootScope()->createScope(name, evictable, limits);
+    return rootScope()->createScope(name, settings, limits);
   }
 
   /**

source/common/stats/isolated_store_impl.cc

@@ -63,13 +63,13 @@ ConstScopeSharedPtr IsolatedStoreImpl::constRootScope() const {
 
 IsolatedStoreImpl::~IsolatedStoreImpl() = default;
 
-ScopeSharedPtr IsolatedScopeImpl::createScope(const std::string& name, bool,
+ScopeSharedPtr IsolatedScopeImpl::createScope(const std::string& name, const EvictionSettings&,
                                               const ScopeStatsLimitSettings& limits) {
   StatNameManagedStorage stat_name_storage(Utility::sanitizeStatsName(name), symbolTable());
-  return scopeFromStatName(stat_name_storage.statName(), false, limits);
+  return scopeFromStatName(stat_name_storage.statName(), {}, limits);
 }
 
-ScopeSharedPtr IsolatedScopeImpl::scopeFromStatName(StatName name, bool,
+ScopeSharedPtr IsolatedScopeImpl::scopeFromStatName(StatName name, const EvictionSettings&,
                                                     const ScopeStatsLimitSettings&) {
   SymbolTable::StoragePtr prefix_name_storage = symbolTable().join({prefix(), name});
   ScopeSharedPtr scope = store_.makeScope(StatName(prefix_name_storage.get()));

source/common/stats/isolated_store_impl.h

@@ -299,9 +299,9 @@ class IsolatedScopeImpl : public Scope {
                                        StatNameTagVectorOptConstRef tags) override {
     return store_.counters_.get(prefix(), name, tags, symbolTable());
   }
-  ScopeSharedPtr createScope(const std::string& name, bool evictable = false,
+  ScopeSharedPtr createScope(const std::string& name, const EvictionSettings& settings = {},
                              const ScopeStatsLimitSettings& limits = {}) override;
-  ScopeSharedPtr scopeFromStatName(StatName name, bool evictable = false,
+  ScopeSharedPtr scopeFromStatName(StatName name, const EvictionSettings& settings = {},
                                    const ScopeStatsLimitSettings& limits = {}) override;
   Gauge& gaugeFromStatNameWithTags(const StatName& name, StatNameTagVectorOptConstRef tags,
                                    Gauge::ImportMode import_mode) override {

source/common/stats/thread_local_store.cc

@@ -38,7 +38,8 @@ ThreadLocalStoreImpl::ThreadLocalStoreImpl(Allocator& alloc)
     well_known_tags_->rememberBuiltin(desc.name_);
   }
   StatNameManagedStorage empty("", alloc.symbolTable());
-  auto new_scope = std::make_shared<ScopeImpl>(*this, StatName(empty.statName()), false);
+  auto new_scope =
+      std::make_shared<ScopeImpl>(*this, StatName(empty.statName()), EvictionSettings{});
   addScope(new_scope);
   default_scope_ = new_scope;
 }
@@ -154,17 +155,18 @@ std::vector<CounterSharedPtr> ThreadLocalStoreImpl::counters() const {
   return ret;
 }
 
-ScopeSharedPtr ThreadLocalStoreImpl::ScopeImpl::createScope(const std::string& name, bool evictable,
+ScopeSharedPtr ThreadLocalStoreImpl::ScopeImpl::createScope(const std::string& name,
+                                                            const EvictionSettings& settings,
                                                             const ScopeStatsLimitSettings& limits) {
   StatNameManagedStorage stat_name_storage(Utility::sanitizeStatsName(name), symbolTable());
-  return scopeFromStatName(stat_name_storage.statName(), evictable, limits);
+  return scopeFromStatName(stat_name_storage.statName(), settings, limits);
 }
 
 ScopeSharedPtr
-ThreadLocalStoreImpl::ScopeImpl::scopeFromStatName(StatName name, bool evictable,
+ThreadLocalStoreImpl::ScopeImpl::scopeFromStatName(StatName name, const EvictionSettings& settings,
                                                    const ScopeStatsLimitSettings& limits) {
   SymbolTable::StoragePtr joined = symbolTable().join({prefix_.statName(), name});
-  auto new_scope = std::make_shared<ScopeImpl>(parent_, StatName(joined.get()), evictable, limits);
+  auto new_scope = std::make_shared<ScopeImpl>(parent_, StatName(joined.get()), settings, limits);
   parent_.addScope(new_scope);
   return new_scope;
 }
@@ -398,9 +400,10 @@ void ThreadLocalStoreImpl::clearHistogramsFromCaches() {
 }
 
 ThreadLocalStoreImpl::ScopeImpl::ScopeImpl(ThreadLocalStoreImpl& parent, StatName prefix,
-                                           bool evictable, const ScopeStatsLimitSettings& limits)
-    : scope_id_(parent.next_scope_id_++), parent_(parent), evictable_(evictable), limits_(limits),
-      prefix_(prefix, parent.alloc_.symbolTable()),
+                                           const EvictionSettings& settings,
+                                           const ScopeStatsLimitSettings& limits)
+    : scope_id_(parent.next_scope_id_++), parent_(parent), eviction_settings_(settings),
+      limits_(limits), prefix_(prefix, parent.alloc_.symbolTable()),
       central_cache_(new CentralCacheEntry(parent.alloc_.symbolTable())) {
   parent_.ensureOverflowStats(limits_);
 }
@@ -1099,28 +1102,34 @@ void ThreadLocalStoreImpl::evictUnused() {
   {
     Thread::LockGuard lock(lock_);
     iterateScopesLockHeld([evicted_metrics](const ScopeImplSharedPtr& scope) -> bool {
-      if (scope->evictable_) {
-        MetricBag metrics(scope->scope_id_);
-        CentralCacheEntrySharedPtr& central_cache = scope->centralCacheMutableNoThreadAnalysis();
-        auto filter_unused = []<typename T>(StatNameHashMap<T>& unused_metrics) {
-          return [&unused_metrics](std::pair<StatName, T> kv) {
-            const auto& [name, metric] = kv;
-            if (metric->used()) {
-              metric->markUnused();
-              return false;
-            } else {
-              unused_metrics.try_emplace(name, metric);
-              return true;
-            }
-          };
+      MetricBag metrics(scope->scope_id_);
+      CentralCacheEntrySharedPtr& central_cache = scope->centralCacheMutableNoThreadAnalysis();
+      auto filter_unused = []<typename T>(StatNameHashMap<T>& unused_metrics) {
+        return [&unused_metrics](std::pair<StatName, T> kv) {
+          const auto& [name, metric] = kv;
+          if (metric->used()) {
+            metric->markUnused();
+            return false;
+          } else {
+            unused_metrics.try_emplace(name, metric);
+            return true;
+          }
         };
+      };
+      if (scope->eviction_settings_.evict_counters) {
         absl::erase_if(central_cache->counters_, filter_unused(metrics.counters_));
+      }
+      if (scope->eviction_settings_.evict_gauges) {
         absl::erase_if(central_cache->gauges_, filter_unused(metrics.gauges_));
-        absl::erase_if(central_cache->text_readouts_, filter_unused(metrics.text_readouts_));
+      }
+      if (scope->eviction_settings_.evict_histograms) {
         absl::erase_if(central_cache->histograms_, filter_unused(metrics.histograms_));
-        if (!metrics.empty()) {
-          evicted_metrics->push_back(std::move(metrics));
-        }
+      }
+      if (scope->eviction_settings_.evict_text_readouts) {
+        absl::erase_if(central_cache->text_readouts_, filter_unused(metrics.text_readouts_));
+      }
+      if (!metrics.empty()) {
+        evicted_metrics->push_back(std::move(metrics));
       }
       return true;
     });

source/common/stats/thread_local_store.h

@@ -287,7 +287,7 @@ class ThreadLocalStoreImpl : Logger::Loggable<Logger::Id::stats>, public StoreRo
   using CentralCacheEntrySharedPtr = RefcountPtr<CentralCacheEntry>;
 
   struct ScopeImpl : public Scope {
-    ScopeImpl(ThreadLocalStoreImpl& parent, StatName prefix, bool evictable,
+    ScopeImpl(ThreadLocalStoreImpl& parent, StatName prefix, const EvictionSettings& settings,
               const ScopeStatsLimitSettings& limits = {});
     ~ScopeImpl() override;
 
@@ -301,9 +301,9 @@ class ThreadLocalStoreImpl : Logger::Loggable<Logger::Id::stats>, public StoreRo
                                              Histogram::Unit unit) override;
     TextReadout& textReadoutFromStatNameWithTags(const StatName& name,
                                                  StatNameTagVectorOptConstRef tags) override;
-    ScopeSharedPtr createScope(const std::string& name, bool evictable = false,
+    ScopeSharedPtr createScope(const std::string& name, const EvictionSettings& settings = {},
                                const ScopeStatsLimitSettings& limits = {}) override;
-    ScopeSharedPtr scopeFromStatName(StatName name, bool evictable = false,
+    ScopeSharedPtr scopeFromStatName(StatName name, const EvictionSettings& settings = {},
                                      const ScopeStatsLimitSettings& limits = {}) override;
     const SymbolTable& constSymbolTable() const final { return parent_.constSymbolTable(); }
     SymbolTable& symbolTable() final { return parent_.symbolTable(); }
@@ -459,7 +459,7 @@ class ThreadLocalStoreImpl : Logger::Loggable<Logger::Id::stats>, public StoreRo
 
     const uint64_t scope_id_;
     ThreadLocalStoreImpl& parent_;
-    const bool evictable_{};
+    const EvictionSettings eviction_settings_;
 
     const ScopeStatsLimitSettings limits_;