stats: add 'detailed' buckets-mode that captures all data from circlhist

changelogs/current.yaml

@@ -172,6 +172,9 @@ new_features:
     <envoy_v3_api_field_extensions.filters.network.http_connection_manager.v3.HttpConnectionManager.HcmAccessLogOptions.flush_log_on_tunnel_successfully_established>`.
     Enabling this option will write a log to all access loggers when HTTP tunnels (e.g. Websocket and CONNECT)
     are successfully established.
+- area: admin
+  change: |
+    Adds a new admin stats html bucket-mode ``detailed`` to generate all recorded buckets and summary percentiles.
 
 deprecated:
 - area: access_log

docs/root/operations/admin.rst

@@ -475,6 +475,26 @@ modify different aspects of the server:
   Buckets do not include values from other buckets with smaller upper bounds;
   the previous bucket's upper bound acts as a lower bound. Compatible with ``usedonly`` and ``filter``.
 
+  .. http:get:: /stats?histogram_buckets=detailed
+
+  Shows histograms as both percentile summary data, and raw bucket
+  data, up to 64 buckets. If the raw bucket data has more than 64
+  buckets, then adjacent buckets are combined until 64 is reached.
+
+  Example output
+  .. code-block:: text
+
+    http.admin.downstream_rq_time:
+      totals=1,0.25:25, 2,0.25:9
+      intervals=1,0.25:2, 2,0.25:3
+      summary=P0(1,1) P25(1.0625,1.034) P50(2.0166,1.068) P75(2.058,2.005) P90(2.083,2.06) P95(2.091,2.08) P99(2.09,2.09) P99.5(2.099,2.098) P99.9(2.099,2.099) P100(2.1,2.1)
+
+  Each bucket is shown as `min_value,width:count`. In the above example there are two
+  buckets. `totals` contains the accumulated data-points since the binary was started.
+  `intervals` shows the new data points since the previous stats flush.
+
+  Compatible with ``usedonly`` and ``filter``.
+
   .. http:get:: /stats?format=html
 
   Renders stats using HTML for a web browser, providing form fields to incrementally
@@ -647,7 +667,55 @@ modify different aspects of the server:
       ]
     }
 
-.. http:get:: /stats?format=prometheus
+  .. http:get:: /stats?format=json&histogram_buckets=detailed
+
+  Shows histograms as both percentile summary data, and raw bucket
+  data, up to 64 buckets. If the raw bucket data has more than 64
+  buckets, then adjacent buckets are combined until 64 is reached.
+
+  Example output:
+
+  .. code-block:: json
+
+    {
+      "stats": [
+        {
+          "histograms": {
+            "supported_percentiles": [0, 25, 50, 75, 90, 95, 99, 99.5, 99.9, 100],
+            "details": [
+              {
+                "name": "http.admin.downstream_rq_time",
+                "percentiles": [
+                  { "interval": null, "cumulative": 1 },
+                  { "interval": null, "cumulative": 1.0351851851851852 },
+                  { "interval": null "cumulative": 1.0703703703703704 },
+                  { "interval": null, "cumulative": 2.0136363636363637 },
+                  { "interval": null "cumulative": 2.0654545454545454 },
+                  { "interval": null "cumulative": 2.0827272727272725 },
+                  { "interval": null "cumulative": 2.0965454545454545 },
+                  { "interval": null, "cumulative": 2.098272727272727 },
+                  { "interval": null, "cumulative": 2.0996545454545457 },
+                  { "interval": null "cumulative": 2.1 }
+                ],
+                "totals": [
+                  { "min_value": 1, "width": 0.25, "count": 25 },
+                  { "min_value": 2, "width": 0.25, "count": 9 }
+                ],
+                "intervals": [
+                  { "min_value": 1, "width": 0.25, "count": 2 },
+                  { "min_value": 2, "width": 0.25, "count": 3 }
+                ],
+              },
+            ]
+          }
+        }
+      ]
+    }
+
+  Compatible with ``usedonly`` and ``filter``.
+
+
+  .. http:get:: /stats?format=prometheus
 
   or alternatively,
 

envoy/stats/histogram.h

@@ -167,6 +167,27 @@ class ParentHistogram : public Histogram {
    * Returns the bucket summary representation.
    */
   virtual std::string bucketSummary() const PURE;
+
+  // Holds detailed value and counts for a histogram bucket.
+  struct Bucket {
+    double min_value_{0}; // Bound of bucket that's closest to zero.
+    double width_{0};
+    uint64_t count_{0};
+  };
+
+  static constexpr uint32_t UnlimitedBuckets = 0;
+
+  /**
+   * @return a vector of histogram buckets collected since binary start or reset.
+   */
+  virtual std::vector<Bucket> detailedTotalBuckets() const PURE;
+
+  /**
+   * @return bucket data collected since the most recent stat sink. Note that
+   *         the number of interval buckets is likely to be much smaller than
+   *         the number of detailed buckets.
+   */
+  virtual std::vector<Bucket> detailedIntervalBuckets() const PURE;
 };
 
 using ParentHistogramSharedPtr = RefcountPtr<ParentHistogram>;

source/common/stats/histogram_impl.h

@@ -49,8 +49,6 @@ class HistogramStatisticsImpl final : public HistogramStatistics, NonCopyable {
       const histogram_t* histogram_ptr, Histogram::Unit unit = Histogram::Unit::Unspecified,
       ConstSupportedBuckets& supported_buckets = HistogramSettingsImpl::defaultBuckets());
 
-  static ConstSupportedBuckets& defaultSupportedBuckets();
-
   void refresh(const histogram_t* new_histogram_ptr);
 
   // HistogramStatistics

source/common/stats/thread_local_store.cc

@@ -958,6 +958,20 @@ std::string ParentHistogramImpl::bucketSummary() const {
   }
 }
 
+std::vector<Stats::ParentHistogram::Bucket>
+ParentHistogramImpl::detailedlBucketsHelper(const histogram_t& histogram) {
+  const uint32_t num_buckets = hist_num_buckets(&histogram);
+  std::vector<Stats::ParentHistogram::Bucket> buckets(num_buckets);
+  hist_bucket_t hist_bucket;
+  for (uint32_t i = 0; i < num_buckets; ++i) {
+    ParentHistogram::Bucket& bucket = buckets[i];
+    hist_bucket_idx_bucket(&histogram, i, &hist_bucket, &bucket.count_);
+    bucket.min_value_ = hist_bucket_to_double(hist_bucket);
+    bucket.width_ = hist_bucket_to_double_bin_width(hist_bucket);
+  }
+  return buckets;
+}
+
 void ParentHistogramImpl::addTlsHistogram(const TlsHistogramSharedPtr& hist_ptr) {
   Thread::LockGuard lock(merge_lock_);
   tls_histograms_.emplace_back(hist_ptr);

source/common/stats/thread_local_store.h

@@ -105,6 +105,12 @@ class ParentHistogramImpl : public MetricImpl<ParentHistogram> {
   }
   std::string quantileSummary() const override;
   std::string bucketSummary() const override;
+  std::vector<Bucket> detailedTotalBuckets() const override {
+    return detailedlBucketsHelper(*cumulative_histogram_);
+  }
+  std::vector<Bucket> detailedIntervalBuckets() const override {
+    return detailedlBucketsHelper(*interval_histogram_);
+  }
 
   // Stats::Metric
   SymbolTable& symbolTable() override;
@@ -121,6 +127,8 @@ class ParentHistogramImpl : public MetricImpl<ParentHistogram> {
 
 private:
   bool usedLockHeld() const ABSL_EXCLUSIVE_LOCKS_REQUIRED(merge_lock_);
+  static std::vector<Stats::ParentHistogram::Bucket>
+  detailedlBucketsHelper(const histogram_t& histogram);
 
   Histogram::Unit unit_;
   ThreadLocalStoreImpl& thread_local_store_;

source/common/stats/utility.h

@@ -231,6 +231,7 @@ TextReadout& textReadoutFromElements(Scope& scope, const ElementVec& elements,
  */
 TextReadout& textReadoutFromStatNames(Scope& scope, const StatNameVec& elements,
                                       StatNameTagVectorOptConstRef tags = absl::nullopt);
+
 } // namespace Utility
 
 /**

source/server/admin/stats_handler.cc

@@ -186,7 +186,7 @@ Admin::UrlHandler StatsHandler::statsHandler(bool active_mode) {
       {Admin::ParamDescriptor::Type::Enum,
        "histogram_buckets",
        "Histogram bucket display mode",
-       {"cumulative", "disjoint", "none"}}};
+       {"cumulative", "disjoint", "detailed", "none"}}};
 
   Admin::ParamDescriptorVec params;
   if (!active_mode) {

source/server/admin/stats_render.cc

@@ -12,9 +12,6 @@ constexpr absl::string_view JsonQuoteCloseBrace = "\"}";
 } // namespace
 
 namespace Envoy {
-
-using ProtoMap = Protobuf::Map<std::string, ProtobufWkt::Value>;
-
 namespace Server {
 
 StatsTextRender::StatsTextRender(const StatsParams& params)
@@ -32,6 +29,11 @@ void StatsTextRender::generate(Buffer::Instance& response, const std::string& na
 
 void StatsTextRender::generate(Buffer::Instance& response, const std::string& name,
                                const Stats::ParentHistogram& histogram) {
+  if (!histogram.used()) {
+    response.addFragments({name, ": No recorded values\n"});
+    return;
+  }
+
   switch (histogram_buckets_mode_) {
   case Utility::HistogramBucketsMode::NoBuckets:
     response.addFragments({name, ": ", histogram.quantileSummary(), "\n"});
@@ -42,11 +44,28 @@ void StatsTextRender::generate(Buffer::Instance& response, const std::string& na
   case Utility::HistogramBucketsMode::Disjoint:
     addDisjointBuckets(name, histogram, response);
     break;
+  case Utility::HistogramBucketsMode::Detailed:
+    response.addFragments({name, ":\n  totals="});
+    addDetail(histogram.detailedTotalBuckets(), response);
+    response.add("\n  intervals=");
+    addDetail(histogram.detailedIntervalBuckets(), response);
+    response.addFragments({"\n  summary=", histogram.quantileSummary(), "\n"});
+    break;
   }
 }
 
 void StatsTextRender::finalize(Buffer::Instance&) {}
 
+void StatsTextRender::addDetail(const std::vector<Stats::ParentHistogram::Bucket>& buckets,
+                                Buffer::Instance& response) {
+  absl::string_view delim = "";
+  for (const Stats::ParentHistogram::Bucket& bucket : buckets) {
+    response.addFragments(
+        {delim, absl::StrCat(bucket.min_value_, ",", bucket.width_, ":", bucket.count_)});
+    delim = ", ";
+  }
+}
+
 // Computes disjoint buckets as text and adds them to the response buffer.
 void StatsTextRender::addDisjointBuckets(const std::string& name,
                                          const Stats::ParentHistogram& histogram,
@@ -126,9 +145,15 @@ void StatsJsonRender::generate(Buffer::Instance& response, const std::string& na
 void StatsJsonRender::generate(Buffer::Instance&, const std::string& name,
                                const Stats::ParentHistogram& histogram) {
   switch (histogram_buckets_mode_) {
-  case Utility::HistogramBucketsMode::NoBuckets:
-    summarizeBuckets(name, histogram);
+  case Utility::HistogramBucketsMode::NoBuckets: {
+    ProtobufWkt::Struct computed_quantile;
+    ProtoMap& computed_quantile_fields = *computed_quantile.mutable_fields();
+    computed_quantile_fields["name"] = ValueUtil::stringValue(name);
+    populateQuantiles(histogram, "supported_quantiles",
+                      computed_quantile_fields["values"].mutable_list_value());
+    *histogram_array_->add_values() = ValueUtil::structValue(computed_quantile);
     break;
+  }
   case Utility::HistogramBucketsMode::Cumulative: {
     const Stats::HistogramStatistics& interval_statistics = histogram.intervalStatistics();
     const std::vector<uint64_t>& interval_buckets = interval_statistics.computedBuckets();
@@ -145,6 +170,32 @@ void StatsJsonRender::generate(Buffer::Instance&, const std::string& name,
     collectBuckets(name, histogram, interval_buckets, cumulative_buckets);
     break;
   }
+  case Utility::HistogramBucketsMode::Detailed: {
+    std::vector<Stats::ParentHistogram::Bucket> buckets = histogram.detailedTotalBuckets();
+    ProtobufWkt::Struct histogram_obj;
+    ProtoMap& histogram_obj_fields = *histogram_obj.mutable_fields();
+    histogram_obj_fields["name"] = ValueUtil::stringValue(histogram.name());
+    populateQuantiles(histogram, "supported_percentiles",
+                      histogram_obj_fields["percentiles"].mutable_list_value());
+    populateDetail("totals", histogram.detailedTotalBuckets(), histogram_obj_fields);
+    populateDetail("intervals", histogram.detailedIntervalBuckets(), histogram_obj_fields);
+    *histogram_array_->add_values() = ValueUtil::structValue(histogram_obj);
+    break;
+  }
+  }
+}
+
+void StatsJsonRender::populateDetail(absl::string_view name,
+                                     const std::vector<Stats::ParentHistogram::Bucket>& buckets,
+                                     ProtoMap& histogram_obj_fields) {
+  ProtobufWkt::ListValue* bucket_array = histogram_obj_fields[name].mutable_list_value();
+  for (const Stats::ParentHistogram::Bucket& bucket : buckets) {
+    ProtobufWkt::Struct bucket_json;
+    ProtoMap& bucket_fields = *bucket_json.mutable_fields();
+    bucket_fields["min_value"] = ValueUtil::numberValue(bucket.min_value_);
+    bucket_fields["width"] = ValueUtil::numberValue(bucket.width_);
+    bucket_fields["count"] = ValueUtil::numberValue(bucket.count_);
+    *bucket_array->add_values() = ValueUtil::structValue(bucket_json);
   }
 }
 
@@ -154,11 +205,18 @@ void StatsJsonRender::finalize(Buffer::Instance& response) {
   if (histogram_array_->values_size() > 0) {
     ProtoMap& histograms_obj_container_fields = *histograms_obj_container_.mutable_fields();
     if (found_used_histogram_) {
-      ASSERT(histogram_buckets_mode_ == Utility::HistogramBucketsMode::NoBuckets);
-      ProtoMap& histograms_obj_fields = *histograms_obj_.mutable_fields();
-      histograms_obj_fields["computed_quantiles"].set_allocated_list_value(
-          histogram_array_.release());
-      histograms_obj_container_fields["histograms"] = ValueUtil::structValue(histograms_obj_);
+      if (histogram_buckets_mode_ == Utility::HistogramBucketsMode::NoBuckets) {
+        ProtoMap& histograms_obj_fields = *histograms_obj_.mutable_fields();
+        histograms_obj_fields["computed_quantiles"].set_allocated_list_value(
+            histogram_array_.release());
+        histograms_obj_container_fields["histograms"] = ValueUtil::structValue(histograms_obj_);
+      } else if (histogram_buckets_mode_ == Utility::HistogramBucketsMode::Detailed) {
+        ProtoMap& histograms_obj_fields = *histograms_obj_.mutable_fields();
+        histograms_obj_fields["details"].set_allocated_list_value(histogram_array_.release());
+        histograms_obj_container_fields["histograms"] = ValueUtil::structValue(histograms_obj_);
+      } else {
+        IS_ENVOY_BUG("reached unexpected buckets mode with found_used_histogram_ set");
+      }
     } else {
       ASSERT(histogram_buckets_mode_ != Utility::HistogramBucketsMode::NoBuckets);
       histograms_obj_container_fields["histograms"].set_allocated_list_value(
@@ -177,38 +235,36 @@ void StatsJsonRender::finalize(Buffer::Instance& response) {
   response.add("]}");
 }
 
+void StatsJsonRender::populateVector(absl::string_view name, const std::vector<double>& values,
+                                     uint32_t multiplier, ProtoMap& histograms_obj_fields) {
+  ProtobufWkt::ListValue* array = histograms_obj_fields[name].mutable_list_value();
+
+  for (double value : values) {
+    *array->add_values() = ValueUtil::numberValue(value * multiplier);
+  }
+}
+
 // Summarizes the buckets in the specified histogram, collecting JSON objects.
 // Note, we do not flush this buffer to the network when it grows large, and
 // if this becomes an issue it should be possible to do, noting that we are
 // one or two levels nesting below the list of scalar stats due to the Envoy
 // stats json schema, where histograms are grouped together.
-void StatsJsonRender::summarizeBuckets(const std::string& name,
-                                       const Stats::ParentHistogram& histogram) {
+void StatsJsonRender::populateQuantiles(const Stats::ParentHistogram& histogram,
+                                        absl::string_view label,
+                                        ProtobufWkt::ListValue* computed_quantile_value_array) {
   if (!found_used_histogram_) {
     // It is not possible for the supported quantiles to differ across histograms, so it is ok
     // to send them once.
     Stats::HistogramStatisticsImpl empty_statistics;
-    ProtoMap& histograms_obj_fields = *histograms_obj_.mutable_fields();
-    ProtobufWkt::ListValue* supported_quantile_array =
-        histograms_obj_fields["supported_quantiles"].mutable_list_value();
-
-    for (double quantile : empty_statistics.supportedQuantiles()) {
-      *supported_quantile_array->add_values() = ValueUtil::numberValue(quantile * 100);
-    }
-
+    ProtoMap& fields = *histograms_obj_.mutable_fields();
+    populateVector(label, empty_statistics.supportedQuantiles(), 100, fields);
     found_used_histogram_ = true;
   }
 
-  ProtobufWkt::Struct computed_quantile;
-  ProtoMap& computed_quantile_fields = *computed_quantile.mutable_fields();
-  computed_quantile_fields["name"] = ValueUtil::stringValue(name);
-
-  ProtobufWkt::ListValue* computed_quantile_value_array =
-      computed_quantile_fields["values"].mutable_list_value();
   const Stats::HistogramStatistics& interval_statistics = histogram.intervalStatistics();
   const std::vector<double>& computed_quantiles = interval_statistics.computedQuantiles();
-  const std::vector<double>& cumulative_quantiles =
-      histogram.cumulativeStatistics().computedQuantiles();
+  const Stats::HistogramStatistics& histogram_stats = histogram.cumulativeStatistics();
+  const std::vector<double>& cumulative_quantiles = histogram_stats.computedQuantiles();
   const size_t min_size = std::min({computed_quantiles.size(), cumulative_quantiles.size(),
                                     interval_statistics.supportedQuantiles().size()});
   ASSERT(min_size == computed_quantiles.size());
@@ -226,7 +282,6 @@ void StatsJsonRender::summarizeBuckets(const std::string& name,
 
     *computed_quantile_value_array->add_values() = ValueUtil::structValue(computed_quantile_value);
   }
-  *histogram_array_->add_values() = ValueUtil::structValue(computed_quantile);
 }
 
 // Collects the buckets from the specified histogram, using either the