stats: add support for histograms in prometheus export

docs/root/intro/version_history.rst

@@ -29,6 +29,7 @@ Version history
 * redis: migrate hash function for host selection to `MurmurHash2 <https://sites.google.com/site/murmurhash>`_ from std::hash. MurmurHash2 is compatible with std::hash in GNU libstdc++ 3.4.20 or above. This is typically the case when compiled on Linux and not macOS.
 * router: added ability to configure a :ref:`retry policy <envoy_api_msg_route.RetryPolicy>` at the
   virtual host level.
+* stats: added support for histograms in prometheus
 * tap: added new alpha :ref:`HTTP tap filter <config_http_filters_tap>`.
 * tls: enabled TLS 1.3 on the server-side (non-FIPS builds).
 * router: added per-route configuration of :ref:`internal redirects <envoy_api_field_route.RouteAction.internal_redirect_action>`.

docs/root/operations/admin.rst

@@ -310,8 +310,7 @@ explanation of the output.
   .. http:get:: /stats/prometheus
 
   Outputs /stats in `Prometheus <https://prometheus.io/docs/instrumenting/exposition_formats/>`_
-  v0.0.4 format. This can be used to integrate with a Prometheus server. Currently, only counters and
-  gauges are output. Histograms will be output in a future update.
+  v0.0.4 format. This can be used to integrate with a Prometheus server.
 
 .. _operations_admin_interface_runtime:
 

include/envoy/stats/histogram.h

@@ -18,9 +18,14 @@ class HistogramStatistics {
   virtual ~HistogramStatistics() {}
 
   /**
-   * Returns summary representation of the histogram.
+   * Returns quantile summary representation of the histogram.
    */
-  virtual std::string summary() const PURE;
+  virtual std::string quantileSummary() const PURE;
+
+  /**
+   * Returns bucket summary representation of the histogram.
+   */
+  virtual std::string bucketSummary() const PURE;
 
   /**
    * Returns supported quantiles.
@@ -31,6 +36,32 @@ class HistogramStatistics {
    * Returns computed quantile values during the period.
    */
   virtual const std::vector<double>& computedQuantiles() const PURE;
+
+  /**
+   * Returns supported buckets. Each value is the upper bound of the bucket
+   * with 0 as the implicit lower bound. For timers, these bucket thresholds
+   * are in milliseconds but the thresholds are applicable to all types of data.
+   */
+  virtual const std::vector<double>& supportedBuckets() const PURE;
+
+  /**
+   * Returns computed bucket values during the period. The vector contains an appoximation
+   * of samples below each quantile bucket defined in supportedBuckets(). This vector is
+   * guaranteed to be the same length as supportedBuckets().
+   */
+  virtual const std::vector<uint64_t>& computedBuckets() const PURE;
+
+  /**
+   * Returns number of values during the period. This number may be an approximation
+   * of the number of samples in the histogram, it is not guaranteed that this will be
+   * 100% the number of samples observed.
+   */
+  virtual double sampleCount() const PURE;
+
+  /**
+   * Returns sum of all values during the period.
+   */
+  virtual double sampleSum() const PURE;
 };
 
 /**
@@ -76,9 +107,14 @@ class ParentHistogram : public virtual Histogram {
   virtual const HistogramStatistics& cumulativeStatistics() const PURE;
 
   /**
-   * Returns the summary representation.
+   * Returns the quantile summary representation.
+   */
+  virtual const std::string quantileSummary() const PURE;
+
+  /**
+   * Returns the bucket summary representation.
    */
-  virtual const std::string summary() const PURE;
+  virtual const std::string bucketSummary() const PURE;
 };
 
 typedef std::shared_ptr<ParentHistogram> ParentHistogramSharedPtr;

source/common/stats/histogram_impl.cc

@@ -14,6 +14,15 @@ HistogramStatisticsImpl::HistogramStatisticsImpl(const histogram_t* histogram_pt
     : computed_quantiles_(supportedQuantiles().size(), 0.0) {
   hist_approx_quantile(histogram_ptr, supportedQuantiles().data(), supportedQuantiles().size(),
                        computed_quantiles_.data());
+
+  sample_count_ = hist_sample_count(histogram_ptr);
+  sample_sum_ = hist_approx_sum(histogram_ptr);
+
+  const std::vector<double>& supported_buckets = supportedBuckets();
+  computed_buckets_.reserve(supported_buckets.size());
+  for (const auto bucket : supported_buckets) {
+    computed_buckets_.emplace_back(hist_approx_count_below(histogram_ptr, bucket));
+  }
 }
 
 const std::vector<double>& HistogramStatisticsImpl::supportedQuantiles() const {
@@ -22,17 +31,33 @@ const std::vector<double>& HistogramStatisticsImpl::supportedQuantiles() const {
   return supported_quantiles;
 }
 
-std::string HistogramStatisticsImpl::summary() const {
+const std::vector<double>& HistogramStatisticsImpl::supportedBuckets() const {
+  static const std::vector<double> supported_buckets = {
+      0.5,  1,    5,     10,    25,    50,     100,    250,     500,    1000,
+      2500, 5000, 10000, 30000, 60000, 300000, 600000, 1800000, 3600000};
+  return supported_buckets;
+}
+
+std::string HistogramStatisticsImpl::quantileSummary() const {
   std::vector<std::string> summary;
-  const std::vector<double>& supported_quantiles_ref = supportedQuantiles();
-  summary.reserve(supported_quantiles_ref.size());
-  for (size_t i = 0; i < supported_quantiles_ref.size(); ++i) {
-    summary.push_back(
-        fmt::format("P{}: {}", 100 * supported_quantiles_ref[i], computed_quantiles_[i]));
+  const std::vector<double>& supported_quantiles = supportedQuantiles();
+  summary.reserve(supported_quantiles.size());
+  for (size_t i = 0; i < supported_quantiles.size(); ++i) {
+    summary.push_back(fmt::format("P{}: {}", 100 * supported_quantiles[i], computed_quantiles_[i]));
   }
   return absl::StrJoin(summary, ", ");
 }
 
+std::string HistogramStatisticsImpl::bucketSummary() const {
+  std::vector<std::string> bucket_summary;
+  const std::vector<double>& supported_buckets = supportedBuckets();
+  bucket_summary.reserve(supported_buckets.size());
+  for (size_t i = 0; i < supported_buckets.size(); ++i) {
+    bucket_summary.push_back(fmt::format("B{}: {}", supported_buckets[i], computed_buckets_[i]));
+  }
+  return absl::StrJoin(bucket_summary, ", ");
+}
+
 /**
  * Clears the old computed values and refreshes it with values computed from passed histogram.
  */
@@ -41,6 +66,17 @@ void HistogramStatisticsImpl::refresh(const histogram_t* new_histogram_ptr) {
   ASSERT(supportedQuantiles().size() == computed_quantiles_.size());
   hist_approx_quantile(new_histogram_ptr, supportedQuantiles().data(), supportedQuantiles().size(),
                        computed_quantiles_.data());
+
+  sample_count_ = hist_sample_count(new_histogram_ptr);
+  sample_sum_ = hist_approx_sum(new_histogram_ptr);
+
+  ASSERT(supportedBuckets().size() == computed_buckets_.size());
+  computed_buckets_.clear();
+  const std::vector<double>& supported_buckets = supportedBuckets();
+  computed_buckets_.reserve(supported_buckets.size());
+  for (const auto bucket : supported_buckets) {
+    computed_buckets_.emplace_back(hist_approx_count_below(new_histogram_ptr, bucket));
+  }
 }
 
 } // namespace Stats

source/common/stats/histogram_impl.h

@@ -31,12 +31,20 @@ class HistogramStatisticsImpl : public HistogramStatistics, NonCopyable {
   void refresh(const histogram_t* new_histogram_ptr);
 
   // HistogramStatistics
-  std::string summary() const override;
+  std::string quantileSummary() const override;
+  std::string bucketSummary() const override;
   const std::vector<double>& supportedQuantiles() const override;
   const std::vector<double>& computedQuantiles() const override { return computed_quantiles_; }
+  const std::vector<double>& supportedBuckets() const override;
+  const std::vector<uint64_t>& computedBuckets() const override { return computed_buckets_; }
+  double sampleCount() const override { return sample_count_; }
+  double sampleSum() const override { return sample_sum_; }
 
 private:
   std::vector<double> computed_quantiles_;
+  std::vector<uint64_t> computed_buckets_;
+  double sample_count_;
+  double sample_sum_;
 };
 
 /**

source/common/stats/thread_local_store.cc

@@ -503,7 +503,7 @@ void ParentHistogramImpl::merge() {
   }
 }
 
-const std::string ParentHistogramImpl::summary() const {
+const std::string ParentHistogramImpl::quantileSummary() const {
   if (used()) {
     std::vector<std::string> summary;
     const std::vector<double>& supported_quantiles_ref = interval_statistics_.supportedQuantiles();
@@ -519,6 +519,22 @@ const std::string ParentHistogramImpl::summary() const {
   }
 }
 
+const std::string ParentHistogramImpl::bucketSummary() const {
+  if (used()) {
+    std::vector<std::string> bucket_summary;
+    const std::vector<double>& supported_buckets = interval_statistics_.supportedBuckets();
+    bucket_summary.reserve(supported_buckets.size());
+    for (size_t i = 0; i < supported_buckets.size(); ++i) {
+      bucket_summary.push_back(fmt::format("B{}({},{})", supported_buckets[i],
+                                           interval_statistics_.computedBuckets()[i],
+                                           cumulative_statistics_.computedBuckets()[i]));
+    }
+    return absl::StrJoin(bucket_summary, " ");
+  } else {
+    return std::string("No recorded values");
+  }
+}
+
 void ParentHistogramImpl::addTlsHistogram(const TlsHistogramSharedPtr& hist_ptr) {
   Thread::LockGuard lock(merge_lock_);
   tls_histograms_.emplace_back(hist_ptr);

source/common/stats/thread_local_store.h

@@ -90,7 +90,8 @@ class ParentHistogramImpl : public ParentHistogram, public MetricImpl {
   const HistogramStatistics& cumulativeStatistics() const override {
     return cumulative_statistics_;
   }
-  const std::string summary() const override;
+  const std::string quantileSummary() const override;
+  const std::string bucketSummary() const override;
 
   // Stats::Metric
   std::string name() const override { return name_; }

source/server/http/admin.cc

@@ -30,6 +30,7 @@
 #include "common/access_log/access_log_impl.h"
 #include "common/buffer/buffer_impl.h"
 #include "common/common/assert.h"
+#include "common/common/empty_string.h"
 #include "common/common/enum_to_int.h"
 #include "common/common/fmt.h"
 #include "common/common/mutex_tracer_impl.h"
@@ -658,7 +659,7 @@ Http::Code AdminImpl::handlerStats(absl::string_view url, Http::HeaderMap& respo
     std::multimap<std::string, std::string> all_histograms;
     for (const Stats::ParentHistogramSharedPtr& histogram : server_.stats().histograms()) {
       if (shouldShowMetric(histogram, used_only, regex)) {
-        all_histograms.emplace(histogram->name(), histogram->summary());
+        all_histograms.emplace(histogram->name(), histogram->quantileSummary());
       }
     }
     for (auto histogram : all_histograms) {
@@ -671,7 +672,7 @@ Http::Code AdminImpl::handlerStats(absl::string_view url, Http::HeaderMap& respo
 Http::Code AdminImpl::handlerPrometheusStats(absl::string_view, Http::HeaderMap&,
                                              Buffer::Instance& response, AdminStream&) {
   PrometheusStatsFormatter::statsAsPrometheus(server_.stats().counters(), server_.stats().gauges(),
-                                              response);
+                                              server_.stats().histograms(), response);
   return Http::Code::OK;
 }
 
@@ -701,11 +702,10 @@ std::string PrometheusStatsFormatter::metricName(const std::string& extractedNam
   return sanitizeName(fmt::format("envoy_{0}", extractedName));
 }
 
-// TODO(ramaraochavali): Add summary histogram output for Prometheus.
-uint64_t
-PrometheusStatsFormatter::statsAsPrometheus(const std::vector<Stats::CounterSharedPtr>& counters,
-                                            const std::vector<Stats::GaugeSharedPtr>& gauges,
-                                            Buffer::Instance& response) {
+uint64_t PrometheusStatsFormatter::statsAsPrometheus(
+    const std::vector<Stats::CounterSharedPtr>& counters,
+    const std::vector<Stats::GaugeSharedPtr>& gauges,
+    const std::vector<Stats::ParentHistogramSharedPtr>& histograms, Buffer::Instance& response) {
   std::unordered_set<std::string> metric_type_tracker;
   for (const auto& counter : counters) {
     const std::string tags = formattedTags(counter->tags());
@@ -726,6 +726,38 @@ PrometheusStatsFormatter::statsAsPrometheus(const std::vector<Stats::CounterShar
     }
     response.add(fmt::format("{0}{{{1}}} {2}\n", metric_name, tags, gauge->value()));
   }
+
+  for (const auto& histogram : histograms) {
+    const std::string tags = formattedTags(histogram->tags());
+    const std::string hist_tags = histogram->tags().empty() ? EMPTY_STRING : (tags + ",");
+
+    const std::string metric_name = metricName(histogram->tagExtractedName());
+    if (metric_type_tracker.find(metric_name) == metric_type_tracker.end()) {
+      metric_type_tracker.insert(metric_name);
+      response.add(fmt::format("# TYPE {0} histogram\n", metric_name));
+    }
+
+    const Stats::HistogramStatistics& stats = histogram->cumulativeStatistics();
+    const std::vector<double>& supported_buckets = stats.supportedBuckets();
+    const std::vector<uint64_t>& computed_buckets = stats.computedBuckets();
+    for (size_t i = 0; i < supported_buckets.size(); ++i) {
+      double bucket = supported_buckets[i];
+      uint64_t value = computed_buckets[i];
+      // We want to print the bucket in a fixed point (non-scientific) format. The fmt library
+      // doesn't have a specific modifier to format as a fixed-point value only so we use the
+      // 'g' operator which prints the number in general fixed point format or scientific format
+      // with precision 50 to round the number up to 32 significant digits in fixed point format
+      // which should cover pretty much all cases
+      response.add(fmt::format("{0}_bucket{{{1}le=\"{2:.32g}\"}} {3}\n", metric_name, hist_tags,
+                               bucket, value));
+    }
+
+    response.add(fmt::format("{0}_bucket{{{1}le=\"+Inf\"}} {2}\n", metric_name, hist_tags,
+                             stats.sampleCount()));
+    response.add(fmt::format("{0}_sum{{{1}}} {2}\n", metric_name, tags, stats.sampleSum()));
+    response.add(fmt::format("{0}_count{{{1}}} {2}\n", metric_name, tags, stats.sampleCount()));
+  }
+
   return metric_type_tracker.size();
 }
 

source/server/http/admin.h

@@ -375,6 +375,7 @@ class PrometheusStatsFormatter {
    */
   static uint64_t statsAsPrometheus(const std::vector<Stats::CounterSharedPtr>& counters,
                                     const std::vector<Stats::GaugeSharedPtr>& gauges,
+                                    const std::vector<Stats::ParentHistogramSharedPtr>& histograms,
                                     Buffer::Instance& response);
   /**
    * Format the given tags, returning a string as a comma-separated list

test/common/stats/thread_local_store_test.cc

@@ -129,13 +129,21 @@ class HistogramTest : public testing::Test {
 
     NameHistogramMap name_histogram_map = makeHistogramMap(histogram_list);
     const ParentHistogramSharedPtr& h1 = name_histogram_map["h1"];
-    EXPECT_EQ(h1->cumulativeStatistics().summary(), h1_cumulative_statistics.summary());
-    EXPECT_EQ(h1->intervalStatistics().summary(), h1_interval_statistics.summary());
+    EXPECT_EQ(h1->cumulativeStatistics().quantileSummary(),
+              h1_cumulative_statistics.quantileSummary());
+    EXPECT_EQ(h1->intervalStatistics().quantileSummary(), h1_interval_statistics.quantileSummary());
+    EXPECT_EQ(h1->cumulativeStatistics().bucketSummary(), h1_cumulative_statistics.bucketSummary());
+    EXPECT_EQ(h1->intervalStatistics().bucketSummary(), h1_interval_statistics.bucketSummary());
 
     if (histogram_list.size() > 1) {
       const ParentHistogramSharedPtr& h2 = name_histogram_map["h2"];
-      EXPECT_EQ(h2->cumulativeStatistics().summary(), h2_cumulative_statistics.summary());
-      EXPECT_EQ(h2->intervalStatistics().summary(), h2_interval_statistics.summary());
+      EXPECT_EQ(h2->cumulativeStatistics().quantileSummary(),
+                h2_cumulative_statistics.quantileSummary());
+      EXPECT_EQ(h2->intervalStatistics().quantileSummary(),
+                h2_interval_statistics.quantileSummary());
+      EXPECT_EQ(h2->cumulativeStatistics().bucketSummary(),
+                h2_cumulative_statistics.bucketSummary());
+      EXPECT_EQ(h2->intervalStatistics().bucketSummary(), h2_interval_statistics.bucketSummary());
     }
 
     h1_interval_values_.clear();
@@ -850,8 +858,18 @@ TEST_F(HistogramTest, BasicHistogramSummaryValidate) {
   const std::string h1_expected_summary =
       "P0: 1, P25: 1.025, P50: 1.05, P75: 1.075, P90: 1.09, P95: 1.095, "
       "P99: 1.099, P99.5: 1.0995, P99.9: 1.0999, P100: 1.1";
-  const std::string h2_expected_summary = "P0: 0, P25: 25, P50: 50, P75: 75, P90: 90, P95: 95, "
-                                          "P99: 99, P99.5: 99.5, P99.9: 99.9, P100: 100";
+  const std::string h2_expected_summary =
+      "P0: 0, P25: 25, P50: 50, P75: 75, P90: 90, P95: 95, P99: 99, "
+      "P99.5: 99.5, P99.9: 99.9, P100: 100";
+
+  const std::string h1_expected_buckets =
+      "B0.5: 0, B1: 0, B5: 1, B10: 1, B25: 1, B50: 1, B100: 1, B250: 1, "
+      "B500: 1, B1000: 1, B2500: 1, B5000: 1, B10000: 1, B30000: 1, B60000: 1, "
+      "B300000: 1, B600000: 1, B1.8e+06: 1, B3.6e+06: 1";
+  const std::string h2_expected_buckets =
+      "B0.5: 1, B1: 1, B5: 5, B10: 10, B25: 25, B50: 50, B100: 100, B250: 100, "
+      "B500: 100, B1000: 100, B2500: 100, B5000: 100, B10000: 100, B30000: 100, "
+      "B60000: 100, B300000: 100, B600000: 100, B1.8e+06: 100, B3.6e+06: 100";
 
   for (size_t i = 0; i < 100; ++i) {
     expectCallAndAccumulate(h2, i);
@@ -860,8 +878,12 @@ TEST_F(HistogramTest, BasicHistogramSummaryValidate) {
   EXPECT_EQ(2, validateMerge());
 
   NameHistogramMap name_histogram_map = makeHistogramMap(store_->histograms());
-  EXPECT_EQ(h1_expected_summary, name_histogram_map["h1"]->cumulativeStatistics().summary());
-  EXPECT_EQ(h2_expected_summary, name_histogram_map["h2"]->cumulativeStatistics().summary());
+  EXPECT_EQ(h1_expected_summary,
+            name_histogram_map["h1"]->cumulativeStatistics().quantileSummary());
+  EXPECT_EQ(h2_expected_summary,
+            name_histogram_map["h2"]->cumulativeStatistics().quantileSummary());
+  EXPECT_EQ(h1_expected_buckets, name_histogram_map["h1"]->cumulativeStatistics().bucketSummary());
+  EXPECT_EQ(h2_expected_buckets, name_histogram_map["h2"]->cumulativeStatistics().bucketSummary());
 }
 
 // Validates the summary after known value merge in to same histogram.
@@ -880,9 +902,15 @@ TEST_F(HistogramTest, BasicHistogramMergeSummary) {
 
   const std::string expected_summary = "P0: 0, P25: 25, P50: 50, P75: 75, P90: 90, P95: 95, P99: "
                                        "99, P99.5: 99.5, P99.9: 99.9, P100: 100";
+  const std::string expected_bucket_summary =
+      "B0.5: 1, B1: 1, B5: 5, B10: 10, B25: 25, B50: 50, B100: 100, B250: 100, "
+      "B500: 100, B1000: 100, B2500: 100, B5000: 100, B10000: 100, B30000: 100, "
+      "B60000: 100, B300000: 100, B600000: 100, B1.8e+06: 100, B3.6e+06: 100";
 
   NameHistogramMap name_histogram_map = makeHistogramMap(store_->histograms());
-  EXPECT_EQ(expected_summary, name_histogram_map["h1"]->cumulativeStatistics().summary());
+  EXPECT_EQ(expected_summary, name_histogram_map["h1"]->cumulativeStatistics().quantileSummary());
+  EXPECT_EQ(expected_bucket_summary,
+            name_histogram_map["h1"]->cumulativeStatistics().bucketSummary());
 }
 
 TEST_F(HistogramTest, BasicHistogramUsed) {