From d790da2df87cf0588efd7609125d2f5a4d80d011 Mon Sep 17 00:00:00 2001
From: Greg Greenway <ggreenway@apple.com>
Date: Fri, 17 Apr 2020 13:13:33 -0700
Subject: [PATCH 1/8] prometheus stats: Correctly group lines of the same
 metric name.

Fixes #10073

Signed-off-by: Greg Greenway <ggreenway@apple.com>
---
 docs/root/version_history/current.rst |   1 +
 source/server/http/admin.cc           | 114 ++++++++-----
 test/server/http/admin_test.cc        | 236 +++++++++++++++++++++++++-
 3 files changed, 297 insertions(+), 54 deletions(-)

diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst
index 915fecc2d7969..9ba4dc52f4f34 100644
--- a/docs/root/version_history/current.rst
+++ b/docs/root/version_history/current.rst
@@ -10,6 +10,7 @@ Changes
   are applied to using :ref:`HTTP headers <config_http_filters_fault_injection_http_header>` to the HTTP fault filter.
 * http: fixed a bug where the upgrade header was not cleared on responses to non-upgrade requests.
   Can be reverted temporarily by setting runtime feature `envoy.reloadable_features.fix_upgrade_response` to false.
+* prometheus stats: fix the sort order of output lines to comply with the standard.
 * tracing: tracing configuration has been made fully dynamic and every HTTP connection manager
   can now have a separate :ref:`tracing provider <envoy_v3_api_field_extensions.filters.network.http_connection_manager.v3.HttpConnectionManager.Tracing.provider>`.
 
diff --git a/source/server/http/admin.cc b/source/server/http/admin.cc
index d2eb06e8b2cd8..95ed3c1d1142a 100644
--- a/source/server/http/admin.cc
+++ b/source/server/http/admin.cc
@@ -1007,52 +1007,65 @@ uint64_t PrometheusStatsFormatter::statsAsPrometheus(
     const std::vector<Stats::GaugeSharedPtr>& gauges,
     const std::vector<Stats::ParentHistogramSharedPtr>& histograms, Buffer::Instance& response,
     const bool used_only, const absl::optional<std::regex>& regex) {
-  std::unordered_set<std::string> metric_type_tracker;
-  for (const auto& counter : counters) {
-    if (!shouldShowMetric(*counter, used_only, regex)) {
-      continue;
-    }
-
-    const std::string tags = formattedTags(counter->tags());
-    const std::string metric_name = metricName(counter->tagExtractedName());
-    if (metric_type_tracker.find(metric_name) == metric_type_tracker.end()) {
-      metric_type_tracker.insert(metric_name);
-      response.add(fmt::format("# TYPE {0} counter\n", metric_name));
-    }
-    response.add(fmt::format("{0}{{{1}}} {2}\n", metric_name, tags, counter->value()));
-  }
 
-  for (const auto& gauge : gauges) {
-    if (!shouldShowMetric(*gauge, used_only, regex)) {
-      continue;
-    }
-
-    const std::string tags = formattedTags(gauge->tags());
-    const std::string metric_name = metricName(gauge->tagExtractedName());
-    if (metric_type_tracker.find(metric_name) == metric_type_tracker.end()) {
-      metric_type_tracker.insert(metric_name);
-      response.add(fmt::format("# TYPE {0} gauge\n", metric_name));
-    }
-    response.add(fmt::format("{0}{{{1}}} {2}\n", metric_name, tags, gauge->value()));
-  }
+  // From
+  // https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#grouping-and-sorting:
+  //
+  // All lines for a given metric must be provided as one single group, with the optional HELP and
+  // TYPE lines first (in no particular order). Beyond that, reproducible sorting in repeated
+  // expositions is preferred but not required, i.e. do not sort if the computational cost is
+  // prohibitive.
+
+  // Processes a metric type (counter, gauge, histogram) by generating all output lines, sorting
+  // them by metric name, and then outputting them in the correct sorted order into response.
+  auto process_type = [&](const auto& metrics, const auto& generate_name_and_output,
+                          absl::string_view type) -> uint64_t {
+    // This is a sorted collection to satisfy the "preferred" ordering from the prometheus
+    // spec: metrics will be sorted by their tags' textual representation, which will be consistent
+    // across calls.
+    std::map<std::string, std::set<std::string>> groups;
+    for (const auto& metric : metrics) {
+      if (!shouldShowMetric(*metric, used_only, regex)) {
+        continue;
+      }
 
-  for (const auto& histogram : histograms) {
-    if (!shouldShowMetric(*histogram, used_only, regex)) {
-      continue;
+      const auto name_output_pair = generate_name_and_output(*metric);
+      groups[name_output_pair.first].emplace(std::move(name_output_pair.second));
     }
 
-    const std::string tags = formattedTags(histogram->tags());
-    const std::string hist_tags = histogram->tags().empty() ? EMPTY_STRING : (tags + ",");
-
-    const std::string metric_name = metricName(histogram->tagExtractedName());
-    if (metric_type_tracker.find(metric_name) == metric_type_tracker.end()) {
-      metric_type_tracker.insert(metric_name);
-      response.add(fmt::format("# TYPE {0} histogram\n", metric_name));
+    for (const auto& group : groups) {
+      response.add(fmt::format("# TYPE {0} {1}\n", group.first, type));
+      for (const auto& output : group.second) {
+        response.add(output);
+      }
+      response.add("\n");
     }
-
-    const Stats::HistogramStatistics& stats = histogram->cumulativeStatistics();
+    return groups.size();
+  };
+
+  // Returns a pair of metric_name and prometheus output line for a counter or a gauge.
+  auto generate_counter_and_gauge_output =
+      [](const auto& metric) -> std::pair<std::string, std::string> {
+    const std::string tags = formattedTags(metric.tags());
+    const std::string metric_name = metricName(metric.tagExtractedName());
+    return std::make_pair(metric_name,
+                          fmt::format("{0}{{{1}}} {2}\n", metric_name, tags, metric.value()));
+  };
+
+  // Returns a pair of metric_name and prometheus output for a histogram. The output is
+  // a multi-line string (with embedded newlines) that contains all the individual bucket counts
+  // and sum/count for a single histogram (metric_name plus all tags).
+  auto generate_histogram_output =
+      [](const Stats::ParentHistogram& histogram) -> std::pair<std::string, std::string> {
+    const std::string tags = formattedTags(histogram.tags());
+    const std::string hist_tags = histogram.tags().empty() ? EMPTY_STRING : (tags + ",");
+
+    const std::string metric_name = metricName(histogram.tagExtractedName());
+
+    const Stats::HistogramStatistics& stats = histogram.cumulativeStatistics();
     const std::vector<double>& supported_buckets = stats.supportedBuckets();
     const std::vector<uint64_t>& computed_buckets = stats.computedBuckets();
+    std::string output;
     for (size_t i = 0; i < supported_buckets.size(); ++i) {
       double bucket = supported_buckets[i];
       uint64_t value = computed_buckets[i];
@@ -1061,17 +1074,24 @@ uint64_t PrometheusStatsFormatter::statsAsPrometheus(
       // 'g' operator which prints the number in general fixed point format or scientific format
       // with precision 50 to round the number up to 32 significant digits in fixed point format
       // which should cover pretty much all cases
-      response.add(fmt::format("{0}_bucket{{{1}le=\"{2:.32g}\"}} {3}\n", metric_name, hist_tags,
-                               bucket, value));
+      output.append(fmt::format("{0}_bucket{{{1}le=\"{2:.32g}\"}} {3}\n", metric_name, hist_tags,
+                                bucket, value));
     }
 
-    response.add(fmt::format("{0}_bucket{{{1}le=\"+Inf\"}} {2}\n", metric_name, hist_tags,
-                             stats.sampleCount()));
-    response.add(fmt::format("{0}_sum{{{1}}} {2:.32g}\n", metric_name, tags, stats.sampleSum()));
-    response.add(fmt::format("{0}_count{{{1}}} {2}\n", metric_name, tags, stats.sampleCount()));
-  }
+    output.append(fmt::format("{0}_bucket{{{1}le=\"+Inf\"}} {2}\n", metric_name, hist_tags,
+                              stats.sampleCount()));
+    output.append(fmt::format("{0}_sum{{{1}}} {2:.32g}\n", metric_name, tags, stats.sampleSum()));
+    output.append(fmt::format("{0}_count{{{1}}} {2}\n", metric_name, tags, stats.sampleCount()));
+
+    return std::make_pair(metric_name, output);
+  };
+
+  uint64_t metric_name_count = 0;
+  metric_name_count += process_type(counters, generate_counter_and_gauge_output, "counter");
+  metric_name_count += process_type(gauges, generate_counter_and_gauge_output, "gauge");
+  metric_name_count += process_type(histograms, generate_histogram_output, "histogram");
 
-  return metric_type_tracker.size();
+  return metric_name_count;
 }
 
 std::string
diff --git a/test/server/http/admin_test.cc b/test/server/http/admin_test.cc
index c4d35b4160025..b409decd37253 100644
--- a/test/server/http/admin_test.cc
+++ b/test/server/http/admin_test.cc
@@ -1584,16 +1584,18 @@ class PrometheusStatsFormatterTest : public testing::Test {
   ~PrometheusStatsFormatterTest() override { clearStorage(); }
 
   void addCounter(const std::string& name, Stats::StatNameTagVector cluster_tags) {
-    Stats::StatNameManagedStorage storage(name, *symbol_table_);
-    Stats::StatName stat_name = storage.statName();
-    counters_.push_back(alloc_.makeCounter(stat_name, stat_name, cluster_tags));
+    Stats::StatNameManagedStorage name_storage(baseName(name, cluster_tags), *symbol_table_);
+    Stats::StatNameManagedStorage tag_extracted_name_storage(name, *symbol_table_);
+    counters_.push_back(alloc_.makeCounter(name_storage.statName(),
+                                           tag_extracted_name_storage.statName(), cluster_tags));
   }
 
   void addGauge(const std::string& name, Stats::StatNameTagVector cluster_tags) {
-    Stats::StatNameManagedStorage storage(name, *symbol_table_);
-    Stats::StatName stat_name = storage.statName();
-    gauges_.push_back(
-        alloc_.makeGauge(stat_name, stat_name, cluster_tags, Stats::Gauge::ImportMode::Accumulate));
+    Stats::StatNameManagedStorage name_storage(baseName(name, cluster_tags), *symbol_table_);
+    Stats::StatNameManagedStorage tag_extracted_name_storage(name, *symbol_table_);
+    gauges_.push_back(alloc_.makeGauge(name_storage.statName(),
+                                       tag_extracted_name_storage.statName(), cluster_tags,
+                                       Stats::Gauge::ImportMode::Accumulate));
   }
 
   void addHistogram(const Stats::ParentHistogramSharedPtr histogram) {
@@ -1607,6 +1609,19 @@ class PrometheusStatsFormatterTest : public testing::Test {
 
   Stats::StatName makeStat(absl::string_view name) { return pool_.add(name); }
 
+  // Format tags into the name to create a unique stat_name for each name:tag combination.
+  // If the same stat_name is passed to makeGauge() or makeCounter(), even with different
+  // tags, a copy of hte previous metric will be returned.
+  std::string baseName(const std::string& name, Stats::StatNameTagVector cluster_tags) {
+    std::string result = name;
+    for (const auto& name_tag : cluster_tags) {
+      // It's difficult to get a raw string out of a StatName, so just use the hash value. It just
+      // needs to be unique; this string is never output or compared in the test.
+      result.append(fmt::format("<{}:{}>", name_tag.first.hash(), name_tag.second.hash()));
+    }
+    return result;
+  }
+
   void clearStorage() {
     pool_.clear();
     counters_.clear();
@@ -1738,6 +1753,7 @@ envoy_histogram1_bucket{le="3600000"} 0
 envoy_histogram1_bucket{le="+Inf"} 0
 envoy_histogram1_sum{} 0
 envoy_histogram1_count{} 0
+
 )EOF";
 
   EXPECT_EQ(expected_output, response.toString());
@@ -1791,6 +1807,7 @@ envoy_histogram1_bucket{le="3600000"} 101100000
 envoy_histogram1_bucket{le="+Inf"} 101100000
 envoy_histogram1_sum{} 105105105000
 envoy_histogram1_count{} 101100000
+
 )EOF";
 
   EXPECT_EQ(expected_output, response.toString());
@@ -1827,12 +1844,16 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithAllMetricTypes) {
 
   const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_cx_total counter
 envoy_cluster_test_1_upstream_cx_total{a_tag_name="a.tag-value"} 0
+
 # TYPE envoy_cluster_test_2_upstream_cx_total counter
 envoy_cluster_test_2_upstream_cx_total{another_tag_name="another_tag-value"} 0
+
 # TYPE envoy_cluster_test_3_upstream_cx_total gauge
 envoy_cluster_test_3_upstream_cx_total{another_tag_name_3="another_tag_3-value"} 0
+
 # TYPE envoy_cluster_test_4_upstream_cx_total gauge
 envoy_cluster_test_4_upstream_cx_total{another_tag_name_4="another_tag_4-value"} 0
+
 # TYPE envoy_cluster_test_1_upstream_rq_time histogram
 envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="0.5"} 0
 envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1"} 0
@@ -1856,6 +1877,205 @@ envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="360
 envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="+Inf"} 7
 envoy_cluster_test_1_upstream_rq_time_sum{key1="value1",key2="value2"} 5532
 envoy_cluster_test_1_upstream_rq_time_count{key1="value1",key2="value2"} 7
+
+)EOF";
+
+  EXPECT_EQ(expected_output, response.toString());
+}
+
+// Test that output groups all metrics of the same name (with different tags) together,
+// as required by the Prometheus exposition format spec. Additionally, groups of metrics
+// should be sorted by their tags; the format specifies that it is preferred that metrics
+// are always grouped in the same order, and sorting is an easy way to ensure this.
+TEST_F(PrometheusStatsFormatterTest, OutputSortedByMetricName) {
+  const std::vector<uint64_t> h1_values = {50, 20, 30, 70, 100, 5000, 200};
+  HistogramWrapper h1_cumulative;
+  h1_cumulative.setHistogramValues(h1_values);
+  Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram());
+
+  // Create the 3 clusters in non-sorted order to exercise the sorting.
+  // Create two of each metric type (counter, gauge, histogram) so that
+  // the output for each needs to be collected together.
+  for (const char* cluster : {"ccc", "aaa", "bbb"}) {
+    const Stats::StatNameTagVector tags{{makeStat("cluster"), makeStat(cluster)}};
+    addCounter("cluster.upstream_cx_total", tags);
+    addCounter("cluster.upstream_cx_connect_fail", tags);
+    addGauge("cluster.upstream_cx_active", tags);
+    addGauge("cluster.upstream_rq_active", tags);
+
+    for (const char* hist_name : {"cluster.upstream_rq_time", "cluster.upstream_response_time"}) {
+      auto histogram1 = makeHistogram();
+      histogram1->name_ = hist_name;
+      histogram1->unit_ = Stats::Histogram::Unit::Milliseconds;
+      histogram1->used_ = true;
+      histogram1->setTags({Stats::Tag{"cluster", cluster}});
+      addHistogram(histogram1);
+      EXPECT_CALL(*histogram1, cumulativeStatistics())
+          .WillOnce(testing::ReturnRef(h1_cumulative_statistics));
+    }
+  }
+
+  Buffer::OwnedImpl response;
+  auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response,
+                                                          false, absl::nullopt);
+  EXPECT_EQ(6UL, size);
+
+  const std::string expected_output = R"EOF(# TYPE envoy_cluster_upstream_cx_connect_fail counter
+envoy_cluster_upstream_cx_connect_fail{cluster="aaa"} 0
+envoy_cluster_upstream_cx_connect_fail{cluster="bbb"} 0
+envoy_cluster_upstream_cx_connect_fail{cluster="ccc"} 0
+
+# TYPE envoy_cluster_upstream_cx_total counter
+envoy_cluster_upstream_cx_total{cluster="aaa"} 0
+envoy_cluster_upstream_cx_total{cluster="bbb"} 0
+envoy_cluster_upstream_cx_total{cluster="ccc"} 0
+
+# TYPE envoy_cluster_upstream_cx_active gauge
+envoy_cluster_upstream_cx_active{cluster="aaa"} 0
+envoy_cluster_upstream_cx_active{cluster="bbb"} 0
+envoy_cluster_upstream_cx_active{cluster="ccc"} 0
+
+# TYPE envoy_cluster_upstream_rq_active gauge
+envoy_cluster_upstream_rq_active{cluster="aaa"} 0
+envoy_cluster_upstream_rq_active{cluster="bbb"} 0
+envoy_cluster_upstream_rq_active{cluster="ccc"} 0
+
+# TYPE envoy_cluster_upstream_response_time histogram
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="0.5"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="1"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="5"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="10"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="25"} 1
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="50"} 2
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="100"} 4
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="250"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="500"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="1000"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="2500"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="5000"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="10000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="30000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="60000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="300000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="600000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="1800000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="3600000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="+Inf"} 7
+envoy_cluster_upstream_response_time_sum{cluster="aaa"} 5532
+envoy_cluster_upstream_response_time_count{cluster="aaa"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="0.5"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="1"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="5"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="10"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="25"} 1
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="50"} 2
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="100"} 4
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="250"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="500"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="1000"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="2500"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="5000"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="10000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="30000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="60000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="300000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="600000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="1800000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="3600000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="+Inf"} 7
+envoy_cluster_upstream_response_time_sum{cluster="bbb"} 5532
+envoy_cluster_upstream_response_time_count{cluster="bbb"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="0.5"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="1"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="5"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="10"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="25"} 1
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="50"} 2
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="100"} 4
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="250"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="500"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="1000"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="2500"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="5000"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="10000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="30000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="60000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="300000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="600000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="1800000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="3600000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="+Inf"} 7
+envoy_cluster_upstream_response_time_sum{cluster="ccc"} 5532
+envoy_cluster_upstream_response_time_count{cluster="ccc"} 7
+
+# TYPE envoy_cluster_upstream_rq_time histogram
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="0.5"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="1"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="5"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="10"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="25"} 1
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="50"} 2
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="100"} 4
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="250"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="500"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="1000"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="2500"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="5000"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="10000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="30000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="60000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="300000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="600000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="1800000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="3600000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="+Inf"} 7
+envoy_cluster_upstream_rq_time_sum{cluster="aaa"} 5532
+envoy_cluster_upstream_rq_time_count{cluster="aaa"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="0.5"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="1"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="5"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="10"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="25"} 1
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="50"} 2
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="100"} 4
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="250"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="500"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="1000"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="2500"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="5000"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="10000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="30000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="60000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="300000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="600000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="1800000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="3600000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="+Inf"} 7
+envoy_cluster_upstream_rq_time_sum{cluster="bbb"} 5532
+envoy_cluster_upstream_rq_time_count{cluster="bbb"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="0.5"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="1"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="5"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="10"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="25"} 1
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="50"} 2
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="100"} 4
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="250"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="500"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="1000"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="2500"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="5000"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="10000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="30000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="60000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="300000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="600000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="1800000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="3600000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="+Inf"} 7
+envoy_cluster_upstream_rq_time_sum{cluster="ccc"} 5532
+envoy_cluster_upstream_rq_time_count{cluster="ccc"} 7
+
 )EOF";
 
   EXPECT_EQ(expected_output, response.toString());
@@ -1913,6 +2133,7 @@ envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="360
 envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="+Inf"} 7
 envoy_cluster_test_1_upstream_rq_time_sum{key1="value1",key2="value2"} 5532
 envoy_cluster_test_1_upstream_rq_time_count{key1="value1",key2="value2"} 7
+
 )EOF";
 
   EXPECT_EQ(expected_output, response.toString());
@@ -1983,6 +2204,7 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithRegexp) {
   const std::string expected_output =
       R"EOF(# TYPE envoy_cluster_test_1_upstream_cx_total counter
 envoy_cluster_test_1_upstream_cx_total{a_tag_name="a.tag-value"} 0
+
 )EOF";
 
   EXPECT_EQ(expected_output, response.toString());

From 48b0f06731cf4db21f3281f0077156c15b2880ca Mon Sep 17 00:00:00 2001
From: Greg Greenway <ggreenway@apple.com>
Date: Fri, 17 Apr 2020 14:57:11 -0700
Subject: [PATCH 2/8] fix spelling

Signed-off-by: Greg Greenway <ggreenway@apple.com>
---
 test/server/http/admin_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/server/http/admin_test.cc b/test/server/http/admin_test.cc
index b409decd37253..fa581407ddd9c 100644
--- a/test/server/http/admin_test.cc
+++ b/test/server/http/admin_test.cc
@@ -1611,7 +1611,7 @@ class PrometheusStatsFormatterTest : public testing::Test {
 
   // Format tags into the name to create a unique stat_name for each name:tag combination.
   // If the same stat_name is passed to makeGauge() or makeCounter(), even with different
-  // tags, a copy of hte previous metric will be returned.
+  // tags, a copy of the previous metric will be returned.
   std::string baseName(const std::string& name, Stats::StatNameTagVector cluster_tags) {
     std::string result = name;
     for (const auto& name_tag : cluster_tags) {

From e4d03749143b1f67c50e67463da76a0c3b355f2f Mon Sep 17 00:00:00 2001
From: Greg Greenway <ggreenway@apple.com>
Date: Thu, 23 Apr 2020 16:51:58 -0700
Subject: [PATCH 3/8] compare StatName instead of strings to avoid doubling
 required memory

Signed-off-by: Greg Greenway <ggreenway@apple.com>
---
 source/server/http/stats_handler.cc    | 98 +++++++++++++++++---------
 test/mocks/stats/mocks.h               | 13 ++++
 test/server/http/stats_handler_test.cc | 60 +++++++---------
 3 files changed, 106 insertions(+), 65 deletions(-)

diff --git a/source/server/http/stats_handler.cc b/source/server/http/stats_handler.cc
index 999e7df1d1b3c..ccb2473ad137f 100644
--- a/source/server/http/stats_handler.cc
+++ b/source/server/http/stats_handler.cc
@@ -191,60 +191,94 @@ uint64_t PrometheusStatsFormatter::statsAsPrometheus(
     const std::vector<Stats::ParentHistogramSharedPtr>& histograms, Buffer::Instance& response,
     const bool used_only, const absl::optional<std::regex>& regex) {
 
-  // From
-  // https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#grouping-and-sorting:
-  //
-  // All lines for a given metric must be provided as one single group, with the optional HELP and
-  // TYPE lines first (in no particular order). Beyond that, reproducible sorting in repeated
-  // expositions is preferred but not required, i.e. do not sort if the computational cost is
-  // prohibitive.
-
-  // Processes a metric type (counter, gauge, histogram) by generating all output lines, sorting
-  // them by metric name, and then outputting them in the correct sorted order into response.
-  auto process_type = [&](const auto& metrics, const auto& generate_name_and_output,
+  /*
+   * From
+   * https:*github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#grouping-and-sorting:
+   *
+   * All lines for a given metric must be provided as one single group, with the optional HELP and
+   * TYPE lines first (in no particular order). Beyond that, reproducible sorting in repeated
+   * expositions is preferred but not required, i.e. do not sort if the computational cost is
+   * prohibitive.
+   */
+
+  /**
+   * Processes a metric type (counter, gauge, histogram) by generating all output lines, sorting
+   * them by tag-extracted metric name, and then outputting them in the correct sorted order into
+   * response.
+   *
+   * @param metrics A vector of pointers to a metric type.
+   * @param generate_output A std::function<std::string(const MetricType& metric, const std::string&
+   *        prefixedTagExtractedName)> which returns the output text for this metric.
+   */
+  auto process_type = [&](const auto& metrics, const auto& generate_output,
                           absl::string_view type) -> uint64_t {
+    using MetricType = typename std::remove_reference<decltype(metrics)>::type::value_type;
+
+    struct MetricLessThan {
+      bool operator()(const MetricType& a, const MetricType& b) const {
+        ASSERT(&a->constSymbolTable() == &b->constSymbolTable());
+        return a->constSymbolTable().lessThan(a->statName(), b->statName());
+      }
+    };
+
     // This is a sorted collection to satisfy the "preferred" ordering from the prometheus
     // spec: metrics will be sorted by their tags' textual representation, which will be consistent
     // across calls.
-    std::map<std::string, std::set<std::string>> groups;
+    using MetricTypeSortedCollection = std::set<MetricType, MetricLessThan>;
+
+    // Return early to avoid crashing when getting the symbol table from the first metric.
+    if (metrics.empty()) {
+      return 0;
+    }
+
+    // There should only be one symbol table for all of the stats in the admin
+    // interface. If this assumption changes, the name comparisons in this function
+    // will have to change to compare to convert all StatNames to strings before
+    // comparison.
+    const Stats::SymbolTable& global_symbol_table = metrics.front()->constSymbolTable();
+
+    // Sorted collection of metrics sorted by their tagExtractedName, to satisfy the requirements
+    // of the exposition format.
+    std::map<Stats::StatName, MetricTypeSortedCollection, Stats::StatNameLessThan> groups(
+        global_symbol_table);
+
     for (const auto& metric : metrics) {
+      ASSERT(&global_symbol_table == &metric->constSymbolTable());
+
       if (!shouldShowMetric(*metric, used_only, regex)) {
         continue;
       }
 
-      const auto name_output_pair = generate_name_and_output(*metric);
-      groups[name_output_pair.first].emplace(std::move(name_output_pair.second));
+      groups[metric->tagExtractedStatName()].emplace(metric);
     }
 
     for (const auto& group : groups) {
-      response.add(fmt::format("# TYPE {0} {1}\n", group.first, type));
-      for (const auto& output : group.second) {
-        response.add(output);
+      const std::string metric_name = metricName(global_symbol_table.toString(group.first));
+      response.add(fmt::format("# TYPE {0} {1}\n", metric_name, type));
+
+      for (const auto& metric : group.second) {
+        response.add(generate_output(*metric, metric_name));
       }
       response.add("\n");
     }
     return groups.size();
   };
 
-  // Returns a pair of metric_name and prometheus output line for a counter or a gauge.
-  auto generate_counter_and_gauge_output =
-      [](const auto& metric) -> std::pair<std::string, std::string> {
+  // Returns the prometheus output line for a counter or a gauge.
+  auto generate_counter_and_gauge_output = [](const auto& metric,
+                                              const std::string& metric_name) -> std::string {
     const std::string tags = formattedTags(metric.tags());
-    const std::string metric_name = metricName(metric.tagExtractedName());
-    return std::make_pair(metric_name,
-                          fmt::format("{0}{{{1}}} {2}\n", metric_name, tags, metric.value()));
+    return fmt::format("{0}{{{1}}} {2}\n", metric_name, tags, metric.value());
   };
 
-  // Returns a pair of metric_name and prometheus output for a histogram. The output is
-  // a multi-line string (with embedded newlines) that contains all the individual bucket counts
-  // and sum/count for a single histogram (metric_name plus all tags).
-  auto generate_histogram_output =
-      [](const Stats::ParentHistogram& histogram) -> std::pair<std::string, std::string> {
+  // Returns the prometheus output for a histogram. The output is a multi-line string (with embedded
+  // newlines) that contains all the individual bucket counts and sum/count for a single histogram
+  // (metric_name plus all tags).
+  auto generate_histogram_output = [](const Stats::ParentHistogram& histogram,
+                                      const std::string& metric_name) -> std::string {
     const std::string tags = formattedTags(histogram.tags());
     const std::string hist_tags = histogram.tags().empty() ? EMPTY_STRING : (tags + ",");
 
-    const std::string metric_name = metricName(histogram.tagExtractedName());
-
     const Stats::HistogramStatistics& stats = histogram.cumulativeStatistics();
     const std::vector<double>& supported_buckets = stats.supportedBuckets();
     const std::vector<uint64_t>& computed_buckets = stats.computedBuckets();
@@ -266,7 +300,7 @@ uint64_t PrometheusStatsFormatter::statsAsPrometheus(
     output.append(fmt::format("{0}_sum{{{1}}} {2:.32g}\n", metric_name, tags, stats.sampleSum()));
     output.append(fmt::format("{0}_count{{{1}}} {2}\n", metric_name, tags, stats.sampleCount()));
 
-    return std::make_pair(metric_name, output);
+    return output;
   };
 
   uint64_t metric_name_count = 0;
@@ -275,7 +309,7 @@ uint64_t PrometheusStatsFormatter::statsAsPrometheus(
   metric_name_count += process_type(histograms, generate_histogram_output, "histogram");
 
   return metric_name_count;
-}
+} // namespace Server
 
 std::string
 StatsHandler::statsAsJson(const std::map<std::string, uint64_t>& all_stats,
diff --git a/test/mocks/stats/mocks.h b/test/mocks/stats/mocks.h
index 06a81e7c8cf52..17491916545f9 100644
--- a/test/mocks/stats/mocks.h
+++ b/test/mocks/stats/mocks.h
@@ -110,12 +110,25 @@ template <class BaseClass> class MockMetric : public BaseClass {
 
   void setTags(const TagVector& tags) {
     tag_pool_.clear();
+    tag_names_and_values_.clear();
     tags_ = tags;
     for (const Tag& tag : tags) {
       tag_names_and_values_.push_back(tag_pool_.add(tag.name_));
       tag_names_and_values_.push_back(tag_pool_.add(tag.value_));
     }
   }
+
+  void setTags(const Stats::StatNameTagVector& tags) {
+    tag_pool_.clear();
+    tag_names_and_values_.clear();
+    tags_.clear();
+    for (const StatNameTag& tag : tags) {
+      tag_names_and_values_.push_back(tag.first);
+      tag_names_and_values_.push_back(tag.second);
+      tags_.push_back(Tag{symbol_table_->toString(tag.first), symbol_table_->toString(tag.second)});
+    }
+  }
+
   void addTag(const Tag& tag) {
     tags_.emplace_back(tag);
     tag_names_and_values_.push_back(tag_pool_.add(tag.name_));
diff --git a/test/server/http/stats_handler_test.cc b/test/server/http/stats_handler_test.cc
index e178b60c21f10..02cc2095967ec 100644
--- a/test/server/http/stats_handler_test.cc
+++ b/test/server/http/stats_handler_test.cc
@@ -618,13 +618,17 @@ class PrometheusStatsFormatterTest : public testing::Test {
                                        Stats::Gauge::ImportMode::Accumulate));
   }
 
-  void addHistogram(const Stats::ParentHistogramSharedPtr histogram) {
-    histograms_.push_back(histogram);
-  }
-
   using MockHistogramSharedPtr = Stats::RefcountPtr<NiceMock<Stats::MockParentHistogram>>;
-  MockHistogramSharedPtr makeHistogram() {
-    return MockHistogramSharedPtr(new NiceMock<Stats::MockParentHistogram>());
+  void addHistogram(MockHistogramSharedPtr histogram) { histograms_.push_back(histogram); }
+
+  MockHistogramSharedPtr makeHistogram(const std::string& name,
+                                       Stats::StatNameTagVector cluster_tags) {
+    auto histogram = MockHistogramSharedPtr(new NiceMock<Stats::MockParentHistogram>());
+    histogram->name_ = baseName(name, cluster_tags);
+    histogram->setTagExtractedName(name);
+    histogram->setTags(cluster_tags);
+    histogram->used_ = true;
+    return histogram;
   }
 
   Stats::StatName makeStat(absl::string_view name) { return pool_.add(name); }
@@ -635,9 +639,8 @@ class PrometheusStatsFormatterTest : public testing::Test {
   std::string baseName(const std::string& name, Stats::StatNameTagVector cluster_tags) {
     std::string result = name;
     for (const auto& name_tag : cluster_tags) {
-      // It's difficult to get a raw string out of a StatName, so just use the hash value. It just
-      // needs to be unique; this string is never output or compared in the test.
-      result.append(fmt::format("<{}:{}>", name_tag.first.hash(), name_tag.second.hash()));
+      result.append(fmt::format("<{}:{}>", symbol_table_->toString(name_tag.first),
+                                symbol_table_->toString(name_tag.second)));
     }
     return result;
   }
@@ -737,9 +740,7 @@ TEST_F(PrometheusStatsFormatterTest, HistogramWithNoValuesAndNoTags) {
   h1_cumulative.setHistogramValues(std::vector<uint64_t>(0));
   Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram());
 
-  auto histogram = makeHistogram();
-  histogram->name_ = "histogram1";
-  histogram->used_ = true;
+  auto histogram = makeHistogram("histogram1", {});
   ON_CALL(*histogram, cumulativeStatistics())
       .WillByDefault(testing::ReturnRef(h1_cumulative_statistics));
 
@@ -791,9 +792,7 @@ TEST_F(PrometheusStatsFormatterTest, HistogramWithHighCounts) {
 
   Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram());
 
-  auto histogram = makeHistogram();
-  histogram->name_ = "histogram1";
-  histogram->used_ = true;
+  auto histogram = makeHistogram("histogram1", {});
   ON_CALL(*histogram, cumulativeStatistics())
       .WillByDefault(testing::ReturnRef(h1_cumulative_statistics));
 
@@ -848,11 +847,10 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithAllMetricTypes) {
   h1_cumulative.setHistogramValues(h1_values);
   Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram());
 
-  auto histogram1 = makeHistogram();
-  histogram1->name_ = "cluster.test_1.upstream_rq_time";
+  auto histogram1 =
+      makeHistogram("cluster.test_1.upstream_rq_time", {{makeStat("key1"), makeStat("value1")},
+                                                        {makeStat("key2"), makeStat("value2")}});
   histogram1->unit_ = Stats::Histogram::Unit::Milliseconds;
-  histogram1->used_ = true;
-  histogram1->setTags({Stats::Tag{"key1", "value1"}, Stats::Tag{"key2", "value2"}});
   addHistogram(histogram1);
   EXPECT_CALL(*histogram1, cumulativeStatistics())
       .WillOnce(testing::ReturnRef(h1_cumulative_statistics));
@@ -924,11 +922,8 @@ TEST_F(PrometheusStatsFormatterTest, OutputSortedByMetricName) {
     addGauge("cluster.upstream_rq_active", tags);
 
     for (const char* hist_name : {"cluster.upstream_rq_time", "cluster.upstream_response_time"}) {
-      auto histogram1 = makeHistogram();
-      histogram1->name_ = hist_name;
+      auto histogram1 = makeHistogram(hist_name, tags);
       histogram1->unit_ = Stats::Histogram::Unit::Milliseconds;
-      histogram1->used_ = true;
-      histogram1->setTags({Stats::Tag{"cluster", cluster}});
       addHistogram(histogram1);
       EXPECT_CALL(*histogram1, cumulativeStatistics())
           .WillOnce(testing::ReturnRef(h1_cumulative_statistics));
@@ -1116,11 +1111,10 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnly) {
   h1_cumulative.setHistogramValues(h1_values);
   Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram());
 
-  auto histogram1 = makeHistogram();
-  histogram1->name_ = "cluster.test_1.upstream_rq_time";
+  auto histogram1 =
+      makeHistogram("cluster.test_1.upstream_rq_time", {{makeStat("key1"), makeStat("value1")},
+                                                        {makeStat("key2"), makeStat("value2")}});
   histogram1->unit_ = Stats::Histogram::Unit::Milliseconds;
-  histogram1->used_ = true;
-  histogram1->setTags({Stats::Tag{"key1", "value1"}, Stats::Tag{"key2", "value2"}});
   addHistogram(histogram1);
   EXPECT_CALL(*histogram1, cumulativeStatistics())
       .WillOnce(testing::ReturnRef(h1_cumulative_statistics));
@@ -1165,11 +1159,11 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnlyHistogram) {
   h1_cumulative.setHistogramValues(h1_values);
   Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram());
 
-  auto histogram1 = makeHistogram();
-  histogram1->name_ = "cluster.test_1.upstream_rq_time";
+  auto histogram1 =
+      makeHistogram("cluster.test_1.upstream_rq_time", {{makeStat("key1"), makeStat("value1")},
+                                                        {makeStat("key2"), makeStat("value2")}});
   histogram1->unit_ = Stats::Histogram::Unit::Milliseconds;
   histogram1->used_ = false;
-  histogram1->setTags({Stats::Tag{"key1", "value1"}, Stats::Tag{"key2", "value2"}});
   addHistogram(histogram1);
 
   {
@@ -1209,10 +1203,10 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithRegexp) {
   h1_cumulative.setHistogramValues(h1_values);
   Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram());
 
-  auto histogram1 = makeHistogram();
-  histogram1->name_ = "cluster.test_1.upstream_rq_time";
+  auto histogram1 =
+      makeHistogram("cluster.test_1.upstream_rq_time", {{makeStat("key1"), makeStat("value1")},
+                                                        {makeStat("key2"), makeStat("value2")}});
   histogram1->unit_ = Stats::Histogram::Unit::Milliseconds;
-  histogram1->setTags({Stats::Tag{"key1", "value1"}, Stats::Tag{"key2", "value2"}});
   addHistogram(histogram1);
 
   Buffer::OwnedImpl response;

From a6ba179b0ad2dde8f94afcd7c5b6c335cf3bb359 Mon Sep 17 00:00:00 2001
From: Greg Greenway <ggreenway@apple.com>
Date: Thu, 23 Apr 2020 17:14:14 -0700
Subject: [PATCH 4/8] fix_format

Signed-off-by: Greg Greenway <ggreenway@apple.com>
---
 docs/root/version_history/current.rst | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/root/version_history/current.rst b/docs/root/version_history/current.rst
index 103c7385d5c69..4a23518ad2a45 100644
--- a/docs/root/version_history/current.rst
+++ b/docs/root/version_history/current.rst
@@ -15,12 +15,12 @@ Changes
   `google.api.HttpBody <https://github.com/googleapis/googleapis/blob/master/google/api/httpbody.proto>`_.
 * http: fixed a bug where the upgrade header was not cleared on responses to non-upgrade requests.
   Can be reverted temporarily by setting runtime feature `envoy.reloadable_features.fix_upgrade_response` to false.
-* prometheus stats: fix the sort order of output lines to comply with the standard.
 * http: remove legacy connection pool code and their runtime features: `envoy.reloadable_features.new_http1_connection_pool_behavior` and
   `envoy.reloadable_features.new_http2_connection_pool_behavior`.
 * logger: added :ref:`--log-format-prefix-with-location <operations_cli>` command line option to prefix '%v' with file path and line number.
 * network filters: added a :ref:`postgres proxy filter <config_network_filters_postgres_proxy>`.
 * network filters: added a :ref:`rocketmq proxy filter <config_network_filters_rocketmq_proxy>`.
+* prometheus stats: fix the sort order of output lines to comply with the standard.
 * request_id: added to :ref:`always_set_request_id_in_response setting <envoy_v3_api_field_extensions.filters.network.http_connection_manager.v3.HttpConnectionManager.always_set_request_id_in_response>`
   to set :ref:`x-request-id <config_http_conn_man_headers_x-request-id>` header in response even if
   tracing is not forced.

From 1e4d334c8585b45ea0ba65da95b35f46d427a8e5 Mon Sep 17 00:00:00 2001
From: Greg Greenway <ggreenway@apple.com>
Date: Fri, 24 Apr 2020 10:03:54 -0700
Subject: [PATCH 5/8] fixes

Signed-off-by: Greg Greenway <ggreenway@apple.com>
---
 include/envoy/stats/refcount_ptr.h  |  2 ++
 source/server/http/stats_handler.cc | 35 +++++++++++++++++++----------
 2 files changed, 25 insertions(+), 12 deletions(-)

diff --git a/include/envoy/stats/refcount_ptr.h b/include/envoy/stats/refcount_ptr.h
index 437f7b40cce24..a72c78ffcf35a 100644
--- a/include/envoy/stats/refcount_ptr.h
+++ b/include/envoy/stats/refcount_ptr.h
@@ -25,6 +25,8 @@ namespace Stats {
 // API in years.
 template <class T> class RefcountPtr {
 public:
+  using element_type = T;
+
   RefcountPtr() : ptr_(nullptr) {}
 
   // Constructing a reference-counted object from a pointer; this is safe to
diff --git a/source/server/http/stats_handler.cc b/source/server/http/stats_handler.cc
index ccb2473ad137f..9648fb6f97d8b 100644
--- a/source/server/http/stats_handler.cc
+++ b/source/server/http/stats_handler.cc
@@ -206,25 +206,31 @@ uint64_t PrometheusStatsFormatter::statsAsPrometheus(
    * them by tag-extracted metric name, and then outputting them in the correct sorted order into
    * response.
    *
-   * @param metrics A vector of pointers to a metric type.
+   * @param metrics A vector of Stats::RefcountPtr to a metric type.
    * @param generate_output A std::function<std::string(const MetricType& metric, const std::string&
    *        prefixedTagExtractedName)> which returns the output text for this metric.
    */
-  auto process_type = [&](const auto& metrics, const auto& generate_output,
-                          absl::string_view type) -> uint64_t {
-    using MetricType = typename std::remove_reference<decltype(metrics)>::type::value_type;
+  auto process_type = [&response, &regex, used_only](const auto& metrics,
+                                                     const auto& generate_output,
+                                                     absl::string_view type) -> uint64_t {
+    // Get the inner element type (MetricType) from a `const
+    // std::vector<Stats::MetricTypeSharedPtr>&`
+    using MetricType =
+        typename std::remove_reference<decltype(metrics)>::type::value_type::element_type;
 
     struct MetricLessThan {
-      bool operator()(const MetricType& a, const MetricType& b) const {
+      bool operator()(const MetricType* a, const MetricType* b) const {
         ASSERT(&a->constSymbolTable() == &b->constSymbolTable());
         return a->constSymbolTable().lessThan(a->statName(), b->statName());
       }
     };
 
-    // This is a sorted collection to satisfy the "preferred" ordering from the prometheus
-    // spec: metrics will be sorted by their tags' textual representation, which will be consistent
-    // across calls.
-    using MetricTypeSortedCollection = std::set<MetricType, MetricLessThan>;
+    // This is an unsorted colllection of dumb-pointers (no need to increment then decrement every
+    // refcount; ownership is held throughout by `metrics`). It is unsorted for efficiency, but will
+    // be sorted before producing the final output to satisfy the "preferred" ordering from the
+    // prometheus spec: metrics will be sorted by their tags' textual representation, which will be
+    // consistent across calls.
+    using MetricTypeUnsortedCollection = std::vector<const MetricType*>;
 
     // Return early to avoid crashing when getting the symbol table from the first metric.
     if (metrics.empty()) {
@@ -239,7 +245,7 @@ uint64_t PrometheusStatsFormatter::statsAsPrometheus(
 
     // Sorted collection of metrics sorted by their tagExtractedName, to satisfy the requirements
     // of the exposition format.
-    std::map<Stats::StatName, MetricTypeSortedCollection, Stats::StatNameLessThan> groups(
+    std::map<Stats::StatName, MetricTypeUnsortedCollection, Stats::StatNameLessThan> groups(
         global_symbol_table);
 
     for (const auto& metric : metrics) {
@@ -249,13 +255,18 @@ uint64_t PrometheusStatsFormatter::statsAsPrometheus(
         continue;
       }
 
-      groups[metric->tagExtractedStatName()].emplace(metric);
+      groups[metric->tagExtractedStatName()].push_back(metric.get());
     }
 
-    for (const auto& group : groups) {
+    for (auto& group : groups) {
       const std::string metric_name = metricName(global_symbol_table.toString(group.first));
       response.add(fmt::format("# TYPE {0} {1}\n", metric_name, type));
 
+      // Sort before producing the final output to satisfy the "preferred" ordering from the
+      // prometheus spec: metrics will be sorted by their tags' textual representation, which will
+      // be consistent across calls.
+      std::sort(group.second.begin(), group.second.end(), MetricLessThan());
+
       for (const auto& metric : group.second) {
         response.add(generate_output(*metric, metric_name));
       }

From b7f3fac131309f3844033441f09338495ae690e0 Mon Sep 17 00:00:00 2001
From: Greg Greenway <ggreenway@apple.com>
Date: Fri, 24 Apr 2020 11:11:13 -0700
Subject: [PATCH 6/8] spelling

Signed-off-by: Greg Greenway <ggreenway@apple.com>
---
 source/server/http/stats_handler.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/source/server/http/stats_handler.cc b/source/server/http/stats_handler.cc
index 9648fb6f97d8b..5ef4bd65ac050 100644
--- a/source/server/http/stats_handler.cc
+++ b/source/server/http/stats_handler.cc
@@ -225,7 +225,7 @@ uint64_t PrometheusStatsFormatter::statsAsPrometheus(
       }
     };
 
-    // This is an unsorted colllection of dumb-pointers (no need to increment then decrement every
+    // This is an unsorted collection of dumb-pointers (no need to increment then decrement every
     // refcount; ownership is held throughout by `metrics`). It is unsorted for efficiency, but will
     // be sorted before producing the final output to satisfy the "preferred" ordering from the
     // prometheus spec: metrics will be sorted by their tags' textual representation, which will be

From c2eb4fb05244686cc1903631ac15f385bea23c36 Mon Sep 17 00:00:00 2001
From: Greg Greenway <ggreenway@apple.com>
Date: Fri, 24 Apr 2020 12:24:26 -0700
Subject: [PATCH 7/8] Convert lambdas to templates for readability

Signed-off-by: Greg Greenway <ggreenway@apple.com>
---
 include/envoy/stats/refcount_ptr.h  |   2 -
 source/server/http/stats_handler.cc | 211 +++++++++++++---------------
 source/server/http/stats_handler.h  |  47 +++++++
 3 files changed, 148 insertions(+), 112 deletions(-)

diff --git a/include/envoy/stats/refcount_ptr.h b/include/envoy/stats/refcount_ptr.h
index a72c78ffcf35a..437f7b40cce24 100644
--- a/include/envoy/stats/refcount_ptr.h
+++ b/include/envoy/stats/refcount_ptr.h
@@ -25,8 +25,6 @@ namespace Stats {
 // API in years.
 template <class T> class RefcountPtr {
 public:
-  using element_type = T;
-
   RefcountPtr() : ptr_(nullptr) {}
 
   // Constructing a reference-counted object from a pointer; this is safe to
diff --git a/source/server/http/stats_handler.cc b/source/server/http/stats_handler.cc
index 5ef4bd65ac050..82dad9f99fc12 100644
--- a/source/server/http/stats_handler.cc
+++ b/source/server/http/stats_handler.cc
@@ -184,12 +184,13 @@ std::string PrometheusStatsFormatter::metricName(const std::string& extracted_na
   return sanitizeName(fmt::format("envoy_{0}", extracted_name));
 }
 
-// TODO(efimki): Add support of text readouts stats.
-uint64_t PrometheusStatsFormatter::statsAsPrometheus(
-    const std::vector<Stats::CounterSharedPtr>& counters,
-    const std::vector<Stats::GaugeSharedPtr>& gauges,
-    const std::vector<Stats::ParentHistogramSharedPtr>& histograms, Buffer::Instance& response,
-    const bool used_only, const absl::optional<std::regex>& regex) {
+template <class StatType>
+uint64_t PrometheusStatsFormatter::outputStatType(
+    Buffer::Instance& response, const bool used_only, const absl::optional<std::regex>& regex,
+    const std::vector<Stats::RefcountPtr<StatType>>& metrics,
+    const std::function<std::string(
+        const StatType& metric, const std::string& prefixed_tag_extracted_name)>& generate_output,
+    absl::string_view type) {
 
   /*
    * From
@@ -201,126 +202,116 @@ uint64_t PrometheusStatsFormatter::statsAsPrometheus(
    * prohibitive.
    */
 
-  /**
-   * Processes a metric type (counter, gauge, histogram) by generating all output lines, sorting
-   * them by tag-extracted metric name, and then outputting them in the correct sorted order into
-   * response.
-   *
-   * @param metrics A vector of Stats::RefcountPtr to a metric type.
-   * @param generate_output A std::function<std::string(const MetricType& metric, const std::string&
-   *        prefixedTagExtractedName)> which returns the output text for this metric.
-   */
-  auto process_type = [&response, &regex, used_only](const auto& metrics,
-                                                     const auto& generate_output,
-                                                     absl::string_view type) -> uint64_t {
-    // Get the inner element type (MetricType) from a `const
-    // std::vector<Stats::MetricTypeSharedPtr>&`
-    using MetricType =
-        typename std::remove_reference<decltype(metrics)>::type::value_type::element_type;
-
-    struct MetricLessThan {
-      bool operator()(const MetricType* a, const MetricType* b) const {
-        ASSERT(&a->constSymbolTable() == &b->constSymbolTable());
-        return a->constSymbolTable().lessThan(a->statName(), b->statName());
-      }
-    };
-
-    // This is an unsorted collection of dumb-pointers (no need to increment then decrement every
-    // refcount; ownership is held throughout by `metrics`). It is unsorted for efficiency, but will
-    // be sorted before producing the final output to satisfy the "preferred" ordering from the
-    // prometheus spec: metrics will be sorted by their tags' textual representation, which will be
-    // consistent across calls.
-    using MetricTypeUnsortedCollection = std::vector<const MetricType*>;
-
-    // Return early to avoid crashing when getting the symbol table from the first metric.
-    if (metrics.empty()) {
-      return 0;
-    }
+  // This is an unsorted collection of dumb-pointers (no need to increment then decrement every
+  // refcount; ownership is held throughout by `metrics`). It is unsorted for efficiency, but will
+  // be sorted before producing the final output to satisfy the "preferred" ordering from the
+  // prometheus spec: metrics will be sorted by their tags' textual representation, which will be
+  // consistent across calls.
+  using StatTypeUnsortedCollection = std::vector<const StatType*>;
 
-    // There should only be one symbol table for all of the stats in the admin
-    // interface. If this assumption changes, the name comparisons in this function
-    // will have to change to compare to convert all StatNames to strings before
-    // comparison.
-    const Stats::SymbolTable& global_symbol_table = metrics.front()->constSymbolTable();
+  // Return early to avoid crashing when getting the symbol table from the first metric.
+  if (metrics.empty()) {
+    return 0;
+  }
 
-    // Sorted collection of metrics sorted by their tagExtractedName, to satisfy the requirements
-    // of the exposition format.
-    std::map<Stats::StatName, MetricTypeUnsortedCollection, Stats::StatNameLessThan> groups(
-        global_symbol_table);
+  // There should only be one symbol table for all of the stats in the admin
+  // interface. If this assumption changes, the name comparisons in this function
+  // will have to change to compare to convert all StatNames to strings before
+  // comparison.
+  const Stats::SymbolTable& global_symbol_table = metrics.front()->constSymbolTable();
 
-    for (const auto& metric : metrics) {
-      ASSERT(&global_symbol_table == &metric->constSymbolTable());
+  // Sorted collection of metrics sorted by their tagExtractedName, to satisfy the requirements
+  // of the exposition format.
+  std::map<Stats::StatName, StatTypeUnsortedCollection, Stats::StatNameLessThan> groups(
+      global_symbol_table);
 
-      if (!shouldShowMetric(*metric, used_only, regex)) {
-        continue;
-      }
+  for (const auto& metric : metrics) {
+    ASSERT(&global_symbol_table == &metric->constSymbolTable());
 
-      groups[metric->tagExtractedStatName()].push_back(metric.get());
+    if (!shouldShowMetric(*metric, used_only, regex)) {
+      continue;
     }
 
-    for (auto& group : groups) {
-      const std::string metric_name = metricName(global_symbol_table.toString(group.first));
-      response.add(fmt::format("# TYPE {0} {1}\n", metric_name, type));
+    groups[metric->tagExtractedStatName()].push_back(metric.get());
+  }
 
-      // Sort before producing the final output to satisfy the "preferred" ordering from the
-      // prometheus spec: metrics will be sorted by their tags' textual representation, which will
-      // be consistent across calls.
-      std::sort(group.second.begin(), group.second.end(), MetricLessThan());
+  for (auto& group : groups) {
+    const std::string prefixed_tag_extracted_name =
+        metricName(global_symbol_table.toString(group.first));
+    response.add(fmt::format("# TYPE {0} {1}\n", prefixed_tag_extracted_name, type));
 
-      for (const auto& metric : group.second) {
-        response.add(generate_output(*metric, metric_name));
-      }
-      response.add("\n");
-    }
-    return groups.size();
-  };
-
-  // Returns the prometheus output line for a counter or a gauge.
-  auto generate_counter_and_gauge_output = [](const auto& metric,
-                                              const std::string& metric_name) -> std::string {
-    const std::string tags = formattedTags(metric.tags());
-    return fmt::format("{0}{{{1}}} {2}\n", metric_name, tags, metric.value());
-  };
-
-  // Returns the prometheus output for a histogram. The output is a multi-line string (with embedded
-  // newlines) that contains all the individual bucket counts and sum/count for a single histogram
-  // (metric_name plus all tags).
-  auto generate_histogram_output = [](const Stats::ParentHistogram& histogram,
-                                      const std::string& metric_name) -> std::string {
-    const std::string tags = formattedTags(histogram.tags());
-    const std::string hist_tags = histogram.tags().empty() ? EMPTY_STRING : (tags + ",");
-
-    const Stats::HistogramStatistics& stats = histogram.cumulativeStatistics();
-    const std::vector<double>& supported_buckets = stats.supportedBuckets();
-    const std::vector<uint64_t>& computed_buckets = stats.computedBuckets();
-    std::string output;
-    for (size_t i = 0; i < supported_buckets.size(); ++i) {
-      double bucket = supported_buckets[i];
-      uint64_t value = computed_buckets[i];
-      // We want to print the bucket in a fixed point (non-scientific) format. The fmt library
-      // doesn't have a specific modifier to format as a fixed-point value only so we use the
-      // 'g' operator which prints the number in general fixed point format or scientific format
-      // with precision 50 to round the number up to 32 significant digits in fixed point format
-      // which should cover pretty much all cases
-      output.append(fmt::format("{0}_bucket{{{1}le=\"{2:.32g}\"}} {3}\n", metric_name, hist_tags,
-                                bucket, value));
+    // Sort before producing the final output to satisfy the "preferred" ordering from the
+    // prometheus spec: metrics will be sorted by their tags' textual representation, which will
+    // be consistent across calls.
+    std::sort(group.second.begin(), group.second.end(), MetricLessThan());
+
+    for (const auto& metric : group.second) {
+      response.add(generate_output(*metric, prefixed_tag_extracted_name));
     }
+    response.add("\n");
+  }
+  return groups.size();
+}
 
-    output.append(fmt::format("{0}_bucket{{{1}le=\"+Inf\"}} {2}\n", metric_name, hist_tags,
-                              stats.sampleCount()));
-    output.append(fmt::format("{0}_sum{{{1}}} {2:.32g}\n", metric_name, tags, stats.sampleSum()));
-    output.append(fmt::format("{0}_count{{{1}}} {2}\n", metric_name, tags, stats.sampleCount()));
+template <class StatType>
+std::string
+PrometheusStatsFormatter::generateNumericOutput(const StatType& metric,
+                                                const std::string& prefixed_tag_extracted_name) {
+  const std::string tags = formattedTags(metric.tags());
+  return fmt::format("{0}{{{1}}} {2}\n", prefixed_tag_extracted_name, tags, metric.value());
+}
 
-    return output;
-  };
+std::string
+PrometheusStatsFormatter::generateHistogramOutput(const Stats::ParentHistogram& histogram,
+                                                  const std::string& prefixed_tag_extracted_name) {
+  const std::string tags = formattedTags(histogram.tags());
+  const std::string hist_tags = histogram.tags().empty() ? EMPTY_STRING : (tags + ",");
+
+  const Stats::HistogramStatistics& stats = histogram.cumulativeStatistics();
+  const std::vector<double>& supported_buckets = stats.supportedBuckets();
+  const std::vector<uint64_t>& computed_buckets = stats.computedBuckets();
+  std::string output;
+  for (size_t i = 0; i < supported_buckets.size(); ++i) {
+    double bucket = supported_buckets[i];
+    uint64_t value = computed_buckets[i];
+    // We want to print the bucket in a fixed point (non-scientific) format. The fmt library
+    // doesn't have a specific modifier to format as a fixed-point value only so we use the
+    // 'g' operator which prints the number in general fixed point format or scientific format
+    // with precision 50 to round the number up to 32 significant digits in fixed point format
+    // which should cover pretty much all cases
+    output.append(fmt::format("{0}_bucket{{{1}le=\"{2:.32g}\"}} {3}\n", prefixed_tag_extracted_name,
+                              hist_tags, bucket, value));
+  }
+
+  output.append(fmt::format("{0}_bucket{{{1}le=\"+Inf\"}} {2}\n", prefixed_tag_extracted_name,
+                            hist_tags, stats.sampleCount()));
+  output.append(fmt::format("{0}_sum{{{1}}} {2:.32g}\n", prefixed_tag_extracted_name, tags,
+                            stats.sampleSum()));
+  output.append(fmt::format("{0}_count{{{1}}} {2}\n", prefixed_tag_extracted_name, tags,
+                            stats.sampleCount()));
+
+  return output;
+};
+
+// TODO(efimki): Add support of text readouts stats.
+uint64_t PrometheusStatsFormatter::statsAsPrometheus(
+    const std::vector<Stats::CounterSharedPtr>& counters,
+    const std::vector<Stats::GaugeSharedPtr>& gauges,
+    const std::vector<Stats::ParentHistogramSharedPtr>& histograms, Buffer::Instance& response,
+    const bool used_only, const absl::optional<std::regex>& regex) {
 
   uint64_t metric_name_count = 0;
-  metric_name_count += process_type(counters, generate_counter_and_gauge_output, "counter");
-  metric_name_count += process_type(gauges, generate_counter_and_gauge_output, "gauge");
-  metric_name_count += process_type(histograms, generate_histogram_output, "histogram");
+  metric_name_count += outputStatType<Stats::Counter>(
+      response, used_only, regex, counters, generateNumericOutput<Stats::Counter>, "counter");
+
+  metric_name_count += outputStatType<Stats::Gauge>(response, used_only, regex, gauges,
+                                                    generateNumericOutput<Stats::Gauge>, "gauge");
+
+  metric_name_count += outputStatType<Stats::ParentHistogram>(
+      response, used_only, regex, histograms, generateHistogramOutput, "histogram");
 
   return metric_name_count;
-} // namespace Server
+}
 
 std::string
 StatsHandler::statsAsJson(const std::map<std::string, uint64_t>& all_stats,
diff --git a/source/server/http/stats_handler.h b/source/server/http/stats_handler.h
index ad4272c1d1931..519f77ec0eebc 100644
--- a/source/server/http/stats_handler.h
+++ b/source/server/http/stats_handler.h
@@ -109,6 +109,53 @@ class PrometheusStatsFormatter {
     return ((!used_only || metric.used()) &&
             (!regex.has_value() || std::regex_search(metric.name(), regex.value())));
   }
+
+  /**
+   * Processes a stat type (counter, gauge, histogram) by generating all output lines, sorting
+   * them by tag-extracted metric name, and then outputting them in the correct sorted order into
+   * response.
+   *
+   * @param response The buffer to put the output into.
+   * @param used_only Whether to only output stats that are used.
+   * @param regex A filter on which stats to output.
+   * @param metrics The metrics to output stats for. This must contain all stats of the given type
+   *        to be included in the same output.
+   * @param generate_output A function which returns the output text for this metric.
+   * @param type The name of the prometheus metric type for used in TYPE annotations.
+   */
+  template <class StatType>
+  static uint64_t outputStatType(
+      Buffer::Instance& response, const bool used_only, const absl::optional<std::regex>& regex,
+      const std::vector<Stats::RefcountPtr<StatType>>& metrics,
+      const std::function<std::string(
+          const StatType& metric, const std::string& prefixed_tag_extracted_name)>& generate_output,
+      absl::string_view type);
+
+  /*
+   * Return the prometheus output for a numeric Stat (Counter or Gauge).
+   */
+  template <class StatType>
+  static std::string generateNumericOutput(const StatType& metric,
+                                           const std::string& prefixed_tag_extracted_name);
+
+  /*
+   * Returns the prometheus output for a histogram. The output is a multi-line string (with embedded
+   * newlines) that contains all the individual bucket counts and sum/count for a single histogram
+   * (metric_name plus all tags).
+   */
+  static std::string generateHistogramOutput(const Stats::ParentHistogram& histogram,
+                                             const std::string& prefixed_tag_extracted_name);
+
+  /*
+   * Comparator for Stats::Metric that does not require a string representation
+   * to make the comparison, for memory efficiency.
+   */
+  struct MetricLessThan {
+    bool operator()(const Stats::Metric* a, const Stats::Metric* b) const {
+      ASSERT(&a->constSymbolTable() == &b->constSymbolTable());
+      return a->constSymbolTable().lessThan(a->statName(), b->statName());
+    }
+  };
 };
 
 } // namespace Server

From 8aee0a6ebcfcade19d16472da605f96477ba8ffa Mon Sep 17 00:00:00 2001
From: Greg Greenway <ggreenway@apple.com>
Date: Fri, 24 Apr 2020 13:14:06 -0700
Subject: [PATCH 8/8] move prom generation into it's own file

Signed-off-by: Greg Greenway <ggreenway@apple.com>
---
 source/server/http/BUILD                  |  12 +
 source/server/http/prometheus_stats.cc    | 218 +++++++
 source/server/http/prometheus_stats.h     |  41 ++
 source/server/http/stats_handler.cc       | 161 +-----
 source/server/http/stats_handler.h        |  92 ---
 test/server/http/BUILD                    |   9 +
 test/server/http/prometheus_stats_test.cc | 667 ++++++++++++++++++++++
 test/server/http/stats_handler_test.cc    | 653 ---------------------
 tools/code_format/check_format.py         |   3 +-
 9 files changed, 950 insertions(+), 906 deletions(-)
 create mode 100644 source/server/http/prometheus_stats.cc
 create mode 100644 source/server/http/prometheus_stats.h
 create mode 100644 test/server/http/prometheus_stats_test.cc

diff --git a/source/server/http/BUILD b/source/server/http/BUILD
index dbbb828c89303..7d46ea39d930b 100644
--- a/source/server/http/BUILD
+++ b/source/server/http/BUILD
@@ -90,6 +90,7 @@ envoy_cc_library(
     srcs = ["stats_handler.cc"],
     hdrs = ["stats_handler.h"],
     deps = [
+        ":prometheus_stats_lib",
         ":utils_lib",
         "//include/envoy/http:codes_interface",
         "//include/envoy/server:admin_interface",
@@ -102,6 +103,17 @@ envoy_cc_library(
     ],
 )
 
+envoy_cc_library(
+    name = "prometheus_stats_lib",
+    srcs = ["prometheus_stats.cc"],
+    hdrs = ["prometheus_stats.h"],
+    deps = [
+        ":utils_lib",
+        "//source/common/buffer:buffer_lib",
+        "//source/common/stats:histogram_lib",
+    ],
+)
+
 envoy_cc_library(
     name = "utils_lib",
     srcs = ["utils.cc"],
diff --git a/source/server/http/prometheus_stats.cc b/source/server/http/prometheus_stats.cc
new file mode 100644
index 0000000000000..e04edeccf9cf3
--- /dev/null
+++ b/source/server/http/prometheus_stats.cc
@@ -0,0 +1,218 @@
+#include "server/http/prometheus_stats.h"
+
+#include "common/common/empty_string.h"
+#include "common/stats/histogram_impl.h"
+
+#include "absl/strings/str_cat.h"
+
+namespace Envoy {
+namespace Server {
+
+namespace {
+
+const std::regex& promRegex() { CONSTRUCT_ON_FIRST_USE(std::regex, "[^a-zA-Z0-9_]"); }
+
+/**
+ * Take a string and sanitize it according to Prometheus conventions.
+ */
+std::string sanitizeName(const std::string& name) {
+  // The name must match the regex [a-zA-Z_][a-zA-Z0-9_]* as required by
+  // prometheus. Refer to https://prometheus.io/docs/concepts/data_model/.
+  std::string stats_name = std::regex_replace(name, promRegex(), "_");
+  if (stats_name[0] >= '0' && stats_name[0] <= '9') {
+    return absl::StrCat("_", stats_name);
+  } else {
+    return stats_name;
+  }
+}
+
+/*
+ * Determine whether a metric has never been emitted and choose to
+ * not show it if we only wanted used metrics.
+ */
+template <class StatType>
+static bool shouldShowMetric(const StatType& metric, const bool used_only,
+                             const absl::optional<std::regex>& regex) {
+  return ((!used_only || metric.used()) &&
+          (!regex.has_value() || std::regex_search(metric.name(), regex.value())));
+}
+
+/*
+ * Comparator for Stats::Metric that does not require a string representation
+ * to make the comparison, for memory efficiency.
+ */
+struct MetricLessThan {
+  bool operator()(const Stats::Metric* a, const Stats::Metric* b) const {
+    ASSERT(&a->constSymbolTable() == &b->constSymbolTable());
+    return a->constSymbolTable().lessThan(a->statName(), b->statName());
+  }
+};
+
+/**
+ * Processes a stat type (counter, gauge, histogram) by generating all output lines, sorting
+ * them by tag-extracted metric name, and then outputting them in the correct sorted order into
+ * response.
+ *
+ * @param response The buffer to put the output into.
+ * @param used_only Whether to only output stats that are used.
+ * @param regex A filter on which stats to output.
+ * @param metrics The metrics to output stats for. This must contain all stats of the given type
+ *        to be included in the same output.
+ * @param generate_output A function which returns the output text for this metric.
+ * @param type The name of the prometheus metric type for used in TYPE annotations.
+ */
+template <class StatType>
+uint64_t outputStatType(
+    Buffer::Instance& response, const bool used_only, const absl::optional<std::regex>& regex,
+    const std::vector<Stats::RefcountPtr<StatType>>& metrics,
+    const std::function<std::string(
+        const StatType& metric, const std::string& prefixed_tag_extracted_name)>& generate_output,
+    absl::string_view type) {
+
+  /*
+   * From
+   * https:*github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#grouping-and-sorting:
+   *
+   * All lines for a given metric must be provided as one single group, with the optional HELP and
+   * TYPE lines first (in no particular order). Beyond that, reproducible sorting in repeated
+   * expositions is preferred but not required, i.e. do not sort if the computational cost is
+   * prohibitive.
+   */
+
+  // This is an unsorted collection of dumb-pointers (no need to increment then decrement every
+  // refcount; ownership is held throughout by `metrics`). It is unsorted for efficiency, but will
+  // be sorted before producing the final output to satisfy the "preferred" ordering from the
+  // prometheus spec: metrics will be sorted by their tags' textual representation, which will be
+  // consistent across calls.
+  using StatTypeUnsortedCollection = std::vector<const StatType*>;
+
+  // Return early to avoid crashing when getting the symbol table from the first metric.
+  if (metrics.empty()) {
+    return 0;
+  }
+
+  // There should only be one symbol table for all of the stats in the admin
+  // interface. If this assumption changes, the name comparisons in this function
+  // will have to change to compare to convert all StatNames to strings before
+  // comparison.
+  const Stats::SymbolTable& global_symbol_table = metrics.front()->constSymbolTable();
+
+  // Sorted collection of metrics sorted by their tagExtractedName, to satisfy the requirements
+  // of the exposition format.
+  std::map<Stats::StatName, StatTypeUnsortedCollection, Stats::StatNameLessThan> groups(
+      global_symbol_table);
+
+  for (const auto& metric : metrics) {
+    ASSERT(&global_symbol_table == &metric->constSymbolTable());
+
+    if (!shouldShowMetric(*metric, used_only, regex)) {
+      continue;
+    }
+
+    groups[metric->tagExtractedStatName()].push_back(metric.get());
+  }
+
+  for (auto& group : groups) {
+    const std::string prefixed_tag_extracted_name =
+        PrometheusStatsFormatter::metricName(global_symbol_table.toString(group.first));
+    response.add(fmt::format("# TYPE {0} {1}\n", prefixed_tag_extracted_name, type));
+
+    // Sort before producing the final output to satisfy the "preferred" ordering from the
+    // prometheus spec: metrics will be sorted by their tags' textual representation, which will
+    // be consistent across calls.
+    std::sort(group.second.begin(), group.second.end(), MetricLessThan());
+
+    for (const auto& metric : group.second) {
+      response.add(generate_output(*metric, prefixed_tag_extracted_name));
+    }
+    response.add("\n");
+  }
+  return groups.size();
+}
+
+/*
+ * Return the prometheus output for a numeric Stat (Counter or Gauge).
+ */
+template <class StatType>
+std::string generateNumericOutput(const StatType& metric,
+                                  const std::string& prefixed_tag_extracted_name) {
+  const std::string tags = PrometheusStatsFormatter::formattedTags(metric.tags());
+  return fmt::format("{0}{{{1}}} {2}\n", prefixed_tag_extracted_name, tags, metric.value());
+}
+
+/*
+ * Returns the prometheus output for a histogram. The output is a multi-line string (with embedded
+ * newlines) that contains all the individual bucket counts and sum/count for a single histogram
+ * (metric_name plus all tags).
+ */
+std::string generateHistogramOutput(const Stats::ParentHistogram& histogram,
+                                    const std::string& prefixed_tag_extracted_name) {
+  const std::string tags = PrometheusStatsFormatter::formattedTags(histogram.tags());
+  const std::string hist_tags = histogram.tags().empty() ? EMPTY_STRING : (tags + ",");
+
+  const Stats::HistogramStatistics& stats = histogram.cumulativeStatistics();
+  const std::vector<double>& supported_buckets = stats.supportedBuckets();
+  const std::vector<uint64_t>& computed_buckets = stats.computedBuckets();
+  std::string output;
+  for (size_t i = 0; i < supported_buckets.size(); ++i) {
+    double bucket = supported_buckets[i];
+    uint64_t value = computed_buckets[i];
+    // We want to print the bucket in a fixed point (non-scientific) format. The fmt library
+    // doesn't have a specific modifier to format as a fixed-point value only so we use the
+    // 'g' operator which prints the number in general fixed point format or scientific format
+    // with precision 50 to round the number up to 32 significant digits in fixed point format
+    // which should cover pretty much all cases
+    output.append(fmt::format("{0}_bucket{{{1}le=\"{2:.32g}\"}} {3}\n", prefixed_tag_extracted_name,
+                              hist_tags, bucket, value));
+  }
+
+  output.append(fmt::format("{0}_bucket{{{1}le=\"+Inf\"}} {2}\n", prefixed_tag_extracted_name,
+                            hist_tags, stats.sampleCount()));
+  output.append(fmt::format("{0}_sum{{{1}}} {2:.32g}\n", prefixed_tag_extracted_name, tags,
+                            stats.sampleSum()));
+  output.append(fmt::format("{0}_count{{{1}}} {2}\n", prefixed_tag_extracted_name, tags,
+                            stats.sampleCount()));
+
+  return output;
+};
+
+} // namespace
+
+std::string PrometheusStatsFormatter::formattedTags(const std::vector<Stats::Tag>& tags) {
+  std::vector<std::string> buf;
+  buf.reserve(tags.size());
+  for (const Stats::Tag& tag : tags) {
+    buf.push_back(fmt::format("{}=\"{}\"", sanitizeName(tag.name_), tag.value_));
+  }
+  return absl::StrJoin(buf, ",");
+}
+
+std::string PrometheusStatsFormatter::metricName(const std::string& extracted_name) {
+  // Add namespacing prefix to avoid conflicts, as per best practice:
+  // https://prometheus.io/docs/practices/naming/#metric-names
+  // Also, naming conventions on https://prometheus.io/docs/concepts/data_model/
+  return sanitizeName(fmt::format("envoy_{0}", extracted_name));
+}
+
+// TODO(efimki): Add support of text readouts stats.
+uint64_t PrometheusStatsFormatter::statsAsPrometheus(
+    const std::vector<Stats::CounterSharedPtr>& counters,
+    const std::vector<Stats::GaugeSharedPtr>& gauges,
+    const std::vector<Stats::ParentHistogramSharedPtr>& histograms, Buffer::Instance& response,
+    const bool used_only, const absl::optional<std::regex>& regex) {
+
+  uint64_t metric_name_count = 0;
+  metric_name_count += outputStatType<Stats::Counter>(
+      response, used_only, regex, counters, generateNumericOutput<Stats::Counter>, "counter");
+
+  metric_name_count += outputStatType<Stats::Gauge>(response, used_only, regex, gauges,
+                                                    generateNumericOutput<Stats::Gauge>, "gauge");
+
+  metric_name_count += outputStatType<Stats::ParentHistogram>(
+      response, used_only, regex, histograms, generateHistogramOutput, "histogram");
+
+  return metric_name_count;
+}
+
+} // namespace Server
+} // namespace Envoy
diff --git a/source/server/http/prometheus_stats.h b/source/server/http/prometheus_stats.h
new file mode 100644
index 0000000000000..e748d051d47bf
--- /dev/null
+++ b/source/server/http/prometheus_stats.h
@@ -0,0 +1,41 @@
+#pragma once
+
+#include <regex>
+#include <string>
+
+#include "envoy/buffer/buffer.h"
+#include "envoy/stats/histogram.h"
+#include "envoy/stats/stats.h"
+
+namespace Envoy {
+namespace Server {
+/**
+ * Formatter for metric/labels exported to Prometheus.
+ *
+ * See: https://prometheus.io/docs/concepts/data_model
+ */
+class PrometheusStatsFormatter {
+public:
+  /**
+   * Extracts counters and gauges and relevant tags, appending them to
+   * the response buffer after sanitizing the metric / label names.
+   * @return uint64_t total number of metric types inserted in response.
+   */
+  static uint64_t statsAsPrometheus(const std::vector<Stats::CounterSharedPtr>& counters,
+                                    const std::vector<Stats::GaugeSharedPtr>& gauges,
+                                    const std::vector<Stats::ParentHistogramSharedPtr>& histograms,
+                                    Buffer::Instance& response, const bool used_only,
+                                    const absl::optional<std::regex>& regex);
+  /**
+   * Format the given tags, returning a string as a comma-separated list
+   * of <tag_name>="<tag_value>" pairs.
+   */
+  static std::string formattedTags(const std::vector<Stats::Tag>& tags);
+  /**
+   * Format the given metric name, prefixed with "envoy_".
+   */
+  static std::string metricName(const std::string& extracted_name);
+};
+
+} // namespace Server
+} // namespace Envoy
diff --git a/source/server/http/stats_handler.cc b/source/server/http/stats_handler.cc
index 82dad9f99fc12..a437bd2ac3952 100644
--- a/source/server/http/stats_handler.cc
+++ b/source/server/http/stats_handler.cc
@@ -5,6 +5,7 @@
 #include "common/http/headers.h"
 #include "common/http/utility.h"
 
+#include "server/http/prometheus_stats.h"
 #include "server/http/utils.h"
 
 namespace Envoy {
@@ -12,10 +13,6 @@ namespace Server {
 
 const uint64_t RecentLookupsCapacity = 100;
 
-namespace {
-const std::regex& promRegex() { CONSTRUCT_ON_FIRST_USE(std::regex, "[^a-zA-Z0-9_]"); }
-} // namespace
-
 Http::Code StatsHandler::handlerResetCounters(absl::string_view, Http::ResponseHeaderMap&,
                                               Buffer::Instance& response, AdminStream&,
                                               Server::Instance& server) {
@@ -157,162 +154,6 @@ Http::Code StatsHandler::handlerPrometheusStats(absl::string_view path_and_query
   return Http::Code::OK;
 }
 
-std::string PrometheusStatsFormatter::sanitizeName(const std::string& name) {
-  // The name must match the regex [a-zA-Z_][a-zA-Z0-9_]* as required by
-  // prometheus. Refer to https://prometheus.io/docs/concepts/data_model/.
-  std::string stats_name = std::regex_replace(name, promRegex(), "_");
-  if (stats_name[0] >= '0' && stats_name[0] <= '9') {
-    return absl::StrCat("_", stats_name);
-  } else {
-    return stats_name;
-  }
-}
-
-std::string PrometheusStatsFormatter::formattedTags(const std::vector<Stats::Tag>& tags) {
-  std::vector<std::string> buf;
-  buf.reserve(tags.size());
-  for (const Stats::Tag& tag : tags) {
-    buf.push_back(fmt::format("{}=\"{}\"", sanitizeName(tag.name_), tag.value_));
-  }
-  return absl::StrJoin(buf, ",");
-}
-
-std::string PrometheusStatsFormatter::metricName(const std::string& extracted_name) {
-  // Add namespacing prefix to avoid conflicts, as per best practice:
-  // https://prometheus.io/docs/practices/naming/#metric-names
-  // Also, naming conventions on https://prometheus.io/docs/concepts/data_model/
-  return sanitizeName(fmt::format("envoy_{0}", extracted_name));
-}
-
-template <class StatType>
-uint64_t PrometheusStatsFormatter::outputStatType(
-    Buffer::Instance& response, const bool used_only, const absl::optional<std::regex>& regex,
-    const std::vector<Stats::RefcountPtr<StatType>>& metrics,
-    const std::function<std::string(
-        const StatType& metric, const std::string& prefixed_tag_extracted_name)>& generate_output,
-    absl::string_view type) {
-
-  /*
-   * From
-   * https:*github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#grouping-and-sorting:
-   *
-   * All lines for a given metric must be provided as one single group, with the optional HELP and
-   * TYPE lines first (in no particular order). Beyond that, reproducible sorting in repeated
-   * expositions is preferred but not required, i.e. do not sort if the computational cost is
-   * prohibitive.
-   */
-
-  // This is an unsorted collection of dumb-pointers (no need to increment then decrement every
-  // refcount; ownership is held throughout by `metrics`). It is unsorted for efficiency, but will
-  // be sorted before producing the final output to satisfy the "preferred" ordering from the
-  // prometheus spec: metrics will be sorted by their tags' textual representation, which will be
-  // consistent across calls.
-  using StatTypeUnsortedCollection = std::vector<const StatType*>;
-
-  // Return early to avoid crashing when getting the symbol table from the first metric.
-  if (metrics.empty()) {
-    return 0;
-  }
-
-  // There should only be one symbol table for all of the stats in the admin
-  // interface. If this assumption changes, the name comparisons in this function
-  // will have to change to compare to convert all StatNames to strings before
-  // comparison.
-  const Stats::SymbolTable& global_symbol_table = metrics.front()->constSymbolTable();
-
-  // Sorted collection of metrics sorted by their tagExtractedName, to satisfy the requirements
-  // of the exposition format.
-  std::map<Stats::StatName, StatTypeUnsortedCollection, Stats::StatNameLessThan> groups(
-      global_symbol_table);
-
-  for (const auto& metric : metrics) {
-    ASSERT(&global_symbol_table == &metric->constSymbolTable());
-
-    if (!shouldShowMetric(*metric, used_only, regex)) {
-      continue;
-    }
-
-    groups[metric->tagExtractedStatName()].push_back(metric.get());
-  }
-
-  for (auto& group : groups) {
-    const std::string prefixed_tag_extracted_name =
-        metricName(global_symbol_table.toString(group.first));
-    response.add(fmt::format("# TYPE {0} {1}\n", prefixed_tag_extracted_name, type));
-
-    // Sort before producing the final output to satisfy the "preferred" ordering from the
-    // prometheus spec: metrics will be sorted by their tags' textual representation, which will
-    // be consistent across calls.
-    std::sort(group.second.begin(), group.second.end(), MetricLessThan());
-
-    for (const auto& metric : group.second) {
-      response.add(generate_output(*metric, prefixed_tag_extracted_name));
-    }
-    response.add("\n");
-  }
-  return groups.size();
-}
-
-template <class StatType>
-std::string
-PrometheusStatsFormatter::generateNumericOutput(const StatType& metric,
-                                                const std::string& prefixed_tag_extracted_name) {
-  const std::string tags = formattedTags(metric.tags());
-  return fmt::format("{0}{{{1}}} {2}\n", prefixed_tag_extracted_name, tags, metric.value());
-}
-
-std::string
-PrometheusStatsFormatter::generateHistogramOutput(const Stats::ParentHistogram& histogram,
-                                                  const std::string& prefixed_tag_extracted_name) {
-  const std::string tags = formattedTags(histogram.tags());
-  const std::string hist_tags = histogram.tags().empty() ? EMPTY_STRING : (tags + ",");
-
-  const Stats::HistogramStatistics& stats = histogram.cumulativeStatistics();
-  const std::vector<double>& supported_buckets = stats.supportedBuckets();
-  const std::vector<uint64_t>& computed_buckets = stats.computedBuckets();
-  std::string output;
-  for (size_t i = 0; i < supported_buckets.size(); ++i) {
-    double bucket = supported_buckets[i];
-    uint64_t value = computed_buckets[i];
-    // We want to print the bucket in a fixed point (non-scientific) format. The fmt library
-    // doesn't have a specific modifier to format as a fixed-point value only so we use the
-    // 'g' operator which prints the number in general fixed point format or scientific format
-    // with precision 50 to round the number up to 32 significant digits in fixed point format
-    // which should cover pretty much all cases
-    output.append(fmt::format("{0}_bucket{{{1}le=\"{2:.32g}\"}} {3}\n", prefixed_tag_extracted_name,
-                              hist_tags, bucket, value));
-  }
-
-  output.append(fmt::format("{0}_bucket{{{1}le=\"+Inf\"}} {2}\n", prefixed_tag_extracted_name,
-                            hist_tags, stats.sampleCount()));
-  output.append(fmt::format("{0}_sum{{{1}}} {2:.32g}\n", prefixed_tag_extracted_name, tags,
-                            stats.sampleSum()));
-  output.append(fmt::format("{0}_count{{{1}}} {2}\n", prefixed_tag_extracted_name, tags,
-                            stats.sampleCount()));
-
-  return output;
-};
-
-// TODO(efimki): Add support of text readouts stats.
-uint64_t PrometheusStatsFormatter::statsAsPrometheus(
-    const std::vector<Stats::CounterSharedPtr>& counters,
-    const std::vector<Stats::GaugeSharedPtr>& gauges,
-    const std::vector<Stats::ParentHistogramSharedPtr>& histograms, Buffer::Instance& response,
-    const bool used_only, const absl::optional<std::regex>& regex) {
-
-  uint64_t metric_name_count = 0;
-  metric_name_count += outputStatType<Stats::Counter>(
-      response, used_only, regex, counters, generateNumericOutput<Stats::Counter>, "counter");
-
-  metric_name_count += outputStatType<Stats::Gauge>(response, used_only, regex, gauges,
-                                                    generateNumericOutput<Stats::Gauge>, "gauge");
-
-  metric_name_count += outputStatType<Stats::ParentHistogram>(
-      response, used_only, regex, histograms, generateHistogramOutput, "histogram");
-
-  return metric_name_count;
-}
-
 std::string
 StatsHandler::statsAsJson(const std::map<std::string, uint64_t>& all_stats,
                           const std::map<std::string, std::string>& text_readouts,
diff --git a/source/server/http/stats_handler.h b/source/server/http/stats_handler.h
index 519f77ec0eebc..4103660689a3b 100644
--- a/source/server/http/stats_handler.h
+++ b/source/server/http/stats_handler.h
@@ -66,97 +66,5 @@ class StatsHandler {
                                  bool pretty_print = false);
 };
 
-/**
- * Formatter for metric/labels exported to Prometheus.
- *
- * See: https://prometheus.io/docs/concepts/data_model
- */
-class PrometheusStatsFormatter {
-public:
-  /**
-   * Extracts counters and gauges and relevant tags, appending them to
-   * the response buffer after sanitizing the metric / label names.
-   * @return uint64_t total number of metric types inserted in response.
-   */
-  static uint64_t statsAsPrometheus(const std::vector<Stats::CounterSharedPtr>& counters,
-                                    const std::vector<Stats::GaugeSharedPtr>& gauges,
-                                    const std::vector<Stats::ParentHistogramSharedPtr>& histograms,
-                                    Buffer::Instance& response, const bool used_only,
-                                    const absl::optional<std::regex>& regex);
-  /**
-   * Format the given tags, returning a string as a comma-separated list
-   * of <tag_name>="<tag_value>" pairs.
-   */
-  static std::string formattedTags(const std::vector<Stats::Tag>& tags);
-  /**
-   * Format the given metric name, prefixed with "envoy_".
-   */
-  static std::string metricName(const std::string& extracted_name);
-
-private:
-  /**
-   * Take a string and sanitize it according to Prometheus conventions.
-   */
-  static std::string sanitizeName(const std::string& name);
-
-  /*
-   * Determine whether a metric has never been emitted and choose to
-   * not show it if we only wanted used metrics.
-   */
-  template <class StatType>
-  static bool shouldShowMetric(const StatType& metric, const bool used_only,
-                               const absl::optional<std::regex>& regex) {
-    return ((!used_only || metric.used()) &&
-            (!regex.has_value() || std::regex_search(metric.name(), regex.value())));
-  }
-
-  /**
-   * Processes a stat type (counter, gauge, histogram) by generating all output lines, sorting
-   * them by tag-extracted metric name, and then outputting them in the correct sorted order into
-   * response.
-   *
-   * @param response The buffer to put the output into.
-   * @param used_only Whether to only output stats that are used.
-   * @param regex A filter on which stats to output.
-   * @param metrics The metrics to output stats for. This must contain all stats of the given type
-   *        to be included in the same output.
-   * @param generate_output A function which returns the output text for this metric.
-   * @param type The name of the prometheus metric type for used in TYPE annotations.
-   */
-  template <class StatType>
-  static uint64_t outputStatType(
-      Buffer::Instance& response, const bool used_only, const absl::optional<std::regex>& regex,
-      const std::vector<Stats::RefcountPtr<StatType>>& metrics,
-      const std::function<std::string(
-          const StatType& metric, const std::string& prefixed_tag_extracted_name)>& generate_output,
-      absl::string_view type);
-
-  /*
-   * Return the prometheus output for a numeric Stat (Counter or Gauge).
-   */
-  template <class StatType>
-  static std::string generateNumericOutput(const StatType& metric,
-                                           const std::string& prefixed_tag_extracted_name);
-
-  /*
-   * Returns the prometheus output for a histogram. The output is a multi-line string (with embedded
-   * newlines) that contains all the individual bucket counts and sum/count for a single histogram
-   * (metric_name plus all tags).
-   */
-  static std::string generateHistogramOutput(const Stats::ParentHistogram& histogram,
-                                             const std::string& prefixed_tag_extracted_name);
-
-  /*
-   * Comparator for Stats::Metric that does not require a string representation
-   * to make the comparison, for memory efficiency.
-   */
-  struct MetricLessThan {
-    bool operator()(const Stats::Metric* a, const Stats::Metric* b) const {
-      ASSERT(&a->constSymbolTable() == &b->constSymbolTable());
-      return a->constSymbolTable().lessThan(a->statName(), b->statName());
-    }
-  };
-};
-
 } // namespace Server
 } // namespace Envoy
diff --git a/test/server/http/BUILD b/test/server/http/BUILD
index 51d73c7346889..9a706a07cc6b0 100644
--- a/test/server/http/BUILD
+++ b/test/server/http/BUILD
@@ -73,6 +73,15 @@ envoy_cc_test(
     ],
 )
 
+envoy_cc_test(
+    name = "prometheus_stats_test",
+    srcs = ["prometheus_stats_test.cc"],
+    deps = [
+        "//source/server/http:prometheus_stats_lib",
+        "//test/test_common:utility_lib",
+    ],
+)
+
 envoy_cc_test(
     name = "config_tracker_impl_test",
     srcs = ["config_tracker_impl_test.cc"],
diff --git a/test/server/http/prometheus_stats_test.cc b/test/server/http/prometheus_stats_test.cc
new file mode 100644
index 0000000000000..fb6f16b958a94
--- /dev/null
+++ b/test/server/http/prometheus_stats_test.cc
@@ -0,0 +1,667 @@
+#include <regex>
+
+#include "server/http/prometheus_stats.h"
+
+#include "test/mocks/stats/mocks.h"
+#include "test/test_common/utility.h"
+
+using testing::NiceMock;
+
+namespace Envoy {
+namespace Server {
+
+class HistogramWrapper {
+public:
+  HistogramWrapper() : histogram_(hist_alloc()) {}
+
+  ~HistogramWrapper() { hist_free(histogram_); }
+
+  const histogram_t* getHistogram() { return histogram_; }
+
+  void setHistogramValues(const std::vector<uint64_t>& values) {
+    for (uint64_t value : values) {
+      hist_insert_intscale(histogram_, value, 0, 1);
+    }
+  }
+
+  void setHistogramValuesWithCounts(const std::vector<std::pair<uint64_t, uint64_t>>& values) {
+    for (std::pair<uint64_t, uint64_t> cv : values) {
+      hist_insert_intscale(histogram_, cv.first, 0, cv.second);
+    }
+  }
+
+private:
+  histogram_t* histogram_;
+};
+
+class PrometheusStatsFormatterTest : public testing::Test {
+protected:
+  PrometheusStatsFormatterTest()
+      : symbol_table_(Stats::SymbolTableCreator::makeSymbolTable()), alloc_(*symbol_table_),
+        pool_(*symbol_table_) {}
+
+  ~PrometheusStatsFormatterTest() override { clearStorage(); }
+
+  void addCounter(const std::string& name, Stats::StatNameTagVector cluster_tags) {
+    Stats::StatNameManagedStorage name_storage(baseName(name, cluster_tags), *symbol_table_);
+    Stats::StatNameManagedStorage tag_extracted_name_storage(name, *symbol_table_);
+    counters_.push_back(alloc_.makeCounter(name_storage.statName(),
+                                           tag_extracted_name_storage.statName(), cluster_tags));
+  }
+
+  void addGauge(const std::string& name, Stats::StatNameTagVector cluster_tags) {
+    Stats::StatNameManagedStorage name_storage(baseName(name, cluster_tags), *symbol_table_);
+    Stats::StatNameManagedStorage tag_extracted_name_storage(name, *symbol_table_);
+    gauges_.push_back(alloc_.makeGauge(name_storage.statName(),
+                                       tag_extracted_name_storage.statName(), cluster_tags,
+                                       Stats::Gauge::ImportMode::Accumulate));
+  }
+
+  using MockHistogramSharedPtr = Stats::RefcountPtr<NiceMock<Stats::MockParentHistogram>>;
+  void addHistogram(MockHistogramSharedPtr histogram) { histograms_.push_back(histogram); }
+
+  MockHistogramSharedPtr makeHistogram(const std::string& name,
+                                       Stats::StatNameTagVector cluster_tags) {
+    auto histogram = MockHistogramSharedPtr(new NiceMock<Stats::MockParentHistogram>());
+    histogram->name_ = baseName(name, cluster_tags);
+    histogram->setTagExtractedName(name);
+    histogram->setTags(cluster_tags);
+    histogram->used_ = true;
+    return histogram;
+  }
+
+  Stats::StatName makeStat(absl::string_view name) { return pool_.add(name); }
+
+  // Format tags into the name to create a unique stat_name for each name:tag combination.
+  // If the same stat_name is passed to makeGauge() or makeCounter(), even with different
+  // tags, a copy of the previous metric will be returned.
+  std::string baseName(const std::string& name, Stats::StatNameTagVector cluster_tags) {
+    std::string result = name;
+    for (const auto& name_tag : cluster_tags) {
+      result.append(fmt::format("<{}:{}>", symbol_table_->toString(name_tag.first),
+                                symbol_table_->toString(name_tag.second)));
+    }
+    return result;
+  }
+
+  void clearStorage() {
+    pool_.clear();
+    counters_.clear();
+    gauges_.clear();
+    histograms_.clear();
+    EXPECT_EQ(0, symbol_table_->numSymbols());
+  }
+
+  Stats::SymbolTablePtr symbol_table_;
+  Stats::AllocatorImpl alloc_;
+  Stats::StatNamePool pool_;
+  std::vector<Stats::CounterSharedPtr> counters_;
+  std::vector<Stats::GaugeSharedPtr> gauges_;
+  std::vector<Stats::ParentHistogramSharedPtr> histograms_;
+};
+
+TEST_F(PrometheusStatsFormatterTest, MetricName) {
+  std::string raw = "vulture.eats-liver";
+  std::string expected = "envoy_vulture_eats_liver";
+  auto actual = PrometheusStatsFormatter::metricName(raw);
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(PrometheusStatsFormatterTest, SanitizeMetricName) {
+  std::string raw = "An.artist.plays-violin@019street";
+  std::string expected = "envoy_An_artist_plays_violin_019street";
+  auto actual = PrometheusStatsFormatter::metricName(raw);
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(PrometheusStatsFormatterTest, SanitizeMetricNameDigitFirst) {
+  std::string raw = "3.artists.play-violin@019street";
+  std::string expected = "envoy_3_artists_play_violin_019street";
+  auto actual = PrometheusStatsFormatter::metricName(raw);
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(PrometheusStatsFormatterTest, FormattedTags) {
+  std::vector<Stats::Tag> tags;
+  Stats::Tag tag1 = {"a.tag-name", "a.tag-value"};
+  Stats::Tag tag2 = {"another_tag_name", "another_tag-value"};
+  tags.push_back(tag1);
+  tags.push_back(tag2);
+  std::string expected = "a_tag_name=\"a.tag-value\",another_tag_name=\"another_tag-value\"";
+  auto actual = PrometheusStatsFormatter::formattedTags(tags);
+  EXPECT_EQ(expected, actual);
+}
+
+TEST_F(PrometheusStatsFormatterTest, MetricNameCollison) {
+
+  // Create two counters and two gauges with each pair having the same name,
+  // but having different tag names and values.
+  //`statsAsPrometheus()` should return two implying it found two unique stat names
+
+  addCounter("cluster.test_cluster_1.upstream_cx_total",
+             {{makeStat("a.tag-name"), makeStat("a.tag-value")}});
+  addCounter("cluster.test_cluster_1.upstream_cx_total",
+             {{makeStat("another_tag_name"), makeStat("another_tag-value")}});
+  addGauge("cluster.test_cluster_2.upstream_cx_total",
+           {{makeStat("another_tag_name_3"), makeStat("another_tag_3-value")}});
+  addGauge("cluster.test_cluster_2.upstream_cx_total",
+           {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}});
+
+  Buffer::OwnedImpl response;
+  auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response,
+                                                          false, absl::nullopt);
+  EXPECT_EQ(2UL, size);
+}
+
+TEST_F(PrometheusStatsFormatterTest, UniqueMetricName) {
+
+  // Create two counters and two gauges, all with unique names.
+  // statsAsPrometheus() should return four implying it found
+  // four unique stat names.
+
+  addCounter("cluster.test_cluster_1.upstream_cx_total",
+             {{makeStat("a.tag-name"), makeStat("a.tag-value")}});
+  addCounter("cluster.test_cluster_2.upstream_cx_total",
+             {{makeStat("another_tag_name"), makeStat("another_tag-value")}});
+  addGauge("cluster.test_cluster_3.upstream_cx_total",
+           {{makeStat("another_tag_name_3"), makeStat("another_tag_3-value")}});
+  addGauge("cluster.test_cluster_4.upstream_cx_total",
+           {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}});
+
+  Buffer::OwnedImpl response;
+  auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response,
+                                                          false, absl::nullopt);
+  EXPECT_EQ(4UL, size);
+}
+
+TEST_F(PrometheusStatsFormatterTest, HistogramWithNoValuesAndNoTags) {
+  HistogramWrapper h1_cumulative;
+  h1_cumulative.setHistogramValues(std::vector<uint64_t>(0));
+  Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram());
+
+  auto histogram = makeHistogram("histogram1", {});
+  ON_CALL(*histogram, cumulativeStatistics())
+      .WillByDefault(testing::ReturnRef(h1_cumulative_statistics));
+
+  addHistogram(histogram);
+
+  Buffer::OwnedImpl response;
+  auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response,
+                                                          false, absl::nullopt);
+  EXPECT_EQ(1UL, size);
+
+  const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram
+envoy_histogram1_bucket{le="0.5"} 0
+envoy_histogram1_bucket{le="1"} 0
+envoy_histogram1_bucket{le="5"} 0
+envoy_histogram1_bucket{le="10"} 0
+envoy_histogram1_bucket{le="25"} 0
+envoy_histogram1_bucket{le="50"} 0
+envoy_histogram1_bucket{le="100"} 0
+envoy_histogram1_bucket{le="250"} 0
+envoy_histogram1_bucket{le="500"} 0
+envoy_histogram1_bucket{le="1000"} 0
+envoy_histogram1_bucket{le="2500"} 0
+envoy_histogram1_bucket{le="5000"} 0
+envoy_histogram1_bucket{le="10000"} 0
+envoy_histogram1_bucket{le="30000"} 0
+envoy_histogram1_bucket{le="60000"} 0
+envoy_histogram1_bucket{le="300000"} 0
+envoy_histogram1_bucket{le="600000"} 0
+envoy_histogram1_bucket{le="1800000"} 0
+envoy_histogram1_bucket{le="3600000"} 0
+envoy_histogram1_bucket{le="+Inf"} 0
+envoy_histogram1_sum{} 0
+envoy_histogram1_count{} 0
+
+)EOF";
+
+  EXPECT_EQ(expected_output, response.toString());
+}
+
+TEST_F(PrometheusStatsFormatterTest, HistogramWithHighCounts) {
+  HistogramWrapper h1_cumulative;
+
+  // Force large counts to prove that the +Inf bucket doesn't overflow to scientific notation.
+  h1_cumulative.setHistogramValuesWithCounts(std::vector<std::pair<uint64_t, uint64_t>>({
+      {1, 100000},
+      {100, 1000000},
+      {1000, 100000000},
+  }));
+
+  Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram());
+
+  auto histogram = makeHistogram("histogram1", {});
+  ON_CALL(*histogram, cumulativeStatistics())
+      .WillByDefault(testing::ReturnRef(h1_cumulative_statistics));
+
+  addHistogram(histogram);
+
+  Buffer::OwnedImpl response;
+  auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response,
+                                                          false, absl::nullopt);
+  EXPECT_EQ(1UL, size);
+
+  const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram
+envoy_histogram1_bucket{le="0.5"} 0
+envoy_histogram1_bucket{le="1"} 0
+envoy_histogram1_bucket{le="5"} 100000
+envoy_histogram1_bucket{le="10"} 100000
+envoy_histogram1_bucket{le="25"} 100000
+envoy_histogram1_bucket{le="50"} 100000
+envoy_histogram1_bucket{le="100"} 100000
+envoy_histogram1_bucket{le="250"} 1100000
+envoy_histogram1_bucket{le="500"} 1100000
+envoy_histogram1_bucket{le="1000"} 1100000
+envoy_histogram1_bucket{le="2500"} 101100000
+envoy_histogram1_bucket{le="5000"} 101100000
+envoy_histogram1_bucket{le="10000"} 101100000
+envoy_histogram1_bucket{le="30000"} 101100000
+envoy_histogram1_bucket{le="60000"} 101100000
+envoy_histogram1_bucket{le="300000"} 101100000
+envoy_histogram1_bucket{le="600000"} 101100000
+envoy_histogram1_bucket{le="1800000"} 101100000
+envoy_histogram1_bucket{le="3600000"} 101100000
+envoy_histogram1_bucket{le="+Inf"} 101100000
+envoy_histogram1_sum{} 105105105000
+envoy_histogram1_count{} 101100000
+
+)EOF";
+
+  EXPECT_EQ(expected_output, response.toString());
+}
+
+TEST_F(PrometheusStatsFormatterTest, OutputWithAllMetricTypes) {
+  addCounter("cluster.test_1.upstream_cx_total",
+             {{makeStat("a.tag-name"), makeStat("a.tag-value")}});
+  addCounter("cluster.test_2.upstream_cx_total",
+             {{makeStat("another_tag_name"), makeStat("another_tag-value")}});
+  addGauge("cluster.test_3.upstream_cx_total",
+           {{makeStat("another_tag_name_3"), makeStat("another_tag_3-value")}});
+  addGauge("cluster.test_4.upstream_cx_total",
+           {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}});
+
+  const std::vector<uint64_t> h1_values = {50, 20, 30, 70, 100, 5000, 200};
+  HistogramWrapper h1_cumulative;
+  h1_cumulative.setHistogramValues(h1_values);
+  Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram());
+
+  auto histogram1 =
+      makeHistogram("cluster.test_1.upstream_rq_time", {{makeStat("key1"), makeStat("value1")},
+                                                        {makeStat("key2"), makeStat("value2")}});
+  histogram1->unit_ = Stats::Histogram::Unit::Milliseconds;
+  addHistogram(histogram1);
+  EXPECT_CALL(*histogram1, cumulativeStatistics())
+      .WillOnce(testing::ReturnRef(h1_cumulative_statistics));
+
+  Buffer::OwnedImpl response;
+  auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response,
+                                                          false, absl::nullopt);
+  EXPECT_EQ(5UL, size);
+
+  const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_cx_total counter
+envoy_cluster_test_1_upstream_cx_total{a_tag_name="a.tag-value"} 0
+
+# TYPE envoy_cluster_test_2_upstream_cx_total counter
+envoy_cluster_test_2_upstream_cx_total{another_tag_name="another_tag-value"} 0
+
+# TYPE envoy_cluster_test_3_upstream_cx_total gauge
+envoy_cluster_test_3_upstream_cx_total{another_tag_name_3="another_tag_3-value"} 0
+
+# TYPE envoy_cluster_test_4_upstream_cx_total gauge
+envoy_cluster_test_4_upstream_cx_total{another_tag_name_4="another_tag_4-value"} 0
+
+# TYPE envoy_cluster_test_1_upstream_rq_time histogram
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="0.5"} 0
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1"} 0
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="5"} 0
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="10"} 0
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="25"} 1
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="50"} 2
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="100"} 4
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="250"} 6
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="500"} 6
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1000"} 6
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="2500"} 6
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="5000"} 6
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="10000"} 7
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="30000"} 7
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="60000"} 7
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="300000"} 7
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="600000"} 7
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1800000"} 7
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="3600000"} 7
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="+Inf"} 7
+envoy_cluster_test_1_upstream_rq_time_sum{key1="value1",key2="value2"} 5532
+envoy_cluster_test_1_upstream_rq_time_count{key1="value1",key2="value2"} 7
+
+)EOF";
+
+  EXPECT_EQ(expected_output, response.toString());
+}
+
+// Test that output groups all metrics of the same name (with different tags) together,
+// as required by the Prometheus exposition format spec. Additionally, groups of metrics
+// should be sorted by their tags; the format specifies that it is preferred that metrics
+// are always grouped in the same order, and sorting is an easy way to ensure this.
+TEST_F(PrometheusStatsFormatterTest, OutputSortedByMetricName) {
+  const std::vector<uint64_t> h1_values = {50, 20, 30, 70, 100, 5000, 200};
+  HistogramWrapper h1_cumulative;
+  h1_cumulative.setHistogramValues(h1_values);
+  Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram());
+
+  // Create the 3 clusters in non-sorted order to exercise the sorting.
+  // Create two of each metric type (counter, gauge, histogram) so that
+  // the output for each needs to be collected together.
+  for (const char* cluster : {"ccc", "aaa", "bbb"}) {
+    const Stats::StatNameTagVector tags{{makeStat("cluster"), makeStat(cluster)}};
+    addCounter("cluster.upstream_cx_total", tags);
+    addCounter("cluster.upstream_cx_connect_fail", tags);
+    addGauge("cluster.upstream_cx_active", tags);
+    addGauge("cluster.upstream_rq_active", tags);
+
+    for (const char* hist_name : {"cluster.upstream_rq_time", "cluster.upstream_response_time"}) {
+      auto histogram1 = makeHistogram(hist_name, tags);
+      histogram1->unit_ = Stats::Histogram::Unit::Milliseconds;
+      addHistogram(histogram1);
+      EXPECT_CALL(*histogram1, cumulativeStatistics())
+          .WillOnce(testing::ReturnRef(h1_cumulative_statistics));
+    }
+  }
+
+  Buffer::OwnedImpl response;
+  auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response,
+                                                          false, absl::nullopt);
+  EXPECT_EQ(6UL, size);
+
+  const std::string expected_output = R"EOF(# TYPE envoy_cluster_upstream_cx_connect_fail counter
+envoy_cluster_upstream_cx_connect_fail{cluster="aaa"} 0
+envoy_cluster_upstream_cx_connect_fail{cluster="bbb"} 0
+envoy_cluster_upstream_cx_connect_fail{cluster="ccc"} 0
+
+# TYPE envoy_cluster_upstream_cx_total counter
+envoy_cluster_upstream_cx_total{cluster="aaa"} 0
+envoy_cluster_upstream_cx_total{cluster="bbb"} 0
+envoy_cluster_upstream_cx_total{cluster="ccc"} 0
+
+# TYPE envoy_cluster_upstream_cx_active gauge
+envoy_cluster_upstream_cx_active{cluster="aaa"} 0
+envoy_cluster_upstream_cx_active{cluster="bbb"} 0
+envoy_cluster_upstream_cx_active{cluster="ccc"} 0
+
+# TYPE envoy_cluster_upstream_rq_active gauge
+envoy_cluster_upstream_rq_active{cluster="aaa"} 0
+envoy_cluster_upstream_rq_active{cluster="bbb"} 0
+envoy_cluster_upstream_rq_active{cluster="ccc"} 0
+
+# TYPE envoy_cluster_upstream_response_time histogram
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="0.5"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="1"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="5"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="10"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="25"} 1
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="50"} 2
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="100"} 4
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="250"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="500"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="1000"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="2500"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="5000"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="10000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="30000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="60000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="300000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="600000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="1800000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="3600000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="+Inf"} 7
+envoy_cluster_upstream_response_time_sum{cluster="aaa"} 5532
+envoy_cluster_upstream_response_time_count{cluster="aaa"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="0.5"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="1"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="5"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="10"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="25"} 1
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="50"} 2
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="100"} 4
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="250"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="500"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="1000"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="2500"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="5000"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="10000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="30000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="60000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="300000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="600000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="1800000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="3600000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="+Inf"} 7
+envoy_cluster_upstream_response_time_sum{cluster="bbb"} 5532
+envoy_cluster_upstream_response_time_count{cluster="bbb"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="0.5"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="1"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="5"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="10"} 0
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="25"} 1
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="50"} 2
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="100"} 4
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="250"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="500"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="1000"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="2500"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="5000"} 6
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="10000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="30000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="60000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="300000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="600000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="1800000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="3600000"} 7
+envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="+Inf"} 7
+envoy_cluster_upstream_response_time_sum{cluster="ccc"} 5532
+envoy_cluster_upstream_response_time_count{cluster="ccc"} 7
+
+# TYPE envoy_cluster_upstream_rq_time histogram
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="0.5"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="1"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="5"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="10"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="25"} 1
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="50"} 2
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="100"} 4
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="250"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="500"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="1000"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="2500"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="5000"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="10000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="30000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="60000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="300000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="600000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="1800000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="3600000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="+Inf"} 7
+envoy_cluster_upstream_rq_time_sum{cluster="aaa"} 5532
+envoy_cluster_upstream_rq_time_count{cluster="aaa"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="0.5"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="1"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="5"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="10"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="25"} 1
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="50"} 2
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="100"} 4
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="250"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="500"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="1000"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="2500"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="5000"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="10000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="30000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="60000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="300000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="600000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="1800000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="3600000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="+Inf"} 7
+envoy_cluster_upstream_rq_time_sum{cluster="bbb"} 5532
+envoy_cluster_upstream_rq_time_count{cluster="bbb"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="0.5"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="1"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="5"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="10"} 0
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="25"} 1
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="50"} 2
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="100"} 4
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="250"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="500"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="1000"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="2500"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="5000"} 6
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="10000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="30000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="60000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="300000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="600000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="1800000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="3600000"} 7
+envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="+Inf"} 7
+envoy_cluster_upstream_rq_time_sum{cluster="ccc"} 5532
+envoy_cluster_upstream_rq_time_count{cluster="ccc"} 7
+
+)EOF";
+
+  EXPECT_EQ(expected_output, response.toString());
+}
+
+TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnly) {
+  addCounter("cluster.test_1.upstream_cx_total",
+             {{makeStat("a.tag-name"), makeStat("a.tag-value")}});
+  addCounter("cluster.test_2.upstream_cx_total",
+             {{makeStat("another_tag_name"), makeStat("another_tag-value")}});
+  addGauge("cluster.test_3.upstream_cx_total",
+           {{makeStat("another_tag_name_3"), makeStat("another_tag_3-value")}});
+  addGauge("cluster.test_4.upstream_cx_total",
+           {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}});
+
+  const std::vector<uint64_t> h1_values = {50, 20, 30, 70, 100, 5000, 200};
+  HistogramWrapper h1_cumulative;
+  h1_cumulative.setHistogramValues(h1_values);
+  Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram());
+
+  auto histogram1 =
+      makeHistogram("cluster.test_1.upstream_rq_time", {{makeStat("key1"), makeStat("value1")},
+                                                        {makeStat("key2"), makeStat("value2")}});
+  histogram1->unit_ = Stats::Histogram::Unit::Milliseconds;
+  addHistogram(histogram1);
+  EXPECT_CALL(*histogram1, cumulativeStatistics())
+      .WillOnce(testing::ReturnRef(h1_cumulative_statistics));
+
+  Buffer::OwnedImpl response;
+  auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response,
+                                                          true, absl::nullopt);
+  EXPECT_EQ(1UL, size);
+
+  const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_rq_time histogram
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="0.5"} 0
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1"} 0
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="5"} 0
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="10"} 0
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="25"} 1
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="50"} 2
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="100"} 4
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="250"} 6
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="500"} 6
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1000"} 6
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="2500"} 6
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="5000"} 6
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="10000"} 7
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="30000"} 7
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="60000"} 7
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="300000"} 7
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="600000"} 7
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1800000"} 7
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="3600000"} 7
+envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="+Inf"} 7
+envoy_cluster_test_1_upstream_rq_time_sum{key1="value1",key2="value2"} 5532
+envoy_cluster_test_1_upstream_rq_time_count{key1="value1",key2="value2"} 7
+
+)EOF";
+
+  EXPECT_EQ(expected_output, response.toString());
+}
+
+TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnlyHistogram) {
+  const std::vector<uint64_t> h1_values = {};
+  HistogramWrapper h1_cumulative;
+  h1_cumulative.setHistogramValues(h1_values);
+  Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram());
+
+  auto histogram1 =
+      makeHistogram("cluster.test_1.upstream_rq_time", {{makeStat("key1"), makeStat("value1")},
+                                                        {makeStat("key2"), makeStat("value2")}});
+  histogram1->unit_ = Stats::Histogram::Unit::Milliseconds;
+  histogram1->used_ = false;
+  addHistogram(histogram1);
+
+  {
+    const bool used_only = true;
+    EXPECT_CALL(*histogram1, cumulativeStatistics()).Times(0);
+
+    Buffer::OwnedImpl response;
+    auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_,
+                                                            response, used_only, absl::nullopt);
+    EXPECT_EQ(0UL, size);
+  }
+
+  {
+    const bool used_only = false;
+    EXPECT_CALL(*histogram1, cumulativeStatistics())
+        .WillOnce(testing::ReturnRef(h1_cumulative_statistics));
+
+    Buffer::OwnedImpl response;
+    auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_,
+                                                            response, used_only, absl::nullopt);
+    EXPECT_EQ(1UL, size);
+  }
+}
+
+TEST_F(PrometheusStatsFormatterTest, OutputWithRegexp) {
+  addCounter("cluster.test_1.upstream_cx_total",
+             {{makeStat("a.tag-name"), makeStat("a.tag-value")}});
+  addCounter("cluster.test_2.upstream_cx_total",
+             {{makeStat("another_tag_name"), makeStat("another_tag-value")}});
+  addGauge("cluster.test_3.upstream_cx_total",
+           {{makeStat("another_tag_name_3"), makeStat("another_tag_3-value")}});
+  addGauge("cluster.test_4.upstream_cx_total",
+           {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}});
+
+  const std::vector<uint64_t> h1_values = {50, 20, 30, 70, 100, 5000, 200};
+  HistogramWrapper h1_cumulative;
+  h1_cumulative.setHistogramValues(h1_values);
+  Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram());
+
+  auto histogram1 =
+      makeHistogram("cluster.test_1.upstream_rq_time", {{makeStat("key1"), makeStat("value1")},
+                                                        {makeStat("key2"), makeStat("value2")}});
+  histogram1->unit_ = Stats::Histogram::Unit::Milliseconds;
+  addHistogram(histogram1);
+
+  Buffer::OwnedImpl response;
+  auto size = PrometheusStatsFormatter::statsAsPrometheus(
+      counters_, gauges_, histograms_, response, false,
+      absl::optional<std::regex>{std::regex("cluster.test_1.upstream_cx_total")});
+  EXPECT_EQ(1UL, size);
+
+  const std::string expected_output =
+      R"EOF(# TYPE envoy_cluster_test_1_upstream_cx_total counter
+envoy_cluster_test_1_upstream_cx_total{a_tag_name="a.tag-value"} 0
+
+)EOF";
+
+  EXPECT_EQ(expected_output, response.toString());
+}
+
+} // namespace Server
+} // namespace Envoy
diff --git a/test/server/http/stats_handler_test.cc b/test/server/http/stats_handler_test.cc
index 02cc2095967ec..6ce1b0cfae488 100644
--- a/test/server/http/stats_handler_test.cc
+++ b/test/server/http/stats_handler_test.cc
@@ -571,658 +571,5 @@ TEST_P(AdminInstanceTest, RecentLookups) {
   // fake symbol table. However we cover this solidly in integration tests.
 }
 
-class HistogramWrapper {
-public:
-  HistogramWrapper() : histogram_(hist_alloc()) {}
-
-  ~HistogramWrapper() { hist_free(histogram_); }
-
-  const histogram_t* getHistogram() { return histogram_; }
-
-  void setHistogramValues(const std::vector<uint64_t>& values) {
-    for (uint64_t value : values) {
-      hist_insert_intscale(histogram_, value, 0, 1);
-    }
-  }
-
-  void setHistogramValuesWithCounts(const std::vector<std::pair<uint64_t, uint64_t>>& values) {
-    for (std::pair<uint64_t, uint64_t> cv : values) {
-      hist_insert_intscale(histogram_, cv.first, 0, cv.second);
-    }
-  }
-
-private:
-  histogram_t* histogram_;
-};
-
-class PrometheusStatsFormatterTest : public testing::Test {
-protected:
-  PrometheusStatsFormatterTest()
-      : symbol_table_(Stats::SymbolTableCreator::makeSymbolTable()), alloc_(*symbol_table_),
-        pool_(*symbol_table_) {}
-
-  ~PrometheusStatsFormatterTest() override { clearStorage(); }
-
-  void addCounter(const std::string& name, Stats::StatNameTagVector cluster_tags) {
-    Stats::StatNameManagedStorage name_storage(baseName(name, cluster_tags), *symbol_table_);
-    Stats::StatNameManagedStorage tag_extracted_name_storage(name, *symbol_table_);
-    counters_.push_back(alloc_.makeCounter(name_storage.statName(),
-                                           tag_extracted_name_storage.statName(), cluster_tags));
-  }
-
-  void addGauge(const std::string& name, Stats::StatNameTagVector cluster_tags) {
-    Stats::StatNameManagedStorage name_storage(baseName(name, cluster_tags), *symbol_table_);
-    Stats::StatNameManagedStorage tag_extracted_name_storage(name, *symbol_table_);
-    gauges_.push_back(alloc_.makeGauge(name_storage.statName(),
-                                       tag_extracted_name_storage.statName(), cluster_tags,
-                                       Stats::Gauge::ImportMode::Accumulate));
-  }
-
-  using MockHistogramSharedPtr = Stats::RefcountPtr<NiceMock<Stats::MockParentHistogram>>;
-  void addHistogram(MockHistogramSharedPtr histogram) { histograms_.push_back(histogram); }
-
-  MockHistogramSharedPtr makeHistogram(const std::string& name,
-                                       Stats::StatNameTagVector cluster_tags) {
-    auto histogram = MockHistogramSharedPtr(new NiceMock<Stats::MockParentHistogram>());
-    histogram->name_ = baseName(name, cluster_tags);
-    histogram->setTagExtractedName(name);
-    histogram->setTags(cluster_tags);
-    histogram->used_ = true;
-    return histogram;
-  }
-
-  Stats::StatName makeStat(absl::string_view name) { return pool_.add(name); }
-
-  // Format tags into the name to create a unique stat_name for each name:tag combination.
-  // If the same stat_name is passed to makeGauge() or makeCounter(), even with different
-  // tags, a copy of the previous metric will be returned.
-  std::string baseName(const std::string& name, Stats::StatNameTagVector cluster_tags) {
-    std::string result = name;
-    for (const auto& name_tag : cluster_tags) {
-      result.append(fmt::format("<{}:{}>", symbol_table_->toString(name_tag.first),
-                                symbol_table_->toString(name_tag.second)));
-    }
-    return result;
-  }
-
-  void clearStorage() {
-    pool_.clear();
-    counters_.clear();
-    gauges_.clear();
-    histograms_.clear();
-    EXPECT_EQ(0, symbol_table_->numSymbols());
-  }
-
-  Stats::SymbolTablePtr symbol_table_;
-  Stats::AllocatorImpl alloc_;
-  Stats::StatNamePool pool_;
-  std::vector<Stats::CounterSharedPtr> counters_;
-  std::vector<Stats::GaugeSharedPtr> gauges_;
-  std::vector<Stats::ParentHistogramSharedPtr> histograms_;
-};
-
-TEST_F(PrometheusStatsFormatterTest, MetricName) {
-  std::string raw = "vulture.eats-liver";
-  std::string expected = "envoy_vulture_eats_liver";
-  auto actual = PrometheusStatsFormatter::metricName(raw);
-  EXPECT_EQ(expected, actual);
-}
-
-TEST_F(PrometheusStatsFormatterTest, SanitizeMetricName) {
-  std::string raw = "An.artist.plays-violin@019street";
-  std::string expected = "envoy_An_artist_plays_violin_019street";
-  auto actual = PrometheusStatsFormatter::metricName(raw);
-  EXPECT_EQ(expected, actual);
-}
-
-TEST_F(PrometheusStatsFormatterTest, SanitizeMetricNameDigitFirst) {
-  std::string raw = "3.artists.play-violin@019street";
-  std::string expected = "envoy_3_artists_play_violin_019street";
-  auto actual = PrometheusStatsFormatter::metricName(raw);
-  EXPECT_EQ(expected, actual);
-}
-
-TEST_F(PrometheusStatsFormatterTest, FormattedTags) {
-  std::vector<Stats::Tag> tags;
-  Stats::Tag tag1 = {"a.tag-name", "a.tag-value"};
-  Stats::Tag tag2 = {"another_tag_name", "another_tag-value"};
-  tags.push_back(tag1);
-  tags.push_back(tag2);
-  std::string expected = "a_tag_name=\"a.tag-value\",another_tag_name=\"another_tag-value\"";
-  auto actual = PrometheusStatsFormatter::formattedTags(tags);
-  EXPECT_EQ(expected, actual);
-}
-
-TEST_F(PrometheusStatsFormatterTest, MetricNameCollison) {
-
-  // Create two counters and two gauges with each pair having the same name,
-  // but having different tag names and values.
-  //`statsAsPrometheus()` should return two implying it found two unique stat names
-
-  addCounter("cluster.test_cluster_1.upstream_cx_total",
-             {{makeStat("a.tag-name"), makeStat("a.tag-value")}});
-  addCounter("cluster.test_cluster_1.upstream_cx_total",
-             {{makeStat("another_tag_name"), makeStat("another_tag-value")}});
-  addGauge("cluster.test_cluster_2.upstream_cx_total",
-           {{makeStat("another_tag_name_3"), makeStat("another_tag_3-value")}});
-  addGauge("cluster.test_cluster_2.upstream_cx_total",
-           {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}});
-
-  Buffer::OwnedImpl response;
-  auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response,
-                                                          false, absl::nullopt);
-  EXPECT_EQ(2UL, size);
-}
-
-TEST_F(PrometheusStatsFormatterTest, UniqueMetricName) {
-
-  // Create two counters and two gauges, all with unique names.
-  // statsAsPrometheus() should return four implying it found
-  // four unique stat names.
-
-  addCounter("cluster.test_cluster_1.upstream_cx_total",
-             {{makeStat("a.tag-name"), makeStat("a.tag-value")}});
-  addCounter("cluster.test_cluster_2.upstream_cx_total",
-             {{makeStat("another_tag_name"), makeStat("another_tag-value")}});
-  addGauge("cluster.test_cluster_3.upstream_cx_total",
-           {{makeStat("another_tag_name_3"), makeStat("another_tag_3-value")}});
-  addGauge("cluster.test_cluster_4.upstream_cx_total",
-           {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}});
-
-  Buffer::OwnedImpl response;
-  auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response,
-                                                          false, absl::nullopt);
-  EXPECT_EQ(4UL, size);
-}
-
-TEST_F(PrometheusStatsFormatterTest, HistogramWithNoValuesAndNoTags) {
-  HistogramWrapper h1_cumulative;
-  h1_cumulative.setHistogramValues(std::vector<uint64_t>(0));
-  Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram());
-
-  auto histogram = makeHistogram("histogram1", {});
-  ON_CALL(*histogram, cumulativeStatistics())
-      .WillByDefault(testing::ReturnRef(h1_cumulative_statistics));
-
-  addHistogram(histogram);
-
-  Buffer::OwnedImpl response;
-  auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response,
-                                                          false, absl::nullopt);
-  EXPECT_EQ(1UL, size);
-
-  const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram
-envoy_histogram1_bucket{le="0.5"} 0
-envoy_histogram1_bucket{le="1"} 0
-envoy_histogram1_bucket{le="5"} 0
-envoy_histogram1_bucket{le="10"} 0
-envoy_histogram1_bucket{le="25"} 0
-envoy_histogram1_bucket{le="50"} 0
-envoy_histogram1_bucket{le="100"} 0
-envoy_histogram1_bucket{le="250"} 0
-envoy_histogram1_bucket{le="500"} 0
-envoy_histogram1_bucket{le="1000"} 0
-envoy_histogram1_bucket{le="2500"} 0
-envoy_histogram1_bucket{le="5000"} 0
-envoy_histogram1_bucket{le="10000"} 0
-envoy_histogram1_bucket{le="30000"} 0
-envoy_histogram1_bucket{le="60000"} 0
-envoy_histogram1_bucket{le="300000"} 0
-envoy_histogram1_bucket{le="600000"} 0
-envoy_histogram1_bucket{le="1800000"} 0
-envoy_histogram1_bucket{le="3600000"} 0
-envoy_histogram1_bucket{le="+Inf"} 0
-envoy_histogram1_sum{} 0
-envoy_histogram1_count{} 0
-
-)EOF";
-
-  EXPECT_EQ(expected_output, response.toString());
-}
-
-TEST_F(PrometheusStatsFormatterTest, HistogramWithHighCounts) {
-  HistogramWrapper h1_cumulative;
-
-  // Force large counts to prove that the +Inf bucket doesn't overflow to scientific notation.
-  h1_cumulative.setHistogramValuesWithCounts(std::vector<std::pair<uint64_t, uint64_t>>({
-      {1, 100000},
-      {100, 1000000},
-      {1000, 100000000},
-  }));
-
-  Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram());
-
-  auto histogram = makeHistogram("histogram1", {});
-  ON_CALL(*histogram, cumulativeStatistics())
-      .WillByDefault(testing::ReturnRef(h1_cumulative_statistics));
-
-  addHistogram(histogram);
-
-  Buffer::OwnedImpl response;
-  auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response,
-                                                          false, absl::nullopt);
-  EXPECT_EQ(1UL, size);
-
-  const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram
-envoy_histogram1_bucket{le="0.5"} 0
-envoy_histogram1_bucket{le="1"} 0
-envoy_histogram1_bucket{le="5"} 100000
-envoy_histogram1_bucket{le="10"} 100000
-envoy_histogram1_bucket{le="25"} 100000
-envoy_histogram1_bucket{le="50"} 100000
-envoy_histogram1_bucket{le="100"} 100000
-envoy_histogram1_bucket{le="250"} 1100000
-envoy_histogram1_bucket{le="500"} 1100000
-envoy_histogram1_bucket{le="1000"} 1100000
-envoy_histogram1_bucket{le="2500"} 101100000
-envoy_histogram1_bucket{le="5000"} 101100000
-envoy_histogram1_bucket{le="10000"} 101100000
-envoy_histogram1_bucket{le="30000"} 101100000
-envoy_histogram1_bucket{le="60000"} 101100000
-envoy_histogram1_bucket{le="300000"} 101100000
-envoy_histogram1_bucket{le="600000"} 101100000
-envoy_histogram1_bucket{le="1800000"} 101100000
-envoy_histogram1_bucket{le="3600000"} 101100000
-envoy_histogram1_bucket{le="+Inf"} 101100000
-envoy_histogram1_sum{} 105105105000
-envoy_histogram1_count{} 101100000
-
-)EOF";
-
-  EXPECT_EQ(expected_output, response.toString());
-}
-
-TEST_F(PrometheusStatsFormatterTest, OutputWithAllMetricTypes) {
-  addCounter("cluster.test_1.upstream_cx_total",
-             {{makeStat("a.tag-name"), makeStat("a.tag-value")}});
-  addCounter("cluster.test_2.upstream_cx_total",
-             {{makeStat("another_tag_name"), makeStat("another_tag-value")}});
-  addGauge("cluster.test_3.upstream_cx_total",
-           {{makeStat("another_tag_name_3"), makeStat("another_tag_3-value")}});
-  addGauge("cluster.test_4.upstream_cx_total",
-           {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}});
-
-  const std::vector<uint64_t> h1_values = {50, 20, 30, 70, 100, 5000, 200};
-  HistogramWrapper h1_cumulative;
-  h1_cumulative.setHistogramValues(h1_values);
-  Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram());
-
-  auto histogram1 =
-      makeHistogram("cluster.test_1.upstream_rq_time", {{makeStat("key1"), makeStat("value1")},
-                                                        {makeStat("key2"), makeStat("value2")}});
-  histogram1->unit_ = Stats::Histogram::Unit::Milliseconds;
-  addHistogram(histogram1);
-  EXPECT_CALL(*histogram1, cumulativeStatistics())
-      .WillOnce(testing::ReturnRef(h1_cumulative_statistics));
-
-  Buffer::OwnedImpl response;
-  auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response,
-                                                          false, absl::nullopt);
-  EXPECT_EQ(5UL, size);
-
-  const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_cx_total counter
-envoy_cluster_test_1_upstream_cx_total{a_tag_name="a.tag-value"} 0
-
-# TYPE envoy_cluster_test_2_upstream_cx_total counter
-envoy_cluster_test_2_upstream_cx_total{another_tag_name="another_tag-value"} 0
-
-# TYPE envoy_cluster_test_3_upstream_cx_total gauge
-envoy_cluster_test_3_upstream_cx_total{another_tag_name_3="another_tag_3-value"} 0
-
-# TYPE envoy_cluster_test_4_upstream_cx_total gauge
-envoy_cluster_test_4_upstream_cx_total{another_tag_name_4="another_tag_4-value"} 0
-
-# TYPE envoy_cluster_test_1_upstream_rq_time histogram
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="0.5"} 0
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1"} 0
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="5"} 0
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="10"} 0
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="25"} 1
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="50"} 2
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="100"} 4
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="250"} 6
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="500"} 6
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1000"} 6
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="2500"} 6
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="5000"} 6
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="10000"} 7
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="30000"} 7
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="60000"} 7
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="300000"} 7
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="600000"} 7
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1800000"} 7
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="3600000"} 7
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="+Inf"} 7
-envoy_cluster_test_1_upstream_rq_time_sum{key1="value1",key2="value2"} 5532
-envoy_cluster_test_1_upstream_rq_time_count{key1="value1",key2="value2"} 7
-
-)EOF";
-
-  EXPECT_EQ(expected_output, response.toString());
-}
-
-// Test that output groups all metrics of the same name (with different tags) together,
-// as required by the Prometheus exposition format spec. Additionally, groups of metrics
-// should be sorted by their tags; the format specifies that it is preferred that metrics
-// are always grouped in the same order, and sorting is an easy way to ensure this.
-TEST_F(PrometheusStatsFormatterTest, OutputSortedByMetricName) {
-  const std::vector<uint64_t> h1_values = {50, 20, 30, 70, 100, 5000, 200};
-  HistogramWrapper h1_cumulative;
-  h1_cumulative.setHistogramValues(h1_values);
-  Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram());
-
-  // Create the 3 clusters in non-sorted order to exercise the sorting.
-  // Create two of each metric type (counter, gauge, histogram) so that
-  // the output for each needs to be collected together.
-  for (const char* cluster : {"ccc", "aaa", "bbb"}) {
-    const Stats::StatNameTagVector tags{{makeStat("cluster"), makeStat(cluster)}};
-    addCounter("cluster.upstream_cx_total", tags);
-    addCounter("cluster.upstream_cx_connect_fail", tags);
-    addGauge("cluster.upstream_cx_active", tags);
-    addGauge("cluster.upstream_rq_active", tags);
-
-    for (const char* hist_name : {"cluster.upstream_rq_time", "cluster.upstream_response_time"}) {
-      auto histogram1 = makeHistogram(hist_name, tags);
-      histogram1->unit_ = Stats::Histogram::Unit::Milliseconds;
-      addHistogram(histogram1);
-      EXPECT_CALL(*histogram1, cumulativeStatistics())
-          .WillOnce(testing::ReturnRef(h1_cumulative_statistics));
-    }
-  }
-
-  Buffer::OwnedImpl response;
-  auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response,
-                                                          false, absl::nullopt);
-  EXPECT_EQ(6UL, size);
-
-  const std::string expected_output = R"EOF(# TYPE envoy_cluster_upstream_cx_connect_fail counter
-envoy_cluster_upstream_cx_connect_fail{cluster="aaa"} 0
-envoy_cluster_upstream_cx_connect_fail{cluster="bbb"} 0
-envoy_cluster_upstream_cx_connect_fail{cluster="ccc"} 0
-
-# TYPE envoy_cluster_upstream_cx_total counter
-envoy_cluster_upstream_cx_total{cluster="aaa"} 0
-envoy_cluster_upstream_cx_total{cluster="bbb"} 0
-envoy_cluster_upstream_cx_total{cluster="ccc"} 0
-
-# TYPE envoy_cluster_upstream_cx_active gauge
-envoy_cluster_upstream_cx_active{cluster="aaa"} 0
-envoy_cluster_upstream_cx_active{cluster="bbb"} 0
-envoy_cluster_upstream_cx_active{cluster="ccc"} 0
-
-# TYPE envoy_cluster_upstream_rq_active gauge
-envoy_cluster_upstream_rq_active{cluster="aaa"} 0
-envoy_cluster_upstream_rq_active{cluster="bbb"} 0
-envoy_cluster_upstream_rq_active{cluster="ccc"} 0
-
-# TYPE envoy_cluster_upstream_response_time histogram
-envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="0.5"} 0
-envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="1"} 0
-envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="5"} 0
-envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="10"} 0
-envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="25"} 1
-envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="50"} 2
-envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="100"} 4
-envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="250"} 6
-envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="500"} 6
-envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="1000"} 6
-envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="2500"} 6
-envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="5000"} 6
-envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="10000"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="30000"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="60000"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="300000"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="600000"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="1800000"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="3600000"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="aaa",le="+Inf"} 7
-envoy_cluster_upstream_response_time_sum{cluster="aaa"} 5532
-envoy_cluster_upstream_response_time_count{cluster="aaa"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="0.5"} 0
-envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="1"} 0
-envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="5"} 0
-envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="10"} 0
-envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="25"} 1
-envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="50"} 2
-envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="100"} 4
-envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="250"} 6
-envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="500"} 6
-envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="1000"} 6
-envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="2500"} 6
-envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="5000"} 6
-envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="10000"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="30000"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="60000"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="300000"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="600000"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="1800000"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="3600000"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="bbb",le="+Inf"} 7
-envoy_cluster_upstream_response_time_sum{cluster="bbb"} 5532
-envoy_cluster_upstream_response_time_count{cluster="bbb"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="0.5"} 0
-envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="1"} 0
-envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="5"} 0
-envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="10"} 0
-envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="25"} 1
-envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="50"} 2
-envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="100"} 4
-envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="250"} 6
-envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="500"} 6
-envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="1000"} 6
-envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="2500"} 6
-envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="5000"} 6
-envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="10000"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="30000"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="60000"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="300000"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="600000"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="1800000"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="3600000"} 7
-envoy_cluster_upstream_response_time_bucket{cluster="ccc",le="+Inf"} 7
-envoy_cluster_upstream_response_time_sum{cluster="ccc"} 5532
-envoy_cluster_upstream_response_time_count{cluster="ccc"} 7
-
-# TYPE envoy_cluster_upstream_rq_time histogram
-envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="0.5"} 0
-envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="1"} 0
-envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="5"} 0
-envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="10"} 0
-envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="25"} 1
-envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="50"} 2
-envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="100"} 4
-envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="250"} 6
-envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="500"} 6
-envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="1000"} 6
-envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="2500"} 6
-envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="5000"} 6
-envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="10000"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="30000"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="60000"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="300000"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="600000"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="1800000"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="3600000"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="aaa",le="+Inf"} 7
-envoy_cluster_upstream_rq_time_sum{cluster="aaa"} 5532
-envoy_cluster_upstream_rq_time_count{cluster="aaa"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="0.5"} 0
-envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="1"} 0
-envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="5"} 0
-envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="10"} 0
-envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="25"} 1
-envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="50"} 2
-envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="100"} 4
-envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="250"} 6
-envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="500"} 6
-envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="1000"} 6
-envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="2500"} 6
-envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="5000"} 6
-envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="10000"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="30000"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="60000"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="300000"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="600000"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="1800000"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="3600000"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="bbb",le="+Inf"} 7
-envoy_cluster_upstream_rq_time_sum{cluster="bbb"} 5532
-envoy_cluster_upstream_rq_time_count{cluster="bbb"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="0.5"} 0
-envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="1"} 0
-envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="5"} 0
-envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="10"} 0
-envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="25"} 1
-envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="50"} 2
-envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="100"} 4
-envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="250"} 6
-envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="500"} 6
-envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="1000"} 6
-envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="2500"} 6
-envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="5000"} 6
-envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="10000"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="30000"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="60000"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="300000"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="600000"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="1800000"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="3600000"} 7
-envoy_cluster_upstream_rq_time_bucket{cluster="ccc",le="+Inf"} 7
-envoy_cluster_upstream_rq_time_sum{cluster="ccc"} 5532
-envoy_cluster_upstream_rq_time_count{cluster="ccc"} 7
-
-)EOF";
-
-  EXPECT_EQ(expected_output, response.toString());
-}
-
-TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnly) {
-  addCounter("cluster.test_1.upstream_cx_total",
-             {{makeStat("a.tag-name"), makeStat("a.tag-value")}});
-  addCounter("cluster.test_2.upstream_cx_total",
-             {{makeStat("another_tag_name"), makeStat("another_tag-value")}});
-  addGauge("cluster.test_3.upstream_cx_total",
-           {{makeStat("another_tag_name_3"), makeStat("another_tag_3-value")}});
-  addGauge("cluster.test_4.upstream_cx_total",
-           {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}});
-
-  const std::vector<uint64_t> h1_values = {50, 20, 30, 70, 100, 5000, 200};
-  HistogramWrapper h1_cumulative;
-  h1_cumulative.setHistogramValues(h1_values);
-  Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram());
-
-  auto histogram1 =
-      makeHistogram("cluster.test_1.upstream_rq_time", {{makeStat("key1"), makeStat("value1")},
-                                                        {makeStat("key2"), makeStat("value2")}});
-  histogram1->unit_ = Stats::Histogram::Unit::Milliseconds;
-  addHistogram(histogram1);
-  EXPECT_CALL(*histogram1, cumulativeStatistics())
-      .WillOnce(testing::ReturnRef(h1_cumulative_statistics));
-
-  Buffer::OwnedImpl response;
-  auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response,
-                                                          true, absl::nullopt);
-  EXPECT_EQ(1UL, size);
-
-  const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_rq_time histogram
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="0.5"} 0
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1"} 0
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="5"} 0
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="10"} 0
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="25"} 1
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="50"} 2
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="100"} 4
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="250"} 6
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="500"} 6
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1000"} 6
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="2500"} 6
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="5000"} 6
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="10000"} 7
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="30000"} 7
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="60000"} 7
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="300000"} 7
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="600000"} 7
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1800000"} 7
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="3600000"} 7
-envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="+Inf"} 7
-envoy_cluster_test_1_upstream_rq_time_sum{key1="value1",key2="value2"} 5532
-envoy_cluster_test_1_upstream_rq_time_count{key1="value1",key2="value2"} 7
-
-)EOF";
-
-  EXPECT_EQ(expected_output, response.toString());
-}
-
-TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnlyHistogram) {
-  const std::vector<uint64_t> h1_values = {};
-  HistogramWrapper h1_cumulative;
-  h1_cumulative.setHistogramValues(h1_values);
-  Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram());
-
-  auto histogram1 =
-      makeHistogram("cluster.test_1.upstream_rq_time", {{makeStat("key1"), makeStat("value1")},
-                                                        {makeStat("key2"), makeStat("value2")}});
-  histogram1->unit_ = Stats::Histogram::Unit::Milliseconds;
-  histogram1->used_ = false;
-  addHistogram(histogram1);
-
-  {
-    const bool used_only = true;
-    EXPECT_CALL(*histogram1, cumulativeStatistics()).Times(0);
-
-    Buffer::OwnedImpl response;
-    auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_,
-                                                            response, used_only, absl::nullopt);
-    EXPECT_EQ(0UL, size);
-  }
-
-  {
-    const bool used_only = false;
-    EXPECT_CALL(*histogram1, cumulativeStatistics())
-        .WillOnce(testing::ReturnRef(h1_cumulative_statistics));
-
-    Buffer::OwnedImpl response;
-    auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_,
-                                                            response, used_only, absl::nullopt);
-    EXPECT_EQ(1UL, size);
-  }
-}
-
-TEST_F(PrometheusStatsFormatterTest, OutputWithRegexp) {
-  addCounter("cluster.test_1.upstream_cx_total",
-             {{makeStat("a.tag-name"), makeStat("a.tag-value")}});
-  addCounter("cluster.test_2.upstream_cx_total",
-             {{makeStat("another_tag_name"), makeStat("another_tag-value")}});
-  addGauge("cluster.test_3.upstream_cx_total",
-           {{makeStat("another_tag_name_3"), makeStat("another_tag_3-value")}});
-  addGauge("cluster.test_4.upstream_cx_total",
-           {{makeStat("another_tag_name_4"), makeStat("another_tag_4-value")}});
-
-  const std::vector<uint64_t> h1_values = {50, 20, 30, 70, 100, 5000, 200};
-  HistogramWrapper h1_cumulative;
-  h1_cumulative.setHistogramValues(h1_values);
-  Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram());
-
-  auto histogram1 =
-      makeHistogram("cluster.test_1.upstream_rq_time", {{makeStat("key1"), makeStat("value1")},
-                                                        {makeStat("key2"), makeStat("value2")}});
-  histogram1->unit_ = Stats::Histogram::Unit::Milliseconds;
-  addHistogram(histogram1);
-
-  Buffer::OwnedImpl response;
-  auto size = PrometheusStatsFormatter::statsAsPrometheus(
-      counters_, gauges_, histograms_, response, false,
-      absl::optional<std::regex>{std::regex("cluster.test_1.upstream_cx_total")});
-  EXPECT_EQ(1UL, size);
-
-  const std::string expected_output =
-      R"EOF(# TYPE envoy_cluster_test_1_upstream_cx_total counter
-envoy_cluster_test_1_upstream_cx_total{a_tag_name="a.tag-value"} 0
-
-)EOF";
-
-  EXPECT_EQ(expected_output, response.toString());
-}
-
 } // namespace Server
 } // namespace Envoy
diff --git a/tools/code_format/check_format.py b/tools/code_format/check_format.py
index c9683f8cce4de..27bc32dbe9267 100755
--- a/tools/code_format/check_format.py
+++ b/tools/code_format/check_format.py
@@ -79,7 +79,8 @@
     "./source/extensions/filters/http/squash/squash_filter.h",
     "./source/extensions/filters/http/squash/squash_filter.cc", "./source/server/http/utils.h",
     "./source/server/http/utils.cc", "./source/server/http/stats_handler.h",
-    "./source/server/http/stats_handler.cc", "./tools/clang_tools/api_booster/main.cc",
+    "./source/server/http/stats_handler.cc", "./source/server/http/prometheus_stats.h",
+    "./source/server/http/prometheus_stats.cc", "./tools/clang_tools/api_booster/main.cc",
     "./tools/clang_tools/api_booster/proto_cxx_utils.cc", "./source/common/common/version.cc")
 
 # Only one C++ file should instantiate grpc_init