diff --git a/docs/root/intro/version_history.rst b/docs/root/intro/version_history.rst index f40ca143c43a0..cc3cb90f95107 100644 --- a/docs/root/intro/version_history.rst +++ b/docs/root/intro/version_history.rst @@ -30,6 +30,8 @@ Version history * router: added ability to configure a :ref:`retry policy ` at the virtual host level. * stats: added support for histograms in prometheus +* stats: added usedonly flag to prometheus stats to only output metrics which have been + updated at least once. * tap: added new alpha :ref:`HTTP tap filter `. * tls: enabled TLS 1.3 on the server-side (non-FIPS builds). * router: added per-route configuration of :ref:`internal redirects `. diff --git a/docs/root/operations/admin.rst b/docs/root/operations/admin.rst index 502312fefbaac..d1e46de993ccf 100644 --- a/docs/root/operations/admin.rst +++ b/docs/root/operations/admin.rst @@ -312,6 +312,10 @@ explanation of the output. Outputs /stats in `Prometheus `_ v0.0.4 format. This can be used to integrate with a Prometheus server. + You can optionally pass the `usedonly` URL query argument to only get statistics that + Envoy has updated (counters incremented at least once, gauges changed at least once, + and histograms added to at least once) + .. _operations_admin_interface_runtime: .. http:get:: /runtime diff --git a/source/server/http/admin.cc b/source/server/http/admin.cc index 1df56ddcf8614..c16b7ae5396ba 100644 --- a/source/server/http/admin.cc +++ b/source/server/http/admin.cc @@ -669,10 +669,12 @@ Http::Code AdminImpl::handlerStats(absl::string_view url, Http::HeaderMap& respo return rc; } -Http::Code AdminImpl::handlerPrometheusStats(absl::string_view, Http::HeaderMap&, +Http::Code AdminImpl::handlerPrometheusStats(absl::string_view path_and_query, Http::HeaderMap&, Buffer::Instance& response, AdminStream&) { + const Http::Utility::QueryParams params = Http::Utility::parseQueryString(path_and_query); + const bool used_only = params.find("usedonly") != params.end(); PrometheusStatsFormatter::statsAsPrometheus(server_.stats().counters(), server_.stats().gauges(), - server_.stats().histograms(), response); + server_.stats().histograms(), response, used_only); return Http::Code::OK; } @@ -705,9 +707,14 @@ std::string PrometheusStatsFormatter::metricName(const std::string& extractedNam uint64_t PrometheusStatsFormatter::statsAsPrometheus( const std::vector& counters, const std::vector& gauges, - const std::vector& histograms, Buffer::Instance& response) { + const std::vector& histograms, Buffer::Instance& response, + const bool used_only) { std::unordered_set metric_type_tracker; for (const auto& counter : counters) { + if (!shouldShowMetric(counter, used_only)) { + continue; + } + const std::string tags = formattedTags(counter->tags()); const std::string metric_name = metricName(counter->tagExtractedName()); if (metric_type_tracker.find(metric_name) == metric_type_tracker.end()) { @@ -718,6 +725,10 @@ uint64_t PrometheusStatsFormatter::statsAsPrometheus( } for (const auto& gauge : gauges) { + if (!shouldShowMetric(gauge, used_only)) { + continue; + } + const std::string tags = formattedTags(gauge->tags()); const std::string metric_name = metricName(gauge->tagExtractedName()); if (metric_type_tracker.find(metric_name) == metric_type_tracker.end()) { @@ -728,6 +739,10 @@ uint64_t PrometheusStatsFormatter::statsAsPrometheus( } for (const auto& histogram : histograms) { + if (!shouldShowMetric(histogram, used_only)) { + continue; + } + const std::string tags = formattedTags(histogram->tags()); const std::string hist_tags = histogram->tags().empty() ? EMPTY_STRING : (tags + ","); diff --git a/source/server/http/admin.h b/source/server/http/admin.h index d9205dba7e0da..05b3c1fa840c5 100644 --- a/source/server/http/admin.h +++ b/source/server/http/admin.h @@ -376,7 +376,7 @@ class PrometheusStatsFormatter { static uint64_t statsAsPrometheus(const std::vector& counters, const std::vector& gauges, const std::vector& histograms, - Buffer::Instance& response); + Buffer::Instance& response, const bool used_only); /** * Format the given tags, returning a string as a comma-separated list * of ="" pairs. @@ -392,6 +392,14 @@ class PrometheusStatsFormatter { * Take a string and sanitize it according to Prometheus conventions. */ static std::string sanitizeName(const std::string& name); + + /* + * Determine whether a metric has never been emitted and choose to + * not show it if we only wanted used metrics. + */ + static bool shouldShowMetric(const std::shared_ptr& metric, const bool used_only) { + return !used_only || metric->used(); + } }; } // namespace Server diff --git a/test/server/http/admin_test.cc b/test/server/http/admin_test.cc index e7ca80fbdf6d1..a13a93629c4a8 100644 --- a/test/server/http/admin_test.cc +++ b/test/server/http/admin_test.cc @@ -1291,7 +1291,7 @@ TEST_F(PrometheusStatsFormatterTest, MetricNameCollison) { Buffer::OwnedImpl response; auto size = - PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response); + PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, false); EXPECT_EQ(2UL, size); } @@ -1311,7 +1311,7 @@ TEST_F(PrometheusStatsFormatterTest, UniqueMetricName) { Buffer::OwnedImpl response; auto size = - PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response); + PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, false); EXPECT_EQ(4UL, size); } @@ -1330,7 +1330,7 @@ TEST_F(PrometheusStatsFormatterTest, HistogramWithNoValuesAndNoTags) { Buffer::OwnedImpl response; auto size = - PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response); + PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, false); EXPECT_EQ(1UL, size); const std::string expected_output = R"EOF(# TYPE envoy_histogram1 histogram @@ -1382,7 +1382,7 @@ TEST_F(PrometheusStatsFormatterTest, OutputWithAllMetricTypes) { Buffer::OwnedImpl response; auto size = - PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response); + PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, false); EXPECT_EQ(5UL, size); const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_cx_total counter @@ -1421,5 +1421,91 @@ envoy_cluster_test_1_upstream_rq_time_count{key1="value1",key2="value2"} 7 EXPECT_EQ(expected_output, response.toString()); } +TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnly) { + addCounter("cluster.test_1.upstream_cx_total", {{"a.tag-name", "a.tag-value"}}); + addCounter("cluster.test_2.upstream_cx_total", {{"another_tag_name", "another_tag-value"}}); + addGauge("cluster.test_3.upstream_cx_total", {{"another_tag_name_3", "another_tag_3-value"}}); + addGauge("cluster.test_4.upstream_cx_total", {{"another_tag_name_4", "another_tag_4-value"}}); + + const std::vector h1_values = {50, 20, 30, 70, 100, 5000, 200}; + HistogramWrapper h1_cumulative; + h1_cumulative.setHistogramValues(h1_values); + Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); + + auto histogram1 = std::make_shared>(); + histogram1->name_ = "cluster.test_1.upstream_rq_time"; + histogram1->used_ = true; + histogram1->tags_ = {Stats::Tag{"key1", "value1"}, Stats::Tag{"key2", "value2"}}; + addHistogram(histogram1); + EXPECT_CALL(*histogram1, cumulativeStatistics()) + .WillOnce(testing::ReturnRef(h1_cumulative_statistics)); + + Buffer::OwnedImpl response; + auto size = + PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, response, true); + EXPECT_EQ(1UL, size); + + const std::string expected_output = R"EOF(# TYPE envoy_cluster_test_1_upstream_rq_time histogram +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="0.5"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="5"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="10"} 0 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="25"} 1 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="50"} 2 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="100"} 4 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="250"} 6 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="500"} 6 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1000"} 6 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="2500"} 6 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="5000"} 6 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="10000"} 7 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="30000"} 7 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="60000"} 7 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="300000"} 7 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="600000"} 7 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="1800000"} 7 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="3600000"} 7 +envoy_cluster_test_1_upstream_rq_time_bucket{key1="value1",key2="value2",le="+Inf"} 7 +envoy_cluster_test_1_upstream_rq_time_sum{key1="value1",key2="value2"} 5532 +envoy_cluster_test_1_upstream_rq_time_count{key1="value1",key2="value2"} 7 +)EOF"; + + EXPECT_EQ(expected_output, response.toString()); +} + +TEST_F(PrometheusStatsFormatterTest, OutputWithUsedOnlyHistogram) { + const std::vector h1_values = {}; + HistogramWrapper h1_cumulative; + h1_cumulative.setHistogramValues(h1_values); + Stats::HistogramStatisticsImpl h1_cumulative_statistics(h1_cumulative.getHistogram()); + + auto histogram1 = std::make_shared>(); + histogram1->name_ = "cluster.test_1.upstream_rq_time"; + histogram1->used_ = false; + histogram1->tags_ = {Stats::Tag{"key1", "value1"}, Stats::Tag{"key2", "value2"}}; + addHistogram(histogram1); + + { + const bool used_only = true; + EXPECT_CALL(*histogram1, cumulativeStatistics()).Times(0); + + Buffer::OwnedImpl response; + auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, + response, used_only); + EXPECT_EQ(0UL, size); + } + + { + const bool used_only = false; + EXPECT_CALL(*histogram1, cumulativeStatistics()) + .WillOnce(testing::ReturnRef(h1_cumulative_statistics)); + + Buffer::OwnedImpl response; + auto size = PrometheusStatsFormatter::statsAsPrometheus(counters_, gauges_, histograms_, + response, used_only); + EXPECT_EQ(1UL, size); + } +} + } // namespace Server } // namespace Envoy