Basic functional options support to metrics assertion in integration tests

Signed-off-by: Marco Pracucci <[email protected]>

Author: pracucci

integration/e2e/composite_service.go

@@ -45,7 +45,7 @@ func (s *CompositeHTTPService) WaitSumMetrics(isExpected func(sums ...float64) b
 	)
 
 	for s.retryBackoff.Reset(); s.retryBackoff.Ongoing(); {
-		sums, err = s.SumMetrics(metricNames...)
+		sums, err = s.SumMetrics(metricNames)
 		if err != nil {
 			return err
 		}
@@ -80,11 +80,11 @@ func (s *CompositeHTTPService) WaitSumMetricWithLabels(isExpected func(sums floa
 }
 
 // SumMetrics returns the sum of the values of each given metric names.
-func (s *CompositeHTTPService) SumMetrics(metricNames ...string) ([]float64, error) {
+func (s *CompositeHTTPService) SumMetrics(metricNames []string, opts ...MetricsOption) ([]float64, error) {
 	sums := make([]float64, len(metricNames))
 
 	for _, service := range s.services {
-		partials, err := service.SumMetrics(metricNames...)
+		partials, err := service.SumMetrics(metricNames, opts...)
 		if err != nil {
 			return nil, err
 		}

integration/e2e/metrics.go

@@ -6,7 +6,7 @@ import (
 	io_prometheus_client "github.com/prometheus/client_model/go"
 )
 
-func getValue(m *io_prometheus_client.Metric) float64 {
+func getMetricValue(m *io_prometheus_client.Metric) float64 {
 	if m.GetGauge() != nil {
 		return m.GetGauge().GetValue()
 	} else if m.GetCounter() != nil {
@@ -20,10 +20,28 @@ func getValue(m *io_prometheus_client.Metric) float64 {
 	}
 }
 
-func sumValues(family *io_prometheus_client.MetricFamily) float64 {
-	sum := 0.0
+func getMetricCount(m *io_prometheus_client.Metric) float64 {
+	if m.GetHistogram() != nil {
+		return float64(m.GetHistogram().GetSampleCount())
+	} else if m.GetSummary() != nil {
+		return float64(m.GetSummary().GetSampleCount())
+	} else {
+		return 0
+	}
+}
+
+func getValues(family *io_prometheus_client.MetricFamily, get GetMetricValueFunc) []float64 {
+	values := make([]float64, 0, len(family.Metric))
 	for _, m := range family.Metric {
-		sum += getValue(m)
+		values = append(values, get(m))
+	}
+	return values
+}
+
+func sumValues(values []float64) float64 {
+	sum := 0.0
+	for _, v := range values {
+		sum += v
 	}
 	return sum
 }

integration/e2e/metrics_options.go

@@ -0,0 +1,33 @@
+package e2e
+
+import io_prometheus_client "github.com/prometheus/client_model/go"
+
+var (
+	DefaultMetricsOptions = MetricsOptions{
+		GetValue: getMetricValue,
+	}
+)
+
+// GetMetricValueFunc defined the signature of a function used to get the metric value.
+type GetMetricValueFunc func(m *io_prometheus_client.Metric) float64
+
+// MetricsOption defined the signature of a function used to manipulate options.
+type MetricsOption func(*MetricsOptions)
+
+// MetricsOptions is the structure holding all options.
+type MetricsOptions struct {
+	GetValue GetMetricValueFunc
+}
+
+// WithMetricCount is an option to get the histogram/summary count as metric value.
+func WithMetricCount(opts *MetricsOptions) {
+	opts.GetValue = getMetricCount
+}
+
+func buildMetricsOptions(opts []MetricsOption) MetricsOptions {
+	result := DefaultMetricsOptions
+	for _, opt := range opts {
+		opt(&result)
+	}
+	return result
+}

integration/e2e/service.go

@@ -522,13 +522,17 @@ func (s *HTTPService) NetworkHTTPEndpointFor(networkName string) string {
 // WaitSumMetrics waits for at least one instance of each given metric names to be present and their sums, returning true
 // when passed to given isExpected(...).
 func (s *HTTPService) WaitSumMetrics(isExpected func(sums ...float64) bool, metricNames ...string) error {
+	return s.WaitSumMetricsWithOptions(isExpected, metricNames)
+}
+
+func (s *HTTPService) WaitSumMetricsWithOptions(isExpected func(sums ...float64) bool, metricNames []string, opts ...MetricsOption) error {
 	var (
 		sums []float64
 		err  error
 	)
 
 	for s.retryBackoff.Reset(); s.retryBackoff.Ongoing(); {
-		sums, err = s.SumMetrics(metricNames...)
+		sums, err = s.SumMetrics(metricNames, opts...)
 		if err != nil {
 			return err
 		}
@@ -544,7 +548,8 @@ func (s *HTTPService) WaitSumMetrics(isExpected func(sums ...float64) bool, metr
 }
 
 // SumMetrics returns the sum of the values of each given metric names.
-func (s *HTTPService) SumMetrics(metricNames ...string) ([]float64, error) {
+func (s *HTTPService) SumMetrics(metricNames []string, opts ...MetricsOption) ([]float64, error) {
+	options := buildMetricsOptions(opts)
 	sums := make([]float64, len(metricNames))
 
 	metrics, err := s.Metrics()
@@ -563,7 +568,7 @@ func (s *HTTPService) SumMetrics(metricNames ...string) ([]float64, error) {
 
 		// Check if the metric is exported.
 		if mf, ok := families[m]; ok {
-			sums[i] = sumValues(mf)
+			sums[i] = sumValues(getValues(mf, options.GetValue))
 			continue
 		}
 		return nil, errors.Errorf("metric %s not found in %s metric page", m, s.name)
@@ -582,7 +587,7 @@ func (s *HTTPService) WaitForMetricWithLabels(okFn func(v float64) bool, metricN
 		}
 
 		for _, m := range ms {
-			if okFn(getValue(m)) {
+			if okFn(getMetricValue(m)) {
 				return nil
 			}
 		}
@@ -602,7 +607,7 @@ func (s *HTTPService) SumMetricWithLabels(metricName string, expectedLabels map[
 	}
 
 	for _, m := range ms {
-		sum += getValue(m)
+		sum += getMetricValue(m)
 	}
 	return sum, nil
 }

integration/query_frontend_test.go

@@ -203,6 +203,11 @@ func runQueryFrontendTest(t *testing.T, testMissingMetricName bool, setup queryF
 	}
 	require.NoError(t, queryFrontend.WaitSumMetrics(e2e.Equals(numUsers*numQueriesPerUser+extra), "cortex_query_frontend_queries_total"))
 
+	// The number of received request is greater then the query requests because include
+	// requests to /metrics and /ready.
+	require.NoError(t, queryFrontend.WaitSumMetricsWithOptions(e2e.Greater(numUsers*numQueriesPerUser), []string{"cortex_request_duration_seconds"}, e2e.WithMetricCount))
+	require.NoError(t, querier.WaitSumMetricsWithOptions(e2e.Greater(numUsers*numQueriesPerUser), []string{"cortex_request_duration_seconds"}, e2e.WithMetricCount))
+
 	// Ensure no service-specific metrics prefix is used by the wrong service.
 	assertServiceMetricsPrefixes(t, Distributor, distributor)
 	assertServiceMetricsPrefixes(t, Ingester, ingester)