diff --git a/exporter/prometheusexporter/end_to_end_test.go b/exporter/prometheusexporter/end_to_end_test.go index 3e728a95a600..8b1b8b116610 100644 --- a/exporter/prometheusexporter/end_to_end_test.go +++ b/exporter/prometheusexporter/end_to_end_test.go @@ -152,6 +152,21 @@ func TestEndToEndSummarySupport(t *testing.T) { `test_jvm_memory_pool_bytes_used.pool="G1 Old Gen". 4.385408e.06.*`, `test_jvm_memory_pool_bytes_used.pool="G1 Survivor Space". 8.388608e.06.*`, `test_jvm_memory_pool_bytes_used.pool="Metaspace". 2.6218176e.07.*`, + `. HELP test_scrape_duration_seconds Duration of the scrape`, + `. TYPE test_scrape_duration_seconds gauge`, + `test_scrape_duration_seconds [0-9.e-]+ [0-9]+`, + `. HELP test_scrape_samples_post_metric_relabeling The number of samples remaining after metric relabeling was applied`, + `. TYPE test_scrape_samples_post_metric_relabeling gauge`, + `test_scrape_samples_post_metric_relabeling 13 .*`, + `. HELP test_scrape_samples_scraped The number of samples the target exposed`, + `. TYPE test_scrape_samples_scraped gauge`, + `test_scrape_samples_scraped 13 .*`, + `. HELP test_scrape_series_added The approximate number of new series in this scrape`, + `. TYPE test_scrape_series_added gauge`, + `test_scrape_series_added 13 .*`, + `. HELP test_up The scraping was successful`, + `. TYPE test_up gauge`, + `test_up 1 .*`, } // 5.5: Perform a complete line by line prefix verification to ensure we extract back the inputs diff --git a/receiver/prometheusreceiver/internal/metricfamily.go b/receiver/prometheusreceiver/internal/metricfamily.go index eca80c3fdeca..e8f200a46f9c 100644 --- a/receiver/prometheusreceiver/internal/metricfamily.go +++ b/receiver/prometheusreceiver/internal/metricfamily.go @@ -15,6 +15,7 @@ package internal import ( + "fmt" "sort" "strings" @@ -22,6 +23,7 @@ import ( "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/textparse" "github.com/prometheus/prometheus/scrape" + "go.uber.org/zap" "google.golang.org/protobuf/types/known/timestamppb" "google.golang.org/protobuf/types/known/wrapperspb" ) @@ -46,7 +48,7 @@ type metricFamily struct { groups map[string]*metricGroup } -func newMetricFamily(metricName string, mc MetadataCache) MetricFamily { +func newMetricFamily(metricName string, mc MetadataCache, logger *zap.Logger) MetricFamily { familyName := normalizeMetricName(metricName) // lookup metadata based on familyName @@ -62,11 +64,17 @@ func newMetricFamily(metricName string, mc MetadataCache) MetricFamily { metadata.Metric = familyName metadata.Type = textparse.MetricTypeUnknown } + } else if !ok && isInternalMetric(metricName) { + metadata = defineInternalMetric(metricName, metadata, logger) + } + ocaMetricType := convToOCAMetricType(metadata.Type) + if ocaMetricType == metricspb.MetricDescriptor_UNSPECIFIED { + logger.Debug(fmt.Sprintf("Invalid metric : %s %+v", metricName, metadata)) } return &metricFamily{ name: familyName, - mtype: convToOCAMetricType(metadata.Type), + mtype: ocaMetricType, mc: mc, droppedTimeseries: 0, labelKeys: make(map[string]bool), @@ -77,6 +85,35 @@ func newMetricFamily(metricName string, mc MetadataCache) MetricFamily { } } +// Define manually the metadata of prometheus scrapper internal metrics +func defineInternalMetric(metricName string, metadata scrape.MetricMetadata, logger *zap.Logger) scrape.MetricMetadata { + if metadata.Metric != "" && metadata.Type != "" && metadata.Help != "" { + logger.Debug("Internal metric seems already fully defined") + return metadata + } + metadata.Metric = metricName + + switch metricName { + case scrapeUpMetricName: + metadata.Type = textparse.MetricTypeGauge + metadata.Help = "The scraping was successful" + case "scrape_duration_seconds": + metadata.Unit = "seconds" + metadata.Type = textparse.MetricTypeGauge + metadata.Help = "Duration of the scrape" + case "scrape_samples_scraped": + metadata.Type = textparse.MetricTypeGauge + metadata.Help = "The number of samples the target exposed" + case "scrape_series_added": + metadata.Type = textparse.MetricTypeGauge + metadata.Help = "The approximate number of new series in this scrape" + case "scrape_samples_post_metric_relabeling": + metadata.Type = textparse.MetricTypeGauge + metadata.Help = "The number of samples remaining after metric relabeling was applied" + } + return metadata +} + func (mf *metricFamily) IsSameFamily(metricName string) bool { // trim known suffix if necessary familyName := normalizeMetricName(metricName) diff --git a/receiver/prometheusreceiver/internal/metricsbuilder.go b/receiver/prometheusreceiver/internal/metricsbuilder.go index 9e02554818d1..7dfef42b6914 100644 --- a/receiver/prometheusreceiver/internal/metricsbuilder.go +++ b/receiver/prometheusreceiver/internal/metricsbuilder.go @@ -114,7 +114,6 @@ func (b *metricBuilder) AddDataPoint(ls labels.Labels, t int64, v float64) error case isInternalMetric(metricName): b.hasInternalMetric = true lm := ls.Map() - delete(lm, model.MetricNameLabel) // See https://www.prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series // up: 1 if the instance is healthy, i.e. reachable, or 0 if the scrape failed. if metricName == scrapeUpMetricName && v != 1.0 { @@ -129,7 +128,6 @@ func (b *metricBuilder) AddDataPoint(ls labels.Labels, t int64, v float64) error zap.String("target_labels", fmt.Sprintf("%v", lm))) } } - return nil case b.useStartTimeMetric && b.matchStartTimeMetric(metricName): b.startTime = v } @@ -143,9 +141,9 @@ func (b *metricBuilder) AddDataPoint(ls labels.Labels, t int64, v float64) error if m != nil { b.metrics = append(b.metrics, m) } - b.currentMf = newMetricFamily(metricName, b.mc) + b.currentMf = newMetricFamily(metricName, b.mc, b.logger) } else if b.currentMf == nil { - b.currentMf = newMetricFamily(metricName, b.mc) + b.currentMf = newMetricFamily(metricName, b.mc, b.logger) } return b.currentMf.Add(metricName, ls, t, v) diff --git a/receiver/prometheusreceiver/internal/metricsbuilder_test.go b/receiver/prometheusreceiver/internal/metricsbuilder_test.go index 1970af746256..0392b5274a6c 100644 --- a/receiver/prometheusreceiver/internal/metricsbuilder_test.go +++ b/receiver/prometheusreceiver/internal/metricsbuilder_test.go @@ -1198,34 +1198,6 @@ func Test_metricBuilder_summary(t *testing.T) { runBuilderTests(t, tests) } -func Test_metricBuilder_skipped(t *testing.T) { - tests := []buildTestData{ - { - name: "skip-internal-metrics", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("scrape_foo", 1), - createDataPoint("up", 1.0), - }, - }, - { - pts: []*testDataPoint{ - createDataPoint("scrape_foo", 2), - createDataPoint("up", 2.0), - }, - }, - }, - wants: [][]*metricspb.Metric{ - {}, - {}, - }, - }, - } - - runBuilderTests(t, tests) -} - func Test_metricBuilder_baddata(t *testing.T) { t.Run("empty-metric-name", func(t *testing.T) { mc := newMockMetadataCache(testMetadata) diff --git a/receiver/prometheusreceiver/internal/otlp_metricfamily_test.go b/receiver/prometheusreceiver/internal/otlp_metricfamily_test.go index 6f8cc98136ed..3e394f90f201 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricfamily_test.go +++ b/receiver/prometheusreceiver/internal/otlp_metricfamily_test.go @@ -21,6 +21,7 @@ import ( "github.com/prometheus/prometheus/pkg/textparse" "github.com/prometheus/prometheus/scrape" "github.com/stretchr/testify/assert" + "go.uber.org/zap" ) type byLookupMetadataCache map[string]scrape.MetricMetadata @@ -89,7 +90,7 @@ func TestIsCumulativeEquivalence(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - mf := newMetricFamily(tt.name, mc).(*metricFamily) + mf := newMetricFamily(tt.name, mc, zap.NewNop()).(*metricFamily) mfp := newMetricFamilyPdata(tt.name, mc).(*metricFamilyPdata) assert.Equal(t, mf.isCumulativeType(), mfp.isCumulativeTypePdata(), "mismatch in isCumulative") assert.Equal(t, mf.isCumulativeType(), tt.want, "isCumulative does not match for regular metricFamily") diff --git a/receiver/prometheusreceiver/metrics_receiver_test.go b/receiver/prometheusreceiver/metrics_receiver_test.go index 17513d6f6c64..411e6960fe73 100644 --- a/receiver/prometheusreceiver/metrics_receiver_test.go +++ b/receiver/prometheusreceiver/metrics_receiver_test.go @@ -32,7 +32,6 @@ import ( metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" resourcepb "github.com/census-instrumentation/opencensus-proto/gen-go/resource/v1" gokitlog "github.com/go-kit/kit/log" - "github.com/golang/protobuf/ptypes/wrappers" promcfg "github.com/prometheus/prometheus/config" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -106,7 +105,10 @@ func (mp *mockPrometheus) Close() { // EndToEnd Test and related // ------------------------- -var srvPlaceHolder = "__SERVER_ADDRESS__" +var ( + srvPlaceHolder = "__SERVER_ADDRESS__" + expectedScrapeMetricCount = 5 +) type testData struct { name string @@ -183,19 +185,197 @@ func verifyNumScrapeResults(t *testing.T, td *testData, mds []*agentmetricspb.Ex } } -func doCompare(name string, t *testing.T, want, got *agentmetricspb.ExportMetricsServiceRequest) { +func doCompare(name string, t *testing.T, want, got *agentmetricspb.ExportMetricsServiceRequest, expectations []testExpectation) { t.Run(name, func(t *testing.T) { - assert.EqualValues(t, want, got) + numScrapeMetrics := countScrapeMetrics(got) + assert.Equal(t, expectedScrapeMetricCount, numScrapeMetrics) + assert.EqualValues(t, want.Node, got.Node) + assert.EqualValues(t, want.Resource, got.Resource) + for _, e := range expectations { + assert.True(t, e(t, got.Metrics)) + } }) } +func getValidScrapes(t *testing.T, mds []*agentmetricspb.ExportMetricsServiceRequest) []*agentmetricspb.ExportMetricsServiceRequest { + out := make([]*agentmetricspb.ExportMetricsServiceRequest, 0) + for _, md := range mds { + // mds will include scrapes that received no metrics but have internal scrape metrics, filter those out + if expectedScrapeMetricCount < len(md.Metrics) && countScrapeMetrics(md) == expectedScrapeMetricCount { + assertUp(t, 1, md) + out = append(out, md) + } else { + assertUp(t, 0, md) + } + } + return out +} + +func assertUp(t *testing.T, expected float64, md *agentmetricspb.ExportMetricsServiceRequest) { + for _, m := range md.Metrics { + if m.GetMetricDescriptor().Name == "up" { + assert.Equal(t, expected, m.Timeseries[0].Points[0].GetDoubleValue()) + return + } + } + t.Error("No 'up' metric found") +} + +func countScrapeMetrics(in *agentmetricspb.ExportMetricsServiceRequest) int { + n := 0 + for _, m := range in.Metrics { + switch m.MetricDescriptor.Name { + case "up", "scrape_duration_seconds", "scrape_samples_scraped", "scrape_samples_post_metric_relabeling", "scrape_series_added": + n++ + default: + } + } + return n +} + +type pointComparator func(*testing.T, *metricspb.Point) bool +type seriesComparator func(*testing.T, *metricspb.TimeSeries) bool +type descriptorComparator func(*testing.T, *metricspb.MetricDescriptor) bool + +type seriesExpectation struct { + series []seriesComparator + points []pointComparator +} + +type testExpectation func(*testing.T, []*metricspb.Metric) bool + +func assertMetricPresent(name string, descriptorExpectations []descriptorComparator, seriesExpectations []seriesExpectation) testExpectation { + return func(t *testing.T, metrics []*metricspb.Metric) bool { + for _, m := range metrics { + if name != m.MetricDescriptor.Name { + continue + } + + for _, de := range descriptorExpectations { + if !de(t, m.MetricDescriptor) { + return false + } + } + + if !assert.Equal(t, len(seriesExpectations), len(m.Timeseries)) { + return false + } + for i, se := range seriesExpectations { + for _, sc := range se.series { + if !sc(t, m.Timeseries[i]) { + return false + } + } + for _, pc := range se.points { + if !pc(t, m.Timeseries[i].Points[0]) { + return false + } + } + } + return true + } + assert.Failf(t, "Unable to match metric expectation", name) + return false + } +} + +func assertMetricAbsent(name string) testExpectation { + return func(t *testing.T, metrics []*metricspb.Metric) bool { + for _, m := range metrics { + if !assert.NotEqual(t, name, m.MetricDescriptor.Name) { + return false + } + } + return true + } +} + +func compareMetricType(typ metricspb.MetricDescriptor_Type) descriptorComparator { + return func(t *testing.T, descriptor *metricspb.MetricDescriptor) bool { + return assert.Equal(t, typ, descriptor.Type) + } +} + +func compareMetricLabelKeys(keys []string) descriptorComparator { + return func(t *testing.T, descriptor *metricspb.MetricDescriptor) bool { + if !assert.Equal(t, len(keys), len(descriptor.LabelKeys)) { + return false + } + for i, k := range keys { + if !assert.Equal(t, k, descriptor.LabelKeys[i].Key) { + return false + } + } + return true + } +} + +func compareSeriesLabelValues(values []string) seriesComparator { + return func(t *testing.T, series *metricspb.TimeSeries) bool { + if !assert.Equal(t, len(values), len(series.LabelValues)) { + return false + } + for i, v := range values { + if !assert.Equal(t, v, series.LabelValues[i].Value) { + return false + } + } + return true + } +} + +func compareSeriesTimestamp(ts *timestamppb.Timestamp) seriesComparator { + return func(t *testing.T, series *metricspb.TimeSeries) bool { + return assert.Equal(t, ts.String(), series.StartTimestamp.String()) + } +} + +func comparePointTimestamp(ts *timestamppb.Timestamp) pointComparator { + return func(t *testing.T, point *metricspb.Point) bool { + return assert.Equal(t, ts.String(), point.Timestamp.String()) + } +} + +func compareDoubleVal(cmp float64) pointComparator { + return func(t *testing.T, pt *metricspb.Point) bool { + return assert.Equal(t, cmp, pt.GetDoubleValue()) + } +} + +func compareHistogram(count int64, sum float64, buckets []int64) pointComparator { + return func(t *testing.T, pt *metricspb.Point) bool { + ret := assert.Equal(t, count, pt.GetDistributionValue().Count) + ret = ret && assert.Equal(t, sum, pt.GetDistributionValue().Sum) + + if ret { + for i, b := range buckets { + ret = ret && assert.Equal(t, b, pt.GetDistributionValue().Buckets[i].Count) + } + } + return ret + } +} + +func compareSummary(count int64, sum float64, quantiles map[float64]float64) pointComparator { + return func(t *testing.T, pt *metricspb.Point) bool { + ret := assert.Equal(t, count, pt.GetSummaryValue().Count.Value) + ret = ret && assert.Equal(t, sum, pt.GetSummaryValue().Sum.Value) + + if ret { + assert.Equal(t, len(quantiles), len(pt.GetSummaryValue().Snapshot.PercentileValues)) + for _, q := range pt.GetSummaryValue().Snapshot.PercentileValues { + assert.Equal(t, quantiles[q.Percentile], q.Value) + } + } + return ret + } +} + // Test data and validation functions for EndToEnd test -// Make sure every page has a gauge, we are relying on it to figure out the starttime if needed +// Make sure every page has a gauge, we are relying on it to figure out the start time if needed -// target1 has one gague, two counts of a same family, one histogram and one summary. We are expecting the first -// successful scrape will produce a metric with only the gauge metric, and the 2nd successful scrape will produce all -// the metrics, with the cumulative types using the firstpage's timestamp as starttime, and values are deltas with the -// one from the first page (summary quantiles will be as it is) +// target1 has one gauge, two counts of a same family, one histogram and one summary. We are expecting the both +// successful scrapes will produce all metrics using the first scrape's timestamp as start time. var target1Page1 = ` # HELP go_threads Number of OS threads created # TYPE go_threads gauge @@ -255,138 +435,90 @@ rpc_duration_seconds_count 1001 func verifyTarget1(t *testing.T, td *testData, mds []*agentmetricspb.ExportMetricsServiceRequest) { verifyNumScrapeResults(t, td, mds) m1 := mds[0] - if l := len(m1.Metrics); l != 4 { - t.Errorf("want 1, but got %v\n", l) + // m1 has 4 metrics + 5 internal scraper metrics + if l := len(m1.Metrics); l != 9 { + t.Errorf("want 9, but got %v\n", l) } ts1 := m1.Metrics[0].Timeseries[0].Points[0].Timestamp - want1 := &agentmetricspb.ExportMetricsServiceRequest{ - Node: td.node, - Resource: td.resource, - Metrics: []*metricspb.Metric{ - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "go_threads", - Description: "Number of OS threads created", - Type: metricspb.MetricDescriptor_GAUGE_DOUBLE}, - Timeseries: []*metricspb.TimeSeries{ - { - Points: []*metricspb.Point{ - {Timestamp: ts1, Value: &metricspb.Point_DoubleValue{DoubleValue: 19.0}}, - }, + e1 := []testExpectation{ + assertMetricPresent("go_threads", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_GAUGE_DOUBLE), + }, + []seriesExpectation{ + { + points: []pointComparator{ + comparePointTimestamp(ts1), + compareDoubleVal(19), }, }, + }), + assertMetricPresent("http_requests_total", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_CUMULATIVE_DOUBLE), + compareMetricLabelKeys([]string{"code", "method"}), }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "http_requests_total", - Description: "The total number of HTTP requests.", - Type: metricspb.MetricDescriptor_CUMULATIVE_DOUBLE, - LabelKeys: []*metricspb.LabelKey{{Key: "code"}, {Key: "method"}}, + []seriesExpectation{ + { + series: []seriesComparator{ + compareSeriesTimestamp(ts1), + compareSeriesLabelValues([]string{"200", "post"}), + }, + points: []pointComparator{ + comparePointTimestamp(ts1), + compareDoubleVal(100), + }, }, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: ts1, - LabelValues: []*metricspb.LabelValue{ - {Value: "200", HasValue: true}, - {Value: "post", HasValue: true}, - }, - Points: []*metricspb.Point{ - {Timestamp: ts1, Value: &metricspb.Point_DoubleValue{DoubleValue: 100.0}}, - }, + { + series: []seriesComparator{ + compareSeriesTimestamp(ts1), + compareSeriesLabelValues([]string{"400", "post"}), }, - { - StartTimestamp: ts1, - LabelValues: []*metricspb.LabelValue{ - {Value: "400", HasValue: true}, - {Value: "post", HasValue: true}, - }, - Points: []*metricspb.Point{ - {Timestamp: ts1, Value: &metricspb.Point_DoubleValue{DoubleValue: 5.0}}, - }, + points: []pointComparator{ + comparePointTimestamp(ts1), + compareDoubleVal(5), }, }, + }), + assertMetricPresent("http_request_duration_seconds", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION), }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "http_request_duration_seconds", - Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, - Description: "A histogram of the request duration.", - Unit: "s", - }, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: ts1, - Points: []*metricspb.Point{ - { - Timestamp: ts1, - Value: &metricspb.Point_DistributionValue{ - DistributionValue: &metricspb.DistributionValue{ - BucketOptions: &metricspb.DistributionValue_BucketOptions{ - Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ - Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ - Bounds: []float64{0.05, 0.5, 1}, - }, - }, - }, - Count: 2500, - Sum: 5000.0, - Buckets: []*metricspb.DistributionValue_Bucket{ - {Count: 1000}, - {Count: 500}, - {Count: 500}, - {Count: 500}, - }, - }}, - }, - }, + []seriesExpectation{ + { + series: []seriesComparator{ + compareSeriesTimestamp(ts1), + }, + points: []pointComparator{ + comparePointTimestamp(ts1), + compareHistogram(2500, 5000, []int64{1000, 500, 500, 500}), }, }, + }), + assertMetricPresent("rpc_duration_seconds", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_SUMMARY), }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "rpc_duration_seconds", - Type: metricspb.MetricDescriptor_SUMMARY, - Description: "A summary of the RPC duration in seconds.", - Unit: "s", - }, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: ts1, - Points: []*metricspb.Point{ - { - Timestamp: ts1, - Value: &metricspb.Point_SummaryValue{ - SummaryValue: &metricspb.SummaryValue{ - Sum: &wrappers.DoubleValue{Value: 5000}, - Count: &wrappers.Int64Value{Value: 1000}, - Snapshot: &metricspb.SummaryValue_Snapshot{ - PercentileValues: []*metricspb.SummaryValue_Snapshot_ValueAtPercentile{ - { - Percentile: 1, - Value: 1, - }, - { - Percentile: 90, - Value: 5, - }, - { - Percentile: 99, - Value: 8, - }, - }, - }, - }, - }, - }, - }, + []seriesExpectation{ + { + series: []seriesComparator{ + compareSeriesTimestamp(ts1), + }, + points: []pointComparator{ + comparePointTimestamp(ts1), + compareSummary(1000, 5000, map[float64]float64{1: 1, 90: 5, 99: 8}), }, }, - }, - }, + }), } - doCompare("scrape1", t, want1, m1) + want1 := &agentmetricspb.ExportMetricsServiceRequest{ + Node: td.node, + Resource: td.resource, + } + + doCompare("scrape1", t, want1, m1, e1) // verify the 2nd metricData m2 := mds[1] @@ -395,137 +527,86 @@ func verifyTarget1(t *testing.T, td *testData, mds []*agentmetricspb.ExportMetri want2 := &agentmetricspb.ExportMetricsServiceRequest{ Node: td.node, Resource: td.resource, - Metrics: []*metricspb.Metric{ - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "go_threads", - Description: "Number of OS threads created", - Type: metricspb.MetricDescriptor_GAUGE_DOUBLE}, - Timeseries: []*metricspb.TimeSeries{ - { - Points: []*metricspb.Point{ - {Timestamp: ts2, Value: &metricspb.Point_DoubleValue{DoubleValue: 18.0}}, - }, + } + + e2 := []testExpectation{ + assertMetricPresent("go_threads", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_GAUGE_DOUBLE), + }, + []seriesExpectation{ + { + points: []pointComparator{ + comparePointTimestamp(ts2), + compareDoubleVal(18), }, }, + }), + assertMetricPresent("http_requests_total", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_CUMULATIVE_DOUBLE), + compareMetricLabelKeys([]string{"code", "method"}), }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "http_requests_total", - Description: "The total number of HTTP requests.", - Type: metricspb.MetricDescriptor_CUMULATIVE_DOUBLE, - LabelKeys: []*metricspb.LabelKey{{Key: "code"}, {Key: "method"}}, + []seriesExpectation{ + { + series: []seriesComparator{ + compareSeriesTimestamp(ts1), + compareSeriesLabelValues([]string{"200", "post"}), + }, + points: []pointComparator{ + comparePointTimestamp(ts2), + compareDoubleVal(199), + }, }, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: ts1, - LabelValues: []*metricspb.LabelValue{ - {Value: "200", HasValue: true}, - {Value: "post", HasValue: true}, - }, - Points: []*metricspb.Point{ - {Timestamp: ts2, Value: &metricspb.Point_DoubleValue{DoubleValue: 199.0}}, - }, + { + series: []seriesComparator{ + compareSeriesTimestamp(ts1), + compareSeriesLabelValues([]string{"400", "post"}), }, - { - StartTimestamp: ts1, - LabelValues: []*metricspb.LabelValue{ - {Value: "400", HasValue: true}, - {Value: "post", HasValue: true}, - }, - Points: []*metricspb.Point{ - {Timestamp: ts2, Value: &metricspb.Point_DoubleValue{DoubleValue: 12.0}}, - }, + points: []pointComparator{ + comparePointTimestamp(ts2), + compareDoubleVal(12), }, }, + }), + assertMetricPresent("http_request_duration_seconds", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION), }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "http_request_duration_seconds", - Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, - Description: "A histogram of the request duration.", - Unit: "s", - }, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: ts1, - Points: []*metricspb.Point{ - { - Timestamp: ts2, - Value: &metricspb.Point_DistributionValue{ - DistributionValue: &metricspb.DistributionValue{ - BucketOptions: &metricspb.DistributionValue_BucketOptions{ - Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ - Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ - Bounds: []float64{0.05, 0.5, 1}, - }, - }, - }, - Count: 2600, - Sum: 5050.0, - Buckets: []*metricspb.DistributionValue_Bucket{ - {Count: 1100}, - {Count: 500}, - {Count: 500}, - {Count: 500}, - }, - }}, - }, - }, + []seriesExpectation{ + { + series: []seriesComparator{ + compareSeriesTimestamp(ts1), + }, + points: []pointComparator{ + comparePointTimestamp(ts2), + compareHistogram(2600, 5050, []int64{1100, 500, 500, 500}), }, }, + }), + assertMetricPresent("rpc_duration_seconds", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_SUMMARY), }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "rpc_duration_seconds", - Type: metricspb.MetricDescriptor_SUMMARY, - Description: "A summary of the RPC duration in seconds.", - Unit: "s", - }, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: ts1, - Points: []*metricspb.Point{ - { - Timestamp: ts2, - Value: &metricspb.Point_SummaryValue{ - SummaryValue: &metricspb.SummaryValue{ - Sum: &wrappers.DoubleValue{Value: 5002}, - Count: &wrappers.Int64Value{Value: 1001}, - Snapshot: &metricspb.SummaryValue_Snapshot{ - PercentileValues: []*metricspb.SummaryValue_Snapshot_ValueAtPercentile{ - { - Percentile: 1, - Value: 1, - }, - { - Percentile: 90, - Value: 6, - }, - { - Percentile: 99, - Value: 8, - }, - }, - }, - }, - }, - }, - }, + []seriesExpectation{ + { + series: []seriesComparator{ + compareSeriesTimestamp(ts1), + }, + points: []pointComparator{ + comparePointTimestamp(ts2), + compareSummary(1001, 5002, map[float64]float64{1: 1, 90: 6, 99: 8}), }, }, - }, - }, + }), } - doCompare("scrape2", t, want2, m2) + doCompare("scrape2", t, want2, m2, e2) } -// target2 is going to have 5 pages, and there's a newly appeared item from the 2nd page. we are expecting the new -// metric will appears on the 3rd scrape, and it uses the timestamp from the 2nd page as starttime, and value is delta -// from the 2nd page. -// with the 4th page, we are simulating a reset (values smaller than previous), cumulative types shall be skipped at -// this run. with the 5th page, cumulative types will be delta with the 4th one +// target2 is going to have 5 pages, and there's a newly added item on the 2nd page. +// with the 4th page, we are simulating a reset (values smaller than previous), start times should be from +// this run for the 4th and 5th scrapes. var target2Page1 = ` # HELP go_threads Number of OS threads created # TYPE go_threads gauge @@ -588,63 +669,60 @@ http_requests_total{method="post",code="500"} 5 func verifyTarget2(t *testing.T, td *testData, mds []*agentmetricspb.ExportMetricsServiceRequest) { verifyNumScrapeResults(t, td, mds) m1 := mds[0] - if l := len(m1.Metrics); l != 2 { - t.Errorf("want 1, but got %v\n", l) + // m1 has 2 metrics + 5 internal scraper metrics + if l := len(m1.Metrics); l != 7 { + t.Errorf("want 7, but got %v\n", l) } ts1 := m1.Metrics[0].Timeseries[0].Points[0].Timestamp want1 := &agentmetricspb.ExportMetricsServiceRequest{ Node: td.node, Resource: td.resource, - Metrics: []*metricspb.Metric{ - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "go_threads", - Description: "Number of OS threads created", - Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, - }, - Timeseries: []*metricspb.TimeSeries{ - { - Points: []*metricspb.Point{ - {Timestamp: ts1, Value: &metricspb.Point_DoubleValue{DoubleValue: 18.0}}, - }, + } + + e1 := []testExpectation{ + assertMetricPresent("go_threads", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_GAUGE_DOUBLE), + }, + []seriesExpectation{ + { + points: []pointComparator{ + comparePointTimestamp(ts1), + compareDoubleVal(18), }, }, + }), + assertMetricPresent("http_requests_total", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_CUMULATIVE_DOUBLE), + compareMetricLabelKeys([]string{"code", "method"}), }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "http_requests_total", - Description: "The total number of HTTP requests.", - Type: metricspb.MetricDescriptor_CUMULATIVE_DOUBLE, - LabelKeys: []*metricspb.LabelKey{{Key: "code"}, {Key: "method"}}, + []seriesExpectation{ + { + series: []seriesComparator{ + compareSeriesTimestamp(ts1), + compareSeriesLabelValues([]string{"200", "post"}), + }, + points: []pointComparator{ + comparePointTimestamp(ts1), + compareDoubleVal(10), + }, }, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: ts1, - LabelValues: []*metricspb.LabelValue{ - {Value: "200", HasValue: true}, - {Value: "post", HasValue: true}, - }, - Points: []*metricspb.Point{ - {Timestamp: ts1, Value: &metricspb.Point_DoubleValue{DoubleValue: 10.0}}, - }, + { + series: []seriesComparator{ + compareSeriesTimestamp(ts1), + compareSeriesLabelValues([]string{"400", "post"}), }, - { - StartTimestamp: ts1, - LabelValues: []*metricspb.LabelValue{ - {Value: "400", HasValue: true}, - {Value: "post", HasValue: true}, - }, - Points: []*metricspb.Point{ - {Timestamp: ts1, Value: &metricspb.Point_DoubleValue{DoubleValue: 50.0}}, - }, + points: []pointComparator{ + comparePointTimestamp(ts1), + compareDoubleVal(50), }, }, - }, - }, + }), } - doCompare("scrape1", t, want1, m1) + doCompare("scrape1", t, want1, m1, e1) // verify the 2nd metricData m2 := mds[1] @@ -653,64 +731,60 @@ func verifyTarget2(t *testing.T, td *testData, mds []*agentmetricspb.ExportMetri want2 := &agentmetricspb.ExportMetricsServiceRequest{ Node: td.node, Resource: td.resource, - Metrics: []*metricspb.Metric{ - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "go_threads", - Description: "Number of OS threads created", - Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, - }, - Timeseries: []*metricspb.TimeSeries{ - { - Points: []*metricspb.Point{ - {Timestamp: ts2, Value: &metricspb.Point_DoubleValue{DoubleValue: 16.0}}, - }, + } + e2 := []testExpectation{ + assertMetricPresent("go_threads", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_GAUGE_DOUBLE), + }, + []seriesExpectation{ + { + points: []pointComparator{ + comparePointTimestamp(ts2), + compareDoubleVal(16), }, }, + }), + assertMetricPresent("http_requests_total", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_CUMULATIVE_DOUBLE), + compareMetricLabelKeys([]string{"code", "method"}), }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "http_requests_total", - Description: "The total number of HTTP requests.", - Type: metricspb.MetricDescriptor_CUMULATIVE_DOUBLE, - LabelKeys: []*metricspb.LabelKey{{Key: "code"}, {Key: "method"}}, + []seriesExpectation{ + { + series: []seriesComparator{ + compareSeriesTimestamp(ts1), + compareSeriesLabelValues([]string{"200", "post"}), + }, + points: []pointComparator{ + comparePointTimestamp(ts2), + compareDoubleVal(50), + }, }, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: ts1, - LabelValues: []*metricspb.LabelValue{ - {Value: "200", HasValue: true}, - {Value: "post", HasValue: true}, - }, - Points: []*metricspb.Point{ - {Timestamp: ts2, Value: &metricspb.Point_DoubleValue{DoubleValue: 50.0}}, - }, + { + series: []seriesComparator{ + compareSeriesTimestamp(ts1), + compareSeriesLabelValues([]string{"400", "post"}), }, - { - StartTimestamp: ts1, - LabelValues: []*metricspb.LabelValue{ - {Value: "400", HasValue: true}, - {Value: "post", HasValue: true}, - }, - Points: []*metricspb.Point{ - {Timestamp: ts2, Value: &metricspb.Point_DoubleValue{DoubleValue: 60.0}}, - }, + points: []pointComparator{ + comparePointTimestamp(ts2), + compareDoubleVal(60), }, - { - StartTimestamp: ts2, - LabelValues: []*metricspb.LabelValue{ - {Value: "500", HasValue: true}, - {Value: "post", HasValue: true}, - }, - Points: []*metricspb.Point{ - {Timestamp: ts2, Value: &metricspb.Point_DoubleValue{DoubleValue: 3.0}}, - }, + }, + { + series: []seriesComparator{ + compareSeriesTimestamp(ts2), + compareSeriesLabelValues([]string{"500", "post"}), + }, + points: []pointComparator{ + comparePointTimestamp(ts2), + compareDoubleVal(3), }, }, - }, - }, + }), } - doCompare("scrape2", t, want2, m2) + + doCompare("scrape2", t, want2, m2, e2) // verify the 3rd metricData, with the new code=500 counter which first appeared on 2nd run m3 := mds[2] @@ -720,66 +794,63 @@ func verifyTarget2(t *testing.T, td *testData, mds []*agentmetricspb.ExportMetri want3 := &agentmetricspb.ExportMetricsServiceRequest{ Node: td.node, Resource: td.resource, - Metrics: []*metricspb.Metric{ - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "go_threads", - Description: "Number of OS threads created", - Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, - }, - Timeseries: []*metricspb.TimeSeries{ - { - Points: []*metricspb.Point{ - {Timestamp: ts3, Value: &metricspb.Point_DoubleValue{DoubleValue: 16.0}}, - }, + } + + e3 := []testExpectation{ + assertMetricPresent("go_threads", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_GAUGE_DOUBLE), + }, + []seriesExpectation{ + { + points: []pointComparator{ + comparePointTimestamp(ts3), + compareDoubleVal(16), }, }, + }), + assertMetricPresent("http_requests_total", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_CUMULATIVE_DOUBLE), + compareMetricLabelKeys([]string{"code", "method"}), }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "http_requests_total", - Description: "The total number of HTTP requests.", - Type: metricspb.MetricDescriptor_CUMULATIVE_DOUBLE, - LabelKeys: []*metricspb.LabelKey{{Key: "code"}, {Key: "method"}}, + []seriesExpectation{ + { + series: []seriesComparator{ + compareSeriesTimestamp(ts1), + compareSeriesLabelValues([]string{"200", "post"}), + }, + points: []pointComparator{ + comparePointTimestamp(ts3), + compareDoubleVal(50), + }, }, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: ts1, - LabelValues: []*metricspb.LabelValue{ - {Value: "200", HasValue: true}, - {Value: "post", HasValue: true}, - }, - Points: []*metricspb.Point{ - {Timestamp: ts3, Value: &metricspb.Point_DoubleValue{DoubleValue: 50.0}}, - }, + { + series: []seriesComparator{ + compareSeriesTimestamp(ts1), + compareSeriesLabelValues([]string{"400", "post"}), }, - { - StartTimestamp: ts1, - LabelValues: []*metricspb.LabelValue{ - {Value: "400", HasValue: true}, - {Value: "post", HasValue: true}, - }, - Points: []*metricspb.Point{ - {Timestamp: ts3, Value: &metricspb.Point_DoubleValue{DoubleValue: 60.0}}, - }, + points: []pointComparator{ + comparePointTimestamp(ts3), + compareDoubleVal(60), }, - { - StartTimestamp: ts2, - LabelValues: []*metricspb.LabelValue{ - {Value: "500", HasValue: true}, - {Value: "post", HasValue: true}, - }, - Points: []*metricspb.Point{ - {Timestamp: ts3, Value: &metricspb.Point_DoubleValue{DoubleValue: 5.0}}, - }, + }, + { + series: []seriesComparator{ + compareSeriesTimestamp(ts2), + compareSeriesLabelValues([]string{"500", "post"}), + }, + points: []pointComparator{ + comparePointTimestamp(ts3), + compareDoubleVal(5), }, }, - }, - }, + }), } - doCompare("scrape3", t, want3, m3) - // verify the 4th metricData which reset happens, all cumulative types shall be absent + doCompare("scrape3", t, want3, m3, e3) + + // verify the 4th metricData which reset happens m4 := mds[3] ts4 := m4.Metrics[0].Timeseries[0].Points[0].Timestamp @@ -789,128 +860,214 @@ func verifyTarget2(t *testing.T, td *testData, mds []*agentmetricspb.ExportMetri Metrics: []*metricspb.Metric{ { MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "go_threads", - Description: "Number of OS threads created", + Name: "up", + Description: "The scraping was successful", Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, + Unit: "bool", }, Timeseries: []*metricspb.TimeSeries{ { Points: []*metricspb.Point{ - {Timestamp: ts4, Value: &metricspb.Point_DoubleValue{DoubleValue: 16.0}}, + {Timestamp: ts2, Value: &metricspb.Point_DoubleValue{DoubleValue: 1.0}}, }, }, }, }, { MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "http_requests_total", - Description: "The total number of HTTP requests.", - Type: metricspb.MetricDescriptor_CUMULATIVE_DOUBLE, - LabelKeys: []*metricspb.LabelKey{{Key: "code"}, {Key: "method"}}, + Name: "scrape_series_added", + Description: "The approximate number of new series in this scrape", + Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, + Unit: "count", }, Timeseries: []*metricspb.TimeSeries{ { - StartTimestamp: ts4, - LabelValues: []*metricspb.LabelValue{ - {Value: "200", HasValue: true}, - {Value: "post", HasValue: true}, - }, Points: []*metricspb.Point{ - {Timestamp: ts4, Value: &metricspb.Point_DoubleValue{DoubleValue: 49.0}}, + {Timestamp: ts2, Value: &metricspb.Point_DoubleValue{DoubleValue: 14.0}}, }, }, + }, + }, + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "scrape_duration_seconds", + Description: "Duration of the scrape", + Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, + Unit: "seconds", + }, + Timeseries: []*metricspb.TimeSeries{ { - StartTimestamp: ts4, - LabelValues: []*metricspb.LabelValue{ - {Value: "400", HasValue: true}, - {Value: "post", HasValue: true}, - }, Points: []*metricspb.Point{ - {Timestamp: ts4, Value: &metricspb.Point_DoubleValue{DoubleValue: 59.0}}, + {Timestamp: ts2, Value: &metricspb.Point_DoubleValue{DoubleValue: 0.0123456}}, }, }, + }, + }, + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "scrape_samples_scraped", + Description: "The number of samples the target exposed", + Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, + Unit: "count", + }, + Timeseries: []*metricspb.TimeSeries{ { - StartTimestamp: ts4, - LabelValues: []*metricspb.LabelValue{ - {Value: "500", HasValue: true}, - {Value: "post", HasValue: true}, - }, Points: []*metricspb.Point{ - {Timestamp: ts4, Value: &metricspb.Point_DoubleValue{DoubleValue: 3.0}}, + {Timestamp: ts2, Value: &metricspb.Point_DoubleValue{DoubleValue: 14.0}}, }, }, }, }, - }, - } - doCompare("scrape4", t, want4, m4) - - // verify the 4th metricData which reset happens, all cumulative types shall be absent - m5 := mds[4] - // its start timestamp shall be from the 3rd run - ts5 := m5.Metrics[0].Timeseries[0].Points[0].Timestamp - - want5 := &agentmetricspb.ExportMetricsServiceRequest{ - Node: td.node, - Resource: td.resource, - Metrics: []*metricspb.Metric{ { MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "go_threads", - Description: "Number of OS threads created", + Name: "scrape_samples_post_metric_relabeling", + Description: "The number of samples remaining after metric relabeling was applied", Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, + Unit: "count", }, Timeseries: []*metricspb.TimeSeries{ { Points: []*metricspb.Point{ - {Timestamp: ts5, Value: &metricspb.Point_DoubleValue{DoubleValue: 16.0}}, + {Timestamp: ts2, Value: &metricspb.Point_DoubleValue{DoubleValue: 14.0}}, }, }, }, }, { MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "http_requests_total", - Description: "The total number of HTTP requests.", - Type: metricspb.MetricDescriptor_CUMULATIVE_DOUBLE, - LabelKeys: []*metricspb.LabelKey{{Key: "code"}, {Key: "method"}}, + Name: "scrape_series_added", + Description: "The approximate number of new series in this scrape", + Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, + Unit: "count", }, Timeseries: []*metricspb.TimeSeries{ { - StartTimestamp: ts4, - LabelValues: []*metricspb.LabelValue{ - {Value: "200", HasValue: true}, - {Value: "post", HasValue: true}, - }, Points: []*metricspb.Point{ - {Timestamp: ts5, Value: &metricspb.Point_DoubleValue{DoubleValue: 50.0}}, + {Timestamp: ts2, Value: &metricspb.Point_DoubleValue{DoubleValue: 14.0}}, }, }, - { - StartTimestamp: ts4, - LabelValues: []*metricspb.LabelValue{ - {Value: "400", HasValue: true}, - {Value: "post", HasValue: true}, - }, - Points: []*metricspb.Point{ - {Timestamp: ts5, Value: &metricspb.Point_DoubleValue{DoubleValue: 59.0}}, - }, + }, + }, + }, + } + + e4 := []testExpectation{ + assertMetricPresent("go_threads", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_GAUGE_DOUBLE), + }, + []seriesExpectation{ + { + points: []pointComparator{ + comparePointTimestamp(ts4), + compareDoubleVal(16), }, - { - StartTimestamp: ts4, - LabelValues: []*metricspb.LabelValue{ - {Value: "500", HasValue: true}, - {Value: "post", HasValue: true}, - }, - Points: []*metricspb.Point{ - {Timestamp: ts5, Value: &metricspb.Point_DoubleValue{DoubleValue: 5.0}}, - }, + }, + }), + assertMetricPresent("http_requests_total", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_CUMULATIVE_DOUBLE), + compareMetricLabelKeys([]string{"code", "method"}), + }, + []seriesExpectation{ + { + series: []seriesComparator{ + compareSeriesTimestamp(ts4), + compareSeriesLabelValues([]string{"200", "post"}), + }, + points: []pointComparator{ + comparePointTimestamp(ts4), + compareDoubleVal(49), + }, + }, + { + series: []seriesComparator{ + compareSeriesTimestamp(ts4), + compareSeriesLabelValues([]string{"400", "post"}), + }, + points: []pointComparator{ + comparePointTimestamp(ts4), + compareDoubleVal(59), + }, + }, + { + series: []seriesComparator{ + compareSeriesTimestamp(ts4), + compareSeriesLabelValues([]string{"500", "post"}), + }, + points: []pointComparator{ + comparePointTimestamp(ts4), + compareDoubleVal(3), }, }, + }), + } + + doCompare("scrape4", t, want4, m4, e4) + + // verify the 5th metricData which reset happens + m5 := mds[4] + // its start timestamp shall be from the 4th run + ts5 := m5.Metrics[0].Timeseries[0].Points[0].Timestamp + + want5 := &agentmetricspb.ExportMetricsServiceRequest{ + Node: td.node, + Resource: td.resource, + } + + e5 := []testExpectation{ + assertMetricPresent("go_threads", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_GAUGE_DOUBLE), }, - }, + []seriesExpectation{ + { + points: []pointComparator{ + comparePointTimestamp(ts5), + compareDoubleVal(16), + }, + }, + }), + assertMetricPresent("http_requests_total", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_CUMULATIVE_DOUBLE), + compareMetricLabelKeys([]string{"code", "method"}), + }, + []seriesExpectation{ + { + series: []seriesComparator{ + compareSeriesTimestamp(ts4), + compareSeriesLabelValues([]string{"200", "post"}), + }, + points: []pointComparator{ + comparePointTimestamp(ts5), + compareDoubleVal(50), + }, + }, + { + series: []seriesComparator{ + compareSeriesTimestamp(ts4), + compareSeriesLabelValues([]string{"400", "post"}), + }, + points: []pointComparator{ + comparePointTimestamp(ts5), + compareDoubleVal(59), + }, + }, + { + series: []seriesComparator{ + compareSeriesTimestamp(ts4), + compareSeriesLabelValues([]string{"500", "post"}), + }, + points: []pointComparator{ + comparePointTimestamp(ts5), + compareDoubleVal(5), + }, + }, + }), } - doCompare("scrape5", t, want5, m5) + + doCompare("scrape5", t, want5, m5, e5) } // target3 for complicated data types, including summaries and histograms. one of the summary and histogram have only @@ -988,138 +1145,76 @@ rpc_duration_seconds_count{foo="no_quantile"} 55 func verifyTarget3(t *testing.T, td *testData, mds []*agentmetricspb.ExportMetricsServiceRequest) { verifyNumScrapeResults(t, td, mds) m1 := mds[0] - if l := len(m1.Metrics); l != 3 { - t.Errorf("want 1, but got %v\n", l) + // m1 has 3 metrics + 5 internal scraper metrics + if l := len(m1.Metrics); l != 8 { + t.Errorf("want 8, but got %v\n", l) } ts1 := m1.Metrics[1].Timeseries[0].Points[0].Timestamp want1 := &agentmetricspb.ExportMetricsServiceRequest{ Node: td.node, Resource: td.resource, - Metrics: []*metricspb.Metric{ - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "go_threads", - Description: "Number of OS threads created", - Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, - }, - Timeseries: []*metricspb.TimeSeries{ - { - Points: []*metricspb.Point{ - {Timestamp: ts1, Value: &metricspb.Point_DoubleValue{DoubleValue: 18.0}}, - }, + } + + e1 := []testExpectation{ + assertMetricPresent("go_threads", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_GAUGE_DOUBLE), + }, + []seriesExpectation{ + { + points: []pointComparator{ + comparePointTimestamp(ts1), + compareDoubleVal(18), }, }, + }), + assertMetricPresent("http_request_duration_seconds", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION), }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "http_request_duration_seconds", - Description: "A histogram of the request duration.", - Unit: "s", - Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, - }, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: ts1, - Points: []*metricspb.Point{ - { - Timestamp: ts1, - Value: &metricspb.Point_DistributionValue{ - DistributionValue: &metricspb.DistributionValue{ - BucketOptions: &metricspb.DistributionValue_BucketOptions{ - Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ - Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ - Bounds: []float64{0.2, 0.5, 1}, - }, - }, - }, - Count: 13003, - Sum: 50000, - Buckets: []*metricspb.DistributionValue_Bucket{ - {Count: 10000}, - {Count: 1000}, - {Count: 1001}, - {Count: 1002}, - }, - }, - }, - }, - }, + []seriesExpectation{ + { + series: []seriesComparator{ + compareSeriesTimestamp(ts1), + }, + points: []pointComparator{ + comparePointTimestamp(ts1), + compareHistogram(13003, 50000, []int64{10000, 1000, 1001, 1002}), }, }, + }), + assertMetricAbsent("corrupted_hist"), + assertMetricPresent("rpc_duration_seconds", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_SUMMARY), + compareMetricLabelKeys([]string{"foo"}), }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "rpc_duration_seconds", - Type: metricspb.MetricDescriptor_SUMMARY, - LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}, - Description: "A summary of the RPC duration in seconds.", - Unit: "s", + []seriesExpectation{ + { + series: []seriesComparator{ + compareSeriesTimestamp(ts1), + compareSeriesLabelValues([]string{"bar"}), + }, + points: []pointComparator{ + comparePointTimestamp(ts1), + compareSummary(900, 8000, map[float64]float64{1: 31, 5: 35, 50: 47, 90: 70, 99: 76}), + }, }, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: ts1, - LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, - Points: []*metricspb.Point{ - { - Timestamp: ts1, - Value: &metricspb.Point_SummaryValue{ - SummaryValue: &metricspb.SummaryValue{ - Sum: &wrappers.DoubleValue{Value: 8000}, - Count: &wrappers.Int64Value{Value: 900}, - Snapshot: &metricspb.SummaryValue_Snapshot{ - PercentileValues: []*metricspb.SummaryValue_Snapshot_ValueAtPercentile{ - { - Percentile: 1, - Value: 31, - }, - { - Percentile: 5, - Value: 35, - }, - { - Percentile: 50, - Value: 47, - }, - { - Percentile: 90, - Value: 70, - }, - { - Percentile: 99, - Value: 76, - }, - }, - }, - }, - }, - }, - }, + { + series: []seriesComparator{ + compareSeriesTimestamp(ts1), + compareSeriesLabelValues([]string{"no_quantile"}), }, - { - StartTimestamp: ts1, - LabelValues: []*metricspb.LabelValue{{Value: "no_quantile", HasValue: true}}, - Points: []*metricspb.Point{ - { - Timestamp: ts1, - Value: &metricspb.Point_SummaryValue{ - SummaryValue: &metricspb.SummaryValue{ - Sum: &wrappers.DoubleValue{Value: 100}, - Count: &wrappers.Int64Value{Value: 50}, - Snapshot: &metricspb.SummaryValue_Snapshot{ - PercentileValues: nil, - }, - }, - }, - }, - }, + points: []pointComparator{ + comparePointTimestamp(ts1), + compareSummary(50, 100, map[float64]float64{}), }, }, - }, - }, + }), } - doCompare("scrape1", t, want1, m1) + doCompare("scrape1", t, want1, m1, e1) // verify the 2nd metricData m2 := mds[1] @@ -1128,130 +1223,67 @@ func verifyTarget3(t *testing.T, td *testData, mds []*agentmetricspb.ExportMetri want2 := &agentmetricspb.ExportMetricsServiceRequest{ Node: td.node, Resource: td.resource, - Metrics: []*metricspb.Metric{ - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "go_threads", - Description: "Number of OS threads created", - Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, - }, - Timeseries: []*metricspb.TimeSeries{ - { - Points: []*metricspb.Point{ - {Timestamp: ts2, Value: &metricspb.Point_DoubleValue{DoubleValue: 16.0}}, - }, + } + + e2 := []testExpectation{ + assertMetricPresent("go_threads", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_GAUGE_DOUBLE), + }, + []seriesExpectation{ + { + points: []pointComparator{ + comparePointTimestamp(ts2), + compareDoubleVal(16), }, }, + }), + assertMetricPresent("http_request_duration_seconds", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION), }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "http_request_duration_seconds", - Description: "A histogram of the request duration.", - Unit: "s", - Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, - }, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: ts1, - Points: []*metricspb.Point{ - { - Timestamp: ts2, - Value: &metricspb.Point_DistributionValue{ - DistributionValue: &metricspb.DistributionValue{ - BucketOptions: &metricspb.DistributionValue_BucketOptions{ - Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ - Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ - Bounds: []float64{0.2, 0.5, 1}, - }, - }, - }, - Count: 14003, - Sum: 50100, - Buckets: []*metricspb.DistributionValue_Bucket{ - {Count: 11000}, - {Count: 1000}, - {Count: 1001}, - {Count: 1002}, - }, - }, - }, - }, - }, + []seriesExpectation{ + { + series: []seriesComparator{ + compareSeriesTimestamp(ts1), + }, + points: []pointComparator{ + comparePointTimestamp(ts2), + compareHistogram(14003, 50100, []int64{11000, 1000, 1001, 1002}), }, }, + }), + assertMetricAbsent("corrupted_hist"), + assertMetricPresent("rpc_duration_seconds", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_SUMMARY), + compareMetricLabelKeys([]string{"foo"}), }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "rpc_duration_seconds", - Type: metricspb.MetricDescriptor_SUMMARY, - LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}, - Description: "A summary of the RPC duration in seconds.", - Unit: "s", + []seriesExpectation{ + { + series: []seriesComparator{ + compareSeriesTimestamp(ts1), + compareSeriesLabelValues([]string{"bar"}), + }, + points: []pointComparator{ + comparePointTimestamp(ts2), + compareSummary(950, 8100, map[float64]float64{1: 32, 5: 35, 50: 47, 90: 70, 99: 77}), + }, }, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: ts1, - LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, - Points: []*metricspb.Point{ - { - Timestamp: ts2, - Value: &metricspb.Point_SummaryValue{ - SummaryValue: &metricspb.SummaryValue{ - Sum: &wrappers.DoubleValue{Value: 8100}, - Count: &wrappers.Int64Value{Value: 950}, - Snapshot: &metricspb.SummaryValue_Snapshot{ - PercentileValues: []*metricspb.SummaryValue_Snapshot_ValueAtPercentile{ - { - Percentile: 1, - Value: 32, - }, - { - Percentile: 5, - Value: 35, - }, - { - Percentile: 50, - Value: 47, - }, - { - Percentile: 90, - Value: 70, - }, - { - Percentile: 99, - Value: 77, - }, - }, - }, - }, - }, - }, - }, + { + series: []seriesComparator{ + compareSeriesTimestamp(ts1), + compareSeriesLabelValues([]string{"no_quantile"}), }, - { - StartTimestamp: ts1, - LabelValues: []*metricspb.LabelValue{{Value: "no_quantile", HasValue: true}}, - Points: []*metricspb.Point{ - { - Timestamp: ts2, - Value: &metricspb.Point_SummaryValue{ - SummaryValue: &metricspb.SummaryValue{ - Sum: &wrappers.DoubleValue{Value: 101}, - Count: &wrappers.Int64Value{Value: 55}, - Snapshot: &metricspb.SummaryValue_Snapshot{ - PercentileValues: nil, - }, - }, - }, - }, - }, + points: []pointComparator{ + comparePointTimestamp(ts2), + compareSummary(55, 101, map[float64]float64{}), }, }, - }, - }, + }), } - doCompare("scrape2", t, want2, m2) + doCompare("scrape2", t, want2, m2, e2) } // TestEndToEnd end to end test executor @@ -1323,7 +1355,8 @@ process_start_time_seconds 400.8 var startTimeMetricPageStartTimestamp = ×tamppb.Timestamp{Seconds: 400, Nanos: 800000000} -const numStartTimeMetricPageTimeseries = 6 +// 6 metrics + 5 internal metrics +const numStartTimeMetricPageTimeseries = 11 func verifyStartTimeMetricPage(t *testing.T, _ *testData, mds []*agentmetricspb.ExportMetricsServiceRequest) { numTimeseries := 0 @@ -1397,7 +1430,8 @@ func testEndToEnd(t *testing.T, targets []*testData, useStartTimeMetric bool) { // loop to validate outputs for each targets for _, target := range targets { t.Run(target.name, func(t *testing.T) { - target.validateFunc(t, target, results[target.name]) + mds := getValidScrapes(t, results[target.name]) + target.validateFunc(t, target, mds) }) } } diff --git a/receiver/prometheusreceiver/metrics_reciever_external_labels_test.go b/receiver/prometheusreceiver/metrics_reciever_external_labels_test.go index 786d135bc36e..658913de0b37 100644 --- a/receiver/prometheusreceiver/metrics_reciever_external_labels_test.go +++ b/receiver/prometheusreceiver/metrics_reciever_external_labels_test.go @@ -87,27 +87,23 @@ func verifyExternalLabels(t *testing.T, td *testData, mds []*agentmetricspb.Expo want := &agentmetricspb.ExportMetricsServiceRequest{ Node: td.node, Resource: td.resource, - Metrics: []*metricspb.Metric{ - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "go_threads", - Description: "Number of OS threads created", - Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, - LabelKeys: []*metricspb.LabelKey{{Key: "key"}}, - }, - Timeseries: []*metricspb.TimeSeries{ - { - Points: []*metricspb.Point{ - {Value: &metricspb.Point_DoubleValue{DoubleValue: 19.0}}, - }, - LabelValues: []*metricspb.LabelValue{ - {Value: "value", HasValue: true}, - }, + } + doCompare("scrape-externalLabels", t, want, mds[0], []testExpectation{ + assertMetricPresent("go_threads", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_GAUGE_DOUBLE), + compareMetricLabelKeys([]string{"key"}), + }, + []seriesExpectation{ + { + series: []seriesComparator{ + compareSeriesLabelValues([]string{"value"}), + }, + points: []pointComparator{ + comparePointTimestamp(mds[0].Metrics[0].Timeseries[0].Points[0].Timestamp), + compareDoubleVal(19), }, }, - }, - }, - } - want.Metrics[0].Timeseries[0].Points[0].Timestamp = mds[0].Metrics[0].Timeseries[0].Points[0].Timestamp - doCompare("scrape-externalLabels", t, want, mds[0]) + }), + }) }