From 1c4423a12eb168ba559a45a2330cc514101c6642 Mon Sep 17 00:00:00 2001 From: Emmanuel T Odeke Date: Thu, 2 Sep 2021 20:09:55 -0700 Subject: [PATCH 01/26] receiver/prometheus: roundtrip Prometheus->Pdata direct conversion without OpenCensus Wire up and use the direct Prometheus->Pdata conversion end to end. With this change the receiver will no longer need OpenCensus. This change will involve more follow-ups that just migrate over the tests, because we don't want a super bloated/massive PR. Fixes #4892 Depends on PR open-telemetry/opentelemetry-collector#3694 Depends on PR open-telemetry/opentelemetry-collector#3695 --- .../internal/metricfamily.go | 413 ----- .../internal/metrics_adjuster.go | 317 ---- .../internal/metrics_adjuster_test.go | 449 ----- .../internal/metricsbuilder.go | 328 ---- .../internal/metricsbuilder_test.go | 1470 ----------------- .../prometheusreceiver/internal/ocastore.go | 27 +- .../internal/otlp_metricfamily.go | 148 +- .../internal/otlp_metricfamily_test.go | 253 +-- .../internal/otlp_metrics_adjuster.go | 435 +++++ .../internal/otlp_metrics_adjuster_test.go | 1006 +++++++++++ .../internal/otlp_metricsbuilder.go | 185 ++- .../internal/otlp_metricsbuilder_test.go | 1199 +++++++++++++- .../internal/otlp_transaction.go | 230 +++ ...ction_test.go => otlp_transaction_test.go} | 38 +- .../internal/prom_to_otlp.go | 4 +- .../internal/prom_to_otlp_test.go | 34 +- .../internal/staleness_end_to_end_test.go | 1 + .../internal/transaction.go | 255 --- .../prometheusreceiver/metrics_receiver.go | 7 +- .../metrics_reciever_external_labels_test.go | 74 +- 20 files changed, 3154 insertions(+), 3719 deletions(-) delete mode 100644 receiver/prometheusreceiver/internal/metricfamily.go delete mode 100644 receiver/prometheusreceiver/internal/metrics_adjuster.go delete mode 100644 receiver/prometheusreceiver/internal/metrics_adjuster_test.go delete mode 100644 receiver/prometheusreceiver/internal/metricsbuilder.go delete mode 100644 receiver/prometheusreceiver/internal/metricsbuilder_test.go create mode 100644 receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go create mode 100644 receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go create mode 100644 receiver/prometheusreceiver/internal/otlp_transaction.go rename receiver/prometheusreceiver/internal/{transaction_test.go => otlp_transaction_test.go} (72%) delete mode 100644 receiver/prometheusreceiver/internal/transaction.go diff --git a/receiver/prometheusreceiver/internal/metricfamily.go b/receiver/prometheusreceiver/internal/metricfamily.go deleted file mode 100644 index 9898a0b9f498c..0000000000000 --- a/receiver/prometheusreceiver/internal/metricfamily.go +++ /dev/null @@ -1,413 +0,0 @@ -// Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package internal - -import ( - "fmt" - "sort" - "strings" - - metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" - "github.com/prometheus/prometheus/pkg/labels" - "github.com/prometheus/prometheus/pkg/textparse" - "github.com/prometheus/prometheus/scrape" - "go.uber.org/zap" - "google.golang.org/protobuf/types/known/timestamppb" - "google.golang.org/protobuf/types/known/wrapperspb" -) - -// MetricFamily is unit which is corresponding to the metrics items which shared the same TYPE/UNIT/... metadata from -// a single scrape. -type MetricFamily interface { - Add(metricName string, ls labels.Labels, t int64, v float64) error - IsSameFamily(metricName string) bool - ToMetric() (*metricspb.Metric, int, int) -} - -type metricFamily struct { - name string - mtype metricspb.MetricDescriptor_Type - mc MetadataCache - droppedTimeseries int - labelKeys map[string]bool - labelKeysOrdered []string - metadata *scrape.MetricMetadata - groupOrders map[string]int - groups map[string]*metricGroup - intervalStartTimeMs int64 -} - -func newMetricFamily(metricName string, mc MetadataCache, logger *zap.Logger, intervalStartTimeMs int64) MetricFamily { - familyName := normalizeMetricName(metricName) - - // lookup metadata based on familyName - metadata, ok := mc.Metadata(familyName) - if !ok && metricName != familyName { - // use the original metricName as metricFamily - familyName = metricName - // perform a 2nd lookup with the original metric name. it can happen if there's a metric which is not histogram - // or summary, but ends with one of those _count/_sum suffixes - metadata, ok = mc.Metadata(metricName) - // still not found, this can happen when metric has no TYPE HINT - if !ok { - metadata.Metric = familyName - metadata.Type = textparse.MetricTypeUnknown - } - } else if !ok && isInternalMetric(metricName) { - metadata = defineInternalMetric(metricName, metadata, logger) - } - ocaMetricType := convToOCAMetricType(metadata.Type) - if ocaMetricType == metricspb.MetricDescriptor_UNSPECIFIED { - logger.Debug(fmt.Sprintf("Invalid metric : %s %+v", metricName, metadata)) - } - - return &metricFamily{ - name: familyName, - mtype: ocaMetricType, - mc: mc, - droppedTimeseries: 0, - labelKeys: make(map[string]bool), - labelKeysOrdered: make([]string, 0), - metadata: &metadata, - groupOrders: make(map[string]int), - groups: make(map[string]*metricGroup), - intervalStartTimeMs: intervalStartTimeMs, - } -} - -// Define manually the metadata of prometheus scrapper internal metrics -func defineInternalMetric(metricName string, metadata scrape.MetricMetadata, logger *zap.Logger) scrape.MetricMetadata { - if metadata.Metric != "" && metadata.Type != "" && metadata.Help != "" { - logger.Debug("Internal metric seems already fully defined") - return metadata - } - metadata.Metric = metricName - - switch metricName { - case scrapeUpMetricName: - metadata.Type = textparse.MetricTypeGauge - metadata.Help = "The scraping was successful" - case "scrape_duration_seconds": - metadata.Unit = "seconds" - metadata.Type = textparse.MetricTypeGauge - metadata.Help = "Duration of the scrape" - case "scrape_samples_scraped": - metadata.Type = textparse.MetricTypeGauge - metadata.Help = "The number of samples the target exposed" - case "scrape_series_added": - metadata.Type = textparse.MetricTypeGauge - metadata.Help = "The approximate number of new series in this scrape" - case "scrape_samples_post_metric_relabeling": - metadata.Type = textparse.MetricTypeGauge - metadata.Help = "The number of samples remaining after metric relabeling was applied" - } - return metadata -} - -func (mf *metricFamily) IsSameFamily(metricName string) bool { - // trim known suffix if necessary - familyName := normalizeMetricName(metricName) - return mf.name == familyName || familyName != metricName && mf.name == metricName -} - -// updateLabelKeys is used to store all the label keys of a same metric family in observed order. since prometheus -// receiver removes any label with empty value before feeding it to an appender, in order to figure out all the labels -// from the same metric family we will need to keep track of what labels have ever been observed. -func (mf *metricFamily) updateLabelKeys(ls labels.Labels) { - for _, l := range ls { - if isUsefulLabel(mf.mtype, l.Name) { - if _, ok := mf.labelKeys[l.Name]; !ok { - mf.labelKeys[l.Name] = true - // use insertion sort to maintain order - i := sort.SearchStrings(mf.labelKeysOrdered, l.Name) - mf.labelKeysOrdered = append(mf.labelKeysOrdered, "") - copy(mf.labelKeysOrdered[i+1:], mf.labelKeysOrdered[i:]) - mf.labelKeysOrdered[i] = l.Name - } - } - } -} - -func (mf *metricFamily) isCumulativeType() bool { - return mf.mtype == metricspb.MetricDescriptor_CUMULATIVE_DOUBLE || - mf.mtype == metricspb.MetricDescriptor_CUMULATIVE_INT64 || - mf.mtype == metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION || - mf.mtype == metricspb.MetricDescriptor_SUMMARY -} - -func (mf *metricFamily) getGroupKey(ls labels.Labels) string { - mf.updateLabelKeys(ls) - return dpgSignature(mf.labelKeysOrdered, ls) -} - -// getGroups to return groups in insertion order -func (mf *metricFamily) getGroups() []*metricGroup { - groups := make([]*metricGroup, len(mf.groupOrders)) - for k, v := range mf.groupOrders { - groups[v] = mf.groups[k] - } - - return groups -} - -func (mf *metricFamily) loadMetricGroupOrCreate(groupKey string, ls labels.Labels, ts int64) *metricGroup { - mg, ok := mf.groups[groupKey] - if !ok { - mg = &metricGroup{ - family: mf, - ts: ts, - ls: ls, - complexValue: make([]*dataPoint, 0), - intervalStartTimeMs: mf.intervalStartTimeMs, - } - mf.groups[groupKey] = mg - // maintaining data insertion order is helpful to generate stable/reproducible metric output - mf.groupOrders[groupKey] = len(mf.groupOrders) - } - return mg -} - -func (mf *metricFamily) getLabelKeys() []*metricspb.LabelKey { - lks := make([]*metricspb.LabelKey, len(mf.labelKeysOrdered)) - for i, k := range mf.labelKeysOrdered { - lks[i] = &metricspb.LabelKey{Key: k} - } - return lks -} - -func (mf *metricFamily) Add(metricName string, ls labels.Labels, t int64, v float64) error { - groupKey := mf.getGroupKey(ls) - mg := mf.loadMetricGroupOrCreate(groupKey, ls, t) - switch mf.mtype { - case metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION: - fallthrough - case metricspb.MetricDescriptor_SUMMARY: - switch { - case strings.HasSuffix(metricName, metricsSuffixSum): - // always use the timestamp from sum (count is ok too), because the startTs from quantiles won't be reliable - // in cases like remote server restart - mg.ts = t - mg.sum = v - mg.hasSum = true - case strings.HasSuffix(metricName, metricsSuffixCount): - mg.count = v - mg.hasCount = true - default: - boundary, err := getBoundary(mf.mtype, ls) - if err != nil { - mf.droppedTimeseries++ - return err - } - mg.complexValue = append(mg.complexValue, &dataPoint{value: v, boundary: boundary}) - } - default: - mg.value = v - } - - return nil -} - -func (mf *metricFamily) ToMetric() (*metricspb.Metric, int, int) { - timeseries := make([]*metricspb.TimeSeries, 0, len(mf.groups)) - switch mf.mtype { - // not supported currently - // case metricspb.MetricDescriptor_GAUGE_DISTRIBUTION: - // return nil - case metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION: - for _, mg := range mf.getGroups() { - tss := mg.toDistributionTimeSeries(mf.labelKeysOrdered) - if tss != nil { - timeseries = append(timeseries, tss) - } else { - mf.droppedTimeseries++ - } - } - case metricspb.MetricDescriptor_SUMMARY: - for _, mg := range mf.getGroups() { - tss := mg.toSummaryTimeSeries(mf.labelKeysOrdered) - if tss != nil { - timeseries = append(timeseries, tss) - } else { - mf.droppedTimeseries++ - } - } - default: - for _, mg := range mf.getGroups() { - tss := mg.toDoubleValueTimeSeries(mf.labelKeysOrdered) - if tss != nil { - timeseries = append(timeseries, tss) - } else { - mf.droppedTimeseries++ - } - } - } - - // note: the total number of timeseries is the length of timeseries plus the number of dropped timeseries. - numTimeseries := len(timeseries) - if numTimeseries != 0 { - return &metricspb.Metric{ - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: mf.name, - Description: mf.metadata.Help, - Unit: heuristicalMetricAndKnownUnits(mf.name, mf.metadata.Unit), - Type: mf.mtype, - LabelKeys: mf.getLabelKeys(), - }, - Timeseries: timeseries, - }, - numTimeseries + mf.droppedTimeseries, - mf.droppedTimeseries - } - return nil, mf.droppedTimeseries, mf.droppedTimeseries -} - -type dataPoint struct { - value float64 - boundary float64 -} - -// metricGroup, represents a single metric of a metric family. for example a histogram metric is usually represent by -// a couple data complexValue (buckets and count/sum), a group of a metric family always share a same set of tags. for -// simple types like counter and gauge, each data point is a group of itself -type metricGroup struct { - family *metricFamily - ts int64 - ls labels.Labels - count float64 - hasCount bool - sum float64 - hasSum bool - value float64 - complexValue []*dataPoint - intervalStartTimeMs int64 -} - -func (mg *metricGroup) sortPoints() { - sort.Slice(mg.complexValue, func(i, j int) bool { - return mg.complexValue[i].boundary < mg.complexValue[j].boundary - }) -} - -func (mg *metricGroup) toDistributionTimeSeries(orderedLabelKeys []string) *metricspb.TimeSeries { - if !(mg.hasCount) || len(mg.complexValue) == 0 { - return nil - } - mg.sortPoints() - // for OCAgent Proto, the bounds won't include +inf - bounds := make([]float64, len(mg.complexValue)-1) - buckets := make([]*metricspb.DistributionValue_Bucket, len(mg.complexValue)) - - for i := 0; i < len(mg.complexValue); i++ { - if i != len(mg.complexValue)-1 { - // not need to add +inf as bound to oc proto - bounds[i] = mg.complexValue[i].boundary - } - adjustedCount := mg.complexValue[i].value - if i != 0 { - adjustedCount -= mg.complexValue[i-1].value - } - buckets[i] = &metricspb.DistributionValue_Bucket{Count: int64(adjustedCount)} - } - - dv := &metricspb.DistributionValue{ - BucketOptions: &metricspb.DistributionValue_BucketOptions{ - Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ - Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ - Bounds: bounds, - }, - }, - }, - Count: int64(mg.count), - Sum: mg.sum, - Buckets: buckets, - // SumOfSquaredDeviation: // there's no way to compute this value from prometheus data - } - - return &metricspb.TimeSeries{ - StartTimestamp: timestampFromMs(mg.ts), - LabelValues: populateLabelValues(orderedLabelKeys, mg.ls), - Points: []*metricspb.Point{ - { - Timestamp: timestampFromMs(mg.ts), - Value: &metricspb.Point_DistributionValue{DistributionValue: dv}, - }, - }, - } -} - -func (mg *metricGroup) toSummaryTimeSeries(orderedLabelKeys []string) *metricspb.TimeSeries { - // expecting count to be provided, however, in the following two cases, they can be missed. - // 1. data is corrupted - // 2. ignored by startValue evaluation - if !(mg.hasCount) { - return nil - } - mg.sortPoints() - percentiles := make([]*metricspb.SummaryValue_Snapshot_ValueAtPercentile, len(mg.complexValue)) - for i, p := range mg.complexValue { - percentiles[i] = - &metricspb.SummaryValue_Snapshot_ValueAtPercentile{Percentile: p.boundary * 100, Value: p.value} - } - - // allow percentiles to be nil when no data provided from prometheus - var snapshot *metricspb.SummaryValue_Snapshot - if len(percentiles) != 0 { - snapshot = &metricspb.SummaryValue_Snapshot{ - PercentileValues: percentiles, - } - } - - // Based on the summary description from https://prometheus.io/docs/concepts/metric_types/#summary - // the quantiles are calculated over a sliding time window, however, the count is the total count of - // observations and the corresponding sum is a sum of all observed values, thus the sum and count used - // at the global level of the metricspb.SummaryValue - - summaryValue := &metricspb.SummaryValue{ - Sum: &wrapperspb.DoubleValue{Value: mg.sum}, - Count: &wrapperspb.Int64Value{Value: int64(mg.count)}, - Snapshot: snapshot, - } - return &metricspb.TimeSeries{ - StartTimestamp: timestampFromMs(mg.ts), - LabelValues: populateLabelValues(orderedLabelKeys, mg.ls), - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(mg.ts), Value: &metricspb.Point_SummaryValue{SummaryValue: summaryValue}}, - }, - } -} - -func (mg *metricGroup) toDoubleValueTimeSeries(orderedLabelKeys []string) *metricspb.TimeSeries { - var startTs *timestamppb.Timestamp - // gauge/undefined types has no start time - if mg.family.isCumulativeType() { - startTs = timestampFromMs(mg.intervalStartTimeMs) - } - - return &metricspb.TimeSeries{ - StartTimestamp: startTs, - Points: []*metricspb.Point{{Timestamp: timestampFromMs(mg.ts), Value: &metricspb.Point_DoubleValue{DoubleValue: mg.value}}}, - LabelValues: populateLabelValues(orderedLabelKeys, mg.ls), - } -} - -func populateLabelValues(orderedKeys []string, ls labels.Labels) []*metricspb.LabelValue { - lvs := make([]*metricspb.LabelValue, len(orderedKeys)) - lmap := ls.Map() - for i, k := range orderedKeys { - value := lmap[k] - lvs[i] = &metricspb.LabelValue{Value: value, HasValue: value != ""} - } - return lvs -} diff --git a/receiver/prometheusreceiver/internal/metrics_adjuster.go b/receiver/prometheusreceiver/internal/metrics_adjuster.go deleted file mode 100644 index 14d69f7e0f612..0000000000000 --- a/receiver/prometheusreceiver/internal/metrics_adjuster.go +++ /dev/null @@ -1,317 +0,0 @@ -// Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package internal - -import ( - "fmt" - "strings" - "sync" - "time" - - metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" - "go.uber.org/zap" -) - -// Notes on garbage collection (gc): -// -// Job-level gc: -// The Prometheus receiver will likely execute in a long running service whose lifetime may exceed -// the lifetimes of many of the jobs that it is collecting from. In order to keep the JobsMap from -// leaking memory for entries of no-longer existing jobs, the JobsMap needs to remove entries that -// haven't been accessed for a long period of time. -// -// Timeseries-level gc: -// Some jobs that the Prometheus receiver is collecting from may export timeseries based on metrics -// from other jobs (e.g. cAdvisor). In order to keep the timeseriesMap from leaking memory for entries -// of no-longer existing jobs, the timeseriesMap for each job needs to remove entries that haven't -// been accessed for a long period of time. -// -// The gc strategy uses a standard mark-and-sweep approach - each time a timeseriesMap is accessed, -// it is marked. Similarly, each time a timeseriesinfo is accessed, it is also marked. -// -// At the end of each JobsMap.get(), if the last time the JobsMap was gc'd exceeds the 'gcInterval', -// the JobsMap is locked and any timeseriesMaps that are unmarked are removed from the JobsMap -// otherwise the timeseriesMap is gc'd -// -// The gc for the timeseriesMap is straightforward - the map is locked and, for each timeseriesinfo -// in the map, if it has not been marked, it is removed otherwise it is unmarked. -// -// Alternative Strategies -// 1. If the job-level gc doesn't run often enough, or runs too often, a separate go routine can -// be spawned at JobMap creation time that gc's at periodic intervals. This approach potentially -// adds more contention and latency to each scrape so the current approach is used. Note that -// the go routine will need to be cancelled upon Shutdown(). -// 2. If the gc of each timeseriesMap during the gc of the JobsMap causes too much contention, -// the gc of timeseriesMaps can be moved to the end of MetricsAdjuster().AdjustMetrics(). This -// approach requires adding 'lastGC' Time and (potentially) a gcInterval duration to -// timeseriesMap so the current approach is used instead. - -// timeseriesinfo contains the information necessary to adjust from the initial point and to detect -// resets. -type timeseriesinfo struct { - mark bool - initial *metricspb.TimeSeries - previous *metricspb.TimeSeries -} - -// timeseriesMap maps from a timeseries instance (metric * label values) to the timeseries info for -// the instance. -type timeseriesMap struct { - sync.RWMutex - mark bool - tsiMap map[string]*timeseriesinfo -} - -// Get the timeseriesinfo for the timeseries associated with the metric and label values. -func (tsm *timeseriesMap) get( - metric *metricspb.Metric, values []*metricspb.LabelValue) *timeseriesinfo { - name := metric.GetMetricDescriptor().GetName() - sig := getTimeseriesSignature(name, values) - tsi, ok := tsm.tsiMap[sig] - if !ok { - tsi = ×eriesinfo{} - tsm.tsiMap[sig] = tsi - } - tsm.mark = true - tsi.mark = true - return tsi -} - -// Remove timeseries that have aged out. -func (tsm *timeseriesMap) gc() { - tsm.Lock() - defer tsm.Unlock() - // this shouldn't happen under the current gc() strategy - if !tsm.mark { - return - } - for ts, tsi := range tsm.tsiMap { - if !tsi.mark { - delete(tsm.tsiMap, ts) - } else { - tsi.mark = false - } - } - tsm.mark = false -} - -func newTimeseriesMap() *timeseriesMap { - return ×eriesMap{mark: true, tsiMap: map[string]*timeseriesinfo{}} -} - -// Create a unique timeseries signature consisting of the metric name and label values. -func getTimeseriesSignature(name string, values []*metricspb.LabelValue) string { - labelValues := make([]string, 0, len(values)) - for _, label := range values { - if label.GetValue() != "" { - labelValues = append(labelValues, label.GetValue()) - } - } - return fmt.Sprintf("%s,%s", name, strings.Join(labelValues, ",")) -} - -// JobsMap maps from a job instance to a map of timeseries instances for the job. -type JobsMap struct { - sync.RWMutex - gcInterval time.Duration - lastGC time.Time - jobsMap map[string]*timeseriesMap -} - -// NewJobsMap creates a new (empty) JobsMap. -func NewJobsMap(gcInterval time.Duration) *JobsMap { - return &JobsMap{gcInterval: gcInterval, lastGC: time.Now(), jobsMap: make(map[string]*timeseriesMap)} -} - -// Remove jobs and timeseries that have aged out. -func (jm *JobsMap) gc() { - jm.Lock() - defer jm.Unlock() - // once the structure is locked, confirm that gc() is still necessary - if time.Since(jm.lastGC) > jm.gcInterval { - for sig, tsm := range jm.jobsMap { - tsm.RLock() - tsmNotMarked := !tsm.mark - tsm.RUnlock() - if tsmNotMarked { - delete(jm.jobsMap, sig) - } else { - tsm.gc() - } - } - jm.lastGC = time.Now() - } -} - -func (jm *JobsMap) maybeGC() { - // speculatively check if gc() is necessary, recheck once the structure is locked - jm.RLock() - defer jm.RUnlock() - if time.Since(jm.lastGC) > jm.gcInterval { - go jm.gc() - } -} - -func (jm *JobsMap) get(job, instance string) *timeseriesMap { - sig := job + ":" + instance - jm.RLock() - tsm, ok := jm.jobsMap[sig] - jm.RUnlock() - defer jm.maybeGC() - if ok { - return tsm - } - jm.Lock() - defer jm.Unlock() - tsm2, ok2 := jm.jobsMap[sig] - if ok2 { - return tsm2 - } - tsm2 = newTimeseriesMap() - jm.jobsMap[sig] = tsm2 - return tsm2 -} - -// MetricsAdjuster takes a map from a metric instance to the initial point in the metrics instance -// and provides AdjustMetrics, which takes a sequence of metrics and adjust their start times based on -// the initial points. -type MetricsAdjuster struct { - tsm *timeseriesMap - logger *zap.Logger -} - -// NewMetricsAdjuster is a constructor for MetricsAdjuster. -func NewMetricsAdjuster(tsm *timeseriesMap, logger *zap.Logger) *MetricsAdjuster { - return &MetricsAdjuster{ - tsm: tsm, - logger: logger, - } -} - -// AdjustMetrics takes a sequence of metrics and adjust their start times based on the initial and -// previous points in the timeseriesMap. -// Returns the total number of timeseries that had reset start times. -func (ma *MetricsAdjuster) AdjustMetrics(metrics []*metricspb.Metric) ([]*metricspb.Metric, int) { - var adjusted = make([]*metricspb.Metric, 0, len(metrics)) - resets := 0 - ma.tsm.Lock() - defer ma.tsm.Unlock() - for _, metric := range metrics { - d := ma.adjustMetric(metric) - resets += d - adjusted = append(adjusted, metric) - } - return adjusted, resets -} - -// Returns the number of timeseries with reset start times. -// -// Types of metrics returned supported by prometheus: -// - MetricDescriptor_GAUGE_DOUBLE -// - MetricDescriptor_GAUGE_DISTRIBUTION -// - MetricDescriptor_CUMULATIVE_DOUBLE -// - MetricDescriptor_CUMULATIVE_DISTRIBUTION -// - MetricDescriptor_SUMMARY -func (ma *MetricsAdjuster) adjustMetric(metric *metricspb.Metric) int { - switch metric.MetricDescriptor.Type { - case metricspb.MetricDescriptor_GAUGE_DOUBLE, metricspb.MetricDescriptor_GAUGE_DISTRIBUTION: - // gauges don't need to be adjusted so no additional processing is necessary - return 0 - default: - return ma.adjustMetricTimeseries(metric) - } -} - -// Returns the number of timeseries that had reset start times. -func (ma *MetricsAdjuster) adjustMetricTimeseries(metric *metricspb.Metric) int { - resets := 0 - filtered := make([]*metricspb.TimeSeries, 0, len(metric.GetTimeseries())) - for _, current := range metric.GetTimeseries() { - tsi := ma.tsm.get(metric, current.GetLabelValues()) - if tsi.initial == nil || !ma.adjustTimeseries(metric.MetricDescriptor.Type, current, tsi.initial, tsi.previous) { - // initial || reset timeseries - tsi.initial = current - resets++ - } - tsi.previous = current - filtered = append(filtered, current) - } - metric.Timeseries = filtered - return resets -} - -// Returns true if 'current' was adjusted and false if 'current' is an the initial occurrence or a -// reset of the timeseries. -func (ma *MetricsAdjuster) adjustTimeseries(metricType metricspb.MetricDescriptor_Type, - current, initial, previous *metricspb.TimeSeries) bool { - if !ma.adjustPoints( - metricType, current.GetPoints(), initial.GetPoints(), previous.GetPoints()) { - return false - } - current.StartTimestamp = initial.StartTimestamp - return true -} - -func (ma *MetricsAdjuster) adjustPoints(metricType metricspb.MetricDescriptor_Type, - current, initial, previous []*metricspb.Point) bool { - if len(current) != 1 || len(initial) != 1 || len(previous) != 1 { - ma.logger.Info("Adjusting Points, all lengths should be 1", - zap.Int("len(current)", len(current)), zap.Int("len(initial)", len(initial)), zap.Int("len(previous)", len(previous))) - return true - } - return ma.isReset(metricType, current[0], previous[0]) -} - -func (ma *MetricsAdjuster) isReset(metricType metricspb.MetricDescriptor_Type, - current, previous *metricspb.Point) bool { - switch metricType { - case metricspb.MetricDescriptor_CUMULATIVE_DOUBLE: - if current.GetDoubleValue() < previous.GetDoubleValue() { - // reset detected - return false - } - case metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION: - // note: sum of squared deviation not currently supported - currentDist := current.GetDistributionValue() - previousDist := previous.GetDistributionValue() - if currentDist == nil || previousDist == nil { - return false - } - if currentDist.Count < previousDist.Count || currentDist.Sum < previousDist.Sum { - // reset detected - return false - } - case metricspb.MetricDescriptor_SUMMARY: - currentSummary := current.GetSummaryValue() - previousSummary := previous.GetSummaryValue() - if currentSummary == nil || previousSummary == nil { - return false - } - if (currentSummary.Count != nil && - previousSummary.Count != nil && - currentSummary.Count.GetValue() < previousSummary.Count.GetValue()) || - - (currentSummary.Sum != nil && - previousSummary.Sum != nil && - currentSummary.Sum.GetValue() < previousSummary.Sum.GetValue()) { - // reset detected - return false - } - default: - // this shouldn't happen - ma.logger.Info("Adjust - skipping unexpected point", zap.String("type", metricType.String())) - } - return true -} diff --git a/receiver/prometheusreceiver/internal/metrics_adjuster_test.go b/receiver/prometheusreceiver/internal/metrics_adjuster_test.go deleted file mode 100644 index e6e45e1b0ef16..0000000000000 --- a/receiver/prometheusreceiver/internal/metrics_adjuster_test.go +++ /dev/null @@ -1,449 +0,0 @@ -// Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package internal - -import ( - "testing" - "time" - - metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap" - - "github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/metricstestutil" -) - -func Test_gauge(t *testing.T) { - script := []*metricsAdjusterTest{{ - "Gauge: round 1 - gauge not adjusted", - []*metricspb.Metric{metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44)))}, - []*metricspb.Metric{metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44)))}, - 0, - }, { - "Gauge: round 2 - gauge not adjusted", - []*metricspb.Metric{metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.Double(t2Ms, 66)))}, - []*metricspb.Metric{metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.Double(t2Ms, 66)))}, - 0, - }, { - "Gauge: round 3 - value less than previous value - gauge is not adjusted", - []*metricspb.Metric{metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t3Ms, 55)))}, - []*metricspb.Metric{metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t3Ms, 55)))}, - 0, - }} - runScript(t, NewJobsMap(time.Minute).get("job", "0"), script) -} - -func Test_gaugeDistribution(t *testing.T) { - script := []*metricsAdjusterTest{{ - "GaugeDist: round 1 - gauge distribution not adjusted", - []*metricspb.Metric{metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7})))}, - []*metricspb.Metric{metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7})))}, - 0, - }, { - "GaugeDist: round 2 - gauge distribution not adjusted", - []*metricspb.Metric{metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.DistPt(t2Ms, bounds0, []int64{6, 5, 8, 11})))}, - []*metricspb.Metric{metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.DistPt(t2Ms, bounds0, []int64{6, 5, 8, 11})))}, - 0, - }, { - "GaugeDist: round 3 - count/sum less than previous - gauge distribution not adjusted", - []*metricspb.Metric{metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.DistPt(t3Ms, bounds0, []int64{2, 0, 1, 5})))}, - []*metricspb.Metric{metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.DistPt(t3Ms, bounds0, []int64{2, 0, 1, 5})))}, - 0, - }} - runScript(t, NewJobsMap(time.Minute).get("job", "0"), script) -} - -func Test_cumulative(t *testing.T) { - script := []*metricsAdjusterTest{{ - "Cumulative: round 1 - initial instance, start time is established", - []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44)))}, - []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44)))}, - 1, - }, { - "Cumulative: round 2 - instance adjusted based on round 1", - []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.Double(t2Ms, 66)))}, - []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t2Ms, 66)))}, - 0, - }, { - "Cumulative: round 3 - instance reset (value less than previous value), start time is reset", - []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t3Ms, 55)))}, - []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t3Ms, 55)))}, - 1, - }, { - "Cumulative: round 4 - instance adjusted based on round 3", - []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.Double(t4Ms, 72)))}, - []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t4Ms, 72)))}, - 0, - }} - runScript(t, NewJobsMap(time.Minute).get("job", "0"), script) -} - -func Test_cumulativeDistribution(t *testing.T) { - script := []*metricsAdjusterTest{{ - "CumulativeDist: round 1 - initial instance, start time is established", - []*metricspb.Metric{metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7})))}, - []*metricspb.Metric{metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7})))}, - 1, - }, { - "CumulativeDist: round 2 - instance adjusted based on round 1", - []*metricspb.Metric{metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.DistPt(t2Ms, bounds0, []int64{6, 3, 4, 8})))}, - []*metricspb.Metric{metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t2Ms, bounds0, []int64{6, 3, 4, 8})))}, - 0, - }, { - "CumulativeDist: round 3 - instance reset (value less than previous value), start time is reset", - []*metricspb.Metric{metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.DistPt(t3Ms, bounds0, []int64{5, 3, 2, 7})))}, - []*metricspb.Metric{metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.DistPt(t3Ms, bounds0, []int64{5, 3, 2, 7})))}, - 1, - }, { - "CumulativeDist: round 4 - instance adjusted based on round 3", - []*metricspb.Metric{metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.DistPt(t4Ms, bounds0, []int64{7, 4, 2, 12})))}, - []*metricspb.Metric{metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.DistPt(t4Ms, bounds0, []int64{7, 4, 2, 12})))}, - 0, - }} - runScript(t, NewJobsMap(time.Minute).get("job", "0"), script) -} - -func Test_summary_no_count(t *testing.T) { - script := []*metricsAdjusterTest{{ - "Summary No Count: round 1 - initial instance, start time is established", - []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.SummPt(t1Ms, 10, 40, percent0, []float64{1, 5, 8})))}, - []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.SummPt(t1Ms, 10, 40, percent0, []float64{1, 5, 8})))}, - 1, - }, { - "Summary No Count: round 2 - instance adjusted based on round 1", - []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.SummPt(t2Ms, 15, 70, percent0, []float64{7, 44, 9})))}, - []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.SummPt(t2Ms, 15, 70, percent0, []float64{7, 44, 9})))}, - 0, - }, { - "Summary No Count: round 3 - instance reset (count less than previous), start time is reset", - []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.SummPt(t3Ms, 12, 66, percent0, []float64{3, 22, 5})))}, - []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.SummPt(t3Ms, 12, 66, percent0, []float64{3, 22, 5})))}, - 1, - }, { - "Summary No Count: round 4 - instance adjusted based on round 3", - []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.SummPt(t4Ms, 14, 96, percent0, []float64{9, 47, 8})))}, - []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.SummPt(t4Ms, 14, 96, percent0, []float64{9, 47, 8})))}, - 0, - }} - - for _, test := range script { - test.metrics[0].GetTimeseries()[0].Points[0].GetSummaryValue().Count = nil - test.adjusted[0].GetTimeseries()[0].Points[0].GetSummaryValue().Count = nil - } - - runScript(t, NewJobsMap(time.Minute).get("job", "0"), script) -} - -func Test_summary(t *testing.T) { - script := []*metricsAdjusterTest{{ - "Summary: round 1 - initial instance, start time is established", - []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.SummPt(t1Ms, 10, 40, percent0, []float64{1, 5, 8})))}, - []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.SummPt(t1Ms, 10, 40, percent0, []float64{1, 5, 8})))}, - 1, - }, { - "Summary: round 2 - instance adjusted based on round 1", - []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.SummPt(t2Ms, 15, 70, percent0, []float64{7, 44, 9})))}, - []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.SummPt(t2Ms, 15, 70, percent0, []float64{7, 44, 9})))}, - 0, - }, { - "Summary: round 3 - instance reset (count less than previous), start time is reset", - []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.SummPt(t3Ms, 12, 66, percent0, []float64{3, 22, 5})))}, - []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.SummPt(t3Ms, 12, 66, percent0, []float64{3, 22, 5})))}, - 1, - }, { - "Summary: round 4 - instance adjusted based on round 3", - []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.SummPt(t4Ms, 14, 96, percent0, []float64{9, 47, 8})))}, - []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.SummPt(t4Ms, 14, 96, percent0, []float64{9, 47, 8})))}, - 0, - }} - runScript(t, NewJobsMap(time.Minute).get("job", "0"), script) -} - -func Test_multiMetrics(t *testing.T) { - script := []*metricsAdjusterTest{{ - "MultiMetrics: round 1 - combined round 1 of individual metrics", - []*metricspb.Metric{ - metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44))), - metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7}))), - metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44))), - metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7}))), - metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.SummPt(t1Ms, 10, 40, percent0, []float64{1, 5, 8}))), - }, - []*metricspb.Metric{ - metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44))), - metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7}))), - metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44))), - metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7}))), - metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.SummPt(t1Ms, 10, 40, percent0, []float64{1, 5, 8}))), - }, - 3, - }, { - "MultiMetrics: round 2 - combined round 2 of individual metrics", - []*metricspb.Metric{ - metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.Double(t2Ms, 66))), - metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.DistPt(t2Ms, bounds0, []int64{6, 5, 8, 11}))), - metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.Double(t2Ms, 66))), - metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.DistPt(t2Ms, bounds0, []int64{6, 3, 4, 8}))), - metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.SummPt(t2Ms, 15, 70, percent0, []float64{7, 44, 9}))), - }, - []*metricspb.Metric{ - metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.Double(t2Ms, 66))), - metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.DistPt(t2Ms, bounds0, []int64{6, 5, 8, 11}))), - metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t2Ms, 66))), - metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t2Ms, bounds0, []int64{6, 3, 4, 8}))), - metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.SummPt(t2Ms, 15, 70, percent0, []float64{7, 44, 9}))), - }, - 0, - }, { - "MultiMetrics: round 3 - combined round 3 of individual metrics", - []*metricspb.Metric{ - metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t3Ms, 55))), - metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.DistPt(t3Ms, bounds0, []int64{2, 0, 1, 5}))), - metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t3Ms, 55))), - metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.DistPt(t3Ms, bounds0, []int64{5, 3, 2, 7}))), - metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.SummPt(t3Ms, 12, 66, percent0, []float64{3, 22, 5}))), - }, - []*metricspb.Metric{ - metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t3Ms, 55))), - metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.DistPt(t3Ms, bounds0, []int64{2, 0, 1, 5}))), - metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t3Ms, 55))), - metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.DistPt(t3Ms, bounds0, []int64{5, 3, 2, 7}))), - metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.SummPt(t3Ms, 12, 66, percent0, []float64{3, 22, 5}))), - }, - 3, - }, { - "MultiMetrics: round 4 - combined round 4 of individual metrics", - []*metricspb.Metric{ - metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.Double(t4Ms, 72))), - metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.DistPt(t4Ms, bounds0, []int64{7, 4, 2, 12}))), - metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.SummPt(t4Ms, 14, 96, percent0, []float64{9, 47, 8}))), - }, - []*metricspb.Metric{ - metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t4Ms, 72))), - metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.DistPt(t4Ms, bounds0, []int64{7, 4, 2, 12}))), - metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.SummPt(t4Ms, 14, 96, percent0, []float64{9, 47, 8}))), - }, - 0, - }} - runScript(t, NewJobsMap(time.Minute).get("job", "0"), script) -} - -func Test_multiTimeseries(t *testing.T) { - script := []*metricsAdjusterTest{{ - "MultiTimeseries: round 1 - initial first instance, start time is established", - []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44)))}, - []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44)))}, - 1, - }, { - "MultiTimeseries: round 2 - first instance adjusted based on round 1, initial second instance", - []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.Double(t2Ms, 66)), metricstestutil.Timeseries(t2Ms, v10v20, metricstestutil.Double(t2Ms, 20)))}, - []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t2Ms, 66)), metricstestutil.Timeseries(t2Ms, v10v20, metricstestutil.Double(t2Ms, 20)))}, - 1, - }, { - "MultiTimeseries: round 3 - first instance adjusted based on round 1, second based on round 2", - []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t3Ms, 88)), metricstestutil.Timeseries(t3Ms, v10v20, metricstestutil.Double(t3Ms, 49)))}, - []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t3Ms, 88)), metricstestutil.Timeseries(t2Ms, v10v20, metricstestutil.Double(t3Ms, 49)))}, - 0, - }, { - "MultiTimeseries: round 4 - first instance reset, second instance adjusted based on round 2, initial third instance", - []*metricspb.Metric{ - metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.Double(t4Ms, 87)), metricstestutil.Timeseries(t4Ms, v10v20, metricstestutil.Double(t4Ms, 57)), metricstestutil.Timeseries(t4Ms, v100v200, metricstestutil.Double(t4Ms, 10)))}, - []*metricspb.Metric{ - metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.Double(t4Ms, 87)), metricstestutil.Timeseries(t2Ms, v10v20, metricstestutil.Double(t4Ms, 57)), metricstestutil.Timeseries(t4Ms, v100v200, metricstestutil.Double(t4Ms, 10)))}, - 2, - }, { - "MultiTimeseries: round 5 - first instance adjusted based on round 4, second on round 2, third on round 4", - []*metricspb.Metric{ - metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t5Ms, v1v2, metricstestutil.Double(t5Ms, 90)), metricstestutil.Timeseries(t5Ms, v10v20, metricstestutil.Double(t5Ms, 65)), metricstestutil.Timeseries(t5Ms, v100v200, metricstestutil.Double(t5Ms, 22)))}, - []*metricspb.Metric{ - metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.Double(t5Ms, 90)), metricstestutil.Timeseries(t2Ms, v10v20, metricstestutil.Double(t5Ms, 65)), metricstestutil.Timeseries(t4Ms, v100v200, metricstestutil.Double(t5Ms, 22)))}, - 0, - }} - runScript(t, NewJobsMap(time.Minute).get("job", "0"), script) -} - -func Test_emptyLabels(t *testing.T) { - script := []*metricsAdjusterTest{{ - "EmptyLabels: round 1 - initial instance, implicitly empty labels, start time is established", - []*metricspb.Metric{metricstestutil.Cumulative(c1, []string{}, metricstestutil.Timeseries(t1Ms, []string{}, metricstestutil.Double(t1Ms, 44)))}, - []*metricspb.Metric{metricstestutil.Cumulative(c1, []string{}, metricstestutil.Timeseries(t1Ms, []string{}, metricstestutil.Double(t1Ms, 44)))}, - 1, - }, { - "EmptyLabels: round 2 - instance adjusted based on round 1", - []*metricspb.Metric{metricstestutil.Cumulative(c1, []string{}, metricstestutil.Timeseries(t2Ms, []string{}, metricstestutil.Double(t2Ms, 66)))}, - []*metricspb.Metric{metricstestutil.Cumulative(c1, []string{}, metricstestutil.Timeseries(t1Ms, []string{}, metricstestutil.Double(t2Ms, 66)))}, - 0, - }, { - "EmptyLabels: round 3 - one explicitly empty label, instance adjusted based on round 1", - []*metricspb.Metric{metricstestutil.Cumulative(c1, k1, metricstestutil.Timeseries(t3Ms, []string{""}, metricstestutil.Double(t3Ms, 77)))}, - []*metricspb.Metric{metricstestutil.Cumulative(c1, k1, metricstestutil.Timeseries(t1Ms, []string{""}, metricstestutil.Double(t3Ms, 77)))}, - 0, - }, { - "EmptyLabels: round 4 - three explicitly empty labels, instance adjusted based on round 1", - []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2k3, metricstestutil.Timeseries(t3Ms, []string{"", "", ""}, metricstestutil.Double(t3Ms, 88)))}, - []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2k3, metricstestutil.Timeseries(t1Ms, []string{"", "", ""}, metricstestutil.Double(t3Ms, 88)))}, - 0, - }} - runScript(t, NewJobsMap(time.Minute).get("job", "0"), script) -} - -func Test_tsGC(t *testing.T) { - script1 := []*metricsAdjusterTest{{ - "TsGC: round 1 - initial instances, start time is established", - []*metricspb.Metric{ - metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44)), metricstestutil.Timeseries(t1Ms, v10v20, metricstestutil.Double(t1Ms, 20))), - metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7})), metricstestutil.Timeseries(t1Ms, v10v20, metricstestutil.DistPt(t1Ms, bounds0, []int64{40, 20, 30, 70}))), - }, - []*metricspb.Metric{ - metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44)), metricstestutil.Timeseries(t1Ms, v10v20, metricstestutil.Double(t1Ms, 20))), - metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7})), metricstestutil.Timeseries(t1Ms, v10v20, metricstestutil.DistPt(t1Ms, bounds0, []int64{40, 20, 30, 70}))), - }, - 4, - }} - - script2 := []*metricsAdjusterTest{{ - "TsGC: round 2 - metrics first timeseries adjusted based on round 2, second timeseries not updated", - []*metricspb.Metric{ - metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.Double(t2Ms, 88))), - metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.DistPt(t2Ms, bounds0, []int64{8, 7, 9, 14}))), - }, - []*metricspb.Metric{ - metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t2Ms, 88))), - metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t2Ms, bounds0, []int64{8, 7, 9, 14}))), - }, - 0, - }} - - script3 := []*metricsAdjusterTest{{ - "TsGC: round 3 - metrics first timeseries adjusted based on round 2, second timeseries empty due to timeseries gc()", - []*metricspb.Metric{ - metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t3Ms, 99)), metricstestutil.Timeseries(t3Ms, v10v20, metricstestutil.Double(t3Ms, 80))), - metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.DistPt(t3Ms, bounds0, []int64{9, 8, 10, 15})), metricstestutil.Timeseries(t3Ms, v10v20, metricstestutil.DistPt(t3Ms, bounds0, []int64{55, 66, 33, 77}))), - }, - []*metricspb.Metric{ - metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t3Ms, 99)), metricstestutil.Timeseries(t3Ms, v10v20, metricstestutil.Double(t3Ms, 80))), - metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t3Ms, bounds0, []int64{9, 8, 10, 15})), metricstestutil.Timeseries(t3Ms, v10v20, metricstestutil.DistPt(t3Ms, bounds0, []int64{55, 66, 33, 77}))), - }, - 2, - }} - - jobsMap := NewJobsMap(time.Minute) - - // run round 1 - runScript(t, jobsMap.get("job", "0"), script1) - // gc the tsmap, unmarking all entries - jobsMap.get("job", "0").gc() - // run round 2 - update metrics first timeseries only - runScript(t, jobsMap.get("job", "0"), script2) - // gc the tsmap, collecting umarked entries - jobsMap.get("job", "0").gc() - // run round 3 - verify that metrics second timeseries have been gc'd - runScript(t, jobsMap.get("job", "0"), script3) -} - -func Test_jobGC(t *testing.T) { - job1Script1 := []*metricsAdjusterTest{{ - "JobGC: job 1, round 1 - initial instances, adjusted should be empty", - []*metricspb.Metric{ - metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44)), metricstestutil.Timeseries(t1Ms, v10v20, metricstestutil.Double(t1Ms, 20))), - metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7})), metricstestutil.Timeseries(t1Ms, v10v20, metricstestutil.DistPt(t1Ms, bounds0, []int64{40, 20, 30, 70}))), - }, - []*metricspb.Metric{ - metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44)), metricstestutil.Timeseries(t1Ms, v10v20, metricstestutil.Double(t1Ms, 20))), - metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7})), metricstestutil.Timeseries(t1Ms, v10v20, metricstestutil.DistPt(t1Ms, bounds0, []int64{40, 20, 30, 70}))), - }, - 4, - }} - - job2Script1 := []*metricsAdjusterTest{{ - "JobGC: job2, round 1 - no metrics adjusted, just trigger gc", - []*metricspb.Metric{}, - []*metricspb.Metric{}, - 0, - }} - - job1Script2 := []*metricsAdjusterTest{{ - "JobGC: job 1, round 2 - metrics timeseries empty due to job-level gc", - []*metricspb.Metric{ - metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.Double(t4Ms, 99)), metricstestutil.Timeseries(t4Ms, v10v20, metricstestutil.Double(t4Ms, 80))), - metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.DistPt(t4Ms, bounds0, []int64{9, 8, 10, 15})), metricstestutil.Timeseries(t4Ms, v10v20, metricstestutil.DistPt(t4Ms, bounds0, []int64{55, 66, 33, 77}))), - }, - []*metricspb.Metric{ - metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.Double(t4Ms, 99)), metricstestutil.Timeseries(t4Ms, v10v20, metricstestutil.Double(t4Ms, 80))), - metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.DistPt(t4Ms, bounds0, []int64{9, 8, 10, 15})), metricstestutil.Timeseries(t4Ms, v10v20, metricstestutil.DistPt(t4Ms, bounds0, []int64{55, 66, 33, 77}))), - }, - 4, - }} - - gcInterval := 10 * time.Millisecond - jobsMap := NewJobsMap(gcInterval) - - // run job 1, round 1 - all entries marked - runScript(t, jobsMap.get("job", "0"), job1Script1) - // sleep longer than gcInterval to enable job gc in the next run - time.Sleep(2 * gcInterval) - // run job 2, round1 - trigger job gc, unmarking all entries - runScript(t, jobsMap.get("job", "1"), job2Script1) - // sleep longer than gcInterval to enable job gc in the next run - time.Sleep(2 * gcInterval) - // re-run job 2, round1 - trigger job gc, removing unmarked entries - runScript(t, jobsMap.get("job", "1"), job2Script1) - // ensure that at least one jobsMap.gc() completed - jobsMap.gc() - // run job 1, round 2 - verify that all job 1 timeseries have been gc'd - runScript(t, jobsMap.get("job", "0"), job1Script2) -} - -var ( - g1 = "gauge1" - gd1 = "gaugedist1" - c1 = "cumulative1" - cd1 = "cumulativedist1" - s1 = "summary1" - k1 = []string{"k1"} - k1k2 = []string{"k1", "k2"} - k1k2k3 = []string{"k1", "k2", "k3"} - v1v2 = []string{"v1", "v2"} - v10v20 = []string{"v10", "v20"} - v100v200 = []string{"v100", "v200"} - bounds0 = []float64{1, 2, 4} - percent0 = []float64{10, 50, 90} - t1Ms = time.Unix(0, 1000000) - t2Ms = time.Unix(0, 2000000) - t3Ms = time.Unix(0, 3000000) - t4Ms = time.Unix(0, 5000000) - t5Ms = time.Unix(0, 5000000) -) - -type metricsAdjusterTest struct { - description string - metrics []*metricspb.Metric - adjusted []*metricspb.Metric - resets int -} - -func runScript(t *testing.T, tsm *timeseriesMap, script []*metricsAdjusterTest) { - l := zap.NewNop() - t.Cleanup(func() { require.NoError(t, l.Sync()) }) // flushes buffer, if any - ma := NewMetricsAdjuster(tsm, l) - - for _, test := range script { - expectedResets := test.resets - adjusted, resets := ma.AdjustMetrics(test.metrics) - assert.EqualValuesf(t, test.adjusted, adjusted, "Test: %v - expected: %v, actual: %v", test.description, test.adjusted, adjusted) - assert.Equalf(t, expectedResets, resets, "Test: %v", test.description) - } -} diff --git a/receiver/prometheusreceiver/internal/metricsbuilder.go b/receiver/prometheusreceiver/internal/metricsbuilder.go deleted file mode 100644 index e2c7fddf5ac63..0000000000000 --- a/receiver/prometheusreceiver/internal/metricsbuilder.go +++ /dev/null @@ -1,328 +0,0 @@ -// Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package internal - -import ( - "errors" - "fmt" - "regexp" - "sort" - "strconv" - "strings" - - metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/pkg/labels" - "github.com/prometheus/prometheus/pkg/textparse" - "go.uber.org/zap" - "google.golang.org/protobuf/types/known/timestamppb" -) - -const ( - metricsSuffixCount = "_count" - metricsSuffixBucket = "_bucket" - metricsSuffixSum = "_sum" - metricSuffixTotal = "_total" - startTimeMetricName = "process_start_time_seconds" - scrapeUpMetricName = "up" -) - -var ( - trimmableSuffixes = []string{metricsSuffixBucket, metricsSuffixCount, metricsSuffixSum, metricSuffixTotal} - errNoDataToBuild = errors.New("there's no data to build") - errNoBoundaryLabel = errors.New("given metricType has no BucketLabel or QuantileLabel") - errEmptyBoundaryLabel = errors.New("BucketLabel or QuantileLabel is empty") -) - -type metricBuilder struct { - hasData bool - hasInternalMetric bool - mc MetadataCache - metrics []*metricspb.Metric - numTimeseries int - droppedTimeseries int - useStartTimeMetric bool - startTimeMetricRegex *regexp.Regexp - startTime float64 - intervalStartTimeMs int64 - logger *zap.Logger - currentMf MetricFamily -} - -// newMetricBuilder creates a MetricBuilder which is allowed to feed all the datapoints from a single prometheus -// scraped page by calling its AddDataPoint function, and turn them into an opencensus data.MetricsData object -// by calling its Build function -func newMetricBuilder(mc MetadataCache, useStartTimeMetric bool, startTimeMetricRegex string, logger *zap.Logger, intervalStartTimeMs int64) *metricBuilder { - var regex *regexp.Regexp - if startTimeMetricRegex != "" { - regex, _ = regexp.Compile(startTimeMetricRegex) - } - return &metricBuilder{ - mc: mc, - metrics: make([]*metricspb.Metric, 0), - logger: logger, - numTimeseries: 0, - droppedTimeseries: 0, - useStartTimeMetric: useStartTimeMetric, - startTimeMetricRegex: regex, - intervalStartTimeMs: intervalStartTimeMs, - } -} - -func (b *metricBuilder) matchStartTimeMetric(metricName string) bool { - if b.startTimeMetricRegex != nil { - return b.startTimeMetricRegex.MatchString(metricName) - } - - return metricName == startTimeMetricName -} - -// AddDataPoint is for feeding prometheus data complexValue in its processing order -func (b *metricBuilder) AddDataPoint(ls labels.Labels, t int64, v float64) error { - // Any datapoint with duplicate labels MUST be rejected per: - // * https://github.com/open-telemetry/wg-prometheus/issues/44 - // * https://github.com/open-telemetry/opentelemetry-collector/issues/3407 - // as Prometheus rejects such too as of version 2.16.0, released on 2020-02-13. - seen := make(map[string]bool) - dupLabels := make([]string, 0, len(ls)) - for _, label := range ls { - if _, ok := seen[label.Name]; ok { - dupLabels = append(dupLabels, label.Name) - } - seen[label.Name] = true - } - if len(dupLabels) != 0 { - sort.Strings(dupLabels) - return fmt.Errorf("invalid sample: non-unique label names: %q", dupLabels) - } - - metricName := ls.Get(model.MetricNameLabel) - switch { - case metricName == "": - b.numTimeseries++ - b.droppedTimeseries++ - return errMetricNameNotFound - case isInternalMetric(metricName): - b.hasInternalMetric = true - lm := ls.Map() - // See https://www.prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series - // up: 1 if the instance is healthy, i.e. reachable, or 0 if the scrape failed. - if metricName == scrapeUpMetricName && v != 1.0 { - if v == 0.0 { - b.logger.Warn("Failed to scrape Prometheus endpoint", - zap.Int64("scrape_timestamp", t), - zap.String("target_labels", fmt.Sprintf("%v", lm))) - } else { - b.logger.Warn("The 'up' metric contains invalid value", - zap.Float64("value", v), - zap.Int64("scrape_timestamp", t), - zap.String("target_labels", fmt.Sprintf("%v", lm))) - } - } - case b.useStartTimeMetric && b.matchStartTimeMetric(metricName): - b.startTime = v - } - - b.hasData = true - - if b.currentMf != nil && !b.currentMf.IsSameFamily(metricName) { - m, ts, dts := b.currentMf.ToMetric() - b.numTimeseries += ts - b.droppedTimeseries += dts - if m != nil { - b.metrics = append(b.metrics, m) - } - b.currentMf = newMetricFamily(metricName, b.mc, b.logger, b.intervalStartTimeMs) - } else if b.currentMf == nil { - b.currentMf = newMetricFamily(metricName, b.mc, b.logger, b.intervalStartTimeMs) - } - - return b.currentMf.Add(metricName, ls, t, v) -} - -// Build an opencensus data.MetricsData based on all added data complexValue. -// The only error returned by this function is errNoDataToBuild. -func (b *metricBuilder) Build() ([]*metricspb.Metric, int, int, error) { - if !b.hasData { - if b.hasInternalMetric { - return make([]*metricspb.Metric, 0), 0, 0, nil - } - return nil, 0, 0, errNoDataToBuild - } - - if b.currentMf != nil { - m, ts, dts := b.currentMf.ToMetric() - b.numTimeseries += ts - b.droppedTimeseries += dts - if m != nil { - b.metrics = append(b.metrics, m) - } - b.currentMf = nil - } - - return b.metrics, b.numTimeseries, b.droppedTimeseries, nil -} - -// TODO: move the following helper functions to a proper place, as they are not called directly in this go file - -func isUsefulLabel(mType metricspb.MetricDescriptor_Type, labelKey string) bool { - switch labelKey { - case model.MetricNameLabel, model.InstanceLabel, model.SchemeLabel, model.MetricsPathLabel, model.JobLabel: - return false - case model.BucketLabel: - return mType != metricspb.MetricDescriptor_GAUGE_DISTRIBUTION && - mType != metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION - case model.QuantileLabel: - return mType != metricspb.MetricDescriptor_SUMMARY - } - return true -} - -// dpgSignature is used to create a key for data complexValue belong to a same group of a metric family -func dpgSignature(orderedKnownLabelKeys []string, ls labels.Labels) string { - size := 0 - for _, k := range orderedKnownLabelKeys { - v := ls.Get(k) - if v == "" { - continue - } - // 2 enclosing quotes + 1 equality sign = 3 extra chars. - // Note: if any character in the label value requires escaping, - // we'll need more space than that, which will lead to some - // extra allocation. - size += 3 + len(k) + len(v) - } - sign := make([]byte, 0, size) - for _, k := range orderedKnownLabelKeys { - v := ls.Get(k) - if v == "" { - continue - } - sign = strconv.AppendQuote(sign, k+"="+v) - } - return string(sign) -} - -func normalizeMetricName(name string) string { - for _, s := range trimmableSuffixes { - if strings.HasSuffix(name, s) && name != s { - return strings.TrimSuffix(name, s) - } - } - return name -} - -func getBoundary(metricType metricspb.MetricDescriptor_Type, labels labels.Labels) (float64, error) { - labelName := "" - switch metricType { - case metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, - metricspb.MetricDescriptor_GAUGE_DISTRIBUTION: - labelName = model.BucketLabel - case metricspb.MetricDescriptor_SUMMARY: - labelName = model.QuantileLabel - default: - return 0, errNoBoundaryLabel - } - - v := labels.Get(labelName) - if v == "" { - return 0, errEmptyBoundaryLabel - } - - return strconv.ParseFloat(v, 64) -} - -func convToOCAMetricType(metricType textparse.MetricType) metricspb.MetricDescriptor_Type { - switch metricType { - case textparse.MetricTypeCounter: - // always use float64, as it's the internal data type used in prometheus - return metricspb.MetricDescriptor_CUMULATIVE_DOUBLE - // textparse.MetricTypeUnknown is converted to gauge by default to fix Prometheus untyped metrics from being dropped - case textparse.MetricTypeGauge, textparse.MetricTypeUnknown: - return metricspb.MetricDescriptor_GAUGE_DOUBLE - case textparse.MetricTypeHistogram: - return metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION - // dropping support for gaugehistogram for now until we have an official spec of its implementation - // a draft can be found in: https://docs.google.com/document/d/1KwV0mAXwwbvvifBvDKH_LU1YjyXE_wxCkHNoCGq1GX0/edit#heading=h.1cvzqd4ksd23 - // case textparse.MetricTypeGaugeHistogram: - // return metricspb.MetricDescriptor_GAUGE_DISTRIBUTION - case textparse.MetricTypeSummary: - return metricspb.MetricDescriptor_SUMMARY - default: - // including: textparse.MetricTypeInfo, textparse.MetricTypeStateset - return metricspb.MetricDescriptor_UNSPECIFIED - } -} - -/* - code borrowed from the original promreceiver -*/ - -func heuristicalMetricAndKnownUnits(metricName, parsedUnit string) string { - if parsedUnit != "" { - return parsedUnit - } - lastUnderscoreIndex := strings.LastIndex(metricName, "_") - if lastUnderscoreIndex <= 0 || lastUnderscoreIndex >= len(metricName)-1 { - return "" - } - - unit := "" - - supposedUnit := metricName[lastUnderscoreIndex+1:] - switch strings.ToLower(supposedUnit) { - case "millisecond", "milliseconds", "ms": - unit = "ms" - case "second", "seconds", "s": - unit = "s" - case "microsecond", "microseconds", "us": - unit = "us" - case "nanosecond", "nanoseconds", "ns": - unit = "ns" - case "byte", "bytes", "by": - unit = "By" - case "bit", "bits": - unit = "Bi" - case "kilogram", "kilograms", "kg": - unit = "kg" - case "gram", "grams", "g": - unit = "g" - case "meter", "meters", "metre", "metres", "m": - unit = "m" - case "kilometer", "kilometers", "kilometre", "kilometres", "km": - unit = "km" - case "milimeter", "milimeters", "milimetre", "milimetres", "mm": - unit = "mm" - case "nanogram", "ng", "nanograms": - unit = "ng" - } - - return unit -} - -func timestampFromMs(timeAtMs int64) *timestamppb.Timestamp { - secs, ns := timeAtMs/1e3, (timeAtMs%1e3)*1e6 - return ×tamppb.Timestamp{ - Seconds: secs, - Nanos: int32(ns), - } -} - -func isInternalMetric(metricName string) bool { - if metricName == scrapeUpMetricName || strings.HasPrefix(metricName, "scrape_") { - return true - } - return false -} diff --git a/receiver/prometheusreceiver/internal/metricsbuilder_test.go b/receiver/prometheusreceiver/internal/metricsbuilder_test.go deleted file mode 100644 index 7bb190ab21f33..0000000000000 --- a/receiver/prometheusreceiver/internal/metricsbuilder_test.go +++ /dev/null @@ -1,1470 +0,0 @@ -// Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package internal - -import ( - "reflect" - "runtime" - "testing" - - metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/pkg/labels" - "github.com/prometheus/prometheus/pkg/textparse" - "github.com/prometheus/prometheus/scrape" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "google.golang.org/protobuf/types/known/wrapperspb" -) - -const startTs = int64(1555366610000) -const interval = int64(15 * 1000) -const defaultBuilderStartTime = float64(1.0) - -var testMetadata = map[string]scrape.MetricMetadata{ - "counter_test": {Metric: "counter_test", Type: textparse.MetricTypeCounter, Help: "", Unit: ""}, - "counter_test2": {Metric: "counter_test2", Type: textparse.MetricTypeCounter, Help: "", Unit: ""}, - "gauge_test": {Metric: "gauge_test", Type: textparse.MetricTypeGauge, Help: "", Unit: ""}, - "gauge_test2": {Metric: "gauge_test2", Type: textparse.MetricTypeGauge, Help: "", Unit: ""}, - "hist_test": {Metric: "hist_test", Type: textparse.MetricTypeHistogram, Help: "", Unit: ""}, - "hist_test2": {Metric: "hist_test2", Type: textparse.MetricTypeHistogram, Help: "", Unit: ""}, - "ghist_test": {Metric: "ghist_test", Type: textparse.MetricTypeGaugeHistogram, Help: "", Unit: ""}, - "summary_test": {Metric: "summary_test", Type: textparse.MetricTypeSummary, Help: "", Unit: ""}, - "summary_test2": {Metric: "summary_test2", Type: textparse.MetricTypeSummary, Help: "", Unit: ""}, - "unknown_test": {Metric: "unknown_test", Type: textparse.MetricTypeUnknown, Help: "", Unit: ""}, - "poor_name_count": {Metric: "poor_name_count", Type: textparse.MetricTypeCounter, Help: "", Unit: ""}, - "up": {Metric: "up", Type: textparse.MetricTypeCounter, Help: "", Unit: ""}, - "scrape_foo": {Metric: "scrape_foo", Type: textparse.MetricTypeCounter, Help: "", Unit: ""}, - "example_process_start_time_seconds": {Metric: "example_process_start_time_seconds", - Type: textparse.MetricTypeGauge, Help: "", Unit: ""}, - "process_start_time_seconds": {Metric: "process_start_time_seconds", - Type: textparse.MetricTypeGauge, Help: "", Unit: ""}, - "badprocess_start_time_seconds": {Metric: "badprocess_start_time_seconds", - Type: textparse.MetricTypeGauge, Help: "", Unit: ""}, -} - -type testDataPoint struct { - lb labels.Labels - t int64 - v float64 -} - -type testScrapedPage struct { - pts []*testDataPoint -} - -type buildTestData struct { - name string - inputs []*testScrapedPage - wants [][]*metricspb.Metric -} - -func createLabels(mFamily string, tagPairs ...string) labels.Labels { - lm := make(map[string]string) - lm[model.MetricNameLabel] = mFamily - if len(tagPairs)%2 != 0 { - panic("tag pairs is not even") - } - - for i := 0; i < len(tagPairs); i += 2 { - lm[tagPairs[i]] = tagPairs[i+1] - } - - return labels.FromMap(lm) -} - -func createDataPoint(mname string, value float64, tagPairs ...string) *testDataPoint { - return &testDataPoint{ - lb: createLabels(mname, tagPairs...), - v: value, - } -} - -func runBuilderTests(t *testing.T, tests []buildTestData) { - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.EqualValues(t, len(tt.wants), len(tt.inputs)) - mc := newMockMetadataCache(testMetadata) - st := startTs - for i, page := range tt.inputs { - b := newMetricBuilder(mc, true, "", testLogger, startTs) - b.startTime = defaultBuilderStartTime // set to a non-zero value - for _, pt := range page.pts { - // set ts for testing - pt.t = st - assert.NoError(t, b.AddDataPoint(pt.lb, pt.t, pt.v)) - } - metrics, _, _, err := b.Build() - assert.NoError(t, err) - assert.EqualValues(t, tt.wants[i], metrics) - st += interval - } - }) - } -} - -func runBuilderStartTimeTests(t *testing.T, tests []buildTestData, - startTimeMetricRegex string, expectedBuilderStartTime float64) { - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - mc := newMockMetadataCache(testMetadata) - st := startTs - for _, page := range tt.inputs { - b := newMetricBuilder(mc, true, startTimeMetricRegex, - testLogger, 0) - b.startTime = defaultBuilderStartTime // set to a non-zero value - for _, pt := range page.pts { - // set ts for testing - pt.t = st - assert.NoError(t, b.AddDataPoint(pt.lb, pt.t, pt.v)) - } - _, _, _, err := b.Build() - assert.NoError(t, err) - assert.EqualValues(t, b.startTime, expectedBuilderStartTime) - st += interval - } - }) - } -} - -func Test_startTimeMetricMatch(t *testing.T) { - matchBuilderStartTime := 123.456 - matchTests := []buildTestData{ - { - name: "prefix_match", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("example_process_start_time_seconds", - matchBuilderStartTime, "foo", "bar"), - }, - }, - }, - }, - { - name: "match", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("process_start_time_seconds", - matchBuilderStartTime, "foo", "bar"), - }, - }, - }, - }, - } - nomatchTests := []buildTestData{ - { - name: "nomatch1", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("_process_start_time_seconds", - matchBuilderStartTime, "foo", "bar"), - }, - }, - }, - }, - { - name: "nomatch2", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("subprocess_start_time_seconds", - matchBuilderStartTime, "foo", "bar"), - }, - }, - }, - }, - } - - runBuilderStartTimeTests(t, matchTests, "^(.+_)*process_start_time_seconds$", matchBuilderStartTime) - runBuilderStartTimeTests(t, nomatchTests, "^(.+_)*process_start_time_seconds$", defaultBuilderStartTime) -} - -func Test_metricBuilder_counters(t *testing.T) { - tests := []buildTestData{ - { - name: "single-item", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("counter_test", 100, "foo", "bar"), - }, - }, - }, - wants: [][]*metricspb.Metric{ - { - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "counter_test", - Type: metricspb.MetricDescriptor_CUMULATIVE_DOUBLE, - LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: timestampFromMs(startTs), - LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 100.0}}, - }, - }, - }, - }, - }, - }, - }, - { - name: "two-items", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("counter_test", 150, "foo", "bar"), - createDataPoint("counter_test", 25, "foo", "other"), - }, - }, - }, - wants: [][]*metricspb.Metric{ - { - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "counter_test", - Type: metricspb.MetricDescriptor_CUMULATIVE_DOUBLE, - LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: timestampFromMs(startTs), - LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 150.0}}, - }, - }, - { - StartTimestamp: timestampFromMs(startTs), - LabelValues: []*metricspb.LabelValue{{Value: "other", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 25.0}}, - }, - }, - }, - }, - }, - }, - }, - { - name: "two-metrics", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("counter_test", 150, "foo", "bar"), - createDataPoint("counter_test", 25, "foo", "other"), - createDataPoint("counter_test2", 100, "foo", "bar"), - }, - }, - }, - wants: [][]*metricspb.Metric{ - { - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "counter_test", - Type: metricspb.MetricDescriptor_CUMULATIVE_DOUBLE, - LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: timestampFromMs(startTs), - LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 150.0}}, - }, - }, - { - StartTimestamp: timestampFromMs(startTs), - LabelValues: []*metricspb.LabelValue{{Value: "other", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 25.0}}, - }, - }, - }, - }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "counter_test2", - Type: metricspb.MetricDescriptor_CUMULATIVE_DOUBLE, - LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: timestampFromMs(startTs), - LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 100.0}}, - }, - }, - }, - }, - }, - }, - }, - { - name: "metrics-with-poor-names", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("poor_name_count", 100, "foo", "bar"), - }, - }, - }, - wants: [][]*metricspb.Metric{ - { - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "poor_name_count", - Type: metricspb.MetricDescriptor_CUMULATIVE_DOUBLE, - LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: timestampFromMs(startTs), - LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 100.0}}, - }, - }, - }, - }, - }, - }, - }, - } - - runBuilderTests(t, tests) -} - -func Test_metricBuilder_gauges(t *testing.T) { - tests := []buildTestData{ - { - name: "one-gauge", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("gauge_test", 100, "foo", "bar"), - }, - }, - { - pts: []*testDataPoint{ - createDataPoint("gauge_test", 90, "foo", "bar"), - }, - }, - }, - wants: [][]*metricspb.Metric{ - { - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "gauge_test", - Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, - LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, - Timeseries: []*metricspb.TimeSeries{ - { - LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 100.0}}, - }, - }, - }, - }, - }, - { - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "gauge_test", - Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, - LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, - Timeseries: []*metricspb.TimeSeries{ - { - LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs + interval), Value: &metricspb.Point_DoubleValue{DoubleValue: 90.0}}, - }, - }, - }, - }, - }, - }, - }, - { - name: "gauge-with-different-tags", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("gauge_test", 100, "foo", "bar"), - createDataPoint("gauge_test", 200, "bar", "foo"), - }, - }, - }, - wants: [][]*metricspb.Metric{ - { - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "gauge_test", - Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, - LabelKeys: []*metricspb.LabelKey{{Key: "bar"}, {Key: "foo"}}}, - Timeseries: []*metricspb.TimeSeries{ - { - LabelValues: []*metricspb.LabelValue{{Value: "", HasValue: false}, {Value: "bar", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 100.0}}, - }, - }, - { - LabelValues: []*metricspb.LabelValue{{Value: "foo", HasValue: true}, {Value: "", HasValue: false}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 200.0}}, - }, - }, - }, - }, - }, - }, - }, - { - // TODO: A decision need to be made. If we want to have the behavior which can generate different tag key - // sets because metrics come and go - name: "gauge-comes-and-go-with-different-tagset", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("gauge_test", 100, "foo", "bar"), - createDataPoint("gauge_test", 200, "bar", "foo"), - }, - }, - { - pts: []*testDataPoint{ - createDataPoint("gauge_test", 20, "foo", "bar"), - }, - }, - }, - wants: [][]*metricspb.Metric{ - { - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "gauge_test", - Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, - LabelKeys: []*metricspb.LabelKey{{Key: "bar"}, {Key: "foo"}}}, - Timeseries: []*metricspb.TimeSeries{ - { - LabelValues: []*metricspb.LabelValue{{Value: "", HasValue: false}, {Value: "bar", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 100.0}}, - }, - }, - { - LabelValues: []*metricspb.LabelValue{{Value: "foo", HasValue: true}, {Value: "", HasValue: false}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 200.0}}, - }, - }, - }, - }, - }, - { - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "gauge_test", - Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, - LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, - Timeseries: []*metricspb.TimeSeries{ - { - LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs + interval), Value: &metricspb.Point_DoubleValue{DoubleValue: 20.0}}, - }, - }, - }, - }, - }, - }, - }, - } - - runBuilderTests(t, tests) -} - -func Test_metricBuilder_untype(t *testing.T) { - tests := []buildTestData{ - { - name: "one-unknown", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("unknown_test", 100, "foo", "bar"), - }, - }, - }, - wants: [][]*metricspb.Metric{ - { - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "unknown_test", - Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, - LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, - Timeseries: []*metricspb.TimeSeries{ - { - LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 100.0}}, - }, - }, - }, - }, - }, - }, - }, - { - name: "no-type-hint", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("something_not_exists", 100, "foo", "bar"), - createDataPoint("theother_not_exists", 200, "foo", "bar"), - createDataPoint("theother_not_exists", 300, "bar", "foo"), - }, - }, - }, - wants: [][]*metricspb.Metric{ - { - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "something_not_exists", - Type: metricspb.MetricDescriptor_UNSPECIFIED, - LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, - Timeseries: []*metricspb.TimeSeries{ - { - LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 100.0}}, - }, - }, - }, - }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "theother_not_exists", - Type: metricspb.MetricDescriptor_UNSPECIFIED, - LabelKeys: []*metricspb.LabelKey{{Key: "bar"}, {Key: "foo"}}}, - Timeseries: []*metricspb.TimeSeries{ - { - LabelValues: []*metricspb.LabelValue{{Value: "", HasValue: false}, {Value: "bar", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 200.0}}, - }, - }, - { - LabelValues: []*metricspb.LabelValue{{Value: "foo", HasValue: true}, {Value: "", HasValue: false}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 300.0}}, - }, - }, - }, - }, - }, - }, - }, - { - name: "untype-metric-poor-names", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("some_count", 100, "foo", "bar"), - }, - }, - }, - wants: [][]*metricspb.Metric{ - { - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "some_count", - Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, - LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, - Timeseries: []*metricspb.TimeSeries{ - { - LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 100.0}}, - }, - }, - }, - }, - }, - }, - }, - } - - runBuilderTests(t, tests) -} - -func Test_metricBuilder_histogram(t *testing.T) { - tests := []buildTestData{ - { - name: "single item", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("hist_test", 1, "foo", "bar", "le", "10"), - createDataPoint("hist_test", 2, "foo", "bar", "le", "20"), - createDataPoint("hist_test", 10, "foo", "bar", "le", "+inf"), - createDataPoint("hist_test_sum", 99, "foo", "bar"), - createDataPoint("hist_test_count", 10, "foo", "bar"), - }, - }, - }, - wants: [][]*metricspb.Metric{ - { - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "hist_test", - Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, - LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: timestampFromMs(startTs), - LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DistributionValue{ - DistributionValue: &metricspb.DistributionValue{ - BucketOptions: &metricspb.DistributionValue_BucketOptions{ - Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ - Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ - Bounds: []float64{10, 20}, - }, - }, - }, - Count: 10, - Sum: 99.0, - Buckets: []*metricspb.DistributionValue_Bucket{{Count: 1}, {Count: 1}, {Count: 8}}, - }}}, - }, - }, - }, - }, - }, - }, - }, - { - name: "multi-groups", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("hist_test", 1, "foo", "bar", "le", "10"), - createDataPoint("hist_test", 2, "foo", "bar", "le", "20"), - createDataPoint("hist_test", 10, "foo", "bar", "le", "+inf"), - createDataPoint("hist_test_sum", 99, "foo", "bar"), - createDataPoint("hist_test_count", 10, "foo", "bar"), - createDataPoint("hist_test", 1, "key2", "v2", "le", "10"), - createDataPoint("hist_test", 2, "key2", "v2", "le", "20"), - createDataPoint("hist_test", 3, "key2", "v2", "le", "+inf"), - createDataPoint("hist_test_sum", 50, "key2", "v2"), - createDataPoint("hist_test_count", 3, "key2", "v2"), - }, - }, - }, - wants: [][]*metricspb.Metric{ - { - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "hist_test", - Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, - LabelKeys: []*metricspb.LabelKey{{Key: "foo"}, {Key: "key2"}}}, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: timestampFromMs(startTs), - LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}, {Value: "", HasValue: false}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DistributionValue{ - DistributionValue: &metricspb.DistributionValue{ - BucketOptions: &metricspb.DistributionValue_BucketOptions{ - Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ - Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ - Bounds: []float64{10, 20}, - }, - }, - }, - Count: 10, - Sum: 99.0, - Buckets: []*metricspb.DistributionValue_Bucket{{Count: 1}, {Count: 1}, {Count: 8}}, - }}}, - }, - }, - { - StartTimestamp: timestampFromMs(startTs), - LabelValues: []*metricspb.LabelValue{{Value: "", HasValue: false}, {Value: "v2", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DistributionValue{ - DistributionValue: &metricspb.DistributionValue{ - BucketOptions: &metricspb.DistributionValue_BucketOptions{ - Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ - Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ - Bounds: []float64{10, 20}, - }, - }, - }, - Count: 3, - Sum: 50.0, - Buckets: []*metricspb.DistributionValue_Bucket{{Count: 1}, {Count: 1}, {Count: 1}}, - }}}, - }, - }, - }, - }, - }, - }, - }, - { - name: "multi-groups-and-families", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("hist_test", 1, "foo", "bar", "le", "10"), - createDataPoint("hist_test", 2, "foo", "bar", "le", "20"), - createDataPoint("hist_test", 10, "foo", "bar", "le", "+inf"), - createDataPoint("hist_test_sum", 99, "foo", "bar"), - createDataPoint("hist_test_count", 10, "foo", "bar"), - createDataPoint("hist_test", 1, "key2", "v2", "le", "10"), - createDataPoint("hist_test", 2, "key2", "v2", "le", "20"), - createDataPoint("hist_test", 3, "key2", "v2", "le", "+inf"), - createDataPoint("hist_test_sum", 50, "key2", "v2"), - createDataPoint("hist_test_count", 3, "key2", "v2"), - createDataPoint("hist_test2", 1, "le", "10"), - createDataPoint("hist_test2", 2, "le", "20"), - createDataPoint("hist_test2", 3, "le", "+inf"), - createDataPoint("hist_test2_sum", 50), - createDataPoint("hist_test2_count", 3), - }, - }, - }, - wants: [][]*metricspb.Metric{ - { - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "hist_test", - Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, - LabelKeys: []*metricspb.LabelKey{{Key: "foo"}, {Key: "key2"}}}, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: timestampFromMs(startTs), - LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}, {Value: "", HasValue: false}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DistributionValue{ - DistributionValue: &metricspb.DistributionValue{ - BucketOptions: &metricspb.DistributionValue_BucketOptions{ - Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ - Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ - Bounds: []float64{10, 20}, - }, - }, - }, - Count: 10, - Sum: 99.0, - Buckets: []*metricspb.DistributionValue_Bucket{{Count: 1}, {Count: 1}, {Count: 8}}, - }}}, - }, - }, - { - StartTimestamp: timestampFromMs(startTs), - LabelValues: []*metricspb.LabelValue{{Value: "", HasValue: false}, {Value: "v2", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DistributionValue{ - DistributionValue: &metricspb.DistributionValue{ - BucketOptions: &metricspb.DistributionValue_BucketOptions{ - Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ - Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ - Bounds: []float64{10, 20}, - }, - }, - }, - Count: 3, - Sum: 50.0, - Buckets: []*metricspb.DistributionValue_Bucket{{Count: 1}, {Count: 1}, {Count: 1}}, - }}}, - }, - }, - }, - }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "hist_test2", - Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, - LabelKeys: []*metricspb.LabelKey{}}, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: timestampFromMs(startTs), - LabelValues: []*metricspb.LabelValue{}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DistributionValue{ - DistributionValue: &metricspb.DistributionValue{ - BucketOptions: &metricspb.DistributionValue_BucketOptions{ - Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ - Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ - Bounds: []float64{10, 20}, - }, - }, - }, - Count: 3, - Sum: 50.0, - Buckets: []*metricspb.DistributionValue_Bucket{{Count: 1}, {Count: 1}, {Count: 1}}, - }}}, - }, - }, - }, - }, - }, - }, - }, - { - name: "unordered-buckets", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("hist_test", 10, "foo", "bar", "le", "+inf"), - createDataPoint("hist_test", 1, "foo", "bar", "le", "10"), - createDataPoint("hist_test", 2, "foo", "bar", "le", "20"), - createDataPoint("hist_test_sum", 99, "foo", "bar"), - createDataPoint("hist_test_count", 10, "foo", "bar"), - }, - }, - }, - wants: [][]*metricspb.Metric{ - { - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "hist_test", - Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, - LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: timestampFromMs(startTs), - LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DistributionValue{ - DistributionValue: &metricspb.DistributionValue{ - BucketOptions: &metricspb.DistributionValue_BucketOptions{ - Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ - Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ - Bounds: []float64{10, 20}, - }, - }, - }, - Count: 10, - Sum: 99.0, - Buckets: []*metricspb.DistributionValue_Bucket{{Count: 1}, {Count: 1}, {Count: 8}}, - }}}, - }, - }, - }, - }, - }, - }, - }, - { - // this won't likely happen in real env, as prometheus wont generate histogram with less than 3 buckets - name: "only-one-bucket", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("hist_test", 3, "le", "+inf"), - createDataPoint("hist_test_count", 3), - createDataPoint("hist_test_sum", 100), - }, - }, - }, - wants: [][]*metricspb.Metric{ - { - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "hist_test", - Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, - LabelKeys: []*metricspb.LabelKey{}}, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: timestampFromMs(startTs), - LabelValues: []*metricspb.LabelValue{}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DistributionValue{ - DistributionValue: &metricspb.DistributionValue{ - BucketOptions: &metricspb.DistributionValue_BucketOptions{ - Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ - Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ - Bounds: []float64{}, - }, - }, - }, - Count: 3, - Sum: 100, - Buckets: []*metricspb.DistributionValue_Bucket{{Count: 3}}, - }}}, - }, - }, - }, - }, - }, - }, - }, - { - // this won't likely happen in real env, as prometheus wont generate histogram with less than 3 buckets - name: "only-one-bucket-noninf", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("hist_test", 3, "le", "20"), - createDataPoint("hist_test_count", 3), - createDataPoint("hist_test_sum", 100), - }, - }, - }, - wants: [][]*metricspb.Metric{ - { - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "hist_test", - Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, - LabelKeys: []*metricspb.LabelKey{}}, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: timestampFromMs(startTs), - LabelValues: []*metricspb.LabelValue{}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DistributionValue{ - DistributionValue: &metricspb.DistributionValue{ - BucketOptions: &metricspb.DistributionValue_BucketOptions{ - Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ - Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ - Bounds: []float64{}, - }, - }, - }, - Count: 3, - Sum: 100, - Buckets: []*metricspb.DistributionValue_Bucket{{Count: 3}}, - }}}, - }, - }, - }, - }, - }, - }, - }, - { - name: "no-sum", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("hist_test", 1, "foo", "bar", "le", "10"), - createDataPoint("hist_test", 2, "foo", "bar", "le", "20"), - createDataPoint("hist_test", 3, "foo", "bar", "le", "+inf"), - createDataPoint("hist_test_count", 3, "foo", "bar"), - }, - }, - }, - wants: [][]*metricspb.Metric{ - { - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "hist_test", - Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, - LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: timestampFromMs(startTs), - LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DistributionValue{ - DistributionValue: &metricspb.DistributionValue{ - BucketOptions: &metricspb.DistributionValue_BucketOptions{ - Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ - Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ - Bounds: []float64{10, 20}, - }, - }, - }, - Count: 3, - Sum: 0, - Buckets: []*metricspb.DistributionValue_Bucket{{Count: 1}, {Count: 1}, {Count: 1}}, - }}}, - }, - }, - }, - }, - }, - }, - }, - { - name: "corrupted-no-buckets", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("hist_test_sum", 99), - createDataPoint("hist_test_count", 10), - }, - }, - }, - wants: [][]*metricspb.Metric{ - {}, - }, - }, - { - name: "corrupted-no-count", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("hist_test", 1, "foo", "bar", "le", "10"), - createDataPoint("hist_test", 2, "foo", "bar", "le", "20"), - createDataPoint("hist_test", 3, "foo", "bar", "le", "+inf"), - createDataPoint("hist_test_sum", 99, "foo", "bar"), - }, - }, - }, - wants: [][]*metricspb.Metric{ - {}, - }, - }, - } - - runBuilderTests(t, tests) -} - -func Test_metricBuilder_summary(t *testing.T) { - tests := []buildTestData{ - { - name: "no-sum-and-count", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("summary_test", 5, "foo", "bar", "quantile", "1"), - }, - }, - }, - wants: [][]*metricspb.Metric{ - {}, - }, - }, - { - name: "no-count", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("summary_test", 1, "foo", "bar", "quantile", "0.5"), - createDataPoint("summary_test", 2, "foo", "bar", "quantile", "0.75"), - createDataPoint("summary_test", 5, "foo", "bar", "quantile", "1"), - createDataPoint("summary_test_sum", 500, "foo", "bar"), - }, - }, - }, - wants: [][]*metricspb.Metric{ - {}, - }, - }, - { - name: "no-sum", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("summary_test", 1, "foo", "bar", "quantile", "0.5"), - createDataPoint("summary_test", 2, "foo", "bar", "quantile", "0.75"), - createDataPoint("summary_test", 5, "foo", "bar", "quantile", "1"), - createDataPoint("summary_test_count", 500, "foo", "bar"), - }, - }, - }, - wants: [][]*metricspb.Metric{ - { - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "summary_test", - Type: metricspb.MetricDescriptor_SUMMARY, - LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: timestampFromMs(startTs), - LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_SummaryValue{ - SummaryValue: &metricspb.SummaryValue{ - Sum: &wrapperspb.DoubleValue{Value: 0.0}, - Count: &wrapperspb.Int64Value{Value: 500}, - Snapshot: &metricspb.SummaryValue_Snapshot{ - PercentileValues: []*metricspb.SummaryValue_Snapshot_ValueAtPercentile{ - {Percentile: 50.0, Value: 1}, - {Percentile: 75.0, Value: 2}, - {Percentile: 100.0, Value: 5}, - }, - }}}}, - }, - }, - }, - }, - }, - }, - }, - { - name: "empty-quantiles", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("summary_test_sum", 100, "foo", "bar"), - createDataPoint("summary_test_count", 500, "foo", "bar"), - }, - }, - }, - wants: [][]*metricspb.Metric{ - { - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "summary_test", - Type: metricspb.MetricDescriptor_SUMMARY, - LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: timestampFromMs(startTs), - LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, - Points: []*metricspb.Point{ - { - Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_SummaryValue{ - SummaryValue: &metricspb.SummaryValue{ - Sum: &wrapperspb.DoubleValue{Value: 100.0}, - Count: &wrapperspb.Int64Value{Value: 500}, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - { - name: "regular-summary", - inputs: []*testScrapedPage{ - { - pts: []*testDataPoint{ - createDataPoint("summary_test", 1, "foo", "bar", "quantile", "0.5"), - createDataPoint("summary_test", 2, "foo", "bar", "quantile", "0.75"), - createDataPoint("summary_test", 5, "foo", "bar", "quantile", "1"), - createDataPoint("summary_test_sum", 100, "foo", "bar"), - createDataPoint("summary_test_count", 500, "foo", "bar"), - }, - }, - }, - wants: [][]*metricspb.Metric{ - { - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "summary_test", - Type: metricspb.MetricDescriptor_SUMMARY, - LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, - Timeseries: []*metricspb.TimeSeries{ - { - StartTimestamp: timestampFromMs(startTs), - LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, - Points: []*metricspb.Point{ - {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_SummaryValue{ - SummaryValue: &metricspb.SummaryValue{ - Sum: &wrapperspb.DoubleValue{Value: 100.0}, - Count: &wrapperspb.Int64Value{Value: 500}, - Snapshot: &metricspb.SummaryValue_Snapshot{ - PercentileValues: []*metricspb.SummaryValue_Snapshot_ValueAtPercentile{ - {Percentile: 50.0, Value: 1}, - {Percentile: 75.0, Value: 2}, - {Percentile: 100.0, Value: 5}, - }, - }}}}, - }, - }, - }, - }, - }, - }, - }, - } - - runBuilderTests(t, tests) -} - -func Test_metricBuilder_baddata(t *testing.T) { - t.Run("empty-metric-name", func(t *testing.T) { - mc := newMockMetadataCache(testMetadata) - b := newMetricBuilder(mc, true, "", testLogger, 0) - b.startTime = 1.0 // set to a non-zero value - if err := b.AddDataPoint(labels.FromStrings("a", "b"), startTs, 123); err != errMetricNameNotFound { - t.Error("expecting errMetricNameNotFound error, but get nil") - return - } - - if _, _, _, err := b.Build(); err != errNoDataToBuild { - t.Error("expecting errNoDataToBuild error, but get nil") - } - }) - - t.Run("histogram-datapoint-no-bucket-label", func(t *testing.T) { - mc := newMockMetadataCache(testMetadata) - b := newMetricBuilder(mc, true, "", testLogger, 0) - b.startTime = 1.0 // set to a non-zero value - if err := b.AddDataPoint(createLabels("hist_test", "k", "v"), startTs, 123); err != errEmptyBoundaryLabel { - t.Error("expecting errEmptyBoundaryLabel error, but get nil") - } - }) - - t.Run("summary-datapoint-no-quantile-label", func(t *testing.T) { - mc := newMockMetadataCache(testMetadata) - b := newMetricBuilder(mc, true, "", testLogger, 0) - b.startTime = 1.0 // set to a non-zero value - if err := b.AddDataPoint(createLabels("summary_test", "k", "v"), startTs, 123); err != errEmptyBoundaryLabel { - t.Error("expecting errEmptyBoundaryLabel error, but get nil") - } - }) - -} - -func Test_isUsefulLabel(t *testing.T) { - type args struct { - mType metricspb.MetricDescriptor_Type - labelKey string - } - tests := []struct { - name string - args args - want bool - }{ - {"metricName", args{metricspb.MetricDescriptor_GAUGE_DOUBLE, model.MetricNameLabel}, false}, - {"instance", args{metricspb.MetricDescriptor_GAUGE_DOUBLE, model.InstanceLabel}, false}, - {"scheme", args{metricspb.MetricDescriptor_GAUGE_DOUBLE, model.SchemeLabel}, false}, - {"metricPath", args{metricspb.MetricDescriptor_GAUGE_DOUBLE, model.MetricsPathLabel}, false}, - {"job", args{metricspb.MetricDescriptor_GAUGE_DOUBLE, model.JobLabel}, false}, - {"bucket", args{metricspb.MetricDescriptor_GAUGE_DOUBLE, model.BucketLabel}, true}, - {"bucketForGaugeDistribution", args{metricspb.MetricDescriptor_GAUGE_DISTRIBUTION, model.BucketLabel}, false}, - {"bucketForCumulativeDistribution", args{metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, model.BucketLabel}, false}, - {"Quantile", args{metricspb.MetricDescriptor_GAUGE_DOUBLE, model.QuantileLabel}, true}, - {"QuantileForSummay", args{metricspb.MetricDescriptor_SUMMARY, model.QuantileLabel}, false}, - {"other", args{metricspb.MetricDescriptor_GAUGE_DOUBLE, "other"}, true}, - {"empty", args{metricspb.MetricDescriptor_GAUGE_DOUBLE, ""}, true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := isUsefulLabel(tt.args.mType, tt.args.labelKey); got != tt.want { - t.Errorf("isUsefulLabel() = %v, want %v", got, tt.want) - } - }) - } -} - -func Benchmark_dpgSignature(b *testing.B) { - knownLabelKeys := []string{"a", "b"} - labels := labels.FromStrings("a", "va", "b", "vb", "x", "xa") - b.ReportAllocs() - for i := 0; i < b.N; i++ { - runtime.KeepAlive(dpgSignature(knownLabelKeys, labels)) - } -} - -func Test_dpgSignature(t *testing.T) { - knownLabelKeys := []string{"a", "b"} - - tests := []struct { - name string - ls labels.Labels - want string - }{ - {"1st label", labels.FromStrings("a", "va"), `"a=va"`}, - {"2nd label", labels.FromStrings("b", "vb"), `"b=vb"`}, - {"two labels", labels.FromStrings("a", "va", "b", "vb"), `"a=va""b=vb"`}, - {"extra label", labels.FromStrings("a", "va", "b", "vb", "x", "xa"), `"a=va""b=vb"`}, - {"different order", labels.FromStrings("b", "vb", "a", "va"), `"a=va""b=vb"`}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := dpgSignature(knownLabelKeys, tt.ls); got != tt.want { - t.Errorf("dpgSignature() = %q, want %q", got, tt.want) - } - }) - } - - // this is important for caching start values, as new metrics with new tag of a same group can come up in a 2nd run, - // however, its order within the group is not predictable. we need to have a way to generate a stable key even if - // the total number of keys changes in between different scrape runs - t.Run("knownLabelKeys updated", func(t *testing.T) { - ls := labels.FromStrings("a", "va") - want := dpgSignature(knownLabelKeys, ls) - got := dpgSignature(append(knownLabelKeys, "c"), ls) - if got != want { - t.Errorf("dpgSignature() = %v, want %v", got, want) - } - }) -} - -func Test_normalizeMetricName(t *testing.T) { - tests := []struct { - name string - mname string - want string - }{ - {"normal", "normal", "normal"}, - {"count", "foo_count", "foo"}, - {"bucket", "foo_bucket", "foo"}, - {"sum", "foo_sum", "foo"}, - {"total", "foo_total", "foo"}, - {"no_prefix", "_sum", "_sum"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := normalizeMetricName(tt.mname); got != tt.want { - t.Errorf("normalizeMetricName() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_getBoundary(t *testing.T) { - ls := labels.FromStrings("le", "100.0", "foo", "bar", "quantile", "0.5") - ls2 := labels.FromStrings("foo", "bar") - ls3 := labels.FromStrings("le", "xyz", "foo", "bar", "quantile", "0.5") - type args struct { - metricType metricspb.MetricDescriptor_Type - labels labels.Labels - } - tests := []struct { - name string - args args - want float64 - wantErr bool - }{ - {"histogram", args{metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, ls}, 100.0, false}, - {"gaugehistogram", args{metricspb.MetricDescriptor_GAUGE_DISTRIBUTION, ls}, 100.0, false}, - {"gaugehistogram_no_label", args{metricspb.MetricDescriptor_GAUGE_DISTRIBUTION, ls2}, 0, true}, - {"gaugehistogram_bad_value", args{metricspb.MetricDescriptor_GAUGE_DISTRIBUTION, ls3}, 0, true}, - {"summary", args{metricspb.MetricDescriptor_SUMMARY, ls}, 0.5, false}, - {"otherType", args{metricspb.MetricDescriptor_GAUGE_DOUBLE, ls}, 0, true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := getBoundary(tt.args.metricType, tt.args.labels) - if (err != nil) != tt.wantErr { - t.Errorf("getBoundary() error = %v, wantErr %v", err, tt.wantErr) - return - } - if got != tt.want { - t.Errorf("getBoundary() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_convToOCAMetricType(t *testing.T) { - tests := []struct { - name string - metricType textparse.MetricType - want metricspb.MetricDescriptor_Type - }{ - {"counter", textparse.MetricTypeCounter, metricspb.MetricDescriptor_CUMULATIVE_DOUBLE}, - {"gauge", textparse.MetricTypeGauge, metricspb.MetricDescriptor_GAUGE_DOUBLE}, - {"histogram", textparse.MetricTypeHistogram, metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION}, - {"guageHistogram", textparse.MetricTypeGaugeHistogram, metricspb.MetricDescriptor_UNSPECIFIED}, - {"summary", textparse.MetricTypeSummary, metricspb.MetricDescriptor_SUMMARY}, - {"info", textparse.MetricTypeInfo, metricspb.MetricDescriptor_UNSPECIFIED}, - {"stateset", textparse.MetricTypeStateset, metricspb.MetricDescriptor_UNSPECIFIED}, - {"unknown", textparse.MetricTypeUnknown, metricspb.MetricDescriptor_GAUGE_DOUBLE}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := convToOCAMetricType(tt.metricType); !reflect.DeepEqual(got, tt.want) { - t.Errorf("convToOCAMetricType() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_heuristicalMetricAndKnownUnits(t *testing.T) { - tests := []struct { - metricName string - parsedUnit string - want string - }{ - {"test", "ms", "ms"}, - {"millisecond", "", ""}, - {"test_millisecond", "", "ms"}, - {"test_milliseconds", "", "ms"}, - {"test_ms", "", "ms"}, - {"test_second", "", "s"}, - {"test_seconds", "", "s"}, - {"test_s", "", "s"}, - {"test_microsecond", "", "us"}, - {"test_microseconds", "", "us"}, - {"test_us", "", "us"}, - {"test_nanosecond", "", "ns"}, - {"test_nanoseconds", "", "ns"}, - {"test_ns", "", "ns"}, - {"test_byte", "", "By"}, - {"test_bytes", "", "By"}, - {"test_by", "", "By"}, - {"test_bit", "", "Bi"}, - {"test_bits", "", "Bi"}, - {"test_kilogram", "", "kg"}, - {"test_kilograms", "", "kg"}, - {"test_kg", "", "kg"}, - {"test_gram", "", "g"}, - {"test_grams", "", "g"}, - {"test_g", "", "g"}, - {"test_nanogram", "", "ng"}, - {"test_nanograms", "", "ng"}, - {"test_ng", "", "ng"}, - {"test_meter", "", "m"}, - {"test_meters", "", "m"}, - {"test_metre", "", "m"}, - {"test_metres", "", "m"}, - {"test_m", "", "m"}, - {"test_kilometer", "", "km"}, - {"test_kilometers", "", "km"}, - {"test_kilometre", "", "km"}, - {"test_kilometres", "", "km"}, - {"test_km", "", "km"}, - {"test_milimeter", "", "mm"}, - {"test_milimeters", "", "mm"}, - {"test_milimetre", "", "mm"}, - {"test_milimetres", "", "mm"}, - {"test_mm", "", "mm"}, - } - for _, tt := range tests { - t.Run(tt.metricName, func(t *testing.T) { - if got := heuristicalMetricAndKnownUnits(tt.metricName, tt.parsedUnit); got != tt.want { - t.Errorf("heuristicalMetricAndKnownUnits() = %v, want %v", got, tt.want) - } - }) - } -} - -// Ensure that we reject duplicate label keys. See https://github.com/open-telemetry/wg-prometheus/issues/44. -func TestMetricBuilderDuplicateLabelKeysAreRejected(t *testing.T) { - mc := newMockMetadataCache(testMetadata) - mb := newMetricBuilder(mc, true, "", testLogger, 0) - - dupLabels := labels.Labels{ - {Name: "__name__", Value: "test"}, - {Name: "a", Value: "1"}, - {Name: "a", Value: "1"}, - {Name: "z", Value: "9"}, - {Name: "z", Value: "1"}, - {Name: "instance", Value: "0.0.0.0:8855"}, - {Name: "job", Value: "test"}, - } - - err := mb.AddDataPoint(dupLabels, 1917, 1.0) - require.NotNil(t, err) - require.Contains(t, err.Error(), `invalid sample: non-unique label names: ["a" "z"]`) -} diff --git a/receiver/prometheusreceiver/internal/ocastore.go b/receiver/prometheusreceiver/internal/ocastore.go index 89672f6179d45..3a995cfb73a46 100644 --- a/receiver/prometheusreceiver/internal/ocastore.go +++ b/receiver/prometheusreceiver/internal/ocastore.go @@ -23,9 +23,10 @@ import ( "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/scrape" "github.com/prometheus/prometheus/storage" + "go.uber.org/zap" + "go.opentelemetry.io/collector/config" "go.opentelemetry.io/collector/consumer" - "go.uber.org/zap" ) const ( @@ -44,7 +45,7 @@ type OcaStore struct { running int32 // access atomically sink consumer.Metrics mc *metadataService - jobsMap *JobsMap + jobsMap *JobsMapPdata useStartTimeMetric bool startTimeMetricRegex string receiverID config.ComponentID @@ -58,7 +59,7 @@ func NewOcaStore( ctx context.Context, sink consumer.Metrics, logger *zap.Logger, - jobsMap *JobsMap, + jobsMap *JobsMapPdata, useStartTimeMetric bool, startTimeMetricRegex string, receiverID config.ComponentID, @@ -87,16 +88,18 @@ func (o *OcaStore) SetScrapeManager(scrapeManager *scrape.Manager) { func (o *OcaStore) Appender(context.Context) storage.Appender { state := atomic.LoadInt32(&o.running) if state == runningStateReady { - return newTransaction( + return newTransactionPdata( o.ctx, - o.jobsMap, - o.useStartTimeMetric, - o.startTimeMetricRegex, - o.receiverID, - o.mc, - o.sink, - o.externalLabels, - o.logger, + &txConfig{ + jobsMap: o.jobsMap, + useStartTimeMetric: o.useStartTimeMetric, + startTimeMetricRegex: o.startTimeMetricRegex, + receiverID: o.receiverID, + ms: o.mc, + sink: o.sink, + externalLabels: o.externalLabels, + logger: o.logger, + }, ) } else if state == runningStateInit { panic("ScrapeManager is not set") diff --git a/receiver/prometheusreceiver/internal/otlp_metricfamily.go b/receiver/prometheusreceiver/internal/otlp_metricfamily.go index 71bf51fc1735c..489a0d172e132 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricfamily.go +++ b/receiver/prometheusreceiver/internal/otlp_metricfamily.go @@ -15,14 +15,24 @@ package internal import ( + "fmt" "sort" "strings" + "time" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/textparse" + "github.com/prometheus/prometheus/scrape" + "go.uber.org/zap" + "go.opentelemetry.io/collector/model/pdata" ) +type dataPoint struct { + value float64 + boundary float64 +} + // MetricFamilyPdata is unit which is corresponding to the metrics items which shared the same TYPE/UNIT/... metadata from // a single scrape. type MetricFamilyPdata interface { @@ -32,26 +42,35 @@ type MetricFamilyPdata interface { } type metricFamilyPdata struct { - // We are composing the already present metricFamily to - // make for a scalable migration, so that we only edit target - // fields progressively, when we are ready to make changes. - metricFamily - mtype pdata.MetricDataType - groups map[string]*metricGroupPdata + mtype pdata.MetricDataType + groups map[string]*metricGroupPdata + name string + mc MetadataCache + droppedTimeseries int + labelKeys map[string]bool + labelKeysOrdered []string + metadata *scrape.MetricMetadata + groupOrders map[string]int + intervalStartTimeMs int64 } // metricGroupPdata, represents a single metric of a metric family. for example a histogram metric is usually represent by // a couple data complexValue (buckets and count/sum), a group of a metric family always share a same set of tags. for // simple types like counter and gauge, each data point is a group of itself type metricGroupPdata struct { - // We are composing the already present metricGroup to - // make for a scalable migration, so that we only edit target - // fields progressively, when we are ready to make changes. - metricGroup - family *metricFamilyPdata + family *metricFamilyPdata + ts int64 + ls labels.Labels + count float64 + hasCount bool + sum float64 + hasSum bool + value float64 + complexValue []*dataPoint + intervalStartTimeMs int64 } -func newMetricFamilyPdata(metricName string, mc MetadataCache, intervalStartTimeMs int64) MetricFamilyPdata { +func newMetricFamilyPdata(metricName string, mc MetadataCache, logger *zap.Logger, intervalStartTimeMs int64) MetricFamilyPdata { familyName := normalizeMetricName(metricName) // lookup metadata based on familyName @@ -67,24 +86,35 @@ func newMetricFamilyPdata(metricName string, mc MetadataCache, intervalStartTime metadata.Metric = familyName metadata.Type = textparse.MetricTypeUnknown } + } else if !ok && isInternalMetric(metricName) { + metadata = defineInternalMetric(metricName, metadata, logger) + } + + mtype := convToPdataMetricType(metadata.Type) + if mtype == pdata.MetricDataTypeNone { + logger.Debug(fmt.Sprintf("Invalid metric : %s %+v", metricName, metadata)) } return &metricFamilyPdata{ - mtype: convToPdataMetricType(metadata.Type), - groups: make(map[string]*metricGroupPdata), - metricFamily: metricFamily{ - name: familyName, - mc: mc, - droppedTimeseries: 0, - labelKeys: make(map[string]bool), - labelKeysOrdered: make([]string, 0), - metadata: &metadata, - groupOrders: make(map[string]int), - intervalStartTimeMs: intervalStartTimeMs, - }, + mtype: mtype, + groups: make(map[string]*metricGroupPdata), + name: familyName, + mc: mc, + droppedTimeseries: 0, + labelKeys: make(map[string]bool), + labelKeysOrdered: make([]string, 0), + metadata: &metadata, + groupOrders: make(map[string]int), + intervalStartTimeMs: intervalStartTimeMs, } } +func (mf *metricFamilyPdata) IsSameFamily(metricName string) bool { + // trim known suffix if necessary + familyName := normalizeMetricName(metricName) + return mf.name == familyName || familyName != metricName && mf.name == metricName +} + // updateLabelKeys is used to store all the label keys of a same metric family in observed order. since prometheus // receiver removes any label with empty value before feeding it to an appender, in order to figure out all the labels // from the same metric family we will need to keep track of what labels have ever been observed. @@ -109,6 +139,12 @@ func (mf *metricFamilyPdata) getGroupKey(ls labels.Labels) string { return dpgSignature(mf.labelKeysOrdered, ls) } +func (mg *metricGroupPdata) sortPoints() { + sort.Slice(mg.complexValue, func(i, j int) bool { + return mg.complexValue[i].boundary < mg.complexValue[j].boundary + }) +} + func (mg *metricGroupPdata) toDistributionPoint(orderedLabelKeys []string, dest *pdata.HistogramDataPointSlice) bool { if !mg.hasCount || len(mg.complexValue) == 0 { return false @@ -139,7 +175,7 @@ func (mg *metricGroupPdata) toDistributionPoint(orderedLabelKeys []string, dest point.SetSum(mg.sum) point.SetBucketCounts(bucketCounts) // The timestamp MUST be in retrieved from milliseconds and converted to nanoseconds. - tsNanos := pdata.Timestamp(mg.ts * 1e6) + tsNanos := timestampFromMs(mg.ts) point.SetStartTimestamp(tsNanos) point.SetTimestamp(tsNanos) populateAttributesPdata(orderedLabelKeys, mg.ls, point.Attributes()) @@ -147,6 +183,11 @@ func (mg *metricGroupPdata) toDistributionPoint(orderedLabelKeys []string, dest return true } +func timestampFromMs(timeAtMs int64) pdata.Timestamp { + secs, ns := timeAtMs/1e3, (timeAtMs%1e3)*1e6 + return pdata.NewTimestampFromTime(time.Unix(secs, ns)) +} + func (mg *metricGroupPdata) toSummaryPoint(orderedLabelKeys []string, dest *pdata.SummaryDataPointSlice) bool { // expecting count to be provided, however, in the following two cases, they can be missed. // 1. data is corrupted @@ -170,9 +211,11 @@ func (mg *metricGroupPdata) toSummaryPoint(orderedLabelKeys []string, dest *pdat // observations and the corresponding sum is a sum of all observed values, thus the sum and count used // at the global level of the metricspb.SummaryValue // The timestamp MUST be in retrieved from milliseconds and converted to nanoseconds. - tsNanos := pdata.Timestamp(mg.ts * 1e6) - point.SetStartTimestamp(tsNanos) + tsNanos := timestampFromMs(mg.ts) point.SetTimestamp(tsNanos) + if mg.family.isCumulativeTypePdata() { + point.SetStartTimestamp(timestampFromMs(mg.intervalStartTimeMs)) + } point.SetSum(mg.sum) point.SetCount(uint64(mg.count)) populateAttributesPdata(orderedLabelKeys, mg.ls, point.Attributes()) @@ -182,10 +225,10 @@ func (mg *metricGroupPdata) toSummaryPoint(orderedLabelKeys []string, dest *pdat func (mg *metricGroupPdata) toNumberDataPoint(orderedLabelKeys []string, dest *pdata.NumberDataPointSlice) bool { var startTsNanos pdata.Timestamp - tsNanos := pdata.Timestamp(mg.ts * 1e6) + tsNanos := timestampFromMs(mg.ts) // gauge/undefined types have no start time. if mg.family.isCumulativeTypePdata() { - startTsNanos = pdata.Timestamp(mg.intervalStartTimeMs * 1e6) + startTsNanos = timestampFromMs(mg.intervalStartTimeMs) } point := dest.AppendEmpty() @@ -217,13 +260,11 @@ func (mf *metricFamilyPdata) loadMetricGroupOrCreate(groupKey string, ls labels. mg, ok := mf.groups[groupKey] if !ok { mg = &metricGroupPdata{ - family: mf, - metricGroup: metricGroup{ - ts: ts, - ls: ls, - complexValue: make([]*dataPoint, 0), - intervalStartTimeMs: mf.intervalStartTimeMs, - }, + family: mf, + ts: ts, + ls: ls, + complexValue: make([]*dataPoint, 0), + intervalStartTimeMs: mf.intervalStartTimeMs, } mf.groups[groupKey] = mg // maintaining data insertion order is helpful to generate stable/reproducible metric output @@ -273,6 +314,9 @@ func (mf *metricFamilyPdata) getGroups() []*metricGroupPdata { func (mf *metricFamilyPdata) ToMetricPdata(metrics *pdata.MetricSlice) (int, int) { metric := pdata.NewMetric() + metric.SetDataType(mf.mtype) + metric.SetName(mf.name) + pointCount := 0 switch mf.mtype { @@ -306,7 +350,8 @@ func (mf *metricFamilyPdata) ToMetricPdata(metrics *pdata.MetricSlice) (int, int } pointCount = sdpL.Len() - default: + default: // Everything else should be set to a Gauge. + metric.SetDataType(pdata.MetricDataTypeGauge) gauge := metric.Gauge() gdpL := gauge.DataPoints() for _, mg := range mf.getGroups() { @@ -326,3 +371,32 @@ func (mf *metricFamilyPdata) ToMetricPdata(metrics *pdata.MetricSlice) (int, int // note: the total number of points is the number of points+droppedTimeseries. return pointCount + mf.droppedTimeseries, mf.droppedTimeseries } + +// Define manually the metadata of prometheus scrapper internal metrics +func defineInternalMetric(metricName string, metadata scrape.MetricMetadata, logger *zap.Logger) scrape.MetricMetadata { + if metadata.Metric != "" && metadata.Type != "" && metadata.Help != "" { + logger.Debug("Internal metric seems already fully defined") + return metadata + } + metadata.Metric = metricName + + switch metricName { + case scrapeUpMetricName: + metadata.Type = textparse.MetricTypeGauge + metadata.Help = "The scraping was successful" + case "scrape_duration_seconds": + metadata.Unit = "seconds" + metadata.Type = textparse.MetricTypeGauge + metadata.Help = "Duration of the scrape" + case "scrape_samples_scraped": + metadata.Type = textparse.MetricTypeGauge + metadata.Help = "The number of samples the target exposed" + case "scrape_series_added": + metadata.Type = textparse.MetricTypeGauge + metadata.Help = "The approximate number of new series in this scrape" + case "scrape_samples_post_metric_relabeling": + metadata.Type = textparse.MetricTypeGauge + metadata.Help = "The number of samples remaining after metric relabeling was applied" + } + return metadata +} diff --git a/receiver/prometheusreceiver/internal/otlp_metricfamily_test.go b/receiver/prometheusreceiver/internal/otlp_metricfamily_test.go index 33592c6717521..b2a4db2b32a2b 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricfamily_test.go +++ b/receiver/prometheusreceiver/internal/otlp_metricfamily_test.go @@ -15,16 +15,14 @@ package internal import ( - "fmt" "testing" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/textparse" "github.com/prometheus/prometheus/scrape" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/model/pdata" - "go.uber.org/zap" ) type byLookupMetadataCache map[string]scrape.MetricMetadata @@ -77,31 +75,6 @@ var mc = byLookupMetadataCache{ }, } -func TestIsCumulativeEquivalence(t *testing.T) { - tests := []struct { - name string - want bool - }{ - {name: "counter", want: true}, - {name: "gauge", want: false}, - {name: "histogram", want: true}, - {name: "gaugehistogram", want: false}, - {name: "does not exist", want: false}, - {name: "unknown", want: false}, - } - - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - mf := newMetricFamily(tt.name, mc, zap.NewNop(), 1).(*metricFamily) - mfp := newMetricFamilyPdata(tt.name, mc, 1).(*metricFamilyPdata) - assert.Equal(t, mf.isCumulativeType(), mfp.isCumulativeTypePdata(), "mismatch in isCumulative") - assert.Equal(t, mf.isCumulativeType(), tt.want, "isCumulative does not match for regular metricFamily") - assert.Equal(t, mfp.isCumulativeTypePdata(), tt.want, "isCumulative does not match for pdata metricFamily") - }) - } -} - func TestMetricGroupData_toDistributionUnitTest(t *testing.T) { type scrape struct { at int64 @@ -145,7 +118,7 @@ func TestMetricGroupData_toDistributionUnitTest(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - mp := newMetricFamilyPdata(tt.metricName, mc, tt.intervalStartTimeMs).(*metricFamilyPdata) + mp := newMetricFamilyPdata(tt.metricName, mc, testLogger, tt.intervalStartTimeMs).(*metricFamilyPdata) for _, tv := range tt.scrapes { require.NoError(t, mp.Add(tv.metric, tt.labels.Copy(), tv.at, tv.value)) } @@ -164,93 +137,6 @@ func TestMetricGroupData_toDistributionUnitTest(t *testing.T) { } } -func TestMetricGroupData_toDistributionPointEquivalence(t *testing.T) { - type scrape struct { - at int64 - value float64 - metric string - } - tests := []struct { - name string - labels labels.Labels - scrapes []*scrape - }{ - { - name: "histogram", - labels: labels.Labels{{Name: "a", Value: "A"}, {Name: "le", Value: "0.75"}, {Name: "b", Value: "B"}}, - scrapes: []*scrape{ - {at: 11, value: 10, metric: "histogram_count"}, - {at: 11, value: 1004.78, metric: "histogram_sum"}, - {at: 13, value: 33.7, metric: "value"}, - }, - }, - } - - for i, tt := range tests { - tt := tt - intervalStartTimeMs := int64(i + 1) - t.Run(tt.name, func(t *testing.T) { - mf := newMetricFamily(tt.name, mc, zap.NewNop(), intervalStartTimeMs).(*metricFamily) - mp := newMetricFamilyPdata(tt.name, mc, intervalStartTimeMs).(*metricFamilyPdata) - for _, tv := range tt.scrapes { - require.NoError(t, mp.Add(tv.metric, tt.labels.Copy(), tv.at, tv.value)) - require.NoError(t, mf.Add(tv.metric, tt.labels.Copy(), tv.at, tv.value)) - } - groupKey := mf.getGroupKey(tt.labels.Copy()) - ocTimeseries := mf.groups[groupKey].toDistributionTimeSeries(mf.labelKeysOrdered) - hdpL := pdata.NewHistogramDataPointSlice() - require.True(t, mp.groups[groupKey].toDistributionPoint(mp.labelKeysOrdered, &hdpL)) - require.Equal(t, len(ocTimeseries.Points), hdpL.Len(), "They should have the exact same number of points") - require.Equal(t, 1, hdpL.Len(), "Exactly one point expected") - ocPoint := ocTimeseries.Points[0] - pdataPoint := hdpL.At(0) - // 1. Ensure that the startTimestamps are equal. - require.Equal(t, ocTimeseries.GetStartTimestamp().AsTime(), pdataPoint.Timestamp().AsTime(), "The timestamp must be equal") - // 2. Ensure that the count is equal. - ocHistogram := ocPoint.GetDistributionValue() - require.Equal(t, ocHistogram.GetCount(), int64(pdataPoint.Count()), "Count must be equal") - // 3. Ensure that the sum is equal. - require.Equal(t, ocHistogram.GetSum(), pdataPoint.Sum(), "Sum must be equal") - // 4. Ensure that the point's timestamp is equal to that from the OpenCensusProto data point. - require.Equal(t, ocPoint.GetTimestamp().AsTime(), pdataPoint.Timestamp().AsTime(), "Point timestamps must be equal") - // 5. Ensure that bucket bounds are the same. - require.Equal(t, len(ocHistogram.GetBuckets()), len(pdataPoint.BucketCounts()), "Bucket counts must have the same length") - var ocBucketCounts []uint64 - for i, bucket := range ocHistogram.GetBuckets() { - ocBucketCounts = append(ocBucketCounts, uint64(bucket.GetCount())) - - // 6. Ensure that the exemplars match. - ocExemplar := bucket.Exemplar - if ocExemplar == nil { - if i >= pdataPoint.Exemplars().Len() { // Both have the exact same number of exemplars. - continue - } - // Otherwise an exemplar is present for the pdata data point but not for the OpenCensus Proto histogram. - t.Fatalf("Exemplar #%d is ONLY present in the pdata point but not in the OpenCensus Proto histogram", i) - } - pdataExemplar := pdataPoint.Exemplars().At(i) - msgPrefix := fmt.Sprintf("Exemplar #%d:: ", i) - require.Equal(t, ocExemplar.Timestamp.AsTime(), pdataExemplar.Timestamp().AsTime(), msgPrefix+"timestamp mismatch") - require.Equal(t, ocExemplar.Value, pdataExemplar.DoubleVal(), msgPrefix+"value mismatch") - pdataExemplarAttachments := make(map[string]string) - pdataExemplar.FilteredAttributes().Range(func(key string, value pdata.AttributeValue) bool { - pdataExemplarAttachments[key] = value.AsString() - return true - }) - require.Equal(t, ocExemplar.Attachments, pdataExemplarAttachments, msgPrefix+"attachments mismatch") - } - // 7. Ensure that bucket bounds are the same. - require.Equal(t, ocBucketCounts, pdataPoint.BucketCounts(), "Bucket counts must be equal") - // 8. Ensure that the labels all match up. - ocStringMap := pdata.NewAttributeMap() - for i, labelValue := range ocTimeseries.LabelValues { - ocStringMap.InsertString(mf.labelKeysOrdered[i], labelValue.Value) - } - require.Equal(t, ocStringMap.Sort(), pdataPoint.Attributes().Sort()) - }) - } -} - func TestMetricGroupData_toSummaryUnitTest(t *testing.T) { type scrape struct { at int64 @@ -342,7 +228,7 @@ func TestMetricGroupData_toSummaryUnitTest(t *testing.T) { qn99.SetQuantile(99) qn99.SetValue(82) point.SetTimestamp(14 * 1e6) // the time in milliseconds -> nanoseconds. - point.SetStartTimestamp(14 * 1e6) + point.SetStartTimestamp(10 * 1e5) attributes := point.Attributes() attributes.InsertString("a", "A") attributes.InsertString("b", "B") @@ -354,7 +240,7 @@ func TestMetricGroupData_toSummaryUnitTest(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - mp := newMetricFamilyPdata(tt.name, mc, 1).(*metricFamilyPdata) + mp := newMetricFamilyPdata(tt.name, mc, testLogger, 1).(*metricFamilyPdata) for _, lbs := range tt.labelsScrapes { for _, scrape := range lbs.scrapes { require.NoError(t, mp.Add(scrape.metric, lbs.labels.Copy(), scrape.at, scrape.value)) @@ -379,77 +265,6 @@ func TestMetricGroupData_toSummaryUnitTest(t *testing.T) { } } -func TestMetricGroupData_toSummaryPointEquivalence(t *testing.T) { - type scrape struct { - at int64 - value float64 - metric string - } - tests := []struct { - name string - labels labels.Labels - scrapes []*scrape - }{ - { - name: "summary", - labels: labels.Labels{{Name: "a", Value: "A"}, {Name: "quantile", Value: "0.75"}, {Name: "b", Value: "B"}}, - scrapes: []*scrape{ - {at: 11, value: 10, metric: "summary_count"}, - {at: 11, value: 1004.78, metric: "summary_sum"}, - {at: 13, value: 33.7, metric: "value"}, - }, - }, - } - - for _, tt := range tests { - tt := tt - t.Run(tt.name, func(t *testing.T) { - mf := newMetricFamily(tt.name, mc, zap.NewNop(), 1).(*metricFamily) - mp := newMetricFamilyPdata(tt.name, mc, 1).(*metricFamilyPdata) - for _, tv := range tt.scrapes { - require.NoError(t, mp.Add(tv.metric, tt.labels.Copy(), tv.at, tv.value)) - require.NoError(t, mf.Add(tv.metric, tt.labels.Copy(), tv.at, tv.value)) - } - groupKey := mf.getGroupKey(tt.labels.Copy()) - ocTimeseries := mf.groups[groupKey].toSummaryTimeSeries(mf.labelKeysOrdered) - sdpL := pdata.NewSummaryDataPointSlice() - require.True(t, mp.groups[groupKey].toSummaryPoint(mp.labelKeysOrdered, &sdpL)) - require.Equal(t, len(ocTimeseries.Points), sdpL.Len(), "They should have the exact same number of points") - require.Equal(t, 1, sdpL.Len(), "Exactly one point expected") - ocPoint := ocTimeseries.Points[0] - pdataPoint := sdpL.At(0) - // 1. Ensure that the startTimestamps are equal. - require.Equal(t, ocTimeseries.GetStartTimestamp().AsTime(), pdataPoint.Timestamp().AsTime(), "The timestamp must be equal") - // 2. Ensure that the count is equal. - ocSummary := ocPoint.GetSummaryValue() - if false { - t.Logf("\nOcSummary: %#v\nPdSummary: %#v\n\nocPoint: %#v\n", ocSummary, pdataPoint, ocPoint.GetSummaryValue()) - return - } - require.Equal(t, ocSummary.GetCount().GetValue(), int64(pdataPoint.Count()), "Count must be equal") - // 3. Ensure that the sum is equal. - require.Equal(t, ocSummary.GetSum().GetValue(), pdataPoint.Sum(), "Sum must be equal") - // 4. Ensure that the point's timestamp is equal to that from the OpenCensusProto data point. - require.Equal(t, ocPoint.GetTimestamp().AsTime(), pdataPoint.Timestamp().AsTime(), "Point timestamps must be equal") - // 5. Ensure that the labels all match up. - ocStringMap := pdata.NewAttributeMap() - for i, labelValue := range ocTimeseries.LabelValues { - ocStringMap.InsertString(mf.labelKeysOrdered[i], labelValue.Value) - } - require.Equal(t, ocStringMap.Sort(), pdataPoint.Attributes().Sort()) - // 6. Ensure that the quantile values all match up. - ocQuantiles := ocSummary.GetSnapshot().GetPercentileValues() - pdataQuantiles := pdataPoint.QuantileValues() - require.Equal(t, len(ocQuantiles), pdataQuantiles.Len()) - for i, ocQuantile := range ocQuantiles { - pdataQuantile := pdataQuantiles.At(i) - require.Equal(t, ocQuantile.Percentile, pdataQuantile.Quantile(), "The quantile percentiles must match") - require.Equal(t, ocQuantile.Value, pdataQuantile.Value(), "The quantile values must match") - } - }) - } -} - func TestMetricGroupData_toNumberDataUnitTest(t *testing.T) { type scrape struct { at int64 @@ -507,7 +322,7 @@ func TestMetricGroupData_toNumberDataUnitTest(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - mp := newMetricFamilyPdata(tt.metricKind, mc, tt.intervalStartTimestampMs).(*metricFamilyPdata) + mp := newMetricFamilyPdata(tt.metricKind, mc, testLogger, tt.intervalStartTimestampMs).(*metricFamilyPdata) for _, tv := range tt.scrapes { require.NoError(t, mp.Add(tv.metric, tt.labels.Copy(), tv.at, tv.value)) } @@ -525,61 +340,3 @@ func TestMetricGroupData_toNumberDataUnitTest(t *testing.T) { }) } } - -func TestMetricGroupData_toNumberDataPointEquivalence(t *testing.T) { - type scrape struct { - at int64 - value float64 - metric string - } - tests := []struct { - name string - labels labels.Labels - scrapes []*scrape - wantValue float64 - }{ - { - name: "counter", - labels: labels.Labels{{Name: "a", Value: "A"}, {Name: "b", Value: "B"}}, - scrapes: []*scrape{ - {at: 13, value: 33.7, metric: "value"}, - }, - wantValue: 33.7, - }, - } - - for i, tt := range tests { - tt := tt - intervalStartTimeMs := int64(11 + i) - t.Run(tt.name, func(t *testing.T) { - mf := newMetricFamily(tt.name, mc, zap.NewNop(), intervalStartTimeMs).(*metricFamily) - mp := newMetricFamilyPdata(tt.name, mc, intervalStartTimeMs).(*metricFamilyPdata) - for _, tv := range tt.scrapes { - require.NoError(t, mp.Add(tv.metric, tt.labels.Copy(), tv.at, tv.value)) - require.NoError(t, mf.Add(tv.metric, tt.labels.Copy(), tv.at, tv.value)) - } - groupKey := mf.getGroupKey(tt.labels.Copy()) - ocTimeseries := mf.groups[groupKey].toDoubleValueTimeSeries(mf.labelKeysOrdered) - ddpL := pdata.NewNumberDataPointSlice() - require.True(t, mp.groups[groupKey].toNumberDataPoint(mp.labelKeysOrdered, &ddpL)) - require.Equal(t, len(ocTimeseries.Points), ddpL.Len(), "They should have the exact same number of points") - require.Equal(t, 1, ddpL.Len(), "Exactly one point expected") - ocPoint := ocTimeseries.Points[0] - pdataPoint := ddpL.At(0) - // 1. Ensure that the startTimestamps are equal. - require.Equal(t, ocTimeseries.GetStartTimestamp().AsTime(), pdataPoint.StartTimestamp().AsTime(), "The timestamp must be equal") - require.Equal(t, intervalStartTimeMs*1e6, pdataPoint.StartTimestamp().AsTime().UnixNano(), "intervalStartTimeMs must be the same") - // 2. Ensure that the value is equal. - require.Equal(t, ocPoint.GetDoubleValue(), pdataPoint.DoubleVal(), "Values must be equal") - require.Equal(t, tt.wantValue, pdataPoint.DoubleVal(), "Values must be equal") - // 4. Ensure that the point's timestamp is equal to that from the OpenCensusProto data point. - require.Equal(t, ocPoint.GetTimestamp().AsTime(), pdataPoint.Timestamp().AsTime(), "Point timestamps must be equal") - // 5. Ensure that the labels all match up. - ocStringMap := pdata.NewAttributeMap() - for i, labelValue := range ocTimeseries.LabelValues { - ocStringMap.InsertString(mf.labelKeysOrdered[i], labelValue.Value) - } - require.Equal(t, ocStringMap.Sort(), pdataPoint.Attributes().Sort()) - }) - } -} diff --git a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go new file mode 100644 index 0000000000000..e10154d1e2da1 --- /dev/null +++ b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go @@ -0,0 +1,435 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "fmt" + "strings" + "sync" + "time" + + "go.opentelemetry.io/collector/model/pdata" + + "go.uber.org/zap" +) + +// Notes on garbage collection (gc): +// +// Job-level gc: +// The Prometheus receiver will likely execute in a long running service whose lifetime may exceed +// the lifetimes of many of the jobs that it is collecting from. In order to keep the JobsMap from +// leaking memory for entries of no-longer existing jobs, the JobsMap needs to remove entries that +// haven't been accessed for a long period of time. +// +// Timeseries-level gc: +// Some jobs that the Prometheus receiver is collecting from may export timeseries based on metrics +// from other jobs (e.g. cAdvisor). In order to keep the timeseriesMap from leaking memory for entries +// of no-longer existing jobs, the timeseriesMap for each job needs to remove entries that haven't +// been accessed for a long period of time. +// +// The gc strategy uses a standard mark-and-sweep approach - each time a timeseriesMap is accessed, +// it is marked. Similarly, each time a timeseriesinfo is accessed, it is also marked. +// +// At the end of each JobsMap.get(), if the last time the JobsMap was gc'd exceeds the 'gcInterval', +// the JobsMap is locked and any timeseriesMaps that are unmarked are removed from the JobsMap +// otherwise the timeseriesMap is gc'd +// +// The gc for the timeseriesMap is straightforward - the map is locked and, for each timeseriesinfo +// in the map, if it has not been marked, it is removed otherwise it is unmarked. +// +// Alternative Strategies +// 1. If the job-level gc doesn't run often enough, or runs too often, a separate go routine can +// be spawned at JobMap creation time that gc's at periodic intervals. This approach potentially +// adds more contention and latency to each scrape so the current approach is used. Note that +// the go routine will need to be cancelled upon Shutdown(). +// 2. If the gc of each timeseriesMap during the gc of the JobsMap causes too much contention, +// the gc of timeseriesMaps can be moved to the end of MetricsAdjuster().AdjustMetrics(). This +// approach requires adding 'lastGC' Time and (potentially) a gcInterval duration to +// timeseriesMap so the current approach is used instead. + +// timeseriesinfo contains the information necessary to adjust from the initial point and to detect +// resets. +type timeseriesinfoPdata struct { + mark bool + initial *pdata.Metric + previous *pdata.Metric +} + +// timeseriesMap maps from a timeseries instance (metric * label values) to the timeseries info for +// the instance. +type timeseriesMapPdata struct { + sync.RWMutex + mark bool + tsiMap map[string]*timeseriesinfoPdata +} + +// Get the timeseriesinfo for the timeseries associated with the metric and label values. +func (tsm *timeseriesMapPdata) get(metric *pdata.Metric, kv pdata.AttributeMap) *timeseriesinfoPdata { + name := metric.Name() + sig := getTimeseriesSignaturePdata(name, kv) + if metric.DataType() == pdata.MetricDataTypeHistogram { + // There are 2 types of Histograms whose aggregation temporality needs distinguishing: + // * CumulativeHistogram + // * GaugeHistogram + aggTemporality := metric.Histogram().AggregationTemporality() + sig += "," + aggTemporality.String() + } + tsi, ok := tsm.tsiMap[sig] + if !ok { + tsi = ×eriesinfoPdata{} + tsm.tsiMap[sig] = tsi + } + tsm.mark = true + tsi.mark = true + return tsi +} + +// Create a unique timeseries signature consisting of the metric name and label values. +func getTimeseriesSignaturePdata(name string, kv pdata.AttributeMap) string { + labelValues := make([]string, 0, kv.Len()) + kv.Sort().Range(func(_ string, attrValue pdata.AttributeValue) bool { + value := attrValue.StringVal() + if value != "" { + labelValues = append(labelValues, value) + } + return true + }) + return fmt.Sprintf("%s,%s", name, strings.Join(labelValues, ",")) +} + +// Remove timeseries that have aged out. +func (tsm *timeseriesMapPdata) gc() { + tsm.Lock() + defer tsm.Unlock() + // this shouldn't happen under the current gc() strategy + if !tsm.mark { + return + } + for ts, tsi := range tsm.tsiMap { + if !tsi.mark { + delete(tsm.tsiMap, ts) + } else { + tsi.mark = false + } + } + tsm.mark = false +} + +func newTimeseriesMapPdata() *timeseriesMapPdata { + return ×eriesMapPdata{mark: true, tsiMap: map[string]*timeseriesinfoPdata{}} +} + +// JobsMapPdata maps from a job instance to a map of timeseriesPdata instances for the job. +type JobsMapPdata struct { + sync.RWMutex + gcInterval time.Duration + lastGC time.Time + jobsMap map[string]*timeseriesMapPdata +} + +// NewJobsMap creates a new (empty) JobsMapPdata. +func NewJobsMapPdata(gcInterval time.Duration) *JobsMapPdata { + return &JobsMapPdata{gcInterval: gcInterval, lastGC: time.Now(), jobsMap: make(map[string]*timeseriesMapPdata)} +} + +// Remove jobs and timeseries that have aged out. +func (jm *JobsMapPdata) gc() { + jm.Lock() + defer jm.Unlock() + // once the structure is locked, confirm that gc() is still necessary + if time.Since(jm.lastGC) > jm.gcInterval { + for sig, tsm := range jm.jobsMap { + tsm.RLock() + tsmNotMarked := !tsm.mark + tsm.RUnlock() + if tsmNotMarked { + delete(jm.jobsMap, sig) + } else { + tsm.gc() + } + } + jm.lastGC = time.Now() + } +} + +func (jm *JobsMapPdata) maybeGC() { + // speculatively check if gc() is necessary, recheck once the structure is locked + jm.RLock() + defer jm.RUnlock() + if time.Since(jm.lastGC) > jm.gcInterval { + go jm.gc() + } +} + +func (jm *JobsMapPdata) get(job, instance string) *timeseriesMapPdata { + sig := job + ":" + instance + jm.RLock() + tsm, ok := jm.jobsMap[sig] + jm.RUnlock() + defer jm.maybeGC() + if ok { + return tsm + } + jm.Lock() + defer jm.Unlock() + tsm2, ok2 := jm.jobsMap[sig] + if ok2 { + return tsm2 + } + tsm2 = newTimeseriesMapPdata() + jm.jobsMap[sig] = tsm2 + return tsm2 +} + +// MetricsAdjusterPdata takes a map from a metric instance to the initial point in the metrics instance +// and provides AdjustMetrics, which takes a sequence of metrics and adjust their start times based on +// the initial points. +type MetricsAdjusterPdata struct { + tsm *timeseriesMapPdata + logger *zap.Logger +} + +// NewMetricsAdjuster is a constructor for MetricsAdjuster. +func NewMetricsAdjusterPdata(tsm *timeseriesMapPdata, logger *zap.Logger) *MetricsAdjusterPdata { + return &MetricsAdjusterPdata{ + tsm: tsm, + logger: logger, + } +} + +// AdjustMetrics takes a sequence of metrics and adjust their start times based on the initial and +// previous points in the timeseriesMap. +// Returns the total number of timeseries that had reset start times. +func (ma *MetricsAdjusterPdata) AdjustMetrics(metricL *pdata.MetricSlice) int { + resets := 0 + ma.tsm.Lock() + defer ma.tsm.Unlock() + for i := 0; i < metricL.Len(); i++ { + metric := metricL.At(i) + resets += ma.adjustMetric(&metric) + } + return resets +} + +// Returns the number of timeseries with reset start times. +func (ma *MetricsAdjusterPdata) adjustMetric(metric *pdata.Metric) int { + switch metric.DataType() { + case pdata.MetricDataTypeGauge: + // gauges don't need to be adjusted so no additional processing is necessary + return 0 + default: + return ma.adjustMetricPoints(metric) + } +} + +// Returns the number of timeseries that had reset start times. +func (ma *MetricsAdjusterPdata) adjustMetricPoints(metric *pdata.Metric) int { + switch dataType := metric.DataType(); dataType { + case pdata.MetricDataTypeGauge: + return ma.adjustMetricGauge(metric) + + case pdata.MetricDataTypeHistogram: + return ma.adjustMetricHistogram(metric) + + case pdata.MetricDataTypeSummary: + return ma.adjustMetricSummary(metric) + + case pdata.MetricDataTypeSum: + return ma.adjustMetricSum(metric) + + default: + // this shouldn't happen + ma.logger.Info("Adjust - skipping unexpected point", zap.String("type", dataType.String())) + return 0 + } +} + +// Returns true if 'current' was adjusted and false if 'current' is an the initial occurrence or a +// reset of the timeseries. +func (ma *MetricsAdjusterPdata) adjustMetricGauge(current *pdata.Metric) (resets int) { + currentPoints := current.Gauge().DataPoints() + + for i := 0; i < currentPoints.Len(); i++ { + currentGauge := currentPoints.At(i) + tsi := ma.tsm.get(current, currentGauge.Attributes()) + previous := tsi.previous + tsi.previous = current + if tsi.initial == nil { + // initial || reset timeseries. + tsi.initial = current + resets++ + } + initialPoints := tsi.initial.Gauge().DataPoints() + previousPoints := previous.Gauge().DataPoints() + if i >= initialPoints.Len() || i >= previousPoints.Len() { + ma.logger.Info("Adjusting Points, all lengths should be equal", + zap.Int("len(current)", currentPoints.Len()), + zap.Int("len(initial)", initialPoints.Len()), + zap.Int("len(previous)", previousPoints.Len())) + // initial || reset timeseries. + tsi.initial = current + resets++ + continue + } + + currentGauge, previousGauge := currentPoints.At(i), previousPoints.At(i) + if currentGauge.DoubleVal() < previousGauge.DoubleVal() { + // reset detected + tsi.initial = current + continue + } + initialGauge := initialPoints.At(i) + currentGauge.SetStartTimestamp(initialGauge.StartTimestamp()) + resets++ + } + return +} + +func (ma *MetricsAdjusterPdata) adjustMetricHistogram(current *pdata.Metric) (resets int) { + histogram := current.Histogram() + if histogram.AggregationTemporality() != pdata.AggregationTemporalityCumulative { + // Only dealing with CumulativeDistributions. + return 0 + } + + // Note: Sum of Squared Deviation not currently supported. + currentPoints := histogram.DataPoints() + + if currentPoints.Len() == 0 { + return 0 + } + + for i := 0; i < currentPoints.Len(); i++ { + currentDist := currentPoints.At(i) + tsi := ma.tsm.get(current, currentDist.Attributes()) + previous := tsi.previous + tsi.previous = current + if tsi.initial == nil { + // initial || reset timeseries. + tsi.initial = current + resets++ + continue + } + initialPoints := tsi.initial.Histogram().DataPoints() + previousPoints := previous.Histogram().DataPoints() + if i >= initialPoints.Len() || i >= previousPoints.Len() { + ma.logger.Info("Adjusting Points, all lengths should be equal", + zap.Int("len(current)", currentPoints.Len()), + zap.Int("len(initial)", initialPoints.Len()), + zap.Int("len(previous)", previousPoints.Len())) + // initial || reset timeseries. + tsi.initial = current + resets++ + continue + } + + previousDist := previousPoints.At(i) + if currentDist.Count() < previousDist.Count() || currentDist.Sum() < previousDist.Sum() { + // reset detected + tsi.initial = current + resets++ + continue + } + initialDist := initialPoints.At(i) + currentDist.SetStartTimestamp(initialDist.StartTimestamp()) + } + return +} + +func (ma *MetricsAdjusterPdata) adjustMetricSum(current *pdata.Metric) (resets int) { + currentPoints := current.Sum().DataPoints() + + for i := 0; i < currentPoints.Len(); i++ { + currentSum := currentPoints.At(i) + tsi := ma.tsm.get(current, currentSum.Attributes()) + previous := tsi.previous + tsi.previous = current + if tsi.initial == nil { + // initial || reset timeseries. + tsi.initial = current + resets++ + continue + } + initialPoints := tsi.initial.Sum().DataPoints() + previousPoints := previous.Sum().DataPoints() + if i >= initialPoints.Len() || i >= previousPoints.Len() { + ma.logger.Info("Adjusting Points, all lengths should be equal", + zap.Int("len(current)", currentPoints.Len()), + zap.Int("len(initial)", initialPoints.Len()), + zap.Int("len(previous)", previousPoints.Len())) + tsi.initial = current + resets++ + continue + } + + previousSum := previousPoints.At(i) + if currentSum.DoubleVal() < previousSum.DoubleVal() { + // reset detected + tsi.initial = current + resets++ + continue + } + initialSum := initialPoints.At(i) + currentSum.SetStartTimestamp(initialSum.StartTimestamp()) + } + + return +} + +func (ma *MetricsAdjusterPdata) adjustMetricSummary(current *pdata.Metric) (resets int) { + currentPoints := current.Summary().DataPoints() + + for i := 0; i < currentPoints.Len(); i++ { + currentSummary := currentPoints.At(i) + tsi := ma.tsm.get(current, currentSummary.Attributes()) + previous := tsi.previous + tsi.previous = current + if tsi.initial == nil { + // initial || reset timeseries. + tsi.initial = current + resets++ + continue + } + initialPoints := tsi.initial.Summary().DataPoints() + previousPoints := previous.Summary().DataPoints() + if i >= initialPoints.Len() || i >= previousPoints.Len() { + ma.logger.Info("Adjusting Points, all lengths should be equal", + zap.Int("len(current)", currentPoints.Len()), + zap.Int("len(initial)", initialPoints.Len()), + zap.Int("len(previous)", previousPoints.Len())) + tsi.initial = current + resets++ + continue + } + + previousSummary := previousPoints.At(i) + if (currentSummary.Count() != 0 && + previousSummary.Count() != 0 && + currentSummary.Count() < previousSummary.Count()) || + + (currentSummary.Sum() != 0 && + previousSummary.Sum() != 0 && + currentSummary.Sum() < previousSummary.Sum()) { + // reset detected + tsi.initial = current + resets++ + continue + } + initialSummary := initialPoints.At(i) + currentSummary.SetStartTimestamp(initialSummary.StartTimestamp()) + } + + return +} diff --git a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go new file mode 100644 index 0000000000000..9cc0373ad8022 --- /dev/null +++ b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go @@ -0,0 +1,1006 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "go.opentelemetry.io/collector/model/pdata" +) + +var ( + t1Ms = pdata.Timestamp(time.Unix(0, 1000000).UnixNano()) + t2Ms = pdata.Timestamp(time.Unix(0, 2000000).UnixNano()) + t3Ms = pdata.Timestamp(time.Unix(0, 3000000).UnixNano()) + t4Ms = pdata.Timestamp(time.Unix(0, 5000000).UnixNano()) + t5Ms = pdata.Timestamp(time.Unix(0, 5000000).UnixNano()) + + bounds0 = []float64{1, 2, 4} + percent0 = []float64{10, 50, 90} + + gd1 = "gaugedist1" + c1 = "cumulative1" + cd1 = "cumulativedist1" + s1 = "summary1" + k1 = []string{"k1"} + k1k2 = []string{"k1", "k2"} + k1k2k3 = []string{"k1", "k2", "k3"} + v1v2 = []string{"v1", "v2"} + v10v20 = []string{"v10", "v20"} + v100v200 = []string{"v100", "v200"} +) + +func Test_gauge(t *testing.T) { + script := []*metricsAdjusterTest{ + { + "Gauge: round 1 - gauge not adjusted", + func() *pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetDataType(pdata.MetricDataTypeGauge) + m0.SetName("gauge1") + g0 := m0.Gauge() + pt0 := g0.DataPoints().AppendEmpty() + pt0.SetStartTimestamp(t1Ms) + pt0.Attributes().InsertString("v1", "v2") + pt0.SetTimestamp(t1Ms) + pt0.SetDoubleVal(44) + return &mL + }(), + func() *pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetDataType(pdata.MetricDataTypeGauge) + m0.SetName("gauge1") + g0 := m0.Gauge() + pt0 := g0.DataPoints().AppendEmpty() + pt0.SetStartTimestamp(t1Ms) + pt0.Attributes().InsertString("v1", "v2") + pt0.SetTimestamp(t1Ms) + pt0.SetDoubleVal(44) + return &mL + }(), + 0, + }, + { + "Gauge: round 2 - gauge not adjusted", + func() *pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetDataType(pdata.MetricDataTypeGauge) + m0.SetName("gauge1") + g0 := m0.Gauge() + pt0 := g0.DataPoints().AppendEmpty() + pt0.SetStartTimestamp(t2Ms) + pt0.Attributes().InsertString("k1", "v1") + pt0.Attributes().InsertString("k2", "v2") + pt0.SetTimestamp(t2Ms) + pt0.SetDoubleVal(66) + + return &mL + }(), + func() *pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetDataType(pdata.MetricDataTypeGauge) + m0.SetName("gauge1") + g0 := m0.Gauge() + pt0 := g0.DataPoints().AppendEmpty() + pt0.SetStartTimestamp(t2Ms) + pt0.Attributes().InsertString("k1", "v1") + pt0.Attributes().InsertString("k2", "v2") + pt0.SetTimestamp(t2Ms) + pt0.SetDoubleVal(66) + return &mL + }(), + 0, + }, + { + "Gauge: round 3 - value less than previous value - gauge is not adjusted", + func() *pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetDataType(pdata.MetricDataTypeGauge) + m0.SetName("gauge1") + g0 := m0.Gauge() + pt0 := g0.DataPoints().AppendEmpty() + pt0.SetStartTimestamp(t3Ms) + pt0.Attributes().InsertString("k1", "v1") + pt0.Attributes().InsertString("k2", "v2") + pt0.SetTimestamp(t3Ms) + pt0.SetDoubleVal(55) + + return &mL + }(), + func() *pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetDataType(pdata.MetricDataTypeGauge) + m0.SetName("gauge1") + g0 := m0.Gauge() + pt0 := g0.DataPoints().AppendEmpty() + pt0.SetStartTimestamp(t3Ms) + pt0.Attributes().InsertString("k1", "v1") + pt0.Attributes().InsertString("k2", "v2") + pt0.SetTimestamp(t3Ms) + pt0.SetDoubleVal(55) + + return &mL + }(), + 0, + }, + } + runScript(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) +} + +func Test_cumulative(t *testing.T) { + script := []*metricsAdjusterTest{ + { + "Cumulative: round 1 - initial instance, start time is established", + func() *pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetDataType(pdata.MetricDataTypeSum) + m0.SetName("cumulative1") + g0 := m0.Sum() + pt0 := g0.DataPoints().AppendEmpty() + pt0.SetStartTimestamp(t1Ms) + pt0.Attributes().InsertString("k1", "v1") + pt0.Attributes().InsertString("k2", "v2") + pt0.SetTimestamp(t1Ms) + pt0.SetDoubleVal(44) + + return &mL + }(), + func() *pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetDataType(pdata.MetricDataTypeSum) + m0.SetName("cumulative1") + g0 := m0.Sum() + pt0 := g0.DataPoints().AppendEmpty() + pt0.SetStartTimestamp(t1Ms) + pt0.Attributes().InsertString("k1", "v1") + pt0.Attributes().InsertString("k2", "v2") + pt0.SetTimestamp(t1Ms) + pt0.SetDoubleVal(44) + + return &mL + }(), + 1, + }, + { + "Cumulative: round 2 - instance adjusted based on round 1", + func() *pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetDataType(pdata.MetricDataTypeSum) + m0.SetName("cumulative1") + g0 := m0.Sum() + pt0 := g0.DataPoints().AppendEmpty() + pt0.SetStartTimestamp(t2Ms) + pt0.Attributes().InsertString("k1", "v1") + pt0.Attributes().InsertString("k2", "v2") + pt0.SetTimestamp(t2Ms) + pt0.SetDoubleVal(66) + + return &mL + }(), + func() *pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetDataType(pdata.MetricDataTypeSum) + m0.SetName("cumulative1") + g0 := m0.Sum() + pt0 := g0.DataPoints().AppendEmpty() + pt0.SetStartTimestamp(t1Ms) + pt0.Attributes().InsertString("k1", "v1") + pt0.Attributes().InsertString("k2", "v2") + pt0.SetTimestamp(t2Ms) + pt0.SetDoubleVal(66) + + return &mL + }(), + 0, + }, + { + "Cumulative: round 3 - instance reset (value less than previous value), start time is reset", + func() *pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetDataType(pdata.MetricDataTypeSum) + m0.SetName("cumulative1") + g0 := m0.Sum() + pt0 := g0.DataPoints().AppendEmpty() + pt0.SetStartTimestamp(t3Ms) + pt0.Attributes().InsertString("k1", "v1") + pt0.Attributes().InsertString("k2", "v2") + pt0.SetTimestamp(t3Ms) + pt0.SetDoubleVal(55) + + return &mL + }(), + func() *pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetDataType(pdata.MetricDataTypeSum) + m0.SetName("cumulative1") + g0 := m0.Sum() + pt0 := g0.DataPoints().AppendEmpty() + pt0.SetStartTimestamp(t3Ms) + pt0.Attributes().InsertString("k1", "v1") + pt0.Attributes().InsertString("k2", "v2") + pt0.SetTimestamp(t3Ms) + pt0.SetDoubleVal(55) + + return &mL + }(), + 1, + }, + { + "Cumulative: round 4 - instance adjusted based on round 3", + func() *pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetDataType(pdata.MetricDataTypeSum) + m0.SetName("cumulative1") + g0 := m0.Sum() + pt0 := g0.DataPoints().AppendEmpty() + pt0.SetStartTimestamp(t4Ms) + pt0.Attributes().InsertString("k1", "v1") + pt0.Attributes().InsertString("k2", "v2") + pt0.SetTimestamp(t4Ms) + pt0.SetDoubleVal(72) + + return &mL + }(), + func() *pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetDataType(pdata.MetricDataTypeSum) + m0.SetName("cumulative1") + g0 := m0.Sum() + pt0 := g0.DataPoints().AppendEmpty() + pt0.SetStartTimestamp(t3Ms) + pt0.Attributes().InsertString("k1", "v1") + pt0.Attributes().InsertString("k2", "v2") + pt0.SetTimestamp(t4Ms) + pt0.SetDoubleVal(72) + + return &mL + }(), + 0, + }, + } + runScript(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) +} + +func populateHistogram(hdp *pdata.HistogramDataPoint, timestamp pdata.Timestamp, bounds []float64, counts []uint64) { + count := uint64(0) + sum := float64(0) + for i, counti := range counts { + if i > 0 { + sum += float64(counti) * bounds[i-1] + } + count += counti + } + hdp.SetBucketCounts(counts) + hdp.SetSum(sum) + hdp.SetCount(count) + hdp.SetTimestamp(timestamp) + hdp.SetExplicitBounds(bounds) +} + +func Test_gaugeDistribution(t *testing.T) { + script := []*metricsAdjusterTest{ + { + "GaugeDist: round 1 - gauge distribution not adjusted", + metricSlice(gaugeDistMetric(gd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7}))), + metricSlice(gaugeDistMetric(gd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7}))), + 0, + }, + { + "GaugeDist: round 2 - gauge distribution not adjusted", + metricSlice(gaugeDistMetric(gd1, k1v1k2v2, t2Ms, distPoint(t2Ms, bounds0, []uint64{6, 5, 8, 11}))), + metricSlice(gaugeDistMetric(gd1, k1v1k2v2, t2Ms, distPoint(t2Ms, bounds0, []uint64{6, 5, 8, 11}))), + 0, + }, + { + "GaugeDist: round 3 - count/sum less than previous - gauge distribution not adjusted", + metricSlice(gaugeDistMetric(gd1, k1v1k2v2, t3Ms, distPoint(t3Ms, bounds0, []uint64{2, 0, 1, 5}))), + metricSlice(gaugeDistMetric(gd1, k1v1k2v2, t3Ms, distPoint(t3Ms, bounds0, []uint64{2, 0, 1, 5}))), + 0, + }, + } + runScript(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) +} + +func populateSummary(sdp *pdata.SummaryDataPoint, timestamp pdata.Timestamp, count uint64, sum float64, quantilePercents, quantileValues []float64) { + quantiles := sdp.QuantileValues() + for i := range quantilePercents { + qv := quantiles.AppendEmpty() + qv.SetQuantile(quantilePercents[i]) + qv.SetValue(quantileValues[i]) + } + sdp.SetCount(count) + sdp.SetTimestamp(timestamp) + sdp.SetSum(sum) +} + +func Test_summary_no_count(t *testing.T) { + script := []*metricsAdjusterTest{ + { + "Summary No Count: round 1 - initial instance, start time is established", + func() *pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetDataType(pdata.MetricDataTypeSummary) + m0.SetName("summary1") + s0 := m0.Summary() + pt0 := s0.DataPoints().AppendEmpty() + pt0.Attributes().InsertString("v1", "v2") + populateSummary(&pt0, t1Ms, 10, 40, percent0, []float64{1, 5, 8}) + return &mL + }(), + func() *pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetDataType(pdata.MetricDataTypeSummary) + m0.SetName("summary1") + s0 := m0.Summary() + pt0 := s0.DataPoints().AppendEmpty() + pt0.Attributes().InsertString("v1", "v2") + populateSummary(&pt0, t1Ms, 10, 40, percent0, []float64{1, 5, 8}) + return &mL + }(), + 1, + }, + { + "Summary No Count: round 2 - instance adjusted based on round 1", + func() *pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetDataType(pdata.MetricDataTypeSummary) + m0.SetName("summary1") + s0 := m0.Summary() + pt0 := s0.DataPoints().AppendEmpty() + pt0.Attributes().InsertString("v1", "v2") + populateSummary(&pt0, t2Ms, 15, 70, percent0, []float64{7, 44, 9}) + return &mL + }(), + func() *pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetDataType(pdata.MetricDataTypeSummary) + m0.SetName("summary1") + s0 := m0.Summary() + pt0 := s0.DataPoints().AppendEmpty() + pt0.Attributes().InsertString("v1", "v2") + populateSummary(&pt0, t2Ms, 15, 70, percent0, []float64{7, 44, 9}) + return &mL + }(), + 0, + }, + { + "Summary No Count: round 3 - instance reset (count less than previous), start time is reset", + func() *pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetDataType(pdata.MetricDataTypeSummary) + m0.SetName("summary1") + s0 := m0.Summary() + pt0 := s0.DataPoints().AppendEmpty() + pt0.Attributes().InsertString("v1", "v2") + populateSummary(&pt0, t3Ms, 12, 66, percent0, []float64{3, 22, 5}) + return &mL + }(), + func() *pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetDataType(pdata.MetricDataTypeSummary) + m0.SetName("summary1") + s0 := m0.Summary() + pt0 := s0.DataPoints().AppendEmpty() + pt0.Attributes().InsertString("v1", "v2") + populateSummary(&pt0, t3Ms, 12, 66, percent0, []float64{3, 22, 5}) + return &mL + }(), + 1, + }, + { + "Summary No Count: round 4 - instance adjusted based on round 3", + func() *pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetDataType(pdata.MetricDataTypeSummary) + m0.SetName("summary1") + s0 := m0.Summary() + pt0 := s0.DataPoints().AppendEmpty() + pt0.Attributes().InsertString("v1", "v2") + populateSummary(&pt0, t4Ms, 14, 96, percent0, []float64{9, 47, 8}) + pt0.SetStartTimestamp(t4Ms) + return &mL + }(), + func() *pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetDataType(pdata.MetricDataTypeSummary) + m0.SetName("summary1") + s0 := m0.Summary() + pt0 := s0.DataPoints().AppendEmpty() + pt0.Attributes().InsertString("v1", "v2") + populateSummary(&pt0, t4Ms, 14, 96, percent0, []float64{9, 47, 8}) + return &mL + }(), + 0, + }, + } + + runScript(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) +} + +func Test_summary(t *testing.T) { + script := []*metricsAdjusterTest{ + { + "Summary: round 1 - initial instance, start time is established", + metricSlice( + summaryMetric(s1, k1v1k2v2, t1Ms, summaryPoint(t1Ms, 10, 40, percent0, []float64{1, 5, 8})), + ), + metricSlice( + summaryMetric(s1, k1v1k2v2, t1Ms, summaryPoint(t1Ms, 10, 40, percent0, []float64{1, 5, 8})), + ), + 1, + }, + { + "Summary: round 2 - instance adjusted based on round 1", + metricSlice( + summaryMetric(s1, k1v1k2v2, t2Ms, summaryPoint(t2Ms, 15, 70, percent0, []float64{7, 44, 9})), + ), + metricSlice( + summaryMetric(s1, k1v1k2v2, t1Ms, summaryPoint(t2Ms, 15, 70, percent0, []float64{7, 44, 9})), + ), + 0, + }, + { + "Summary: round 3 - instance reset (count less than previous), start time is reset", + metricSlice( + summaryMetric(s1, k1v1k2v2, t3Ms, summaryPoint(t3Ms, 12, 66, percent0, []float64{3, 22, 5})), + ), + metricSlice( + summaryMetric(s1, k1v1k2v2, t3Ms, summaryPoint(t3Ms, 12, 66, percent0, []float64{3, 22, 5})), + ), + 1, + }, + { + "Summary: round 4 - instance adjusted based on round 3", + metricSlice( + summaryMetric(s1, k1v1k2v2, t4Ms, summaryPoint(t4Ms, 14, 96, percent0, []float64{9, 47, 8})), + ), + metricSlice( + summaryMetric(s1, k1v1k2v2, t3Ms, summaryPoint(t4Ms, 14, 96, percent0, []float64{9, 47, 8})), + ), + 0, + }, + } + + runScript(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) +} + +func distPoint(ts pdata.Timestamp, bounds []float64, counts []uint64) *pdata.HistogramDataPoint { + hdp := pdata.NewHistogramDataPoint() + hdp.SetExplicitBounds(bounds) + hdp.SetBucketCounts(counts) + hdp.SetTimestamp(ts) + var sum float64 + var count uint64 + for i, bcount := range counts { + count += bcount + if i > 0 { + sum += float64(bcount) * bounds[i-1] + } + } + hdp.SetCount(count) + hdp.SetSum(sum) + + return &hdp +} + +type kv struct { + key, value string +} + +func gaugeDistMetric(name string, kvp []*kv, startTs pdata.Timestamp, points ...*pdata.HistogramDataPoint) *pdata.Metric { + hMetric := histogramMetric(name, kvp, startTs, points...) + hMetric.Histogram().SetAggregationTemporality(pdata.AggregationTemporalityDelta) + return hMetric +} + +func histogramMetric(name string, kvp []*kv, startTs pdata.Timestamp, points ...*pdata.HistogramDataPoint) *pdata.Metric { + metric := pdata.NewMetric() + metric.SetName(name) + metric.SetDataType(pdata.MetricDataTypeHistogram) + histogram := metric.Histogram() + histogram.SetAggregationTemporality(pdata.AggregationTemporalityCumulative) + + destPointL := histogram.DataPoints() + // By default the AggregationTemporality is Cumulative until it'll be changed by the caller. + for _, point := range points { + destPoint := destPointL.AppendEmpty() + point.CopyTo(destPoint) + point.SetStartTimestamp(startTs) + attrs := destPoint.Attributes() + for _, kv := range kvp { + attrs.InsertString(kv.key, kv.value) + } + } + return &metric +} + +func doublePoint(ts pdata.Timestamp, value float64) *pdata.NumberDataPoint { + ndp := pdata.NewNumberDataPoint() + ndp.SetTimestamp(ts) + ndp.SetDoubleVal(value) + + return &ndp +} + +func gaugeMetric(name string, kvp []*kv, startTs pdata.Timestamp, points ...*pdata.NumberDataPoint) *pdata.Metric { + metric := pdata.NewMetric() + metric.SetName(name) + metric.SetDataType(pdata.MetricDataTypeGauge) + + destPointL := metric.Gauge().DataPoints() + for _, point := range points { + destPoint := destPointL.AppendEmpty() + point.CopyTo(destPoint) + point.SetStartTimestamp(startTs) + attrs := destPoint.Attributes() + for _, kv := range kvp { + attrs.InsertString(kv.key, kv.value) + } + } + return &metric +} + +func summaryPoint(ts pdata.Timestamp, count uint64, sum float64, quantiles, values []float64) *pdata.SummaryDataPoint { + sdp := pdata.NewSummaryDataPoint() + sdp.SetTimestamp(ts) + sdp.SetCount(count) + sdp.SetSum(sum) + qvL := sdp.QuantileValues() + for i := 0; i < len(quantiles); i++ { + qvi := qvL.AppendEmpty() + qvi.SetQuantile(quantiles[i]) + qvi.SetValue(values[i]) + } + return &sdp +} + +func summaryMetric(name string, kvp []*kv, startTs pdata.Timestamp, points ...*pdata.SummaryDataPoint) *pdata.Metric { + metric := pdata.NewMetric() + metric.SetName(name) + metric.SetDataType(pdata.MetricDataTypeSummary) + + destPointL := metric.Summary().DataPoints() + for _, point := range points { + destPoint := destPointL.AppendEmpty() + point.CopyTo(destPoint) + point.SetStartTimestamp(startTs) + attrs := destPoint.Attributes() + for _, kv := range kvp { + attrs.InsertString(kv.key, kv.value) + } + } + return &metric +} + +func sumMetric(name string, kvp []*kv, startTs pdata.Timestamp, points ...*pdata.NumberDataPoint) *pdata.Metric { + metric := pdata.NewMetric() + metric.SetName(name) + metric.SetDataType(pdata.MetricDataTypeSum) + + destPointL := metric.Sum().DataPoints() + for _, point := range points { + destPoint := destPointL.AppendEmpty() + point.CopyTo(destPoint) + point.SetStartTimestamp(startTs) + attrs := destPoint.Attributes() + for _, kv := range kvp { + attrs.InsertString(kv.key, kv.value) + } + } + return &metric +} + +func metricSlice(metrics ...*pdata.Metric) *pdata.MetricSlice { + ms := pdata.NewMetricSlice() + for _, metric := range metrics { + destMetric := ms.AppendEmpty() + metric.CopyTo(destMetric) + } + return &ms +} + +var ( + k1v1k2v2 = []*kv{ + {"k1", "v1"}, + {"k2", "v2"}, + } + + k1v10k2v20 = []*kv{ + {"k1", "v10"}, + {"k2", "v20"}, + } + + k1v100k2v200 = []*kv{ + {"k1", "v100"}, + {"k2", "v200"}, + } +) + +func Test_cumulativeDistribution(t *testing.T) { + script := []*metricsAdjusterTest{ + { + "CumulativeDist: round 1 - initial instance, start time is established", + metricSlice(histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7}))), + metricSlice(histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7}))), + 1, + }, { + "CumulativeDist: round 2 - instance adjusted based on round 1", + metricSlice(histogramMetric(cd1, k1v1k2v2, t2Ms, distPoint(t2Ms, bounds0, []uint64{6, 3, 4, 8}))), + metricSlice(histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t2Ms, bounds0, []uint64{6, 3, 4, 8}))), + 0, + }, { + "CumulativeDist: round 3 - instance reset (value less than previous value), start time is reset", + metricSlice(histogramMetric(cd1, k1v1k2v2, t3Ms, distPoint(t3Ms, bounds0, []uint64{5, 3, 2, 7}))), + metricSlice(histogramMetric(cd1, k1v1k2v2, t3Ms, distPoint(t3Ms, bounds0, []uint64{5, 3, 2, 7}))), + 1, + }, { + "CumulativeDist: round 4 - instance adjusted based on round 3", + metricSlice(histogramMetric(cd1, k1v1k2v2, t4Ms, distPoint(t4Ms, bounds0, []uint64{7, 4, 2, 12}))), + metricSlice(histogramMetric(cd1, k1v1k2v2, t3Ms, distPoint(t4Ms, bounds0, []uint64{7, 4, 2, 12}))), + 0, + }, + } + runScript(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) +} + +func Test_multiMetrics(t *testing.T) { + g1 := "gauge1" + script := []*metricsAdjusterTest{ + { + "MultiMetrics: round 1 - combined round 1 of individual metrics", + metricSlice( + gaugeMetric(g1, k1v1k2v2, t1Ms, doublePoint(t1Ms, 44)), + gaugeDistMetric(gd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7})), + sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t1Ms, 44)), + histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7})), + summaryMetric(s1, k1v1k2v2, t1Ms, summaryPoint(t1Ms, 10, 40, percent0, []float64{1, 5, 8})), + ), + metricSlice( + gaugeMetric(g1, k1v1k2v2, t1Ms, doublePoint(t1Ms, 44)), + gaugeDistMetric(gd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7})), + sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t1Ms, 44)), + histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7})), + summaryMetric(s1, k1v1k2v2, t1Ms, summaryPoint(t1Ms, 10, 40, percent0, []float64{1, 5, 8})), + ), + 3, + }, { + "MultiMetrics: round 2 - combined round 2 of individual metrics", + metricSlice( + gaugeMetric(g1, k1v1k2v2, t2Ms, doublePoint(t2Ms, 66)), + gaugeDistMetric(gd1, k1v1k2v2, t2Ms, distPoint(t2Ms, bounds0, []uint64{6, 5, 8, 11})), + sumMetric(c1, k1v1k2v2, t2Ms, doublePoint(t2Ms, 66)), + histogramMetric(cd1, k1v1k2v2, t2Ms, distPoint(t2Ms, bounds0, []uint64{6, 3, 4, 8})), + summaryMetric(s1, k1v1k2v2, t2Ms, summaryPoint(t2Ms, 15, 70, percent0, []float64{7, 44, 9})), + ), + metricSlice( + gaugeMetric(g1, k1v1k2v2, t2Ms, doublePoint(t2Ms, 66)), + gaugeDistMetric(gd1, k1v1k2v2, t2Ms, distPoint(t2Ms, bounds0, []uint64{6, 5, 8, 11})), + sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t2Ms, 66)), + histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t2Ms, bounds0, []uint64{6, 3, 4, 8})), + summaryMetric(s1, k1v1k2v2, t1Ms, summaryPoint(t2Ms, 15, 70, percent0, []float64{7, 44, 9})), + ), + 0, + }, { + "MultiMetrics: round 3 - combined round 3 of individual metrics", + metricSlice( + gaugeMetric(g1, k1v1k2v2, t3Ms, doublePoint(t3Ms, 55)), + gaugeDistMetric(gd1, k1v1k2v2, t3Ms, distPoint(t3Ms, bounds0, []uint64{2, 0, 1, 5})), + sumMetric(c1, k1v1k2v2, t3Ms, doublePoint(t3Ms, 55)), + histogramMetric(cd1, k1v1k2v2, t3Ms, distPoint(t3Ms, bounds0, []uint64{5, 3, 2, 7})), + summaryMetric(s1, k1v1k2v2, t3Ms, summaryPoint(t3Ms, 12, 66, percent0, []float64{3, 22, 5})), + ), + metricSlice( + gaugeMetric(g1, k1v1k2v2, t3Ms, doublePoint(t3Ms, 55)), + gaugeDistMetric(gd1, k1v1k2v2, t3Ms, distPoint(t3Ms, bounds0, []uint64{2, 0, 1, 5})), + sumMetric(c1, k1v1k2v2, t3Ms, doublePoint(t3Ms, 55)), + histogramMetric(cd1, k1v1k2v2, t3Ms, distPoint(t3Ms, bounds0, []uint64{5, 3, 2, 7})), + summaryMetric(s1, k1v1k2v2, t3Ms, summaryPoint(t3Ms, 12, 66, percent0, []float64{3, 22, 5})), + ), + 3, + }, { + "MultiMetrics: round 4 - combined round 4 of individual metrics", + metricSlice( + sumMetric(c1, k1v1k2v2, t4Ms, doublePoint(t4Ms, 72)), + histogramMetric(cd1, k1v1k2v2, t4Ms, distPoint(t4Ms, bounds0, []uint64{7, 4, 2, 12})), + summaryMetric(s1, k1v1k2v2, t4Ms, summaryPoint(t4Ms, 14, 96, percent0, []float64{9, 47, 8})), + ), + metricSlice( + sumMetric(c1, k1v1k2v2, t3Ms, doublePoint(t4Ms, 72)), + histogramMetric(cd1, k1v1k2v2, t3Ms, distPoint(t4Ms, bounds0, []uint64{7, 4, 2, 12})), + summaryMetric(s1, k1v1k2v2, t3Ms, summaryPoint(t4Ms, 14, 96, percent0, []float64{9, 47, 8})), + ), + 0, + }, + } + runScript(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) +} + +func Test_multiTimeseries(t *testing.T) { + script := []*metricsAdjusterTest{ + { + "MultiTimeseries: round 1 - initial first instance, start time is established", + metricSlice(sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t1Ms, 44))), + metricSlice(sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t1Ms, 44))), + 1, + }, { + "MultiTimeseries: round 2 - first instance adjusted based on round 1, initial second instance", + metricSlice( + sumMetric(c1, k1v1k2v2, t2Ms, doublePoint(t2Ms, 66)), + sumMetric(c1, k1v10k2v20, t2Ms, doublePoint(t2Ms, 20.0)), + ), + metricSlice( + sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t2Ms, 66)), + sumMetric(c1, k1v10k2v20, t2Ms, doublePoint(t2Ms, 20.0)), + ), + 1, + }, { + "MultiTimeseries: round 3 - first instance adjusted based on round 1, second based on round 2", + metricSlice( + sumMetric(c1, k1v1k2v2, t3Ms, doublePoint(t3Ms, 88.0)), + sumMetric(c1, k1v10k2v20, t3Ms, doublePoint(t3Ms, 49.0)), + ), + metricSlice( + sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t3Ms, 88.0)), + sumMetric(c1, k1v10k2v20, t2Ms, doublePoint(t3Ms, 49.0)), + ), + 0, + }, { + "MultiTimeseries: round 4 - first instance reset, second instance adjusted based on round 2, initial third instance", + metricSlice( + sumMetric(c1, k1v1k2v2, t4Ms, doublePoint(t4Ms, 87.0)), + sumMetric(c1, k1v10k2v20, t4Ms, doublePoint(t4Ms, 57.0)), + sumMetric(c1, k1v100k2v200, t4Ms, doublePoint(t4Ms, 10.0)), + ), + metricSlice( + sumMetric(c1, k1v1k2v2, t4Ms, doublePoint(t4Ms, 87.0)), + sumMetric(c1, k1v10k2v20, t2Ms, doublePoint(t4Ms, 57.0)), + sumMetric(c1, k1v100k2v200, t4Ms, doublePoint(t4Ms, 10.0)), + ), + 2, + }, { + "MultiTimeseries: round 5 - first instance adjusted based on round 4, second on round 2, third on round 4", + metricSlice( + sumMetric(c1, k1v1k2v2, t5Ms, doublePoint(t5Ms, 90.0)), + sumMetric(c1, k1v10k2v20, t5Ms, doublePoint(t5Ms, 65.0)), + sumMetric(c1, k1v100k2v200, t5Ms, doublePoint(t5Ms, 22.0)), + ), + metricSlice( + sumMetric(c1, k1v1k2v2, t4Ms, doublePoint(t5Ms, 90.0)), + sumMetric(c1, k1v10k2v20, t2Ms, doublePoint(t5Ms, 65.0)), + sumMetric(c1, k1v100k2v200, t4Ms, doublePoint(t5Ms, 22.0)), + ), + 0, + }, + } + runScript(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) +} + +var ( + emptyLabels = []*kv{} + k1vEmpty = []*kv{{"k1", ""}} + k1vEmptyk2vEmptyk3vEmpty = []*kv{{"k1", ""}, {"k2", ""}, {"k3", ""}} +) + +func Test_emptyLabels(t *testing.T) { + script := []*metricsAdjusterTest{ + { + "EmptyLabels: round 1 - initial instance, implicitly empty labels, start time is established", + metricSlice(sumMetric(c1, emptyLabels, t1Ms, doublePoint(t1Ms, 44))), + metricSlice(sumMetric(c1, emptyLabels, t1Ms, doublePoint(t1Ms, 44))), + 1, + }, { + "EmptyLabels: round 2 - instance adjusted based on round 1", + metricSlice(sumMetric(c1, emptyLabels, t2Ms, doublePoint(t2Ms, 66))), + metricSlice(sumMetric(c1, emptyLabels, t1Ms, doublePoint(t2Ms, 66))), + 0, + }, { + "EmptyLabels: round 3 - one explicitly empty label, instance adjusted based on round 1", + metricSlice(sumMetric(c1, k1vEmpty, t3Ms, doublePoint(t3Ms, 77))), + metricSlice(sumMetric(c1, k1vEmpty, t1Ms, doublePoint(t3Ms, 77))), + 0, + }, { + "EmptyLabels: round 4 - three explicitly empty labels, instance adjusted based on round 1", + metricSlice(sumMetric(c1, k1vEmptyk2vEmptyk3vEmpty, t3Ms, doublePoint(t3Ms, 88))), + metricSlice(sumMetric(c1, k1vEmptyk2vEmptyk3vEmpty, t1Ms, doublePoint(t3Ms, 88))), + 0, + }, + } + runScript(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) +} + +func Test_tsGC(t *testing.T) { + script1 := []*metricsAdjusterTest{ + { + "TsGC: round 1 - initial instances, start time is established", + metricSlice( + sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t1Ms, 44)), + sumMetric(c1, k1v10k2v20, t1Ms, doublePoint(t1Ms, 20)), + histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7})), + histogramMetric(cd1, k1v10k2v20, t1Ms, distPoint(t1Ms, bounds0, []uint64{40, 20, 30, 70})), + ), + metricSlice( + sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t1Ms, 44)), + sumMetric(c1, k1v10k2v20, t1Ms, doublePoint(t1Ms, 20)), + histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7})), + histogramMetric(cd1, k1v10k2v20, t1Ms, distPoint(t1Ms, bounds0, []uint64{40, 20, 30, 70})), + ), + 4, + }, + } + + script2 := []*metricsAdjusterTest{ + { + "TsGC: round 2 - metrics first timeseries adjusted based on round 2, second timeseries not updated", + metricSlice( + sumMetric(c1, k1v1k2v2, t2Ms, doublePoint(t2Ms, 88)), + histogramMetric(cd1, k1v1k2v2, t2Ms, distPoint(t2Ms, bounds0, []uint64{8, 7, 9, 14})), + ), + metricSlice( + sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t2Ms, 88)), + histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t2Ms, bounds0, []uint64{8, 7, 9, 14})), + ), + 0, + }, + } + + script3 := []*metricsAdjusterTest{ + { + "TsGC: round 3 - metrics first timeseries adjusted based on round 2, second timeseries empty due to timeseries gc()", + metricSlice( + sumMetric(c1, k1v1k2v2, t3Ms, doublePoint(t3Ms, 99)), + sumMetric(c1, k1v10k2v20, t3Ms, doublePoint(t3Ms, 80)), + histogramMetric(cd1, k1v1k2v2, t3Ms, distPoint(t3Ms, bounds0, []uint64{9, 8, 10, 15})), + histogramMetric(cd1, k1v10k2v20, t3Ms, distPoint(t3Ms, bounds0, []uint64{55, 66, 33, 77})), + ), + metricSlice( + sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t3Ms, 99)), + sumMetric(c1, k1v10k2v20, t3Ms, doublePoint(t3Ms, 80)), + histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t3Ms, bounds0, []uint64{9, 8, 10, 15})), + histogramMetric(cd1, k1v10k2v20, t3Ms, distPoint(t3Ms, bounds0, []uint64{55, 66, 33, 77})), + ), + 2, + }, + } + + jobsMap := NewJobsMapPdata(time.Minute) + + // run round 1 + runScript(t, jobsMap.get("job", "0"), script1) + // gc the tsmap, unmarking all entries + jobsMap.get("job", "0").gc() + // run round 2 - update metrics first timeseries only + runScript(t, jobsMap.get("job", "0"), script2) + // gc the tsmap, collecting umarked entries + jobsMap.get("job", "0").gc() + // run round 3 - verify that metrics second timeseries have been gc'd + runScript(t, jobsMap.get("job", "0"), script3) +} + +func Test_jobGC(t *testing.T) { + job1Script1 := []*metricsAdjusterTest{ + { + "JobGC: job 1, round 1 - initial instances, adjusted should be empty", + metricSlice( + sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t1Ms, 44)), + sumMetric(c1, k1v10k2v20, t1Ms, doublePoint(t1Ms, 20)), + histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7})), + histogramMetric(cd1, k1v10k2v20, t1Ms, distPoint(t1Ms, bounds0, []uint64{40, 20, 30, 70})), + ), + metricSlice( + sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t1Ms, 44)), + sumMetric(c1, k1v10k2v20, t1Ms, doublePoint(t1Ms, 20)), + histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7})), + histogramMetric(cd1, k1v10k2v20, t1Ms, distPoint(t1Ms, bounds0, []uint64{40, 20, 30, 70})), + ), + 4, + }, + } + + emptyMetricSlice := func() *pdata.MetricSlice { ms := pdata.NewMetricSlice(); return &ms } + job2Script1 := []*metricsAdjusterTest{ + { + "JobGC: job2, round 1 - no metrics adjusted, just trigger gc", + emptyMetricSlice(), + emptyMetricSlice(), + 0, + }, + } + + job1Script2 := []*metricsAdjusterTest{ + { + "JobGC: job 1, round 2 - metrics timeseries empty due to job-level gc", + metricSlice( + sumMetric(c1, k1v1k2v2, t4Ms, doublePoint(t4Ms, 99)), + sumMetric(c1, k1v10k2v20, t4Ms, doublePoint(t4Ms, 80)), + histogramMetric(cd1, k1v1k2v2, t4Ms, distPoint(t4Ms, bounds0, []uint64{9, 8, 10, 15})), + histogramMetric(cd1, k1v10k2v20, t4Ms, distPoint(t4Ms, bounds0, []uint64{55, 66, 33, 77})), + ), + metricSlice( + sumMetric(c1, k1v1k2v2, t4Ms, doublePoint(t4Ms, 99)), + sumMetric(c1, k1v10k2v20, t4Ms, doublePoint(t4Ms, 80)), + histogramMetric(cd1, k1v1k2v2, t4Ms, distPoint(t4Ms, bounds0, []uint64{9, 8, 10, 15})), + histogramMetric(cd1, k1v10k2v20, t4Ms, distPoint(t4Ms, bounds0, []uint64{55, 66, 33, 77})), + ), + 4, + }, + } + + gcInterval := 10 * time.Millisecond + jobsMap := NewJobsMapPdata(gcInterval) + + // run job 1, round 1 - all entries marked + runScript(t, jobsMap.get("job", "0"), job1Script1) + // sleep longer than gcInterval to enable job gc in the next run + time.Sleep(2 * gcInterval) + // run job 2, round1 - trigger job gc, unmarking all entries + runScript(t, jobsMap.get("job", "1"), job2Script1) + // sleep longer than gcInterval to enable job gc in the next run + time.Sleep(2 * gcInterval) + // re-run job 2, round1 - trigger job gc, removing unmarked entries + runScript(t, jobsMap.get("job", "1"), job2Script1) + // ensure that at least one jobsMap.gc() completed + jobsMap.gc() + // run job 1, round 2 - verify that all job 1 timeseries have been gc'd + runScript(t, jobsMap.get("job", "0"), job1Script2) + return +} + +type metricsAdjusterTest struct { + description string + metrics *pdata.MetricSlice + adjusted *pdata.MetricSlice + resets int +} + +func runScript(t *testing.T, tsm *timeseriesMapPdata, script []*metricsAdjusterTest) { + l := zap.NewNop() + t.Cleanup(func() { require.NoError(t, l.Sync()) }) // flushes buffer, if any + ma := NewMetricsAdjusterPdata(tsm, l) + + for _, test := range script { + expectedResets := test.resets + resets := ma.AdjustMetrics(test.metrics) + adjusted := test.metrics + assert.EqualValuesf(t, test.adjusted, adjusted, "Test: %v - expected: %v, actual: %v", test.description, test.adjusted, adjusted) + assert.Equalf(t, expectedResets, resets, "Test: %v", test.description) + } +} diff --git a/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go b/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go index 9aa519ffd26b3..f177bf8e0c333 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go +++ b/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go @@ -15,18 +15,120 @@ package internal import ( + "errors" "fmt" "regexp" "sort" "strconv" + "strings" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/textparse" - "go.opentelemetry.io/collector/model/pdata" "go.uber.org/zap" + + "go.opentelemetry.io/collector/model/pdata" ) +const ( + metricsSuffixCount = "_count" + metricsSuffixBucket = "_bucket" + metricsSuffixSum = "_sum" + metricSuffixTotal = "_total" + startTimeMetricName = "process_start_time_seconds" + scrapeUpMetricName = "up" +) + +var ( + trimmableSuffixes = []string{metricsSuffixBucket, metricsSuffixCount, metricsSuffixSum, metricSuffixTotal} + errNoDataToBuild = errors.New("there's no data to build") + errNoBoundaryLabel = errors.New("given metricType has no BucketLabel or QuantileLabel") + errEmptyBoundaryLabel = errors.New("BucketLabel or QuantileLabel is empty") +) + +// dpgSignature is used to create a key for data complexValue belong to a same group of a metric family +func dpgSignature(orderedKnownLabelKeys []string, ls labels.Labels) string { + size := 0 + for _, k := range orderedKnownLabelKeys { + v := ls.Get(k) + if v == "" { + continue + } + // 2 enclosing quotes + 1 equality sign = 3 extra chars. + // Note: if any character in the label value requires escaping, + // we'll need more space than that, which will lead to some + // extra allocation. + size += 3 + len(k) + len(v) + } + sign := make([]byte, 0, size) + for _, k := range orderedKnownLabelKeys { + v := ls.Get(k) + if v == "" { + continue + } + sign = strconv.AppendQuote(sign, k+"="+v) + } + return string(sign) +} + +func normalizeMetricName(name string) string { + for _, s := range trimmableSuffixes { + if strings.HasSuffix(name, s) && name != s { + return strings.TrimSuffix(name, s) + } + } + return name +} + +// Code borrowed from the original promreceiver +func heuristicalMetricAndKnownUnits(metricName, parsedUnit string) string { + if parsedUnit != "" { + return parsedUnit + } + lastUnderscoreIndex := strings.LastIndex(metricName, "_") + if lastUnderscoreIndex <= 0 || lastUnderscoreIndex >= len(metricName)-1 { + return "" + } + + unit := "" + + supposedUnit := metricName[lastUnderscoreIndex+1:] + switch strings.ToLower(supposedUnit) { + case "millisecond", "milliseconds", "ms": + unit = "ms" + case "second", "seconds", "s": + unit = "s" + case "microsecond", "microseconds", "us": + unit = "us" + case "nanosecond", "nanoseconds", "ns": + unit = "ns" + case "byte", "bytes", "by": + unit = "By" + case "bit", "bits": + unit = "Bi" + case "kilogram", "kilograms", "kg": + unit = "kg" + case "gram", "grams", "g": + unit = "g" + case "meter", "meters", "metre", "metres", "m": + unit = "m" + case "kilometer", "kilometers", "kilometre", "kilometres", "km": + unit = "km" + case "milimeter", "milimeters", "milimetre", "milimetres", "mm": + unit = "mm" + case "nanogram", "ng", "nanograms": + unit = "ng" + } + + return unit +} + +func isInternalMetric(metricName string) bool { + if metricName == scrapeUpMetricName || strings.HasPrefix(metricName, "scrape_") { + return true + } + return false +} func isUsefulLabelPdata(mType pdata.MetricDataType, labelKey string) bool { switch labelKey { case model.MetricNameLabel, model.InstanceLabel, model.SchemeLabel, model.MetricsPathLabel, model.JobLabel: @@ -60,13 +162,13 @@ func getBoundaryPdata(metricType pdata.MetricDataType, labels labels.Labels) (fl func convToPdataMetricType(metricType textparse.MetricType) pdata.MetricDataType { switch metricType { - case textparse.MetricTypeCounter: + case textparse.MetricTypeCounter: // metricspb.MetricDescriptor_CUMULATIVE_DOUBLE // always use float64, as it's the internal data type used in prometheus return pdata.MetricDataTypeSum // textparse.MetricTypeUnknown is converted to gauge by default to fix Prometheus untyped metrics from being dropped - case textparse.MetricTypeGauge, textparse.MetricTypeUnknown: + case textparse.MetricTypeGauge, textparse.MetricTypeUnknown: // metricspb.MetricDescriptor_GAUGE_DOUBLE return pdata.MetricDataTypeGauge - case textparse.MetricTypeHistogram: + case textparse.MetricTypeHistogram: // metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION return pdata.MetricDataTypeHistogram // dropping support for gaugehistogram for now until we have an official spec of its implementation // a draft can be found in: https://docs.google.com/document/d/1KwV0mAXwwbvvifBvDKH_LU1YjyXE_wxCkHNoCGq1GX0/edit#heading=h.1cvzqd4ksd23 @@ -81,35 +183,47 @@ func convToPdataMetricType(metricType textparse.MetricType) pdata.MetricDataType } type metricBuilderPdata struct { - *metricBuilder - metrics pdata.MetricSlice - currentMf MetricFamilyPdata + metrics pdata.MetricSlice + currentMf MetricFamilyPdata + hasData bool + hasInternalMetric bool + mc MetadataCache + numTimeseries int + droppedTimeseries int + useStartTimeMetric bool + startTimeMetricRegex *regexp.Regexp + startTime float64 + intervalStartTimeMs int64 + logger *zap.Logger } // newMetricBuilder creates a MetricBuilder which is allowed to feed all the datapoints from a single prometheus -// scraped page by calling its AddDataPoint function, and turn them into an opencensus data.MetricsData object +// scraped page by calling its AddDataPoint function, and turn them into a pdata.Metrics object. // by calling its Build function -func newMetricBuilderPdata(mc MetadataCache, useStartTimeMetric bool, startTimeMetricRegex string, logger *zap.Logger) *metricBuilderPdata { +func newMetricBuilderPdata(mc MetadataCache, useStartTimeMetric bool, startTimeMetricRegex string, logger *zap.Logger, intervalStartTimeMs int64) *metricBuilderPdata { var regex *regexp.Regexp if startTimeMetricRegex != "" { regex, _ = regexp.Compile(startTimeMetricRegex) } return &metricBuilderPdata{ - metrics: pdata.NewMetricSlice(), - metricBuilder: &metricBuilder{ - mc: mc, - logger: logger, - numTimeseries: 0, - droppedTimeseries: 0, - useStartTimeMetric: useStartTimeMetric, - startTimeMetricRegex: regex, - }, + metrics: pdata.NewMetricSlice(), + mc: mc, + logger: logger, + numTimeseries: 0, + droppedTimeseries: 0, + useStartTimeMetric: useStartTimeMetric, + startTimeMetricRegex: regex, + intervalStartTimeMs: intervalStartTimeMs, } } -// This code is used in follow-up changes but golangci-lint is so pedantic. -var _ = newMetricBuilderPdata -var _ = (*metricBuilderPdata)(nil).AddDataPoint +func (b *metricBuilderPdata) matchStartTimeMetric(metricName string) bool { + if b.startTimeMetricRegex != nil { + return b.startTimeMetricRegex.MatchString(metricName) + } + + return metricName == startTimeMetricName +} // AddDataPoint is for feeding prometheus data complexValue in its processing order func (b *metricBuilderPdata) AddDataPoint(ls labels.Labels, t int64, v float64) error { @@ -160,13 +274,34 @@ func (b *metricBuilderPdata) AddDataPoint(ls labels.Labels, t int64, v float64) b.hasData = true if b.currentMf != nil && !b.currentMf.IsSameFamily(metricName) { + nTs, nDts := b.currentMf.ToMetricPdata(&b.metrics) + b.numTimeseries += nTs + b.droppedTimeseries += nDts + b.currentMf = newMetricFamilyPdata(metricName, b.mc, b.logger, b.intervalStartTimeMs) + } else if b.currentMf == nil { + b.currentMf = newMetricFamilyPdata(metricName, b.mc, b.logger, b.intervalStartTimeMs) + } + + return b.currentMf.Add(metricName, ls, t, v) +} + +// Build an pdata.MetricSlice based on all added data complexValue. +// The only error returned by this function is errNoDataToBuild. +func (b *metricBuilderPdata) Build() (*pdata.MetricSlice, int, int, error) { + if !b.hasData { + if b.hasInternalMetric { + metricsL := pdata.NewMetricSlice() + return &metricsL, 0, 0, nil + } + return nil, 0, 0, errNoDataToBuild + } + + if b.currentMf != nil { ts, dts := b.currentMf.ToMetricPdata(&b.metrics) b.numTimeseries += ts b.droppedTimeseries += dts - b.currentMf = newMetricFamilyPdata(metricName, b.mc, b.intervalStartTimeMs) - } else if b.currentMf == nil { - b.currentMf = newMetricFamilyPdata(metricName, b.mc, b.intervalStartTimeMs) + b.currentMf = nil } - return b.currentMf.Add(metricName, ls, t, v) + return &b.metrics, b.numTimeseries, b.droppedTimeseries, nil } diff --git a/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go b/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go index bfff50709a0e6..9fcb42ad105a8 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go +++ b/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go @@ -15,96 +15,208 @@ package internal import ( + "runtime" "testing" - metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/textparse" + "github.com/prometheus/prometheus/scrape" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/model/pdata" ) -func TestGetBoundaryEquivalence(t *testing.T) { - cases := []struct { - name string - mtype metricspb.MetricDescriptor_Type - pmtype pdata.MetricDataType - labels labels.Labels - wantValue float64 - wantErr string - }{ - { - name: "cumulative histogram with bucket label", - mtype: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, - pmtype: pdata.MetricDataTypeHistogram, - labels: labels.Labels{ - {Name: model.BucketLabel, Value: "0.256"}, - }, - wantValue: 0.256, - }, - { - name: "gauge histogram with bucket label", - mtype: metricspb.MetricDescriptor_GAUGE_DISTRIBUTION, - pmtype: pdata.MetricDataTypeHistogram, - labels: labels.Labels{ - {Name: model.BucketLabel, Value: "11.71"}, - }, - wantValue: 11.71, - }, +const startTs = int64(1555366610000) +const interval = int64(15 * 1000) +const defaultBuilderStartTime = float64(1.0) + +var testMetadata = map[string]scrape.MetricMetadata{ + "counter_test": {Metric: "counter_test", Type: textparse.MetricTypeCounter, Help: "", Unit: ""}, + "counter_test2": {Metric: "counter_test2", Type: textparse.MetricTypeCounter, Help: "", Unit: ""}, + "gauge_test": {Metric: "gauge_test", Type: textparse.MetricTypeGauge, Help: "", Unit: ""}, + "gauge_test2": {Metric: "gauge_test2", Type: textparse.MetricTypeGauge, Help: "", Unit: ""}, + "hist_test": {Metric: "hist_test", Type: textparse.MetricTypeHistogram, Help: "", Unit: ""}, + "hist_test2": {Metric: "hist_test2", Type: textparse.MetricTypeHistogram, Help: "", Unit: ""}, + "ghist_test": {Metric: "ghist_test", Type: textparse.MetricTypeGaugeHistogram, Help: "", Unit: ""}, + "summary_test": {Metric: "summary_test", Type: textparse.MetricTypeSummary, Help: "", Unit: ""}, + "summary_test2": {Metric: "summary_test2", Type: textparse.MetricTypeSummary, Help: "", Unit: ""}, + "unknown_test": {Metric: "unknown_test", Type: textparse.MetricTypeUnknown, Help: "", Unit: ""}, + "poor_name_count": {Metric: "poor_name_count", Type: textparse.MetricTypeCounter, Help: "", Unit: ""}, + "up": {Metric: "up", Type: textparse.MetricTypeCounter, Help: "", Unit: ""}, + "scrape_foo": {Metric: "scrape_foo", Type: textparse.MetricTypeCounter, Help: "", Unit: ""}, + "example_process_start_time_seconds": {Metric: "example_process_start_time_seconds", + Type: textparse.MetricTypeGauge, Help: "", Unit: ""}, + "process_start_time_seconds": {Metric: "process_start_time_seconds", + Type: textparse.MetricTypeGauge, Help: "", Unit: ""}, + "badprocess_start_time_seconds": {Metric: "badprocess_start_time_seconds", + Type: textparse.MetricTypeGauge, Help: "", Unit: ""}, +} + +type testDataPoint struct { + lb labels.Labels + t int64 + v float64 +} + +type testScrapedPage struct { + pts []*testDataPoint +} + +func createLabels(mFamily string, tagPairs ...string) labels.Labels { + lm := make(map[string]string) + lm[model.MetricNameLabel] = mFamily + if len(tagPairs)%2 != 0 { + panic("tag pairs is not even") + } + + for i := 0; i < len(tagPairs); i += 2 { + lm[tagPairs[i]] = tagPairs[i+1] + } + + return labels.FromMap(lm) +} + +func createDataPoint(mname string, value float64, tagPairs ...string) *testDataPoint { + return &testDataPoint{ + lb: createLabels(mname, tagPairs...), + v: value, + } +} + +func runBuilderStartTimeTests(t *testing.T, tests []buildTestDataPdata, + startTimeMetricRegex string, expectedBuilderStartTime float64) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mc := newMockMetadataCache(testMetadata) + st := startTs + for _, page := range tt.inputs { + b := newMetricBuilderPdata(mc, true, startTimeMetricRegex, testLogger, 0) + b.startTime = defaultBuilderStartTime // set to a non-zero value + for _, pt := range page.pts { + // set ts for testing + pt.t = st + assert.NoError(t, b.AddDataPoint(pt.lb, pt.t, pt.v)) + } + _, _, _, err := b.Build() + assert.NoError(t, err) + assert.EqualValues(t, b.startTime, expectedBuilderStartTime) + st += interval + } + }) + } +} + +func Test_startTimeMetricMatch(t *testing.T) { + matchBuilderStartTime := 123.456 + matchTests := []buildTestDataPdata{ { - name: "summary with bucket label", - mtype: metricspb.MetricDescriptor_SUMMARY, - pmtype: pdata.MetricDataTypeSummary, - labels: labels.Labels{ - {Name: model.BucketLabel, Value: "11.71"}, + name: "prefix_match", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("example_process_start_time_seconds", + matchBuilderStartTime, "foo", "bar"), + }, + }, }, - wantErr: "QuantileLabel is empty", }, { - name: "summary with quantile label", - mtype: metricspb.MetricDescriptor_SUMMARY, - pmtype: pdata.MetricDataTypeSummary, - labels: labels.Labels{ - {Name: model.QuantileLabel, Value: "92.88"}, + name: "match", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("process_start_time_seconds", + matchBuilderStartTime, "foo", "bar"), + }, + }, }, - wantValue: 92.88, }, + } + nomatchTests := []buildTestDataPdata{ { - name: "gauge histogram mismatched with bucket label", - mtype: metricspb.MetricDescriptor_SUMMARY, - pmtype: pdata.MetricDataTypeSummary, - labels: labels.Labels{ - {Name: model.BucketLabel, Value: "11.71"}, + name: "nomatch1", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("_process_start_time_seconds", + matchBuilderStartTime, "foo", "bar"), + }, + }, }, - wantErr: "QuantileLabel is empty", }, { - name: "other data types without matches", - mtype: metricspb.MetricDescriptor_GAUGE_DOUBLE, - pmtype: pdata.MetricDataTypeGauge, - labels: labels.Labels{ - {Name: model.BucketLabel, Value: "11.71"}, + name: "nomatch2", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("subprocess_start_time_seconds", + matchBuilderStartTime, "foo", "bar"), + }, + }, }, - wantErr: "given metricType has no BucketLabel or QuantileLabel", }, } - for _, tt := range cases { - tt := tt - t.Run(tt.name, func(t *testing.T) { - oldBoundary, oerr := getBoundary(tt.mtype, tt.labels) - pdataBoundary, perr := getBoundaryPdata(tt.pmtype, tt.labels) - assert.Equal(t, oldBoundary, pdataBoundary, "Both boundary values MUST be equal") - assert.Equal(t, oldBoundary, tt.wantValue, "Mismatched boundary messages") - assert.Equal(t, oerr, perr, "The exact same error MUST be returned from both boundary helpers") + runBuilderStartTimeTests(t, matchTests, "^(.+_)*process_start_time_seconds$", matchBuilderStartTime) + runBuilderStartTimeTests(t, nomatchTests, "^(.+_)*process_start_time_seconds$", defaultBuilderStartTime) +} - if tt.wantErr != "" { - require.NotEqual(t, oerr, "expected an error from old style boundary retrieval") - require.NotEqual(t, perr, "expected an error from new style boundary retrieval") - require.Contains(t, oerr.Error(), tt.wantErr) - require.Contains(t, perr.Error(), tt.wantErr) +func Test_heuristicalMetricAndKnownUnits(t *testing.T) { + tests := []struct { + metricName string + parsedUnit string + want string + }{ + {"test", "ms", "ms"}, + {"millisecond", "", ""}, + {"test_millisecond", "", "ms"}, + {"test_milliseconds", "", "ms"}, + {"test_ms", "", "ms"}, + {"test_second", "", "s"}, + {"test_seconds", "", "s"}, + {"test_s", "", "s"}, + {"test_microsecond", "", "us"}, + {"test_microseconds", "", "us"}, + {"test_us", "", "us"}, + {"test_nanosecond", "", "ns"}, + {"test_nanoseconds", "", "ns"}, + {"test_ns", "", "ns"}, + {"test_byte", "", "By"}, + {"test_bytes", "", "By"}, + {"test_by", "", "By"}, + {"test_bit", "", "Bi"}, + {"test_bits", "", "Bi"}, + {"test_kilogram", "", "kg"}, + {"test_kilograms", "", "kg"}, + {"test_kg", "", "kg"}, + {"test_gram", "", "g"}, + {"test_grams", "", "g"}, + {"test_g", "", "g"}, + {"test_nanogram", "", "ng"}, + {"test_nanograms", "", "ng"}, + {"test_ng", "", "ng"}, + {"test_meter", "", "m"}, + {"test_meters", "", "m"}, + {"test_metre", "", "m"}, + {"test_metres", "", "m"}, + {"test_m", "", "m"}, + {"test_kilometer", "", "km"}, + {"test_kilometers", "", "km"}, + {"test_kilometre", "", "km"}, + {"test_kilometres", "", "km"}, + {"test_km", "", "km"}, + {"test_milimeter", "", "mm"}, + {"test_milimeters", "", "mm"}, + {"test_milimetre", "", "mm"}, + {"test_milimetres", "", "mm"}, + {"test_mm", "", "mm"}, + } + for _, tt := range tests { + t.Run(tt.metricName, func(t *testing.T) { + if got := heuristicalMetricAndKnownUnits(tt.metricName, tt.parsedUnit); got != tt.want { + t.Errorf("heuristicalMetricAndKnownUnits() = %v, want %v", got, tt.want) } }) } @@ -197,8 +309,8 @@ func TestConvToPdataMetricType(t *testing.T) { }, { name: "textparse.gauge", - mtype: textparse.MetricTypeCounter, - want: pdata.MetricDataTypeSum, + mtype: textparse.MetricTypeGauge, + want: pdata.MetricDataTypeGauge, }, { name: "textparse.unknown", @@ -231,7 +343,7 @@ func TestConvToPdataMetricType(t *testing.T) { tt := tt t.Run(tt.name, func(t *testing.T) { got := convToPdataMetricType(tt.mtype) - require.Equal(t, got, tt.want) + require.Equal(t, got.String(), tt.want.String()) }) } } @@ -325,3 +437,954 @@ func TestIsUsefulLabelPdata(t *testing.T) { }) } } + +type buildTestDataPdata struct { + name string + inputs []*testScrapedPage + wants func() []*pdata.MetricSlice +} + +func Test_OTLPMetricBuilder_counters(t *testing.T) { + tests := []buildTestDataPdata{ + { + name: "single-item", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("counter_test", 100, "foo", "bar"), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetName("counter_test") + m0.SetDataType(pdata.MetricDataTypeSum) + sum := m0.Sum() + pt0 := sum.DataPoints().AppendEmpty() + pt0.SetDoubleVal(100.0) + pt0.SetStartTimestamp(startTsNanos) + pt0.SetTimestamp(startTsNanos) + pt0.Attributes().InsertString("foo", "bar") + + return []*pdata.MetricSlice{&mL} + }, + }, + { + name: "two-items", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("counter_test", 150, "foo", "bar"), + createDataPoint("counter_test", 25, "foo", "other"), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetName("counter_test") + m0.SetDataType(pdata.MetricDataTypeSum) + sum := m0.Sum() + pt0 := sum.DataPoints().AppendEmpty() + pt0.SetDoubleVal(150.0) + pt0.SetStartTimestamp(startTsNanos) + pt0.SetTimestamp(startTsNanos) + pt0.Attributes().InsertString("foo", "bar") + + pt1 := sum.DataPoints().AppendEmpty() + pt1.SetDoubleVal(25.0) + pt1.SetStartTimestamp(startTsNanos) + pt1.SetTimestamp(startTsNanos) + pt1.Attributes().InsertString("foo", "other") + + return []*pdata.MetricSlice{&mL} + }, + }, + { + name: "two-metrics", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("counter_test", 150, "foo", "bar"), + createDataPoint("counter_test", 25, "foo", "other"), + createDataPoint("counter_test2", 100, "foo", "bar"), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL0 := pdata.NewMetricSlice() + m0 := mL0.AppendEmpty() + m0.SetName("counter_test") + m0.SetDataType(pdata.MetricDataTypeSum) + sum0 := m0.Sum() + pt0 := sum0.DataPoints().AppendEmpty() + pt0.SetDoubleVal(150.0) + pt0.SetStartTimestamp(startTsNanos) + pt0.SetTimestamp(startTsNanos) + pt0.Attributes().InsertString("foo", "bar") + + pt1 := sum0.DataPoints().AppendEmpty() + pt1.SetDoubleVal(25.0) + pt1.SetStartTimestamp(startTsNanos) + pt1.SetTimestamp(startTsNanos) + pt1.Attributes().InsertString("foo", "other") + + m1 := mL0.AppendEmpty() + m1.SetName("counter_test2") + m1.SetDataType(pdata.MetricDataTypeSum) + sum1 := m1.Sum() + pt2 := sum1.DataPoints().AppendEmpty() + pt2.SetDoubleVal(100.0) + pt2.SetStartTimestamp(startTsNanos) + pt2.SetTimestamp(startTsNanos) + pt2.Attributes().InsertString("foo", "bar") + + return []*pdata.MetricSlice{&mL0} + }, + }, + { + name: "metrics-with-poor-names", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("poor_name_count", 100, "foo", "bar"), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL := pdata.NewMetricSlice() + m0 := mL.AppendEmpty() + m0.SetName("poor_name_count") + m0.SetDataType(pdata.MetricDataTypeSum) + sum := m0.Sum() + pt0 := sum.DataPoints().AppendEmpty() + pt0.SetDoubleVal(100.0) + pt0.SetStartTimestamp(startTsNanos) + pt0.SetTimestamp(startTsNanos) + pt0.Attributes().InsertString("foo", "bar") + + return []*pdata.MetricSlice{&mL} + }, + }, + } + + runBuilderTestsPdata(t, tests) +} + +func runBuilderTestsPdata(t *testing.T, tests []buildTestDataPdata) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + wants := tt.wants() + assert.EqualValues(t, len(wants), len(tt.inputs)) + mc := newMockMetadataCache(testMetadata) + st := startTs + for i, page := range tt.inputs { + b := newMetricBuilderPdata(mc, true, "", testLogger, startTs) + b.startTime = defaultBuilderStartTime // set to a non-zero value + b.intervalStartTimeMs = startTs + for _, pt := range page.pts { + // set ts for testing + pt.t = st + assert.NoError(t, b.AddDataPoint(pt.lb, pt.t, pt.v)) + } + metrics, _, _, err := b.Build() + assert.NoError(t, err) + assert.EqualValues(t, wants[i], metrics) + st += interval + } + }) + } +} + +var ( + startTsNanos = pdata.Timestamp(startTs * 1e6) + startTsPlusIntervalNanos = pdata.Timestamp((startTs + interval) * 1e6) +) + +func Test_OTLPMetricBuilder_gauges(t *testing.T) { + tests := []buildTestDataPdata{ + { + name: "one-gauge", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("gauge_test", 100, "foo", "bar"), + }, + }, + { + pts: []*testDataPoint{ + createDataPoint("gauge_test", 90, "foo", "bar"), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL0 := pdata.NewMetricSlice() + m0 := mL0.AppendEmpty() + m0.SetName("gauge_test") + m0.SetDataType(pdata.MetricDataTypeGauge) + gauge0 := m0.Gauge() + pt0 := gauge0.DataPoints().AppendEmpty() + pt0.SetDoubleVal(100.0) + pt0.SetStartTimestamp(0) + pt0.SetTimestamp(startTsNanos) + pt0.Attributes().InsertString("foo", "bar") + + mL1 := pdata.NewMetricSlice() + m1 := mL1.AppendEmpty() + m1.SetName("gauge_test") + m1.SetDataType(pdata.MetricDataTypeGauge) + gauge1 := m1.Gauge() + pt1 := gauge1.DataPoints().AppendEmpty() + pt1.SetDoubleVal(90.0) + pt1.SetStartTimestamp(0) + pt1.SetTimestamp(startTsPlusIntervalNanos) + pt1.Attributes().InsertString("foo", "bar") + + return []*pdata.MetricSlice{&mL0, &mL1} + }, + }, + { + name: "gauge-with-different-tags", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("gauge_test", 100, "foo", "bar"), + createDataPoint("gauge_test", 200, "bar", "foo"), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL0 := pdata.NewMetricSlice() + m0 := mL0.AppendEmpty() + m0.SetName("gauge_test") + m0.SetDataType(pdata.MetricDataTypeGauge) + gauge0 := m0.Gauge() + pt0 := gauge0.DataPoints().AppendEmpty() + pt0.SetDoubleVal(100.0) + pt0.SetStartTimestamp(0) + pt0.SetTimestamp(startTsNanos) + pt0.Attributes().InsertString("bar", "") + pt0.Attributes().InsertString("foo", "bar") + + pt1 := gauge0.DataPoints().AppendEmpty() + pt1.SetDoubleVal(200.0) + pt1.SetStartTimestamp(0) + pt1.SetTimestamp(startTsNanos) + pt1.Attributes().InsertString("bar", "foo") + pt1.Attributes().InsertString("foo", "") + + return []*pdata.MetricSlice{&mL0} + }, + }, + { + // TODO: A decision need to be made. If we want to have the behavior which can generate different tag key + // sets because metrics come and go + name: "gauge-comes-and-go-with-different-tagset", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("gauge_test", 100, "foo", "bar"), + createDataPoint("gauge_test", 200, "bar", "foo"), + }, + }, + { + pts: []*testDataPoint{ + createDataPoint("gauge_test", 20, "foo", "bar"), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL0 := pdata.NewMetricSlice() + m0 := mL0.AppendEmpty() + m0.SetName("gauge_test") + m0.SetDataType(pdata.MetricDataTypeGauge) + gauge0 := m0.Gauge() + pt0 := gauge0.DataPoints().AppendEmpty() + pt0.SetDoubleVal(100.0) + pt0.SetStartTimestamp(0) + pt0.SetTimestamp(startTsNanos) + pt0.Attributes().InsertString("bar", "") + pt0.Attributes().InsertString("foo", "bar") + + pt1 := gauge0.DataPoints().AppendEmpty() + pt1.SetDoubleVal(200.0) + pt1.SetStartTimestamp(0) + pt1.SetTimestamp(startTsNanos) + pt1.Attributes().InsertString("bar", "foo") + pt1.Attributes().InsertString("foo", "") + + mL1 := pdata.NewMetricSlice() + m1 := mL1.AppendEmpty() + m1.SetName("gauge_test") + m1.SetDataType(pdata.MetricDataTypeGauge) + gauge1 := m1.Gauge() + pt2 := gauge1.DataPoints().AppendEmpty() + pt2.SetDoubleVal(20.0) + pt2.SetStartTimestamp(0) + pt2.SetTimestamp(startTsPlusIntervalNanos) + pt2.Attributes().InsertString("foo", "bar") + + return []*pdata.MetricSlice{&mL0, &mL1} + }, + }, + } + + runBuilderTestsPdata(t, tests) +} + +func Test_OTLPMetricBuilder_untype(t *testing.T) { + tests := []buildTestDataPdata{ + { + name: "one-unknown", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("unknown_test", 100, "foo", "bar"), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL0 := pdata.NewMetricSlice() + m0 := mL0.AppendEmpty() + m0.SetName("unknown_test") + m0.SetDataType(pdata.MetricDataTypeGauge) + gauge0 := m0.Gauge() + pt0 := gauge0.DataPoints().AppendEmpty() + pt0.SetDoubleVal(100.0) + pt0.SetStartTimestamp(0) + pt0.SetTimestamp(startTsNanos) + pt0.Attributes().InsertString("foo", "bar") + + return []*pdata.MetricSlice{&mL0} + }, + }, + { + name: "no-type-hint", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("something_not_exists", 100, "foo", "bar"), + createDataPoint("theother_not_exists", 200, "foo", "bar"), + createDataPoint("theother_not_exists", 300, "bar", "foo"), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL0 := pdata.NewMetricSlice() + m0 := mL0.AppendEmpty() + m0.SetName("something_not_exists") + m0.SetDataType(pdata.MetricDataTypeGauge) + gauge0 := m0.Gauge() + pt0 := gauge0.DataPoints().AppendEmpty() + pt0.SetDoubleVal(100.0) + pt0.SetTimestamp(startTsNanos) + pt0.Attributes().InsertString("foo", "bar") + + m1 := mL0.AppendEmpty() + m1.SetName("theother_not_exists") + m1.SetDataType(pdata.MetricDataTypeGauge) + gauge1 := m1.Gauge() + pt1 := gauge1.DataPoints().AppendEmpty() + pt1.SetDoubleVal(200.0) + pt1.SetTimestamp(startTsNanos) + pt1.Attributes().InsertString("bar", "") + pt1.Attributes().InsertString("foo", "bar") + + pt2 := gauge1.DataPoints().AppendEmpty() + pt2.SetDoubleVal(300.0) + pt2.SetTimestamp(startTsNanos) + pt2.Attributes().InsertString("bar", "foo") + pt2.Attributes().InsertString("foo", "") + + return []*pdata.MetricSlice{&mL0} + }, + }, + { + name: "untype-metric-poor-names", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("some_count", 100, "foo", "bar"), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL0 := pdata.NewMetricSlice() + m0 := mL0.AppendEmpty() + m0.SetName("some_count") + m0.SetDataType(pdata.MetricDataTypeGauge) + gauge0 := m0.Gauge() + pt0 := gauge0.DataPoints().AppendEmpty() + pt0.SetDoubleVal(100.0) + pt0.SetTimestamp(startTsNanos) + pt0.Attributes().InsertString("foo", "bar") + + return []*pdata.MetricSlice{&mL0} + }, + }, + } + + runBuilderTestsPdata(t, tests) +} + +func Test_OTLPMetricBuilder_histogram(t *testing.T) { + tests := []buildTestDataPdata{ + { + name: "single item", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("hist_test", 1, "foo", "bar", "le", "10"), + createDataPoint("hist_test", 2, "foo", "bar", "le", "20"), + createDataPoint("hist_test", 10, "foo", "bar", "le", "+inf"), + createDataPoint("hist_test_sum", 99, "foo", "bar"), + createDataPoint("hist_test_count", 10, "foo", "bar"), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL0 := pdata.NewMetricSlice() + m0 := mL0.AppendEmpty() + m0.SetName("hist_test") + m0.SetDataType(pdata.MetricDataTypeHistogram) + hist0 := m0.Histogram() + pt0 := hist0.DataPoints().AppendEmpty() + pt0.SetCount(10) + pt0.SetSum(99) + pt0.SetExplicitBounds([]float64{10, 20}) + pt0.SetBucketCounts([]uint64{1, 1, 8}) + pt0.SetTimestamp(startTsNanos) + pt0.SetStartTimestamp(startTsNanos) + pt0.Attributes().InsertString("foo", "bar") + + return []*pdata.MetricSlice{&mL0} + }, + }, + { + name: "multi-groups", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("hist_test", 1, "foo", "bar", "le", "10"), + createDataPoint("hist_test", 2, "foo", "bar", "le", "20"), + createDataPoint("hist_test", 10, "foo", "bar", "le", "+inf"), + createDataPoint("hist_test_sum", 99, "foo", "bar"), + createDataPoint("hist_test_count", 10, "foo", "bar"), + createDataPoint("hist_test", 1, "key2", "v2", "le", "10"), + createDataPoint("hist_test", 2, "key2", "v2", "le", "20"), + createDataPoint("hist_test", 3, "key2", "v2", "le", "+inf"), + createDataPoint("hist_test_sum", 50, "key2", "v2"), + createDataPoint("hist_test_count", 3, "key2", "v2"), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL0 := pdata.NewMetricSlice() + m0 := mL0.AppendEmpty() + m0.SetName("hist_test") + m0.SetDataType(pdata.MetricDataTypeHistogram) + hist0 := m0.Histogram() + pt0 := hist0.DataPoints().AppendEmpty() + pt0.SetCount(10) + pt0.SetSum(99) + pt0.SetExplicitBounds([]float64{10, 20}) + pt0.SetBucketCounts([]uint64{1, 1, 8}) + pt0.SetTimestamp(startTsNanos) + pt0.SetStartTimestamp(startTsNanos) + pt0.Attributes().InsertString("foo", "bar") + pt0.Attributes().InsertString("key2", "") + + pt1 := hist0.DataPoints().AppendEmpty() + pt1.SetCount(3) + pt1.SetSum(50) + pt1.SetExplicitBounds([]float64{10, 20}) + pt1.SetBucketCounts([]uint64{1, 1, 1}) + pt1.SetTimestamp(startTsNanos) + pt1.SetStartTimestamp(startTsNanos) + pt1.Attributes().InsertString("foo", "") + pt1.Attributes().InsertString("key2", "v2") + + return []*pdata.MetricSlice{&mL0} + }, + }, + { + name: "multi-groups-and-families", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("hist_test", 1, "foo", "bar", "le", "10"), + createDataPoint("hist_test", 2, "foo", "bar", "le", "20"), + createDataPoint("hist_test", 10, "foo", "bar", "le", "+inf"), + createDataPoint("hist_test_sum", 99, "foo", "bar"), + createDataPoint("hist_test_count", 10, "foo", "bar"), + createDataPoint("hist_test", 1, "key2", "v2", "le", "10"), + createDataPoint("hist_test", 2, "key2", "v2", "le", "20"), + createDataPoint("hist_test", 3, "key2", "v2", "le", "+inf"), + createDataPoint("hist_test_sum", 50, "key2", "v2"), + createDataPoint("hist_test_count", 3, "key2", "v2"), + createDataPoint("hist_test2", 1, "le", "10"), + createDataPoint("hist_test2", 2, "le", "20"), + createDataPoint("hist_test2", 3, "le", "+inf"), + createDataPoint("hist_test2_sum", 50), + createDataPoint("hist_test2_count", 3), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL0 := pdata.NewMetricSlice() + m0 := mL0.AppendEmpty() + m0.SetName("hist_test") + m0.SetDataType(pdata.MetricDataTypeHistogram) + hist0 := m0.Histogram() + pt0 := hist0.DataPoints().AppendEmpty() + pt0.SetCount(10) + pt0.SetSum(99) + pt0.SetExplicitBounds([]float64{10, 20}) + pt0.SetBucketCounts([]uint64{1, 1, 8}) + pt0.SetTimestamp(startTsNanos) + pt0.SetStartTimestamp(startTsNanos) + pt0.Attributes().InsertString("foo", "bar") + pt0.Attributes().InsertString("key2", "") + + pt1 := hist0.DataPoints().AppendEmpty() + pt1.SetCount(3) + pt1.SetSum(50) + pt1.SetExplicitBounds([]float64{10, 20}) + pt1.SetBucketCounts([]uint64{1, 1, 1}) + pt1.SetTimestamp(startTsNanos) + pt1.SetStartTimestamp(startTsNanos) + pt1.Attributes().InsertString("foo", "") + pt1.Attributes().InsertString("key2", "v2") + + m1 := mL0.AppendEmpty() + m1.SetName("hist_test2") + m1.SetDataType(pdata.MetricDataTypeHistogram) + hist1 := m1.Histogram() + pt2 := hist1.DataPoints().AppendEmpty() + pt2.SetCount(3) + pt2.SetSum(50) + pt2.SetExplicitBounds([]float64{10, 20}) + pt2.SetBucketCounts([]uint64{1, 1, 1}) + pt2.SetTimestamp(startTsNanos) + pt2.SetStartTimestamp(startTsNanos) + + return []*pdata.MetricSlice{&mL0} + }, + }, + { + name: "unordered-buckets", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("hist_test", 10, "foo", "bar", "le", "+inf"), + createDataPoint("hist_test", 1, "foo", "bar", "le", "10"), + createDataPoint("hist_test", 2, "foo", "bar", "le", "20"), + createDataPoint("hist_test_sum", 99, "foo", "bar"), + createDataPoint("hist_test_count", 10, "foo", "bar"), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL0 := pdata.NewMetricSlice() + m0 := mL0.AppendEmpty() + m0.SetName("hist_test") + m0.SetDataType(pdata.MetricDataTypeHistogram) + hist0 := m0.Histogram() + pt0 := hist0.DataPoints().AppendEmpty() + pt0.SetCount(10) + pt0.SetSum(99) + pt0.SetExplicitBounds([]float64{10, 20}) + pt0.SetBucketCounts([]uint64{1, 1, 8}) + pt0.SetTimestamp(startTsNanos) + pt0.SetStartTimestamp(startTsNanos) + pt0.Attributes().InsertString("foo", "bar") + + return []*pdata.MetricSlice{&mL0} + }, + }, + { + // this won't likely happen in real env, as prometheus wont generate histogram with less than 3 buckets + name: "only-one-bucket", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("hist_test", 3, "le", "+inf"), + createDataPoint("hist_test_count", 3), + createDataPoint("hist_test_sum", 100), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL0 := pdata.NewMetricSlice() + m0 := mL0.AppendEmpty() + m0.SetName("hist_test") + m0.SetDataType(pdata.MetricDataTypeHistogram) + hist0 := m0.Histogram() + pt0 := hist0.DataPoints().AppendEmpty() + pt0.SetCount(3) + pt0.SetSum(100) + pt0.SetExplicitBounds([]float64{}) + pt0.SetBucketCounts([]uint64{3}) + pt0.SetTimestamp(startTsNanos) + pt0.SetStartTimestamp(startTsNanos) + + return []*pdata.MetricSlice{&mL0} + }, + }, + { + // this won't likely happen in real env, as prometheus wont generate histogram with less than 3 buckets + name: "only-one-bucket-noninf", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("hist_test", 3, "le", "20"), + createDataPoint("hist_test_count", 3), + createDataPoint("hist_test_sum", 100), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL0 := pdata.NewMetricSlice() + m0 := mL0.AppendEmpty() + m0.SetName("hist_test") + m0.SetDataType(pdata.MetricDataTypeHistogram) + hist0 := m0.Histogram() + pt0 := hist0.DataPoints().AppendEmpty() + pt0.SetCount(3) + pt0.SetSum(100) + pt0.SetExplicitBounds([]float64{}) + pt0.SetBucketCounts([]uint64{3}) + pt0.SetTimestamp(startTsNanos) + pt0.SetStartTimestamp(startTsNanos) + + return []*pdata.MetricSlice{&mL0} + }, + }, + { + name: "no-sum", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("hist_test", 1, "foo", "bar", "le", "10"), + createDataPoint("hist_test", 2, "foo", "bar", "le", "20"), + createDataPoint("hist_test", 3, "foo", "bar", "le", "+inf"), + createDataPoint("hist_test_count", 3, "foo", "bar"), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL0 := pdata.NewMetricSlice() + m0 := mL0.AppendEmpty() + m0.SetName("hist_test") + m0.SetDataType(pdata.MetricDataTypeHistogram) + hist0 := m0.Histogram() + pt0 := hist0.DataPoints().AppendEmpty() + pt0.SetCount(3) + pt0.SetSum(0) + pt0.SetExplicitBounds([]float64{10, 20}) + pt0.SetBucketCounts([]uint64{1, 1, 1}) + pt0.SetTimestamp(startTsNanos) + pt0.SetStartTimestamp(startTsNanos) + pt0.Attributes().InsertString("foo", "bar") + + return []*pdata.MetricSlice{&mL0} + }, + }, + { + name: "corrupted-no-buckets", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("hist_test_sum", 99), + createDataPoint("hist_test_count", 10), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL0 := pdata.NewMetricSlice() + return []*pdata.MetricSlice{&mL0} + }, + }, + { + name: "corrupted-no-count", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("hist_test", 1, "foo", "bar", "le", "10"), + createDataPoint("hist_test", 2, "foo", "bar", "le", "20"), + createDataPoint("hist_test", 3, "foo", "bar", "le", "+inf"), + createDataPoint("hist_test_sum", 99, "foo", "bar"), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL0 := pdata.NewMetricSlice() + return []*pdata.MetricSlice{&mL0} + }, + }, + } + + runBuilderTestsPdata(t, tests) +} + +func Test_metricBuilder_summary(t *testing.T) { + tests := []buildTestDataPdata{ + { + name: "no-sum-and-count", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("summary_test", 5, "foo", "bar", "quantile", "1"), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL0 := pdata.NewMetricSlice() + return []*pdata.MetricSlice{&mL0} + }, + }, + { + name: "no-count", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("summary_test", 1, "foo", "bar", "quantile", "0.5"), + createDataPoint("summary_test", 2, "foo", "bar", "quantile", "0.75"), + createDataPoint("summary_test", 5, "foo", "bar", "quantile", "1"), + createDataPoint("summary_test_sum", 500, "foo", "bar"), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL0 := pdata.NewMetricSlice() + return []*pdata.MetricSlice{&mL0} + }, + }, + { + name: "no-sum", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("summary_test", 1, "foo", "bar", "quantile", "0.5"), + createDataPoint("summary_test", 2, "foo", "bar", "quantile", "0.75"), + createDataPoint("summary_test", 5, "foo", "bar", "quantile", "1"), + createDataPoint("summary_test_count", 500, "foo", "bar"), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL0 := pdata.NewMetricSlice() + m0 := mL0.AppendEmpty() + m0.SetName("summary_test") + m0.SetDataType(pdata.MetricDataTypeSummary) + sum0 := m0.Summary() + pt0 := sum0.DataPoints().AppendEmpty() + pt0.SetTimestamp(startTsNanos) + pt0.SetStartTimestamp(startTsNanos) + pt0.SetCount(500) + pt0.SetSum(0.0) + pt0.Attributes().InsertString("foo", "bar") + qvL := pt0.QuantileValues() + q50 := qvL.AppendEmpty() + q50.SetQuantile(50) + q50.SetValue(1.0) + q75 := qvL.AppendEmpty() + q75.SetQuantile(75) + q75.SetValue(2.0) + q100 := qvL.AppendEmpty() + q100.SetQuantile(100) + q100.SetValue(5.0) + + return []*pdata.MetricSlice{&mL0} + }, + }, + { + name: "empty-quantiles", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("summary_test_sum", 100, "foo", "bar"), + createDataPoint("summary_test_count", 500, "foo", "bar"), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL0 := pdata.NewMetricSlice() + m0 := mL0.AppendEmpty() + m0.SetName("summary_test") + m0.SetDataType(pdata.MetricDataTypeSummary) + sum0 := m0.Summary() + pt0 := sum0.DataPoints().AppendEmpty() + pt0.SetStartTimestamp(startTsNanos) + pt0.SetTimestamp(startTsNanos) + pt0.SetCount(500) + pt0.SetSum(100.0) + pt0.Attributes().InsertString("foo", "bar") + + return []*pdata.MetricSlice{&mL0} + }, + }, + { + name: "regular-summary", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("summary_test", 1, "foo", "bar", "quantile", "0.5"), + createDataPoint("summary_test", 2, "foo", "bar", "quantile", "0.75"), + createDataPoint("summary_test", 5, "foo", "bar", "quantile", "1"), + createDataPoint("summary_test_sum", 100, "foo", "bar"), + createDataPoint("summary_test_count", 500, "foo", "bar"), + }, + }, + }, + wants: func() []*pdata.MetricSlice { + mL0 := pdata.NewMetricSlice() + m0 := mL0.AppendEmpty() + m0.SetName("summary_test") + m0.SetDataType(pdata.MetricDataTypeSummary) + sum0 := m0.Summary() + pt0 := sum0.DataPoints().AppendEmpty() + pt0.SetStartTimestamp(startTsNanos) + pt0.SetTimestamp(startTsNanos) + pt0.SetCount(500) + pt0.SetSum(100.0) + pt0.Attributes().InsertString("foo", "bar") + qvL := pt0.QuantileValues() + q50 := qvL.AppendEmpty() + q50.SetQuantile(50) + q50.SetValue(1.0) + q75 := qvL.AppendEmpty() + q75.SetQuantile(75) + q75.SetValue(2.0) + q100 := qvL.AppendEmpty() + q100.SetQuantile(100) + q100.SetValue(5.0) + + return []*pdata.MetricSlice{&mL0} + }, + }, + } + + runBuilderTestsPdata(t, tests) +} + +// Ensure that we reject duplicate label keys. See https://github.com/open-telemetry/wg-prometheus/issues/44. +func TestMetricBuilderDuplicateLabelKeysAreRejected(t *testing.T) { + mc := newMockMetadataCache(testMetadata) + mb := newMetricBuilderPdata(mc, true, "", testLogger, 0) + + dupLabels := labels.Labels{ + {Name: "__name__", Value: "test"}, + {Name: "a", Value: "1"}, + {Name: "a", Value: "1"}, + {Name: "z", Value: "9"}, + {Name: "z", Value: "1"}, + {Name: "instance", Value: "0.0.0.0:8855"}, + {Name: "job", Value: "test"}, + } + + err := mb.AddDataPoint(dupLabels, 1917, 1.0) + require.NotNil(t, err) + require.Contains(t, err.Error(), `invalid sample: non-unique label names: ["a" "z"]`) +} + +func Test_metricBuilder_baddata(t *testing.T) { + t.Run("empty-metric-name", func(t *testing.T) { + mc := newMockMetadataCache(testMetadata) + b := newMetricBuilderPdata(mc, true, "", testLogger, 0) + b.startTime = 1.0 // set to a non-zero value + if err := b.AddDataPoint(labels.FromStrings("a", "b"), startTs, 123); err != errMetricNameNotFound { + t.Error("expecting errMetricNameNotFound error, but get nil") + return + } + + if _, _, _, err := b.Build(); err != errNoDataToBuild { + t.Error("expecting errNoDataToBuild error, but get nil") + } + }) + + t.Run("histogram-datapoint-no-bucket-label", func(t *testing.T) { + mc := newMockMetadataCache(testMetadata) + b := newMetricBuilderPdata(mc, true, "", testLogger, 0) + b.startTime = 1.0 // set to a non-zero value + if err := b.AddDataPoint(createLabels("hist_test", "k", "v"), startTs, 123); err != errEmptyBoundaryLabel { + t.Error("expecting errEmptyBoundaryLabel error, but get nil") + } + }) + + t.Run("summary-datapoint-no-quantile-label", func(t *testing.T) { + mc := newMockMetadataCache(testMetadata) + b := newMetricBuilderPdata(mc, true, "", testLogger, 0) + b.startTime = 1.0 // set to a non-zero value + if err := b.AddDataPoint(createLabels("summary_test", "k", "v"), startTs, 123); err != errEmptyBoundaryLabel { + t.Error("expecting errEmptyBoundaryLabel error, but get nil") + } + }) +} + +func Benchmark_dpgSignature(b *testing.B) { + knownLabelKeys := []string{"a", "b"} + labels := labels.FromStrings("a", "va", "b", "vb", "x", "xa") + b.ReportAllocs() + for i := 0; i < b.N; i++ { + runtime.KeepAlive(dpgSignature(knownLabelKeys, labels)) + } +} + +func Test_dpgSignature(t *testing.T) { + knownLabelKeys := []string{"a", "b"} + + tests := []struct { + name string + ls labels.Labels + want string + }{ + {"1st label", labels.FromStrings("a", "va"), `"a=va"`}, + {"2nd label", labels.FromStrings("b", "vb"), `"b=vb"`}, + {"two labels", labels.FromStrings("a", "va", "b", "vb"), `"a=va""b=vb"`}, + {"extra label", labels.FromStrings("a", "va", "b", "vb", "x", "xa"), `"a=va""b=vb"`}, + {"different order", labels.FromStrings("b", "vb", "a", "va"), `"a=va""b=vb"`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := dpgSignature(knownLabelKeys, tt.ls); got != tt.want { + t.Errorf("dpgSignature() = %q, want %q", got, tt.want) + } + }) + } + + // this is important for caching start values, as new metrics with new tag of a same group can come up in a 2nd run, + // however, its order within the group is not predictable. we need to have a way to generate a stable key even if + // the total number of keys changes in between different scrape runs + t.Run("knownLabelKeys updated", func(t *testing.T) { + ls := labels.FromStrings("a", "va") + want := dpgSignature(knownLabelKeys, ls) + got := dpgSignature(append(knownLabelKeys, "c"), ls) + if got != want { + t.Errorf("dpgSignature() = %v, want %v", got, want) + } + }) +} + +func Test_normalizeMetricName(t *testing.T) { + tests := []struct { + name string + mname string + want string + }{ + {"normal", "normal", "normal"}, + {"count", "foo_count", "foo"}, + {"bucket", "foo_bucket", "foo"}, + {"sum", "foo_sum", "foo"}, + {"total", "foo_total", "foo"}, + {"no_prefix", "_sum", "_sum"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := normalizeMetricName(tt.mname); got != tt.want { + t.Errorf("normalizeMetricName() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/receiver/prometheusreceiver/internal/otlp_transaction.go b/receiver/prometheusreceiver/internal/otlp_transaction.go new file mode 100644 index 0000000000000..8d3fcd780774a --- /dev/null +++ b/receiver/prometheusreceiver/internal/otlp_transaction.go @@ -0,0 +1,230 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "context" + "errors" + "sync/atomic" + "time" + + "go.uber.org/zap" + + "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/model/pdata" + "go.opentelemetry.io/collector/obsreport" + + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/exemplar" + "github.com/prometheus/prometheus/pkg/labels" +) + +const ( + portAttr = "port" + schemeAttr = "scheme" + jobAttr = "job" + instanceAttr = "instance" + + transport = "http" + dataformat = "prometheus" +) + +var errMetricNameNotFound = errors.New("metricName not found from labels") +var errTransactionAborted = errors.New("transaction aborted") +var errNoJobInstance = errors.New("job or instance cannot be found from labels") +var errNoStartTimeMetrics = errors.New("process_start_time_seconds metric is missing") + +type transactionPdata struct { + id int64 + isNew bool + ctx context.Context + useStartTimeMetric bool + startTimeMetricRegex string + sink consumer.Metrics + metadataService *metadataService + externalLabels labels.Labels + nodeResource *pdata.Resource + logger *zap.Logger + receiverID config.ComponentID + metricBuilder *metricBuilderPdata + job, instance string + jobsMap *JobsMapPdata + obsrecv *obsreport.Receiver + startTimeMs int64 +} + +type txConfig struct { + jobsMap *JobsMapPdata + useStartTimeMetric bool + startTimeMetricRegex string + receiverID config.ComponentID + ms *metadataService + sink consumer.Metrics + externalLabels labels.Labels + logger *zap.Logger +} + +func newTransactionPdata(ctx context.Context, txc *txConfig) *transactionPdata { + return &transactionPdata{ + id: atomic.AddInt64(&idSeq, 1), + ctx: ctx, + isNew: true, + sink: txc.sink, + jobsMap: txc.jobsMap, + useStartTimeMetric: txc.useStartTimeMetric, + startTimeMetricRegex: txc.startTimeMetricRegex, + receiverID: txc.receiverID, + metadataService: txc.ms, + externalLabels: txc.externalLabels, + logger: txc.logger, + obsrecv: obsreport.NewReceiver(obsreport.ReceiverSettings{ReceiverID: txc.receiverID, Transport: transport}), + } +} + +// Append always returns 0 to disable label caching. +func (t *transactionPdata) Append(ref uint64, labels labels.Labels, atMs int64, value float64) (pointCount uint64, err error) { + select { + case <-t.ctx.Done(): + return 0, errTransactionAborted + default: + } + + if len(t.externalLabels) != 0 { + labels = append(labels, t.externalLabels...) + } + + if t.isNew { + if err := t.initTransaction(labels); err != nil { + return 0, err + } + } + + return 0, t.metricBuilder.AddDataPoint(labels, atMs, value) +} + +func (t *transactionPdata) AppendExemplar(ref uint64, l labels.Labels, e exemplar.Exemplar) (uint64, error) { + return 0, nil +} + +func (t *transactionPdata) initTransaction(labels labels.Labels) error { + job, instance := labels.Get(model.JobLabel), labels.Get(model.InstanceLabel) + if job == "" || instance == "" { + return errNoJobInstance + } + metadataCache, err := t.metadataService.Get(job, instance) + if err != nil { + return err + } + if t.jobsMap != nil { + t.job = job + t.instance = instance + } + t.nodeResource = createNodeAndResourcePdata(job, instance, metadataCache.SharedLabels().Get(model.SchemeLabel)) + t.metricBuilder = newMetricBuilderPdata(metadataCache, t.useStartTimeMetric, t.startTimeMetricRegex, t.logger, t.startTimeMs) + t.isNew = false + return nil +} + +func (t *transactionPdata) Commit() error { + if t.isNew { + return nil + } + + t.startTimeMs = -1 + + ctx := t.obsrecv.StartMetricsOp(t.ctx) + metricsL, numPoints, _, err := t.metricBuilder.Build() + if err != nil { + t.obsrecv.EndMetricsOp(ctx, dataformat, 0, err) + return err + } + + if t.useStartTimeMetric { + if t.metricBuilder.startTime == 0.0 { + err = errNoStartTimeMetrics + t.obsrecv.EndMetricsOp(ctx, dataformat, 0, err) + return err + } + // Otherwise adjust the startTimestamp for all the metrics. + adjustStartTimestampPdata(t.metricBuilder.startTime, metricsL) + } else { + // TODO: Derive numPoints in this case. + _ = NewMetricsAdjusterPdata(t.jobsMap.get(t.job, t.instance), t.logger).AdjustMetrics(metricsL) + } + + if metricsL.Len() > 0 { + metrics := t.metricSliceToMetrics(metricsL) + t.sink.ConsumeMetrics(ctx, *metrics) + } + + t.obsrecv.EndMetricsOp(ctx, dataformat, numPoints, nil) + return nil +} + +func (t *transactionPdata) Rollback() error { + t.startTimeMs = -1 + return nil +} + +func timestampFromFloat64(ts float64) pdata.Timestamp { + secs := int64(ts) + nanos := int64((ts - float64(secs)) * 1e9) + return pdata.NewTimestampFromTime(time.Unix(secs, nanos)) +} + +func adjustStartTimestampPdata(startTime float64, metricsL *pdata.MetricSlice) { + startTimeTs := timestampFromFloat64(startTime) + for i := 0; i < metricsL.Len(); i++ { + metric := metricsL.At(i) + switch metric.DataType() { + case pdata.MetricDataTypeGauge: + continue + + case pdata.MetricDataTypeSum: + dataPoints := metric.Sum().DataPoints() + for i := 0; i < dataPoints.Len(); i++ { + dataPoint := dataPoints.At(i) + dataPoint.SetStartTimestamp(startTimeTs) + } + + case pdata.MetricDataTypeSummary: + dataPoints := metric.Summary().DataPoints() + for i := 0; i < dataPoints.Len(); i++ { + dataPoint := dataPoints.At(i) + dataPoint.SetStartTimestamp(startTimeTs) + } + + case pdata.MetricDataTypeHistogram: + dataPoints := metric.Histogram().DataPoints() + for i := 0; i < dataPoints.Len(); i++ { + dataPoint := dataPoints.At(i) + dataPoint.SetStartTimestamp(startTimeTs) + } + + default: + panic("Unknown type:: " + metric.DataType().String()) + } + } +} + +func (t *transactionPdata) metricSliceToMetrics(metricsL *pdata.MetricSlice) *pdata.Metrics { + metrics := pdata.NewMetrics() + rms := metrics.ResourceMetrics().AppendEmpty() + ilm := rms.InstrumentationLibraryMetrics().AppendEmpty() + metricsL.CopyTo(ilm.Metrics()) + t.nodeResource.CopyTo(rms.Resource()) + return &metrics +} diff --git a/receiver/prometheusreceiver/internal/transaction_test.go b/receiver/prometheusreceiver/internal/otlp_transaction_test.go similarity index 72% rename from receiver/prometheusreceiver/internal/transaction_test.go rename to receiver/prometheusreceiver/internal/otlp_transaction_test.go index 87c69b49dbf6a..26cf0dc2378ce 100644 --- a/receiver/prometheusreceiver/internal/transaction_test.go +++ b/receiver/prometheusreceiver/internal/otlp_transaction_test.go @@ -19,16 +19,13 @@ import ( "testing" "time" - agentmetricspb "github.com/census-instrumentation/opencensus-proto/gen-go/agent/metrics/v1" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/scrape" "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/config" "go.opentelemetry.io/collector/consumer/consumertest" - "google.golang.org/protobuf/proto" - - "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus" ) func Test_transaction(t *testing.T) { @@ -65,15 +62,15 @@ func Test_transaction(t *testing.T) { t.Run("Commit Without Adding", func(t *testing.T) { nomc := consumertest.NewNop() - tr := newTransaction(context.Background(), nil, true, "", rID, ms, nomc, nil, testLogger) + tr := newTransactionPdata(context.Background(), &txConfig{nil, true, "", rID, ms, nomc, nil, testLogger}) if got := tr.Commit(); got != nil { t.Errorf("expecting nil from Commit() but got err %v", got) } }) - t.Run("Rollback dose nothing", func(t *testing.T) { + t.Run("Rollback does nothing", func(t *testing.T) { nomc := consumertest.NewNop() - tr := newTransaction(context.Background(), nil, true, "", rID, ms, nomc, nil, testLogger) + tr := newTransactionPdata(context.Background(), &txConfig{nil, true, "", rID, ms, nomc, nil, testLogger}) if got := tr.Rollback(); got != nil { t.Errorf("expecting nil from Rollback() but got err %v", got) } @@ -82,7 +79,7 @@ func Test_transaction(t *testing.T) { badLabels := labels.Labels([]labels.Label{{Name: "foo", Value: "bar"}}) t.Run("Add One No Target", func(t *testing.T) { nomc := consumertest.NewNop() - tr := newTransaction(context.Background(), nil, true, "", rID, ms, nomc, nil, testLogger) + tr := newTransactionPdata(context.Background(), &txConfig{nil, true, "", rID, ms, nomc, nil, testLogger}) if _, got := tr.Append(0, badLabels, time.Now().Unix()*1000, 1.0); got == nil { t.Errorf("expecting error from Add() but got nil") } @@ -94,7 +91,7 @@ func Test_transaction(t *testing.T) { {Name: "foo", Value: "bar"}}) t.Run("Add One Job not found", func(t *testing.T) { nomc := consumertest.NewNop() - tr := newTransaction(context.Background(), nil, true, "", rID, ms, nomc, nil, testLogger) + tr := newTransactionPdata(context.Background(), &txConfig{nil, true, "", rID, ms, nomc, nil, testLogger}) if _, got := tr.Append(0, jobNotFoundLb, time.Now().Unix()*1000, 1.0); got == nil { t.Errorf("expecting error from Add() but got nil") } @@ -105,7 +102,7 @@ func Test_transaction(t *testing.T) { {Name: "__name__", Value: "foo"}}) t.Run("Add One Good", func(t *testing.T) { sink := new(consumertest.MetricsSink) - tr := newTransaction(context.Background(), nil, true, "", rID, ms, sink, nil, testLogger) + tr := newTransactionPdata(context.Background(), &txConfig{nil, true, "", rID, ms, sink, nil, testLogger}) if _, got := tr.Append(0, goodLabels, time.Now().Unix()*1000, 1.0); got != nil { t.Errorf("expecting error == nil from Add() but got: %v\n", got) } @@ -113,33 +110,20 @@ func Test_transaction(t *testing.T) { if got := tr.Commit(); got != nil { t.Errorf("expecting nil from Commit() but got err %v", got) } - expectedNode, expectedResource := createNodeAndResource("test", "localhost:8080", "http") + expectedNodeResource := createNodeAndResourcePdata("test", "localhost:8080", "http") mds := sink.AllMetrics() if len(mds) != 1 { t.Fatalf("wanted one batch, got %v\n", sink.AllMetrics()) } - var ocmds []*agentmetricspb.ExportMetricsServiceRequest - rms := mds[0].ResourceMetrics() - for i := 0; i < rms.Len(); i++ { - ocmd := &agentmetricspb.ExportMetricsServiceRequest{} - ocmd.Node, ocmd.Resource, ocmd.Metrics = opencensus.ResourceMetricsToOC(rms.At(i)) - ocmds = append(ocmds, ocmd) - } - require.Len(t, ocmds, 1) - if !proto.Equal(ocmds[0].Node, expectedNode) { - t.Errorf("generated node %v and expected node %v is different\n", ocmds[0].Node, expectedNode) - } - if !proto.Equal(ocmds[0].Resource, expectedResource) { - t.Errorf("generated resource %v and expected resource %v is different\n", ocmds[0].Resource, expectedResource) - } - + gotNodeResource := mds[0].ResourceMetrics().At(0).Resource() + require.Equal(t, *expectedNodeResource, gotNodeResource, "Resources do not match") // TODO: re-enable this when handle unspecified OC type // assert.Len(t, ocmds[0].Metrics, 1) }) t.Run("Error when start time is zero", func(t *testing.T) { sink := new(consumertest.MetricsSink) - tr := newTransaction(context.Background(), nil, true, "", rID, ms, sink, nil, testLogger) + tr := newTransactionPdata(context.Background(), &txConfig{nil, true, "", rID, ms, sink, nil, testLogger}) if _, got := tr.Append(0, goodLabels, time.Now().Unix()*1000, 1.0); got != nil { t.Errorf("expecting error == nil from Add() but got: %v\n", got) } diff --git a/receiver/prometheusreceiver/internal/prom_to_otlp.go b/receiver/prometheusreceiver/internal/prom_to_otlp.go index 3ce2440759c32..19ffc17b2797a 100644 --- a/receiver/prometheusreceiver/internal/prom_to_otlp.go +++ b/receiver/prometheusreceiver/internal/prom_to_otlp.go @@ -21,7 +21,7 @@ import ( conventions "go.opentelemetry.io/collector/model/semconv/v1.5.0" ) -func createNodeAndResourcePdata(job, instance, scheme string) pdata.Resource { +func createNodeAndResourcePdata(job, instance, scheme string) *pdata.Resource { host, port, err := net.SplitHostPort(instance) if err != nil { host = instance @@ -35,5 +35,5 @@ func createNodeAndResourcePdata(job, instance, scheme string) pdata.Resource { attrs.UpsertString(portAttr, port) attrs.UpsertString(schemeAttr, scheme) - return resource + return &resource } diff --git a/receiver/prometheusreceiver/internal/prom_to_otlp_test.go b/receiver/prometheusreceiver/internal/prom_to_otlp_test.go index 4038cde1302cb..fe61e5ab486d5 100644 --- a/receiver/prometheusreceiver/internal/prom_to_otlp_test.go +++ b/receiver/prometheusreceiver/internal/prom_to_otlp_test.go @@ -17,42 +17,16 @@ package internal import ( "testing" - metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" "github.com/stretchr/testify/require" - "go.opentelemetry.io/collector/model/pdata" - "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus" + "go.opentelemetry.io/collector/model/pdata" ) -// Parity test to ensure that createNodeAndResource produces identical results to createNodeAndResourcePdata. -func TestCreateNodeAndResourceEquivalence(t *testing.T) { - job, instance, scheme := "converter", "ocmetrics", "http" - ocNode, ocResource := createNodeAndResource(job, instance, scheme) - mdFromOC := opencensus.OCToMetrics(ocNode, ocResource, - // We need to pass in a dummy set of metrics - // just to populate and allow for full conversion. - []*metricspb.Metric{ - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "m1", - Description: "d1", - Unit: "By", - }, - }, - }, - ) - - fromOCResource := mdFromOC.ResourceMetrics().At(0).Resource().Attributes().Sort() - byDirectOTLPResource := createNodeAndResourcePdata(job, instance, scheme).Attributes().Sort() - - require.Equal(t, byDirectOTLPResource, fromOCResource) -} - type jobInstanceDefinition struct { job, instance, host, scheme, port string } -func makeResourceWithJobInstanceScheme(def *jobInstanceDefinition) pdata.Resource { +func makeResourceWithJobInstanceScheme(def *jobInstanceDefinition) *pdata.Resource { resource := pdata.NewResource() attrs := resource.Attributes() // Using hardcoded values to assert on outward expectations so that @@ -63,7 +37,7 @@ func makeResourceWithJobInstanceScheme(def *jobInstanceDefinition) pdata.Resourc attrs.UpsertString("instance", def.instance) attrs.UpsertString("port", def.port) attrs.UpsertString("scheme", def.scheme) - return resource + return &resource } func TestCreateNodeAndResourcePromToOTLP(t *testing.T) { @@ -71,7 +45,7 @@ func TestCreateNodeAndResourcePromToOTLP(t *testing.T) { name, job string instance string scheme string - want pdata.Resource + want *pdata.Resource }{ { name: "all attributes proper", diff --git a/receiver/prometheusreceiver/internal/staleness_end_to_end_test.go b/receiver/prometheusreceiver/internal/staleness_end_to_end_test.go index 6d8bd4f797ba7..fe243ecd929d8 100644 --- a/receiver/prometheusreceiver/internal/staleness_end_to_end_test.go +++ b/receiver/prometheusreceiver/internal/staleness_end_to_end_test.go @@ -201,6 +201,7 @@ service: // 6. Assert that we encounter the stale markers aka special NaNs for the various time series. staleMarkerCount := 0 totalSamples := 0 + require.True(t, len(wReqL) > 0, "Expecting at least one WriteRequest") for i, wReq := range wReqL { name := fmt.Sprintf("WriteRequest#%d", i) require.True(t, len(wReq.Timeseries) > 0, "Expecting at least 1 timeSeries for:: "+name) diff --git a/receiver/prometheusreceiver/internal/transaction.go b/receiver/prometheusreceiver/internal/transaction.go deleted file mode 100644 index f790a2795115e..0000000000000 --- a/receiver/prometheusreceiver/internal/transaction.go +++ /dev/null @@ -1,255 +0,0 @@ -// Copyright The OpenTelemetry Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package internal - -import ( - "context" - "errors" - "net" - "sync/atomic" - - commonpb "github.com/census-instrumentation/opencensus-proto/gen-go/agent/common/v1" - metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" - resourcepb "github.com/census-instrumentation/opencensus-proto/gen-go/resource/v1" - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/pkg/exemplar" - "github.com/prometheus/prometheus/pkg/labels" - "github.com/prometheus/prometheus/storage" - "go.opentelemetry.io/collector/config" - "go.opentelemetry.io/collector/consumer" - "go.opentelemetry.io/collector/obsreport" - "go.uber.org/zap" - "google.golang.org/protobuf/types/known/timestamppb" - - "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus" -) - -const ( - portAttr = "port" - schemeAttr = "scheme" - jobAttr = "job" - instanceAttr = "instance" - - transport = "http" - dataformat = "prometheus" -) - -var errMetricNameNotFound = errors.New("metricName not found from labels") -var errTransactionAborted = errors.New("transaction aborted") -var errNoJobInstance = errors.New("job or instance cannot be found from labels") -var errNoStartTimeMetrics = errors.New("process_start_time_seconds metric is missing") - -// A transaction is corresponding to an individual scrape operation or stale report. -// That said, whenever prometheus receiver scrapped a target metric endpoint a page of raw metrics is returned, -// a transaction, which acts as appender, is created to process this page of data, the scrapeLoop will call the Add or -// AddFast method to insert metrics data points, when finished either Commit, which means success, is called and data -// will be flush to the downstream consumer, or Rollback, which means discard all the data, is called and all data -// points are discarded. -type transaction struct { - id int64 - ctx context.Context - isNew bool - sink consumer.Metrics - job string - instance string - jobsMap *JobsMap - useStartTimeMetric bool - startTimeMetricRegex string - ms *metadataService - node *commonpb.Node - resource *resourcepb.Resource - metricBuilder *metricBuilder - externalLabels labels.Labels - logger *zap.Logger - obsrecv *obsreport.Receiver - startTimeMs int64 -} - -func newTransaction( - ctx context.Context, - jobsMap *JobsMap, - useStartTimeMetric bool, - startTimeMetricRegex string, - receiverID config.ComponentID, - ms *metadataService, - sink consumer.Metrics, - externalLabels labels.Labels, - logger *zap.Logger) *transaction { - return &transaction{ - id: atomic.AddInt64(&idSeq, 1), - ctx: ctx, - isNew: true, - sink: sink, - jobsMap: jobsMap, - useStartTimeMetric: useStartTimeMetric, - startTimeMetricRegex: startTimeMetricRegex, - ms: ms, - externalLabels: externalLabels, - logger: logger, - obsrecv: obsreport.NewReceiver(obsreport.ReceiverSettings{ReceiverID: receiverID, Transport: transport}), - startTimeMs: -1, - } -} - -// ensure *transaction has implemented the storage.Appender interface -var _ storage.Appender = (*transaction)(nil) - -// Append always returns 0 to disable label caching. -func (tr *transaction) Append(ref uint64, ls labels.Labels, t int64, v float64) (uint64, error) { - if tr.startTimeMs < 0 { - tr.startTimeMs = t - } - - select { - case <-tr.ctx.Done(): - return 0, errTransactionAborted - default: - } - if len(tr.externalLabels) > 0 { - // TODO(jbd): Improve the allocs. - ls = append(ls, tr.externalLabels...) - } - if tr.isNew { - if err := tr.initTransaction(ls); err != nil { - return 0, err - } - } - return 0, tr.metricBuilder.AddDataPoint(ls, t, v) -} - -func (tr *transaction) AppendExemplar(ref uint64, l labels.Labels, e exemplar.Exemplar) (uint64, error) { - return 0, nil -} - -// AddFast always returns error since caching is not supported by Add() function. -func (tr *transaction) AddFast(_ uint64, _ int64, _ float64) error { - return storage.ErrNotFound -} - -func (tr *transaction) initTransaction(ls labels.Labels) error { - job, instance := ls.Get(model.JobLabel), ls.Get(model.InstanceLabel) - if job == "" || instance == "" { - return errNoJobInstance - } - // discover the binding target when this method is called for the first time during a transaction - mc, err := tr.ms.Get(job, instance) - if err != nil { - return err - } - if tr.jobsMap != nil { - tr.job = job - tr.instance = instance - } - tr.node, tr.resource = createNodeAndResource(job, instance, mc.SharedLabels().Get(model.SchemeLabel)) - tr.metricBuilder = newMetricBuilder(mc, tr.useStartTimeMetric, tr.startTimeMetricRegex, tr.logger, tr.startTimeMs) - tr.isNew = false - return nil -} - -// Commit submits metrics data to consumers. -func (tr *transaction) Commit() error { - if tr.isNew { - // In a situation like not able to connect to the remote server, scrapeloop will still commit even if it had - // never added any data points, that the transaction has not been initialized. - return nil - } - - tr.startTimeMs = -1 - - ctx := tr.obsrecv.StartMetricsOp(tr.ctx) - metrics, _, _, err := tr.metricBuilder.Build() - if err != nil { - // Only error by Build() is errNoDataToBuild, with numReceivedPoints set to zero. - tr.obsrecv.EndMetricsOp(ctx, dataformat, 0, err) - return err - } - - if tr.useStartTimeMetric { - // startTime is mandatory in this case, but may be zero when the - // process_start_time_seconds metric is missing from the target endpoint. - if tr.metricBuilder.startTime == 0.0 { - // Since we are unable to adjust metrics properly, we will drop them - // and return an error. - err = errNoStartTimeMetrics - tr.obsrecv.EndMetricsOp(ctx, dataformat, 0, err) - return err - } - - adjustStartTimestamp(tr.metricBuilder.startTime, metrics) - } else { - // AdjustMetrics - jobsMap has to be non-nil in this case. - // Note: metrics could be empty after adjustment, which needs to be checked before passing it on to ConsumeMetrics() - metrics, _ = NewMetricsAdjuster(tr.jobsMap.get(tr.job, tr.instance), tr.logger).AdjustMetrics(metrics) - } - - numPoints := 0 - if len(metrics) > 0 { - md := opencensus.OCToMetrics(tr.node, tr.resource, metrics) - numPoints = md.DataPointCount() - err = tr.sink.ConsumeMetrics(ctx, md) - } - tr.obsrecv.EndMetricsOp(ctx, dataformat, numPoints, err) - return err -} - -func (tr *transaction) Rollback() error { - tr.startTimeMs = -1 - return nil -} - -func adjustStartTimestamp(startTime float64, metrics []*metricspb.Metric) { - startTimeTs := timestampFromFloat64(startTime) - for _, metric := range metrics { - switch metric.GetMetricDescriptor().GetType() { - case metricspb.MetricDescriptor_GAUGE_DOUBLE, metricspb.MetricDescriptor_GAUGE_DISTRIBUTION: - continue - default: - for _, ts := range metric.GetTimeseries() { - ts.StartTimestamp = startTimeTs - } - } - } -} - -func timestampFromFloat64(ts float64) *timestamppb.Timestamp { - secs := int64(ts) - nanos := int64((ts - float64(secs)) * 1e9) - return ×tamppb.Timestamp{ - Seconds: secs, - Nanos: int32(nanos), - } -} - -func createNodeAndResource(job, instance, scheme string) (*commonpb.Node, *resourcepb.Resource) { - host, port, err := net.SplitHostPort(instance) - if err != nil { - host = instance - } - node := &commonpb.Node{ - ServiceInfo: &commonpb.ServiceInfo{Name: job}, - Identifier: &commonpb.ProcessIdentifier{ - HostName: host, - }, - } - resource := &resourcepb.Resource{ - Labels: map[string]string{ - jobAttr: job, - instanceAttr: instance, - portAttr: port, - schemeAttr: scheme, - }, - } - return node, resource -} diff --git a/receiver/prometheusreceiver/metrics_receiver.go b/receiver/prometheusreceiver/metrics_receiver.go index 74883b64f4343..6350cf987cb28 100644 --- a/receiver/prometheusreceiver/metrics_receiver.go +++ b/receiver/prometheusreceiver/metrics_receiver.go @@ -20,9 +20,10 @@ import ( "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/scrape" + "go.uber.org/zap" + "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/consumer" - "go.uber.org/zap" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/internal" ) @@ -71,9 +72,9 @@ func (r *pReceiver) Start(_ context.Context, host component.Host) error { } }() - var jobsMap *internal.JobsMap + var jobsMap *internal.JobsMapPdata if !r.cfg.UseStartTimeMetric { - jobsMap = internal.NewJobsMap(2 * time.Minute) + jobsMap = internal.NewJobsMapPdata(2 * time.Minute) } // Per component.Component Start instructions, for async operations we should not use the // incoming context, it may get cancelled. diff --git a/receiver/prometheusreceiver/metrics_reciever_external_labels_test.go b/receiver/prometheusreceiver/metrics_reciever_external_labels_test.go index 85d266cd79070..9c5ce69cf0543 100644 --- a/receiver/prometheusreceiver/metrics_reciever_external_labels_test.go +++ b/receiver/prometheusreceiver/metrics_reciever_external_labels_test.go @@ -18,15 +18,14 @@ import ( "context" "testing" - agentmetricspb "github.com/census-instrumentation/opencensus-proto/gen-go/agent/metrics/v1" - metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" "github.com/prometheus/prometheus/pkg/labels" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component/componenttest" "go.opentelemetry.io/collector/config" "go.opentelemetry.io/collector/consumer/consumertest" - - "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus" + "go.opentelemetry.io/collector/model/pdata" ) const targetExternalLabels = ` @@ -62,48 +61,53 @@ func TestExternalLabels(t *testing.T) { mp.wg.Wait() metrics := cms.AllMetrics() - results := make(map[string][]*agentmetricspb.ExportMetricsServiceRequest) + // split and store results by target name + results := make(map[string][]*pdata.MetricSlice) for _, md := range metrics { rms := md.ResourceMetrics() for i := 0; i < rms.Len(); i++ { - ocmd := &agentmetricspb.ExportMetricsServiceRequest{} - ocmd.Node, ocmd.Resource, ocmd.Metrics = opencensus.ResourceMetricsToOC(rms.At(i)) - result, ok := results[ocmd.Node.ServiceInfo.Name] - if !ok { - result = make([]*agentmetricspb.ExportMetricsServiceRequest, 0) + rmi := rms.At(i) + serviceNameAttr, ok := rmi.Resource().Attributes().Get("service.name") + assert.True(t, ok, `expected "service.name" as a known attribute`) + serviceName := serviceNameAttr.StringVal() + ilmL := rmi.InstrumentationLibraryMetrics() + for j := 0; j < ilmL.Len(); j++ { + ilm := ilmL.At(j) + metricL := ilm.Metrics() + results[serviceName] = append(results[serviceName], &metricL) } - results[ocmd.Node.ServiceInfo.Name] = append(result, ocmd) } - } for _, target := range targets { target.validateFunc(t, target, results[target.name]) } } -func verifyExternalLabels(t *testing.T, td *testData, mds []*agentmetricspb.ExportMetricsServiceRequest) { +func verifyExternalLabels(t *testing.T, td *testData, mds []*pdata.MetricSlice) { verifyNumScrapeResults(t, td, mds) - - want := &agentmetricspb.ExportMetricsServiceRequest{ - Node: td.node, - Resource: td.resource, - } - doCompare("scrape-externalLabels", t, want, mds[0], []testExpectation{ - assertMetricPresent("go_threads", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_GAUGE_DOUBLE), - compareMetricLabelKeys([]string{"key"}), - }, - []seriesExpectation{ - { - series: []seriesComparator{ - compareSeriesLabelValues([]string{"value"}), - }, - points: []pointComparator{ - comparePointTimestamp(mds[0].Metrics[0].Timeseries[0].Points[0].Timestamp), - compareDoubleVal(19), - }, + // TODO: Translate me. + /* + want := &agentmetricspb.ExportMetricsServiceRequest{ + Node: td.node, + Resource: td.resource, + } + doCompare("scrape-externalLabels", t, want, mds[0], []testExpectation{ + assertMetricPresent("go_threads", + []descriptorComparator{ + compareMetricType(metricspb.MetricDescriptor_GAUGE_DOUBLE), + compareMetricLabelKeys([]string{"key"}), }, - }), - }) + []seriesExpectation{ + { + series: []seriesComparator{ + compareSeriesLabelValues([]string{"value"}), + }, + points: []pointComparator{ + comparePointTimestamp(mds[0].Metrics[0].Timeseries[0].Points[0].Timestamp), + compareDoubleVal(19), + }, + }, + }), + }) + */ } From 5cf4213f950bcc09c83435954282a6924aeb62bd Mon Sep 17 00:00:00 2001 From: Emmanuel T Odeke Date: Mon, 13 Sep 2021 16:48:39 -0600 Subject: [PATCH 02/26] Retrofit top level package for pdata comparisons --- .../metrics_receiver_test.go | 10 ++----- .../metrics_reciever_external_labels_test.go | 28 +++++++++++++------ 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/receiver/prometheusreceiver/metrics_receiver_test.go b/receiver/prometheusreceiver/metrics_receiver_test.go index 19a819238c4a7..1993e5039d2b5 100644 --- a/receiver/prometheusreceiver/metrics_receiver_test.go +++ b/receiver/prometheusreceiver/metrics_receiver_test.go @@ -190,15 +190,9 @@ func verifyNumScrapeResults(t *testing.T, td *testData, mds []*agentmetricspb.Ex } } -func doCompare(name string, t *testing.T, want, got *agentmetricspb.ExportMetricsServiceRequest, expectations []testExpectation) { +func doCompare(name string, t *testing.T, want, got pdata.ResourceMetrics) { t.Run(name, func(t *testing.T) { - numScrapeMetrics := countScrapeMetrics(got) - assert.Equal(t, expectedScrapeMetricCount, numScrapeMetrics) - assert.EqualValues(t, want.Node, got.Node) - assert.EqualValues(t, want.Resource, got.Resource) - for _, e := range expectations { - assert.True(t, e(t, got.Metrics)) - } + assert.True(t, want, got) }) } diff --git a/receiver/prometheusreceiver/metrics_reciever_external_labels_test.go b/receiver/prometheusreceiver/metrics_reciever_external_labels_test.go index 9c5ce69cf0543..8cc16ce8b8594 100644 --- a/receiver/prometheusreceiver/metrics_reciever_external_labels_test.go +++ b/receiver/prometheusreceiver/metrics_reciever_external_labels_test.go @@ -62,7 +62,7 @@ func TestExternalLabels(t *testing.T) { metrics := cms.AllMetrics() // split and store results by target name - results := make(map[string][]*pdata.MetricSlice) + results := make(map[string][]pdata.Metric) for _, md := range metrics { rms := md.ResourceMetrics() for i := 0; i < rms.Len(); i++ { @@ -70,12 +70,7 @@ func TestExternalLabels(t *testing.T) { serviceNameAttr, ok := rmi.Resource().Attributes().Get("service.name") assert.True(t, ok, `expected "service.name" as a known attribute`) serviceName := serviceNameAttr.StringVal() - ilmL := rmi.InstrumentationLibraryMetrics() - for j := 0; j < ilmL.Len(); j++ { - ilm := ilmL.At(j) - metricL := ilm.Metrics() - results[serviceName] = append(results[serviceName], &metricL) - } + results[serviceName] = append(results[serviceName], md) } } for _, target := range targets { @@ -83,8 +78,25 @@ func TestExternalLabels(t *testing.T) { } } -func verifyExternalLabels(t *testing.T, td *testData, mds []*pdata.MetricSlice) { +func makeMetric(td *testData, metricL []pdata.Metric) pdata.ResourceMetrics { + rms := pdata.NewResourceMetrics() + td.resource.CopyTo(rms.Resource()) + ilm := rms.InstrumentationLibraryMetrics().AppendEmpty() + destMetricL := ilm.Metrics() + for _, metric := range metricL { + destMetric := destMetricL.AppendMetric() + metric.CopyTo(destMetric) + } + return rms +} + +func verifyExternalLabels(t *testing.T, td *testData, mds []pdata.Metric) { verifyNumScrapeResults(t, td, mds) + + want := makeMetric(td, []pdata.Metric{ + internal.GaugeMetric("go_threads", k1v2, t1Ms, internal.DoublePoint(t1Ms, 19)) + }) + // TODO: Translate me. /* want := &agentmetricspb.ExportMetricsServiceRequest{ From 959a3ede5aa9c85fca966244d9f8248c570caf11 Mon Sep 17 00:00:00 2001 From: Emmanuel T Odeke Date: Mon, 13 Sep 2021 17:21:21 -0600 Subject: [PATCH 03/26] Reuse for pdata metrics creators from PR #5231 --- .../metricstestutil/metricsutil_pdata.go | 143 ++++++++++++++++++ .../internal/otlp_metrics_adjuster_test.go | 137 ++--------------- 2 files changed, 156 insertions(+), 124 deletions(-) create mode 100644 internal/coreinternal/metricstestutil/metricsutil_pdata.go diff --git a/internal/coreinternal/metricstestutil/metricsutil_pdata.go b/internal/coreinternal/metricstestutil/metricsutil_pdata.go new file mode 100644 index 0000000000000..882446dcf1692 --- /dev/null +++ b/internal/coreinternal/metricstestutil/metricsutil_pdata.go @@ -0,0 +1,143 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metricstestutil + +import "go.opentelemetry.io/collector/model/pdata" + +type KV struct { + Key, Value string +} + +func DistPointPdata(ts pdata.Timestamp, bounds []float64, counts []uint64) *pdata.HistogramDataPoint { + hdp := pdata.NewHistogramDataPoint() + hdp.SetExplicitBounds(bounds) + hdp.SetBucketCounts(counts) + hdp.SetTimestamp(ts) + var sum float64 + var count uint64 + for i, bcount := range counts { + count += bcount + if i > 0 { + sum += float64(bcount) * bounds[i-1] + } + } + hdp.SetCount(count) + hdp.SetSum(sum) + + return &hdp +} + +func GaugeDistMetricPdata(name string, kvp []*KV, startTs pdata.Timestamp, points ...*pdata.HistogramDataPoint) *pdata.Metric { + hMetric := CumulativeDistMetricPdata(name, kvp, startTs, points...) + hMetric.Histogram().SetAggregationTemporality(pdata.AggregationTemporalityDelta) + return hMetric +} + +func CumulativeDistMetricPdata(name string, kvp []*KV, startTs pdata.Timestamp, points ...*pdata.HistogramDataPoint) *pdata.Metric { + metric := pdata.NewMetric() + metric.SetName(name) + metric.SetDataType(pdata.MetricDataTypeHistogram) + histogram := metric.Histogram() + histogram.SetAggregationTemporality(pdata.AggregationTemporalityCumulative) + + destPointL := histogram.DataPoints() + // By default the AggregationTemporality is Cumulative until it'll be changed by the caller. + for _, point := range points { + destPoint := destPointL.AppendEmpty() + point.CopyTo(destPoint) + point.SetStartTimestamp(startTs) + attrs := destPoint.Attributes() + for _, kv := range kvp { + attrs.InsertString(kv.Key, kv.Value) + } + } + return &metric +} + +func DoublePointPdata(ts pdata.Timestamp, value float64) *pdata.NumberDataPoint { + ndp := pdata.NewNumberDataPoint() + ndp.SetTimestamp(ts) + ndp.SetDoubleVal(value) + + return &ndp +} + +func GaugeMetricPdata(name string, kvp []*KV, startTs pdata.Timestamp, points ...*pdata.NumberDataPoint) *pdata.Metric { + metric := pdata.NewMetric() + metric.SetName(name) + metric.SetDataType(pdata.MetricDataTypeGauge) + + destPointL := metric.Gauge().DataPoints() + for _, point := range points { + destPoint := destPointL.AppendEmpty() + point.CopyTo(destPoint) + point.SetStartTimestamp(startTs) + attrs := destPoint.Attributes() + for _, kv := range kvp { + attrs.InsertString(kv.Key, kv.Value) + } + } + return &metric +} + +func SummaryPointPdata(ts pdata.Timestamp, count uint64, sum float64, quantiles, values []float64) *pdata.SummaryDataPoint { + sdp := pdata.NewSummaryDataPoint() + sdp.SetTimestamp(ts) + sdp.SetCount(count) + sdp.SetSum(sum) + qvL := sdp.QuantileValues() + for i := 0; i < len(quantiles); i++ { + qvi := qvL.AppendEmpty() + qvi.SetQuantile(quantiles[i]) + qvi.SetValue(values[i]) + } + return &sdp +} + +func SummaryMetricPdata(name string, kvp []*KV, startTs pdata.Timestamp, points ...*pdata.SummaryDataPoint) *pdata.Metric { + metric := pdata.NewMetric() + metric.SetName(name) + metric.SetDataType(pdata.MetricDataTypeSummary) + + destPointL := metric.Summary().DataPoints() + for _, point := range points { + destPoint := destPointL.AppendEmpty() + point.CopyTo(destPoint) + point.SetStartTimestamp(startTs) + attrs := destPoint.Attributes() + for _, kv := range kvp { + attrs.InsertString(kv.Key, kv.Value) + } + } + return &metric +} + +func SumMetricPdata(name string, kvp []*KV, startTs pdata.Timestamp, points ...*pdata.NumberDataPoint) *pdata.Metric { + metric := pdata.NewMetric() + metric.SetName(name) + metric.SetDataType(pdata.MetricDataTypeSum) + + destPointL := metric.Sum().DataPoints() + for _, point := range points { + destPoint := destPointL.AppendEmpty() + point.CopyTo(destPoint) + point.SetStartTimestamp(startTs) + attrs := destPoint.Attributes() + for _, kv := range kvp { + attrs.InsertString(kv.Key, kv.Value) + } + } + return &metric +} diff --git a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go index 9cc0373ad8022..d46887cc12fa8 100644 --- a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go +++ b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go @@ -23,6 +23,8 @@ import ( "go.uber.org/zap" "go.opentelemetry.io/collector/model/pdata" + + "github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/metricstestutil" ) var ( @@ -503,131 +505,18 @@ func Test_summary(t *testing.T) { runScript(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) } -func distPoint(ts pdata.Timestamp, bounds []float64, counts []uint64) *pdata.HistogramDataPoint { - hdp := pdata.NewHistogramDataPoint() - hdp.SetExplicitBounds(bounds) - hdp.SetBucketCounts(counts) - hdp.SetTimestamp(ts) - var sum float64 - var count uint64 - for i, bcount := range counts { - count += bcount - if i > 0 { - sum += float64(bcount) * bounds[i-1] - } - } - hdp.SetCount(count) - hdp.SetSum(sum) - - return &hdp -} - -type kv struct { - key, value string -} - -func gaugeDistMetric(name string, kvp []*kv, startTs pdata.Timestamp, points ...*pdata.HistogramDataPoint) *pdata.Metric { - hMetric := histogramMetric(name, kvp, startTs, points...) - hMetric.Histogram().SetAggregationTemporality(pdata.AggregationTemporalityDelta) - return hMetric -} - -func histogramMetric(name string, kvp []*kv, startTs pdata.Timestamp, points ...*pdata.HistogramDataPoint) *pdata.Metric { - metric := pdata.NewMetric() - metric.SetName(name) - metric.SetDataType(pdata.MetricDataTypeHistogram) - histogram := metric.Histogram() - histogram.SetAggregationTemporality(pdata.AggregationTemporalityCumulative) - - destPointL := histogram.DataPoints() - // By default the AggregationTemporality is Cumulative until it'll be changed by the caller. - for _, point := range points { - destPoint := destPointL.AppendEmpty() - point.CopyTo(destPoint) - point.SetStartTimestamp(startTs) - attrs := destPoint.Attributes() - for _, kv := range kvp { - attrs.InsertString(kv.key, kv.value) - } - } - return &metric -} +type kv = metricstestutil.KV -func doublePoint(ts pdata.Timestamp, value float64) *pdata.NumberDataPoint { - ndp := pdata.NewNumberDataPoint() - ndp.SetTimestamp(ts) - ndp.SetDoubleVal(value) - - return &ndp -} - -func gaugeMetric(name string, kvp []*kv, startTs pdata.Timestamp, points ...*pdata.NumberDataPoint) *pdata.Metric { - metric := pdata.NewMetric() - metric.SetName(name) - metric.SetDataType(pdata.MetricDataTypeGauge) - - destPointL := metric.Gauge().DataPoints() - for _, point := range points { - destPoint := destPointL.AppendEmpty() - point.CopyTo(destPoint) - point.SetStartTimestamp(startTs) - attrs := destPoint.Attributes() - for _, kv := range kvp { - attrs.InsertString(kv.key, kv.value) - } - } - return &metric -} - -func summaryPoint(ts pdata.Timestamp, count uint64, sum float64, quantiles, values []float64) *pdata.SummaryDataPoint { - sdp := pdata.NewSummaryDataPoint() - sdp.SetTimestamp(ts) - sdp.SetCount(count) - sdp.SetSum(sum) - qvL := sdp.QuantileValues() - for i := 0; i < len(quantiles); i++ { - qvi := qvL.AppendEmpty() - qvi.SetQuantile(quantiles[i]) - qvi.SetValue(values[i]) - } - return &sdp -} - -func summaryMetric(name string, kvp []*kv, startTs pdata.Timestamp, points ...*pdata.SummaryDataPoint) *pdata.Metric { - metric := pdata.NewMetric() - metric.SetName(name) - metric.SetDataType(pdata.MetricDataTypeSummary) - - destPointL := metric.Summary().DataPoints() - for _, point := range points { - destPoint := destPointL.AppendEmpty() - point.CopyTo(destPoint) - point.SetStartTimestamp(startTs) - attrs := destPoint.Attributes() - for _, kv := range kvp { - attrs.InsertString(kv.key, kv.value) - } - } - return &metric -} - -func sumMetric(name string, kvp []*kv, startTs pdata.Timestamp, points ...*pdata.NumberDataPoint) *pdata.Metric { - metric := pdata.NewMetric() - metric.SetName(name) - metric.SetDataType(pdata.MetricDataTypeSum) - - destPointL := metric.Sum().DataPoints() - for _, point := range points { - destPoint := destPointL.AppendEmpty() - point.CopyTo(destPoint) - point.SetStartTimestamp(startTs) - attrs := destPoint.Attributes() - for _, kv := range kvp { - attrs.InsertString(kv.key, kv.value) - } - } - return &metric -} +var ( + distPoint = metricstestutil.DistPointPdata + gaugeDistMetric = metricstestutil.GaugeDistMetricPdata + histogramMetric = metricstestutil.CumulativeDistMetricPdata + doublePoint = metricstestutil.DoublePointPdata + gaugeMetric = metricstestutil.GaugeMetricPdata + summaryPoint = metricstestutil.SummaryPointPdata + summaryMetric = metricstestutil.SummaryMetricPdata + sumMetric = metricstestutil.SumMetricPdata +) func metricSlice(metrics ...*pdata.Metric) *pdata.MetricSlice { ms := pdata.NewMetricSlice() From de159f6879080a6b33697e00a8bb53642b7671da Mon Sep 17 00:00:00 2001 From: Emmanuel T Odeke Date: Tue, 14 Sep 2021 16:02:05 -0600 Subject: [PATCH 04/26] Shed all in-code direct references to OpenCensus --- receiver/prometheusreceiver/go.mod | 6 +- receiver/prometheusreceiver/go.sum | 1 - .../internal/otlp_metricfamily.go | 6 +- .../metrics_receiver_test.go | 1381 ++++++----------- .../metrics_reciever_external_labels_test.go | 54 +- 5 files changed, 468 insertions(+), 980 deletions(-) diff --git a/receiver/prometheusreceiver/go.mod b/receiver/prometheusreceiver/go.mod index 66eeba85c78eb..65af6d4562a4e 100644 --- a/receiver/prometheusreceiver/go.mod +++ b/receiver/prometheusreceiver/go.mod @@ -3,13 +3,11 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/receiver/promet go 1.17 require ( - github.com/census-instrumentation/opencensus-proto v0.3.0 github.com/go-kit/kit v0.11.0 github.com/gogo/protobuf v1.3.2 github.com/golang/snappy v0.0.4 github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.35.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.35.0 - github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus v0.35.0 github.com/prometheus/common v0.30.0 github.com/prometheus/prometheus v1.8.2-0.20210621150501-ff58416a0b02 github.com/stretchr/testify v1.7.0 @@ -39,6 +37,7 @@ require ( github.com/aws/aws-sdk-go v1.38.68 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.1.1 // indirect + github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/containerd/containerd v1.4.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -66,7 +65,6 @@ require ( github.com/googleapis/gax-go/v2 v2.0.5 // indirect github.com/googleapis/gnostic v0.4.1 // indirect github.com/gophercloud/gophercloud v0.18.0 // indirect - github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/hashicorp/consul/api v1.8.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-hclog v0.12.2 // indirect @@ -156,5 +154,3 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prome replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal => ../../internal/coreinternal replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry => ../../pkg/resourcetotelemetry - -replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus => ../../pkg/translator/opencensus diff --git a/receiver/prometheusreceiver/go.sum b/receiver/prometheusreceiver/go.sum index abb98901f296d..f6120aecce745 100644 --- a/receiver/prometheusreceiver/go.sum +++ b/receiver/prometheusreceiver/go.sum @@ -562,7 +562,6 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.14.4/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0= -github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= diff --git a/receiver/prometheusreceiver/internal/otlp_metricfamily.go b/receiver/prometheusreceiver/internal/otlp_metricfamily.go index 489a0d172e132..b1cb238f73237 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricfamily.go +++ b/receiver/prometheusreceiver/internal/otlp_metricfamily.go @@ -176,7 +176,9 @@ func (mg *metricGroupPdata) toDistributionPoint(orderedLabelKeys []string, dest point.SetBucketCounts(bucketCounts) // The timestamp MUST be in retrieved from milliseconds and converted to nanoseconds. tsNanos := timestampFromMs(mg.ts) - point.SetStartTimestamp(tsNanos) + if mg.family.isCumulativeTypePdata() { + point.SetStartTimestamp(timestampFromMs(mg.intervalStartTimeMs)) + } point.SetTimestamp(tsNanos) populateAttributesPdata(orderedLabelKeys, mg.ls, point.Attributes()) @@ -322,6 +324,7 @@ func (mf *metricFamilyPdata) ToMetricPdata(metrics *pdata.MetricSlice) (int, int switch mf.mtype { case pdata.MetricDataTypeHistogram: histogram := metric.Histogram() + histogram.SetAggregationTemporality(pdata.AggregationTemporalityCumulative) hdpL := histogram.DataPoints() for _, mg := range mf.getGroups() { if !mg.toDistributionPoint(mf.labelKeysOrdered, &hdpL) { @@ -342,6 +345,7 @@ func (mf *metricFamilyPdata) ToMetricPdata(metrics *pdata.MetricSlice) (int, int case pdata.MetricDataTypeSum: sum := metric.Sum() + sum.SetAggregationTemporality(pdata.AggregationTemporalityCumulative) sdpL := sum.DataPoints() for _, mg := range mf.getGroups() { if !mg.toNumberDataPoint(mf.labelKeysOrdered, &sdpL) { diff --git a/receiver/prometheusreceiver/metrics_receiver_test.go b/receiver/prometheusreceiver/metrics_receiver_test.go index 1993e5039d2b5..fc98efc13f156 100644 --- a/receiver/prometheusreceiver/metrics_receiver_test.go +++ b/receiver/prometheusreceiver/metrics_receiver_test.go @@ -26,11 +26,8 @@ import ( "sync" "sync/atomic" "testing" + "time" - commonpb "github.com/census-instrumentation/opencensus-proto/gen-go/agent/common/v1" - agentmetricspb "github.com/census-instrumentation/opencensus-proto/gen-go/agent/metrics/v1" - metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" - resourcepb "github.com/census-instrumentation/opencensus-proto/gen-go/resource/v1" gokitlog "github.com/go-kit/kit/log" promcfg "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/scrape" @@ -39,11 +36,10 @@ import ( "go.opentelemetry.io/collector/component/componenttest" "go.opentelemetry.io/collector/config" "go.opentelemetry.io/collector/consumer/consumertest" + "go.opentelemetry.io/collector/model/pdata" "go.uber.org/zap" "google.golang.org/protobuf/types/known/timestamppb" "gopkg.in/yaml.v2" - - "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus" ) var logger = zap.NewNop() @@ -118,9 +114,20 @@ var ( type testData struct { name string pages []mockPrometheusResponse - node *commonpb.Node - resource *resourcepb.Resource - validateFunc func(t *testing.T, td *testData, result []*agentmetricspb.ExportMetricsServiceRequest) + resource pdata.Resource + validateFunc func(t *testing.T, td *testData, rmsL []pdata.ResourceMetrics) +} + +// Given that pdata doesn't allow us to trivially construct pdata.Metrics, +// we can extract pdata.ResourceMetrics from each value. +func resourceMetricsFromMetrics(metricsL []pdata.Metrics) (rmsL []pdata.ResourceMetrics) { + for _, metrics := range metricsL { + irmsL := metrics.ResourceMetrics() + for i := 0; i < irmsL.Len(); i++ { + rmsL = append(rmsL, irmsL.At(i)) + } + } + return rmsL } // setupMockPrometheus to create a mocked prometheus based on targets, returning the server and a prometheus exporting @@ -155,22 +162,15 @@ func setupMockPrometheus(tds ...*testData) (*mockPrometheus, *promcfg.Config, er // update node value (will use for validation) for _, t := range tds { - t.node = &commonpb.Node{ - Identifier: &commonpb.ProcessIdentifier{ - HostName: host, - }, - ServiceInfo: &commonpb.ServiceInfo{ - Name: t.name, - }, - } - t.resource = &resourcepb.Resource{ - Labels: map[string]string{ - "instance": u.Host, - "job": t.name, - "scheme": "http", - "port": port, - }, - } + rsc := pdata.NewResource() + attrs := rsc.Attributes() + attrs.InsertString("instance", u.Host) + attrs.InsertString("scheme", "http") + attrs.InsertString("port", port) + attrs.InsertString("job", t.name) + attrs.InsertString("host.name", host) + attrs.InsertString("service.name", t.name) + t.resource = rsc } cfgStr := strings.ReplaceAll(string(cfg), srvPlaceHolder, u.Host) @@ -178,196 +178,73 @@ func setupMockPrometheus(tds ...*testData) (*mockPrometheus, *promcfg.Config, er return mp, pCfg, err } -func verifyNumScrapeResults(t *testing.T, td *testData, mds []*agentmetricspb.ExportMetricsServiceRequest) { +func verifyNumScrapeResults(t *testing.T, td *testData, metricsL []pdata.ResourceMetrics) { want := 0 for _, p := range td.pages { if p.code == 200 { want++ } } - if l := len(mds); l != want { - t.Fatalf("want %d, but got %d\n", want, l) + if l := len(metricsL); l != want { + t.Errorf("want %d, but got %d\n", want, l) } } -func doCompare(name string, t *testing.T, want, got pdata.ResourceMetrics) { +func doCompare(t *testing.T, name string, want, got pdata.ResourceMetrics) { + // Ensure that the resource attributes can be deterministically compared. + got.Resource().Attributes().Sort() + want.Resource().Attributes().Sort() + t.Run(name, func(t *testing.T) { - assert.True(t, want, got) + assert.Equal(t, want.Resource(), got.Resource(), "Resource mismatch") + assert.Equal(t, want.SchemaUrl(), got.SchemaUrl(), "Resource mismatch") + assert.Equal(t, want.InstrumentationLibraryMetrics(), got.InstrumentationLibraryMetrics()) }) } -func getValidScrapes(t *testing.T, mds []*agentmetricspb.ExportMetricsServiceRequest) []*agentmetricspb.ExportMetricsServiceRequest { - out := make([]*agentmetricspb.ExportMetricsServiceRequest, 0) - for _, md := range mds { - // mds will include scrapes that received no metrics but have internal scrape metrics, filter those out - if expectedScrapeMetricCount < len(md.Metrics) && countScrapeMetrics(md) == expectedScrapeMetricCount { - assertUp(t, 1, md) - out = append(out, md) - } else { - assertUp(t, 0, md) - } - } - return out -} - -func assertUp(t *testing.T, expected float64, md *agentmetricspb.ExportMetricsServiceRequest) { - for _, m := range md.Metrics { - if m.GetMetricDescriptor().Name == "up" { - assert.Equal(t, expected, m.Timeseries[0].Points[0].GetDoubleValue()) - return - } - } - t.Error("No 'up' metric found") -} - -func countScrapeMetrics(in *agentmetricspb.ExportMetricsServiceRequest) int { - n := 0 - for _, m := range in.Metrics { - switch m.MetricDescriptor.Name { - case "up", "scrape_duration_seconds", "scrape_samples_scraped", "scrape_samples_post_metric_relabeling", "scrape_series_added": - n++ - default: - } - } - return n -} - -type pointComparator func(*testing.T, *metricspb.Point) bool -type seriesComparator func(*testing.T, *metricspb.TimeSeries) bool -type descriptorComparator func(*testing.T, *metricspb.MetricDescriptor) bool - -type seriesExpectation struct { - series []seriesComparator - points []pointComparator -} - -type testExpectation func(*testing.T, []*metricspb.Metric) bool - -func assertMetricPresent(name string, descriptorExpectations []descriptorComparator, seriesExpectations []seriesExpectation) testExpectation { - return func(t *testing.T, metrics []*metricspb.Metric) bool { - for _, m := range metrics { - if name != m.MetricDescriptor.Name { - continue - } - - for _, de := range descriptorExpectations { - if !de(t, m.MetricDescriptor) { - return false - } - } - - if !assert.Equal(t, len(seriesExpectations), len(m.Timeseries)) { - return false - } - for i, se := range seriesExpectations { - for _, sc := range se.series { - if !sc(t, m.Timeseries[i]) { - return false - } - } - for _, pc := range se.points { - if !pc(t, m.Timeseries[i].Points[0]) { - return false - } - } - } - return true - } - assert.Failf(t, "Unable to match metric expectation", name) - return false - } -} - -func assertMetricAbsent(name string) testExpectation { - return func(t *testing.T, metrics []*metricspb.Metric) bool { - for _, m := range metrics { - if !assert.NotEqual(t, name, m.MetricDescriptor.Name) { - return false +func getValidScrapes(t *testing.T, metricsL []pdata.ResourceMetrics) (out []pdata.ResourceMetrics) { + return metricsL + panic("FIX ME") + /* + for _, metrics := range metricsL { + // mds will include scrapes that received no metrics but have internal scrape metrics, filter those out + if expectedScrapeMetricCount < len(md.Metrics) && countScrapeMetrics(md) == expectedScrapeMetricCount { + assertUp(t, 1, md) + out = append(out, md) + } else { + assertUp(t, 0, md) } } - return true - } + return out + */ } -func compareMetricType(typ metricspb.MetricDescriptor_Type) descriptorComparator { - return func(t *testing.T, descriptor *metricspb.MetricDescriptor) bool { - return assert.Equal(t, typ, descriptor.Type) - } -} - -func compareMetricLabelKeys(keys []string) descriptorComparator { - return func(t *testing.T, descriptor *metricspb.MetricDescriptor) bool { - if !assert.Equal(t, len(keys), len(descriptor.LabelKeys)) { - return false - } - for i, k := range keys { - if !assert.Equal(t, k, descriptor.LabelKeys[i].Key) { - return false +func assertUp(t *testing.T, expected float64, metrics pdata.ResourceMetrics) { + panic("FIX ME") + /* + for _, m := range md.Metrics { + if m.GetMetricDescriptor().Name == "up" { + assert.Equal(t, expected, m.Timeseries[0].Points[0].GetDoubleValue()) + return } } - return true - } + t.Error("No 'up' metric found") + */ } -func compareSeriesLabelValues(values []string) seriesComparator { - return func(t *testing.T, series *metricspb.TimeSeries) bool { - if !assert.Equal(t, len(values), len(series.LabelValues)) { - return false - } - for i, v := range values { - if !assert.Equal(t, v, series.LabelValues[i].Value) { - return false +func countScrapeMetrics(in pdata.ResourceMetrics) int { + panic("FIX ME") + /* + n := 0 + for _, m := range in.Metrics { + switch m.MetricDescriptor.Name { + case "up", "scrape_duration_seconds", "scrape_samples_scraped", "scrape_samples_post_metric_relabeling", "scrape_series_added": + n++ + default: } } - return true - } -} - -func compareSeriesTimestamp(ts *timestamppb.Timestamp) seriesComparator { - return func(t *testing.T, series *metricspb.TimeSeries) bool { - return assert.Equal(t, ts.String(), series.StartTimestamp.String()) - } -} - -func comparePointTimestamp(ts *timestamppb.Timestamp) pointComparator { - return func(t *testing.T, point *metricspb.Point) bool { - return assert.Equal(t, ts.String(), point.Timestamp.String()) - } -} - -func compareDoubleVal(cmp float64) pointComparator { - return func(t *testing.T, pt *metricspb.Point) bool { - return assert.Equal(t, cmp, pt.GetDoubleValue()) - } -} - -func compareHistogram(count int64, sum float64, buckets []int64) pointComparator { - return func(t *testing.T, pt *metricspb.Point) bool { - ret := assert.Equal(t, count, pt.GetDistributionValue().Count) - ret = ret && assert.Equal(t, sum, pt.GetDistributionValue().Sum) - - if ret { - for i, b := range buckets { - ret = ret && assert.Equal(t, b, pt.GetDistributionValue().Buckets[i].Count) - } - } - return ret - } -} - -func compareSummary(count int64, sum float64, quantiles map[float64]float64) pointComparator { - return func(t *testing.T, pt *metricspb.Point) bool { - ret := assert.Equal(t, count, pt.GetSummaryValue().Count.Value) - ret = ret && assert.Equal(t, sum, pt.GetSummaryValue().Sum.Value) - - if ret { - assert.Equal(t, len(quantiles), len(pt.GetSummaryValue().Snapshot.PercentileValues)) - for _, q := range pt.GetSummaryValue().Snapshot.PercentileValues { - assert.Equal(t, quantiles[q.Percentile], q.Value) - } - } - return ret - } + return n + */ } // Test data and validation functions for EndToEnd test @@ -431,179 +308,216 @@ rpc_duration_seconds_sum 5002 rpc_duration_seconds_count 1001 ` -func verifyTarget1(t *testing.T, td *testData, mds []*agentmetricspb.ExportMetricsServiceRequest) { - verifyNumScrapeResults(t, td, mds) - if len(mds) < 1 { +func dataPointCount(rms pdata.ResourceMetrics) int { + return rms.InstrumentationLibraryMetrics().At(0).Metrics().Len() +} + +func extractFirstTimestamp(rms pdata.ResourceMetrics) (startTimestamp, timestamp pdata.Timestamp) { + startTs, ts, _ := extractFirstTimestampDataType(rms) + return startTs, ts +} + +func extractFirstTimestampDataType(rms pdata.ResourceMetrics) (startTimestamp, timestamp pdata.Timestamp, _ pdata.MetricDataType) { + metric := rms.InstrumentationLibraryMetrics().At(0).Metrics().At(0) + switch dataType := metric.DataType(); dataType { + case pdata.MetricDataTypeGauge: + point0 := metric.Gauge().DataPoints().At(0) + return point0.StartTimestamp(), point0.Timestamp(), dataType + case pdata.MetricDataTypeSum: + point0 := metric.Sum().DataPoints().At(0) + return point0.StartTimestamp(), point0.Timestamp(), dataType + case pdata.MetricDataTypeHistogram: + point0 := metric.Histogram().DataPoints().At(0) + return point0.StartTimestamp(), point0.Timestamp(), dataType + case pdata.MetricDataTypeSummary: + point0 := metric.Summary().DataPoints().At(0) + return point0.StartTimestamp(), point0.Timestamp(), dataType + default: + panic("Unhandled " + metric.DataType().String()) + } +} + +const ts1 = pdata.Timestamp(1e9) +const ts2 = pdata.Timestamp(2e9) + +func verifyTarget1(t *testing.T, td *testData, metricsL []pdata.ResourceMetrics) { + verifyNumScrapeResults(t, td, metricsL) + if len(metricsL) < 1 { t.Fatal("At least one metric request should be present") } - m1 := mds[0] - // m1 has 4 metrics + 5 internal scraper metrics - if l := len(m1.Metrics); l != 9 { - t.Errorf("want 9, but got %v\n", l) + + // Extract the first startTimestamp. + got1 := metricsL[0] + + // got 1 has 4 metrics + 5 internal scraper metrics + if g, w := dataPointCount(got1), 9; g != w { + t.Fatalf("got %d, want %d", g, w) } + startTs1, ts1 := extractFirstTimestamp(got1) + + want1 := makeMetrics( + td, + makeGaugeMetric("go_threads", startTs1, makeDoublePoint(ts1, 19.0)), + makeSumMetric("http_requests_total", startTs1, + makeDoublePoint(ts1, 100.0, kv{"code", "200"}, kv{"method", "post"}), + makeDoublePoint(ts1, 3, kv{"code", "400"}, kv{"method", "post"}), + ), + makeCumulativeDistMetric("http_request_duration_seconds", startTs1, + makeDistPoint(ts1, 2500, 5000, []uint64{1000, 500, 500, 500}, []float64{0.05, 0.5, 1}), + ), + makeSummaryMetric("rpc_duration_seconds", startTs1, + makeSummaryPoint(ts1, 1000, 5000, map[float64]float64{1: 1, 90: 5, 99: 8}), + ), + makeGaugeMetric("up", startTs1, makeDoublePoint(ts1, 1.0)), + makeGaugeMetric("scrape_duration_seconds", startTs1, makeDoublePoint(ts1, 0.001662471)), + makeGaugeMetric("scrape_samples_scraped", startTs1, makeDoublePoint(ts1, 18.0)), + makeGaugeMetric("scrape_samples_post_metric_relabeling", startTs1, makeDoublePoint(ts1, 18.0)), + makeGaugeMetric("scrape_series_added", startTs1, makeDoublePoint(ts1, 18.0)), + ) + + doCompare(t, "scrape1", want1, got1) + + got2 := metricsL[1] + startTs2, ts2 := extractFirstTimestamp(got2) + // Verify the 2nd data. + want2 := makeMetrics( + td, + makeGaugeMetric("go_threads", startTs2, makeDoublePoint(ts2, 18.0)), + makeSumMetric("http_requests_total", startTs2, + makeDoublePoint(ts2, 199, kv{"code", "200"}, kv{"method", "post"}), + makeDoublePoint(ts2, 12, kv{"code", "400"}, kv{"method", "post"}), + ), + makeCumulativeDistMetric("http_requests_duration_seconds", startTs2, + makeDistPoint(ts2, 2600, 5050, []uint64{1100, 500, 500, 500}, []float64{0.05, 0.5, 1}), + ), + makeSummaryMetric("rpc_duration_seconds", startTs2, + makeSummaryPoint(ts2, 1000, 5000, map[float64]float64{1: 1, 90: 6, 99: 8}), + ), + makeGaugeMetric("up", startTs1, makeDoublePoint(ts1, 1.0)), + makeGaugeMetric("scrape_duration_seconds", startTs1, makeDoublePoint(ts1, 0.001662471)), + makeGaugeMetric("scrape_samples_scraped", startTs1, makeDoublePoint(ts1, 18.0)), + makeGaugeMetric("scrape_samples_post_metric_relabeling", startTs1, makeDoublePoint(ts1, 18.0)), + makeGaugeMetric("scrape_series_added", startTs1, makeDoublePoint(ts1, 18.0)), + ) + + doCompare(t, "scrape2", want2, got2) +} - ts1 := m1.Metrics[0].Timeseries[0].Points[0].Timestamp - e1 := []testExpectation{ - assertMetricPresent("go_threads", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_GAUGE_DOUBLE), - }, - []seriesExpectation{ - { - points: []pointComparator{ - comparePointTimestamp(ts1), - compareDoubleVal(19), - }, - }, - }), - assertMetricPresent("http_requests_total", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_CUMULATIVE_DOUBLE), - compareMetricLabelKeys([]string{"code", "method"}), - }, - []seriesExpectation{ - { - series: []seriesComparator{ - compareSeriesTimestamp(ts1), - compareSeriesLabelValues([]string{"200", "post"}), - }, - points: []pointComparator{ - comparePointTimestamp(ts1), - compareDoubleVal(100), - }, - }, - { - series: []seriesComparator{ - compareSeriesTimestamp(ts1), - compareSeriesLabelValues([]string{"400", "post"}), - }, - points: []pointComparator{ - comparePointTimestamp(ts1), - compareDoubleVal(5), - }, - }, - }), - assertMetricPresent("http_request_duration_seconds", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION), - }, - []seriesExpectation{ - { - series: []seriesComparator{ - compareSeriesTimestamp(ts1), - }, - points: []pointComparator{ - comparePointTimestamp(ts1), - compareHistogram(2500, 5000, []int64{1000, 500, 500, 500}), - }, - }, - }), - assertMetricPresent("rpc_duration_seconds", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_SUMMARY), - }, - []seriesExpectation{ - { - series: []seriesComparator{ - compareSeriesTimestamp(ts1), - }, - points: []pointComparator{ - comparePointTimestamp(ts1), - compareSummary(1000, 5000, map[float64]float64{1: 1, 90: 5, 99: 8}), - }, - }, - }), +type kv struct { + key, value string +} + +func makeDoublePoint(ts pdata.Timestamp, value float64, kvp ...kv) pdata.NumberDataPoint { + ndp := pdata.NewNumberDataPoint() + ndp.SetTimestamp(ts) + ndp.SetDoubleVal(value) + + attrs := ndp.Attributes() + for _, kv := range kvp { + attrs.InsertString(kv.key, kv.value) + } + return ndp +} + +func makeGaugeMetric(name string, startTs pdata.Timestamp, points ...pdata.NumberDataPoint) pdata.Metric { + metric := pdata.NewMetric() + metric.SetName(name) + metric.SetDataType(pdata.MetricDataTypeGauge) + + destPointL := metric.Gauge().DataPoints() + for _, point := range points { + destPoint := destPointL.AppendEmpty() + point.CopyTo(destPoint) + destPoint.SetStartTimestamp(startTs) + } + return metric +} + +func makeSummaryPoint(ts pdata.Timestamp, count uint64, sum float64, qvm map[float64]float64, kvp ...kv) pdata.SummaryDataPoint { + sdp := pdata.NewSummaryDataPoint() + sdp.SetTimestamp(ts) + sdp.SetCount(count) + sdp.SetSum(sum) + qvL := sdp.QuantileValues() + for quantile, value := range qvm { + qvi := qvL.AppendEmpty() + qvi.SetQuantile(quantile) + qvi.SetValue(value) } - want1 := &agentmetricspb.ExportMetricsServiceRequest{ - Node: td.node, - Resource: td.resource, + attrs := sdp.Attributes() + for _, kv := range kvp { + attrs.InsertString(kv.key, kv.value) } + return sdp +} - doCompare("scrape1", t, want1, m1, e1) +func makeSummaryMetric(name string, startTs pdata.Timestamp, points ...pdata.SummaryDataPoint) pdata.Metric { + metric := pdata.NewMetric() + metric.SetName(name) + metric.SetDataType(pdata.MetricDataTypeSummary) - // verify the 2nd metricData - m2 := mds[1] - ts2 := m2.Metrics[0].Timeseries[0].Points[0].Timestamp + destPointL := metric.Summary().DataPoints() + for _, point := range points { + destPoint := destPointL.AppendEmpty() + point.CopyTo(destPoint) + destPoint.SetStartTimestamp(startTs) + } + return metric +} - want2 := &agentmetricspb.ExportMetricsServiceRequest{ - Node: td.node, - Resource: td.resource, +func makeSumMetric(name string, startTs pdata.Timestamp, points ...pdata.NumberDataPoint) pdata.Metric { + metric := pdata.NewMetric() + metric.SetName(name) + metric.SetDataType(pdata.MetricDataTypeSum) + sum := metric.Sum() + sum.SetAggregationTemporality(pdata.AggregationTemporalityCumulative) + destPointL := sum.DataPoints() + + for _, point := range points { + destPoint := destPointL.AppendEmpty() + point.CopyTo(destPoint) + destPoint.SetStartTimestamp(startTs) } + return metric +} - e2 := []testExpectation{ - assertMetricPresent("go_threads", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_GAUGE_DOUBLE), - }, - []seriesExpectation{ - { - points: []pointComparator{ - comparePointTimestamp(ts2), - compareDoubleVal(18), - }, - }, - }), - assertMetricPresent("http_requests_total", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_CUMULATIVE_DOUBLE), - compareMetricLabelKeys([]string{"code", "method"}), - }, - []seriesExpectation{ - { - series: []seriesComparator{ - compareSeriesTimestamp(ts1), - compareSeriesLabelValues([]string{"200", "post"}), - }, - points: []pointComparator{ - comparePointTimestamp(ts2), - compareDoubleVal(199), - }, - }, - { - series: []seriesComparator{ - compareSeriesTimestamp(ts1), - compareSeriesLabelValues([]string{"400", "post"}), - }, - points: []pointComparator{ - comparePointTimestamp(ts2), - compareDoubleVal(12), - }, - }, - }), - assertMetricPresent("http_request_duration_seconds", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION), - }, - []seriesExpectation{ - { - series: []seriesComparator{ - compareSeriesTimestamp(ts1), - }, - points: []pointComparator{ - comparePointTimestamp(ts2), - compareHistogram(2600, 5050, []int64{1100, 500, 500, 500}), - }, - }, - }), - assertMetricPresent("rpc_duration_seconds", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_SUMMARY), - }, - []seriesExpectation{ - { - series: []seriesComparator{ - compareSeriesTimestamp(ts1), - }, - points: []pointComparator{ - comparePointTimestamp(ts2), - compareSummary(1001, 5002, map[float64]float64{1: 1, 90: 6, 99: 8}), - }, - }, - }), +func makeDistPoint(ts pdata.Timestamp, count uint64, sum float64, counts []uint64, bounds []float64, kvp ...kv) pdata.HistogramDataPoint { + hdp := pdata.NewHistogramDataPoint() + hdp.SetExplicitBounds(bounds) + hdp.SetBucketCounts(counts) + hdp.SetTimestamp(ts) + hdp.SetCount(count) + hdp.SetSum(sum) + + attrs := hdp.Attributes() + for _, kv := range kvp { + attrs.InsertString(kv.key, kv.value) } + return hdp +} - doCompare("scrape2", t, want2, m2, e2) +func makeGaugeDistMetric(name string, startTs pdata.Timestamp, points ...pdata.HistogramDataPoint) pdata.Metric { + hMetric := makeCumulativeDistMetric(name, startTs, points...) + hMetric.Histogram().SetAggregationTemporality(pdata.AggregationTemporalityDelta) + return hMetric +} + +func makeCumulativeDistMetric(name string, startTs pdata.Timestamp, points ...pdata.HistogramDataPoint) pdata.Metric { + metric := pdata.NewMetric() + metric.SetName(name) + metric.SetDataType(pdata.MetricDataTypeHistogram) + histogram := metric.Histogram() + histogram.SetAggregationTemporality(pdata.AggregationTemporalityCumulative) + + destPointL := histogram.DataPoints() + // By default the AggregationTemporality is Cumulative until it'll be changed by the caller. + for _, point := range points { + destPoint := destPointL.AppendEmpty() + point.CopyTo(destPoint) + destPoint.SetStartTimestamp(startTs) + } + return metric } // target2 is going to have 5 pages, and there's a newly added item on the 2nd page. @@ -668,408 +582,110 @@ http_requests_total{method="post",code="400"} 59 http_requests_total{method="post",code="500"} 5 ` -func verifyTarget2(t *testing.T, td *testData, mds []*agentmetricspb.ExportMetricsServiceRequest) { - verifyNumScrapeResults(t, td, mds) - m1 := mds[0] - // m1 has 2 metrics + 5 internal scraper metrics - if l := len(m1.Metrics); l != 7 { - t.Errorf("want 7, but got %v\n", l) - } - - ts1 := m1.Metrics[0].Timeseries[0].Points[0].Timestamp - want1 := &agentmetricspb.ExportMetricsServiceRequest{ - Node: td.node, - Resource: td.resource, - } - - e1 := []testExpectation{ - assertMetricPresent("go_threads", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_GAUGE_DOUBLE), - }, - []seriesExpectation{ - { - points: []pointComparator{ - comparePointTimestamp(ts1), - compareDoubleVal(18), - }, - }, - }), - assertMetricPresent("http_requests_total", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_CUMULATIVE_DOUBLE), - compareMetricLabelKeys([]string{"code", "method"}), - }, - []seriesExpectation{ - { - series: []seriesComparator{ - compareSeriesTimestamp(ts1), - compareSeriesLabelValues([]string{"200", "post"}), - }, - points: []pointComparator{ - comparePointTimestamp(ts1), - compareDoubleVal(10), - }, - }, - { - series: []seriesComparator{ - compareSeriesTimestamp(ts1), - compareSeriesLabelValues([]string{"400", "post"}), - }, - points: []pointComparator{ - comparePointTimestamp(ts1), - compareDoubleVal(50), - }, - }, - }), - } - - doCompare("scrape1", t, want1, m1, e1) - - // verify the 2nd metricData - m2 := mds[1] - ts2 := m2.Metrics[0].Timeseries[0].Points[0].Timestamp - - want2 := &agentmetricspb.ExportMetricsServiceRequest{ - Node: td.node, - Resource: td.resource, - } - e2 := []testExpectation{ - assertMetricPresent("go_threads", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_GAUGE_DOUBLE), - }, - []seriesExpectation{ - { - points: []pointComparator{ - comparePointTimestamp(ts2), - compareDoubleVal(16), - }, - }, - }), - assertMetricPresent("http_requests_total", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_CUMULATIVE_DOUBLE), - compareMetricLabelKeys([]string{"code", "method"}), - }, - []seriesExpectation{ - { - series: []seriesComparator{ - compareSeriesTimestamp(ts1), - compareSeriesLabelValues([]string{"200", "post"}), - }, - points: []pointComparator{ - comparePointTimestamp(ts2), - compareDoubleVal(50), - }, - }, - { - series: []seriesComparator{ - compareSeriesTimestamp(ts1), - compareSeriesLabelValues([]string{"400", "post"}), - }, - points: []pointComparator{ - comparePointTimestamp(ts2), - compareDoubleVal(60), - }, - }, - { - series: []seriesComparator{ - compareSeriesTimestamp(ts2), - compareSeriesLabelValues([]string{"500", "post"}), - }, - points: []pointComparator{ - comparePointTimestamp(ts2), - compareDoubleVal(3), - }, - }, - }), - } - - doCompare("scrape2", t, want2, m2, e2) - - // verify the 3rd metricData, with the new code=500 counter which first appeared on 2nd run - m3 := mds[2] - // its start timestamp shall be from the 2nd run - ts3 := m3.Metrics[0].Timeseries[0].Points[0].Timestamp - - want3 := &agentmetricspb.ExportMetricsServiceRequest{ - Node: td.node, - Resource: td.resource, - } - - e3 := []testExpectation{ - assertMetricPresent("go_threads", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_GAUGE_DOUBLE), - }, - []seriesExpectation{ - { - points: []pointComparator{ - comparePointTimestamp(ts3), - compareDoubleVal(16), - }, - }, - }), - assertMetricPresent("http_requests_total", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_CUMULATIVE_DOUBLE), - compareMetricLabelKeys([]string{"code", "method"}), - }, - []seriesExpectation{ - { - series: []seriesComparator{ - compareSeriesTimestamp(ts1), - compareSeriesLabelValues([]string{"200", "post"}), - }, - points: []pointComparator{ - comparePointTimestamp(ts3), - compareDoubleVal(50), - }, - }, - { - series: []seriesComparator{ - compareSeriesTimestamp(ts1), - compareSeriesLabelValues([]string{"400", "post"}), - }, - points: []pointComparator{ - comparePointTimestamp(ts3), - compareDoubleVal(60), - }, - }, - { - series: []seriesComparator{ - compareSeriesTimestamp(ts2), - compareSeriesLabelValues([]string{"500", "post"}), - }, - points: []pointComparator{ - comparePointTimestamp(ts3), - compareDoubleVal(5), - }, - }, - }), - } - - doCompare("scrape3", t, want3, m3, e3) - - // verify the 4th metricData which reset happens - m4 := mds[3] - ts4 := m4.Metrics[0].Timeseries[0].Points[0].Timestamp - - want4 := &agentmetricspb.ExportMetricsServiceRequest{ - Node: td.node, - Resource: td.resource, - Metrics: []*metricspb.Metric{ - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "up", - Description: "The scraping was successful", - Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, - Unit: "bool", - }, - Timeseries: []*metricspb.TimeSeries{ - { - Points: []*metricspb.Point{ - {Timestamp: ts2, Value: &metricspb.Point_DoubleValue{DoubleValue: 1.0}}, - }, - }, - }, - }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "scrape_series_added", - Description: "The approximate number of new series in this scrape", - Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, - Unit: "count", - }, - Timeseries: []*metricspb.TimeSeries{ - { - Points: []*metricspb.Point{ - {Timestamp: ts2, Value: &metricspb.Point_DoubleValue{DoubleValue: 14.0}}, - }, - }, - }, - }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "scrape_duration_seconds", - Description: "Duration of the scrape", - Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, - Unit: "seconds", - }, - Timeseries: []*metricspb.TimeSeries{ - { - Points: []*metricspb.Point{ - {Timestamp: ts2, Value: &metricspb.Point_DoubleValue{DoubleValue: 0.0123456}}, - }, - }, - }, - }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "scrape_samples_scraped", - Description: "The number of samples the target exposed", - Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, - Unit: "count", - }, - Timeseries: []*metricspb.TimeSeries{ - { - Points: []*metricspb.Point{ - {Timestamp: ts2, Value: &metricspb.Point_DoubleValue{DoubleValue: 14.0}}, - }, - }, - }, - }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "scrape_samples_post_metric_relabeling", - Description: "The number of samples remaining after metric relabeling was applied", - Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, - Unit: "count", - }, - Timeseries: []*metricspb.TimeSeries{ - { - Points: []*metricspb.Point{ - {Timestamp: ts2, Value: &metricspb.Point_DoubleValue{DoubleValue: 14.0}}, - }, - }, - }, - }, - { - MetricDescriptor: &metricspb.MetricDescriptor{ - Name: "scrape_series_added", - Description: "The approximate number of new series in this scrape", - Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, - Unit: "count", - }, - Timeseries: []*metricspb.TimeSeries{ - { - Points: []*metricspb.Point{ - {Timestamp: ts2, Value: &metricspb.Point_DoubleValue{DoubleValue: 14.0}}, - }, - }, - }, - }, - }, - } - - e4 := []testExpectation{ - assertMetricPresent("go_threads", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_GAUGE_DOUBLE), - }, - []seriesExpectation{ - { - points: []pointComparator{ - comparePointTimestamp(ts4), - compareDoubleVal(16), - }, - }, - }), - assertMetricPresent("http_requests_total", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_CUMULATIVE_DOUBLE), - compareMetricLabelKeys([]string{"code", "method"}), - }, - []seriesExpectation{ - { - series: []seriesComparator{ - compareSeriesTimestamp(ts4), - compareSeriesLabelValues([]string{"200", "post"}), - }, - points: []pointComparator{ - comparePointTimestamp(ts4), - compareDoubleVal(49), - }, - }, - { - series: []seriesComparator{ - compareSeriesTimestamp(ts4), - compareSeriesLabelValues([]string{"400", "post"}), - }, - points: []pointComparator{ - comparePointTimestamp(ts4), - compareDoubleVal(59), - }, - }, - { - series: []seriesComparator{ - compareSeriesTimestamp(ts4), - compareSeriesLabelValues([]string{"500", "post"}), - }, - points: []pointComparator{ - comparePointTimestamp(ts4), - compareDoubleVal(3), - }, - }, - }), +func verifyTarget2(t *testing.T, td *testData, metricsL []pdata.ResourceMetrics) { + verifyNumScrapeResults(t, td, metricsL) + if len(metricsL) < 1 { + t.Fatal("At least one metric request should be present") } - doCompare("scrape4", t, want4, m4, e4) - - // verify the 5th metricData which reset happens - m5 := mds[4] - // its start timestamp shall be from the 4th run - ts5 := m5.Metrics[0].Timeseries[0].Points[0].Timestamp + // Extract the first startTimestamp. + got1 := metricsL[0] - want5 := &agentmetricspb.ExportMetricsServiceRequest{ - Node: td.node, - Resource: td.resource, + // got has 2 metrics + 5 internal scraper metrics + if g, w := dataPointCount(got1), 7; g != w { + t.Fatalf("got %d, want %d", g, w) } - - e5 := []testExpectation{ - assertMetricPresent("go_threads", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_GAUGE_DOUBLE), - }, - []seriesExpectation{ - { - points: []pointComparator{ - comparePointTimestamp(ts5), - compareDoubleVal(16), - }, - }, - }), - assertMetricPresent("http_requests_total", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_CUMULATIVE_DOUBLE), - compareMetricLabelKeys([]string{"code", "method"}), - }, - []seriesExpectation{ - { - series: []seriesComparator{ - compareSeriesTimestamp(ts4), - compareSeriesLabelValues([]string{"200", "post"}), - }, - points: []pointComparator{ - comparePointTimestamp(ts5), - compareDoubleVal(50), - }, - }, - { - series: []seriesComparator{ - compareSeriesTimestamp(ts4), - compareSeriesLabelValues([]string{"400", "post"}), - }, - points: []pointComparator{ - comparePointTimestamp(ts5), - compareDoubleVal(59), - }, - }, - { - series: []seriesComparator{ - compareSeriesTimestamp(ts4), - compareSeriesLabelValues([]string{"500", "post"}), - }, - points: []pointComparator{ - comparePointTimestamp(ts5), - compareDoubleVal(5), - }, - }, - }), - } - - doCompare("scrape5", t, want5, m5, e5) + startTs1, ts1 := extractFirstTimestamp(got1) + + want1 := makeMetrics( + td, + makeGaugeMetric("go_threads", startTs1, makeDoublePoint(ts1, 18.0)), + makeSumMetric("http_requests_total", startTs1, + makeDoublePoint(ts1, 10, kv{"code", "200"}, kv{"method", "post"}), + makeDoublePoint(ts1, 50, kv{"code", "400"}, kv{"method", "post"}), + ), + makeGaugeMetric("up", startTs1, makeDoublePoint(ts1, 1.0)), + makeGaugeMetric("scrape_duration_seconds", startTs1, makeDoublePoint(ts1, 0.001662471)), + makeGaugeMetric("scrape_samples_scraped", startTs1, makeDoublePoint(ts1, 14.0)), + makeGaugeMetric("scrape_samples_post_metric_relabeling", startTs1, makeDoublePoint(ts1, 14.0)), + makeGaugeMetric("scrape_series_added", startTs1, makeDoublePoint(ts1, 14.0)), + ) + doCompare(t, "scrape1", want1, got1) + + // Verify the 2nd data. + got2 := metricsL[1] + startTs2, ts2 := extractFirstTimestamp(got2) + want2 := makeMetrics( + td, + makeGaugeMetric("go_threads", startTs2, makeDoublePoint(ts2, 16.0)), + makeSumMetric("http_requests_total", startTs2, + makeDoublePoint(ts2, 50, kv{"code", "200"}, kv{"method", "post"}), + makeDoublePoint(ts2, 60, kv{"code", "400"}, kv{"method", "post"}), + makeDoublePoint(ts2, 3, kv{"code", "500"}, kv{"method", "post"}), + ), + makeGaugeMetric("up", startTs1, makeDoublePoint(ts1, 1.0)), + makeGaugeMetric("scrape_duration_seconds", startTs1, makeDoublePoint(ts1, 0.001662471)), + makeGaugeMetric("scrape_samples_scraped", startTs1, makeDoublePoint(ts1, 14.0)), + makeGaugeMetric("scrape_samples_post_metric_relabeling", startTs1, makeDoublePoint(ts1, 14.0)), + makeGaugeMetric("scrape_series_added", startTs1, makeDoublePoint(ts1, 14.0)), + ) + + doCompare(t, "scrape2", want2, got2) + + // Verify the 3rd, with the new coce=500 counter which first appeared on the 2nd run. + got3 := metricsL[2] + startTs3, ts3 := extractFirstTimestamp(got3) + want3 := makeMetrics( + td, + makeGaugeMetric("go_threads", startTs3, makeDoublePoint(ts3, 16.0)), + makeSumMetric("http_requests_total", startTs3, + makeDoublePoint(ts1, 50, kv{"code", "200"}, kv{"method", "post"}), + makeDoublePoint(ts3, 60, kv{"code", "400"}, kv{"method", "post"}), + makeDoublePoint(ts3, 5, kv{"code", "500"}, kv{"method", "post"}), + ), + makeGaugeMetric("up", startTs1, makeDoublePoint(ts1, 1.0)), + makeGaugeMetric("scrape_duration_seconds", startTs1, makeDoublePoint(ts1, 0.001662471)), + makeGaugeMetric("scrape_samples_scraped", startTs1, makeDoublePoint(ts1, 14.0)), + makeGaugeMetric("scrape_samples_post_metric_relabeling", startTs1, makeDoublePoint(ts1, 14.0)), + makeGaugeMetric("scrape_series_added", startTs1, makeDoublePoint(ts1, 14.0)), + ) + doCompare(t, "scrape3", want3, got3) + + got4 := metricsL[3] + startTs4, ts4 := extractFirstTimestamp(got4) + want4 := makeMetrics( + td, + makeGaugeMetric("go_threads", startTs4, makeDoublePoint(ts4, 16.0)), + makeSumMetric("http_requests_total", startTs3, + makeDoublePoint(ts4, 50, kv{"code", "200"}, kv{"method", "post"}), + makeDoublePoint(ts4, 60, kv{"code", "400"}, kv{"method", "post"}), + makeDoublePoint(ts4, 5, kv{"code", "500"}, kv{"method", "post"}), + ), + makeGaugeMetric("up", startTs4, makeDoublePoint(ts4, 1.0)), + makeGaugeMetric("scrape_duration_seconds", startTs4, makeDoublePoint(ts4, 0.001662471)), + makeGaugeMetric("scrape_samples_scraped", startTs4, makeDoublePoint(ts4, 14.0)), + makeGaugeMetric("scrape_samples_post_metric_relabeling", startTs4, makeDoublePoint(ts4, 14.0)), + makeGaugeMetric("scrape_series_added", startTs4, makeDoublePoint(ts4, 14.0)), + ) + doCompare(t, "scrape4", want4, got4) + + got5 := metricsL[4] + startTs5, ts5 := extractFirstTimestamp(got5) + want5 := makeMetrics( + td, + makeGaugeMetric("go_threads", startTs5, makeDoublePoint(ts5, 16.0)), + makeSumMetric("http_requests_total", startTs5, + makeDoublePoint(ts4, 49, kv{"code", "200"}, kv{"method", "post"}), + makeDoublePoint(ts4, 60, kv{"code", "400"}, kv{"method", "post"}), + makeDoublePoint(ts3, 3, kv{"code", "500"}, kv{"method", "post"}), + ), + makeGaugeMetric("up", startTs1, makeDoublePoint(ts1, 1.0)), + makeGaugeMetric("scrape_duration_seconds", startTs1, makeDoublePoint(ts1, 0.001662471)), + makeGaugeMetric("scrape_samples_scraped", startTs1, makeDoublePoint(ts1, 4.0)), + makeGaugeMetric("scrape_samples_post_metric_relabeling", startTs1, makeDoublePoint(ts1, 4.0)), + makeGaugeMetric("scrape_series_added", startTs1, makeDoublePoint(ts1, 4.0)), + ) + doCompare(t, "scrape5", want5, got5) } // target3 for complicated data types, including summaries and histograms. one of the summary and histogram have only @@ -1144,148 +760,61 @@ rpc_duration_seconds_sum{foo="no_quantile"} 101 rpc_duration_seconds_count{foo="no_quantile"} 55 ` -func verifyTarget3(t *testing.T, td *testData, mds []*agentmetricspb.ExportMetricsServiceRequest) { - verifyNumScrapeResults(t, td, mds) - m1 := mds[0] - // m1 has 3 metrics + 5 internal scraper metrics - if l := len(m1.Metrics); l != 8 { - t.Errorf("want 8, but got %v\n", l) - } - - ts1 := m1.Metrics[1].Timeseries[0].Points[0].Timestamp - want1 := &agentmetricspb.ExportMetricsServiceRequest{ - Node: td.node, - Resource: td.resource, - } - - e1 := []testExpectation{ - assertMetricPresent("go_threads", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_GAUGE_DOUBLE), - }, - []seriesExpectation{ - { - points: []pointComparator{ - comparePointTimestamp(ts1), - compareDoubleVal(18), - }, - }, - }), - assertMetricPresent("http_request_duration_seconds", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION), - }, - []seriesExpectation{ - { - series: []seriesComparator{ - compareSeriesTimestamp(ts1), - }, - points: []pointComparator{ - comparePointTimestamp(ts1), - compareHistogram(13003, 50000, []int64{10000, 1000, 1001, 1002}), - }, - }, - }), - assertMetricAbsent("corrupted_hist"), - assertMetricPresent("rpc_duration_seconds", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_SUMMARY), - compareMetricLabelKeys([]string{"foo"}), - }, - []seriesExpectation{ - { - series: []seriesComparator{ - compareSeriesTimestamp(ts1), - compareSeriesLabelValues([]string{"bar"}), - }, - points: []pointComparator{ - comparePointTimestamp(ts1), - compareSummary(900, 8000, map[float64]float64{1: 31, 5: 35, 50: 47, 90: 70, 99: 76}), - }, - }, - { - series: []seriesComparator{ - compareSeriesTimestamp(ts1), - compareSeriesLabelValues([]string{"no_quantile"}), - }, - points: []pointComparator{ - comparePointTimestamp(ts1), - compareSummary(50, 100, map[float64]float64{}), - }, - }, - }), +func verifyTarget3(t *testing.T, td *testData, metricsL []pdata.ResourceMetrics) { + verifyNumScrapeResults(t, td, metricsL) + if len(metricsL) < 1 { + t.Fatal("At least one metric request should be present") } - doCompare("scrape1", t, want1, m1, e1) - - // verify the 2nd metricData - m2 := mds[1] - ts2 := m2.Metrics[0].Timeseries[0].Points[0].Timestamp + // Extract the first startTimestamp. + got1 := metricsL[0] - want2 := &agentmetricspb.ExportMetricsServiceRequest{ - Node: td.node, - Resource: td.resource, - } - - e2 := []testExpectation{ - assertMetricPresent("go_threads", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_GAUGE_DOUBLE), - }, - []seriesExpectation{ - { - points: []pointComparator{ - comparePointTimestamp(ts2), - compareDoubleVal(16), - }, - }, - }), - assertMetricPresent("http_request_duration_seconds", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION), - }, - []seriesExpectation{ - { - series: []seriesComparator{ - compareSeriesTimestamp(ts1), - }, - points: []pointComparator{ - comparePointTimestamp(ts2), - compareHistogram(14003, 50100, []int64{11000, 1000, 1001, 1002}), - }, - }, - }), - assertMetricAbsent("corrupted_hist"), - assertMetricPresent("rpc_duration_seconds", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_SUMMARY), - compareMetricLabelKeys([]string{"foo"}), - }, - []seriesExpectation{ - { - series: []seriesComparator{ - compareSeriesTimestamp(ts1), - compareSeriesLabelValues([]string{"bar"}), - }, - points: []pointComparator{ - comparePointTimestamp(ts2), - compareSummary(950, 8100, map[float64]float64{1: 32, 5: 35, 50: 47, 90: 70, 99: 77}), - }, - }, - { - series: []seriesComparator{ - compareSeriesTimestamp(ts1), - compareSeriesLabelValues([]string{"no_quantile"}), - }, - points: []pointComparator{ - comparePointTimestamp(ts2), - compareSummary(55, 101, map[float64]float64{}), - }, - }, - }), + // got has 8 metrics + 5 internal scraper metrics + if g, w := dataPointCount(got1), 8; g != w { + t.Fatalf("got %d, want %d", g, w) } - - doCompare("scrape2", t, want2, m2, e2) + startTs1, ts1 := extractFirstTimestamp(got1) + + want1 := makeMetrics( + td, + makeGaugeMetric("go_threads", startTs1, makeDoublePoint(ts1, 18.0)), + makeCumulativeDistMetric("http_request_duration_seconds", startTs1, + makeDistPoint(ts1, 13003, 50000, []uint64{10000, 1000, 1001, 1002}, []float64{0.2, 0.5, 1}), + ), + makeSummaryMetric("rpc_duration_seconds", startTs1, + makeSummaryPoint(ts1, 900, 8000, map[float64]float64{1: 31, 5: 35, 50: 47, 90: 70, 99: 76}, kv{"foo", "bar"}), + makeSummaryPoint(ts1, 50, 100, nil, kv{"foo", "no_quantile"}), + ), + /* + makeGaugeMetric("up", startTs1, makeDoublePoint(ts1, 1.0)), + makeGaugeMetric("scrape_duration_seconds", startTs1, makeDoublePoint(ts1, 0.001662471)), + makeGaugeMetric("scrape_samples_scraped", startTs1, makeDoublePoint(ts1, 18.0)), + makeGaugeMetric("scrape_samples_post_metric_relabeling", startTs1, makeDoublePoint(ts1, 18.0)), + makeGaugeMetric("scrape_series_added", startTs1, makeDoublePoint(ts1, 18.0)), + */ + ) + doCompare(t, "scrape1", want1, got1) + + // Verify the 2nd data. + got2 := metricsL[1] + startTs2, ts2 := extractFirstTimestamp(got2) + want2 := makeMetrics( + td, + makeGaugeMetric("go_threads", startTs2, makeDoublePoint(ts2, 16.0)), + makeCumulativeDistMetric("http_request_duration_seconds", startTs2, + makeDistPoint(ts2, 14003, 50100, []uint64{11000, 1000, 1001, 1002}, []float64{0.2, 0.5, 1}), + ), + makeSummaryMetric("rpc_duration_seconds", startTs2, + makeSummaryPoint(ts2, 950, 8100, map[float64]float64{1: 32, 5: 35, 50: 47, 90: 70, 99: 77}, kv{"foo", "bar"}), + makeSummaryPoint(ts1, 55, 101, nil, kv{"foo", "no_quantile"}), + ), + makeGaugeMetric("up", startTs1, makeDoublePoint(ts1, 1.0)), + makeGaugeMetric("scrape_duration_seconds", startTs1, makeDoublePoint(ts1, 0.001662471)), + makeGaugeMetric("scrape_samples_scraped", startTs1, makeDoublePoint(ts1, 18.0)), + makeGaugeMetric("scrape_samples_post_metric_relabeling", startTs1, makeDoublePoint(ts1, 18.0)), + makeGaugeMetric("scrape_series_added", startTs1, makeDoublePoint(ts1, 18.0)), + ) + doCompare(t, "scrape2", want2, got2) } // TestEndToEnd end to end test executor @@ -1360,21 +889,27 @@ var startTimeMetricPageStartTimestamp = ×tamppb.Timestamp{Seconds: 400, Nan // 6 metrics + 5 internal metrics const numStartTimeMetricPageTimeseries = 11 -func verifyStartTimeMetricPage(t *testing.T, _ *testData, mds []*agentmetricspb.ExportMetricsServiceRequest) { +const timestampZero pdata.Timestamp = 0 + +func verifyStartTimeMetricPage(t *testing.T, _ *testData, metricsL []pdata.ResourceMetrics) { numTimeseries := 0 - for _, cmd := range mds { - for _, metric := range cmd.Metrics { - timestamp := startTimeMetricPageStartTimestamp - switch metric.GetMetricDescriptor().GetType() { - case metricspb.MetricDescriptor_GAUGE_DOUBLE, metricspb.MetricDescriptor_GAUGE_DISTRIBUTION: - timestamp = nil - } - for _, ts := range metric.GetTimeseries() { - assert.Equal(t, timestamp.AsTime(), ts.GetStartTimestamp().AsTime(), ts.String()) - numTimeseries++ + for _, rms := range metricsL { + _, ts, dataType := extractFirstTimestampDataType(rms) + want := pdata.NewTimestampFromTime(time.Unix(400, 800000000)) + switch dataType { + case pdata.MetricDataTypeGauge: + want = timestampZero + + case pdata.MetricDataTypeHistogram: + histogram := rms.InstrumentationLibraryMetrics().At(0).Metrics().At(0).Histogram() + if histogram.AggregationTemporality() == pdata.AggregationTemporalityDelta { + want = timestampZero } } + assert.Equal(t, want, ts, "Timestamp mismatch") + numTimeseries++ } + assert.Equal(t, numStartTimeMetricPageTimeseries, numTimeseries) } @@ -1414,29 +949,16 @@ func testEndToEnd(t *testing.T, targets []*testData, useStartTimeMetric bool) { // wait for all provided data to be scraped mp.wg.Wait() - metrics := cms.AllMetrics() // split and store results by target name - results := make(map[string][]*agentmetricspb.ExportMetricsServiceRequest) - for _, md := range metrics { - rms := md.ResourceMetrics() - for i := 0; i < rms.Len(); i++ { - ocmd := &agentmetricspb.ExportMetricsServiceRequest{} - ocmd.Node, ocmd.Resource, ocmd.Metrics = opencensus.ResourceMetricsToOC(rms.At(i)) - result, ok := results[ocmd.Node.ServiceInfo.Name] - if !ok { - result = make([]*agentmetricspb.ExportMetricsServiceRequest, 0) - } - results[ocmd.Node.ServiceInfo.Name] = append(result, ocmd) - } - } + results := metricsGroupedByServiceName(t, cms.AllMetrics()) lres, lep := len(results), len(mp.endpoints) assert.Equalf(t, lep, lres, "want %d targets, but got %v\n", lep, lres) // Skipping the validate loop below, because it falsely assumed that // staleness markers would not be returned, yet the tests are a bit rigid. - if true { + if false { t.Log(`Skipping the "up" metric checks as they seem to be spuriously failing after staleness marker insertions`) return } @@ -1515,6 +1037,22 @@ func TestStartTimeMetricRegex(t *testing.T) { } } +func metricsGroupedByServiceName(t *testing.T, metricsL []pdata.Metrics) map[string][]pdata.ResourceMetrics { + // split and store results by target name + splits := make(map[string][]pdata.ResourceMetrics) + for _, metrics := range metricsL { + rms := metrics.ResourceMetrics() + for i := 0; i < rms.Len(); i++ { + rmi := rms.At(i) + serviceNameAttr, ok := rmi.Resource().Attributes().Get("service.name") + assert.True(t, ok, `expected "service.name" as a known attribute`) + serviceName := serviceNameAttr.StringVal() + splits[serviceName] = append(splits[serviceName], rmi) + } + } + return splits +} + func testEndToEndRegex(t *testing.T, targets []*testData, useStartTimeMetric bool, startTimeMetricRegex string) { // 1. setup mock server mp, cfg, err := setupMockPrometheus(targets...) @@ -1533,22 +1071,7 @@ func testEndToEndRegex(t *testing.T, targets []*testData, useStartTimeMetric boo // wait for all provided data to be scraped mp.wg.Wait() - metrics := cms.AllMetrics() - - // split and store results by target name - results := make(map[string][]*agentmetricspb.ExportMetricsServiceRequest) - for _, md := range metrics { - rms := md.ResourceMetrics() - for i := 0; i < rms.Len(); i++ { - ocmd := &agentmetricspb.ExportMetricsServiceRequest{} - ocmd.Node, ocmd.Resource, ocmd.Metrics = opencensus.ResourceMetricsToOC(rms.At(i)) - result, ok := results[ocmd.Node.ServiceInfo.Name] - if !ok { - result = make([]*agentmetricspb.ExportMetricsServiceRequest, 0) - } - results[ocmd.Node.ServiceInfo.Name] = append(result, ocmd) - } - } + results := metricsGroupedByServiceName(t, cms.AllMetrics()) lres, lep := len(results), len(mp.endpoints) assert.Equalf(t, lep, lres, "want %d targets, but got %v\n", lep, lres) diff --git a/receiver/prometheusreceiver/metrics_reciever_external_labels_test.go b/receiver/prometheusreceiver/metrics_reciever_external_labels_test.go index 8cc16ce8b8594..6b831847d3bf4 100644 --- a/receiver/prometheusreceiver/metrics_reciever_external_labels_test.go +++ b/receiver/prometheusreceiver/metrics_reciever_external_labels_test.go @@ -19,7 +19,6 @@ import ( "testing" "github.com/prometheus/prometheus/pkg/labels" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/component/componenttest" @@ -59,67 +58,34 @@ func TestExternalLabels(t *testing.T) { t.Cleanup(func() { require.NoError(t, receiver.Shutdown(ctx)) }) mp.wg.Wait() - metrics := cms.AllMetrics() - // split and store results by target name - results := make(map[string][]pdata.Metric) - for _, md := range metrics { - rms := md.ResourceMetrics() - for i := 0; i < rms.Len(); i++ { - rmi := rms.At(i) - serviceNameAttr, ok := rmi.Resource().Attributes().Get("service.name") - assert.True(t, ok, `expected "service.name" as a known attribute`) - serviceName := serviceNameAttr.StringVal() - results[serviceName] = append(results[serviceName], md) - } - } + results := metricsGroupedByServiceName(t, cms.AllMetrics()) for _, target := range targets { target.validateFunc(t, target, results[target.name]) } } -func makeMetric(td *testData, metricL []pdata.Metric) pdata.ResourceMetrics { +func makeMetrics(td *testData, metricL ...pdata.Metric) pdata.ResourceMetrics { rms := pdata.NewResourceMetrics() td.resource.CopyTo(rms.Resource()) ilm := rms.InstrumentationLibraryMetrics().AppendEmpty() destMetricL := ilm.Metrics() for _, metric := range metricL { - destMetric := destMetricL.AppendMetric() + destMetric := destMetricL.AppendEmpty() metric.CopyTo(destMetric) } return rms } -func verifyExternalLabels(t *testing.T, td *testData, mds []pdata.Metric) { - verifyNumScrapeResults(t, td, mds) - - want := makeMetric(td, []pdata.Metric{ - internal.GaugeMetric("go_threads", k1v2, t1Ms, internal.DoublePoint(t1Ms, 19)) - }) +func verifyExternalLabels(t *testing.T, td *testData, got []pdata.ResourceMetrics) { + verifyNumScrapeResults(t, td, got) + panic("FIX ME") - // TODO: Translate me. /* - want := &agentmetricspb.ExportMetricsServiceRequest{ - Node: td.node, - Resource: td.resource, - } - doCompare("scrape-externalLabels", t, want, mds[0], []testExpectation{ - assertMetricPresent("go_threads", - []descriptorComparator{ - compareMetricType(metricspb.MetricDescriptor_GAUGE_DOUBLE), - compareMetricLabelKeys([]string{"key"}), - }, - []seriesExpectation{ - { - series: []seriesComparator{ - compareSeriesLabelValues([]string{"value"}), - }, - points: []pointComparator{ - comparePointTimestamp(mds[0].Metrics[0].Timeseries[0].Points[0].Timestamp), - compareDoubleVal(19), - }, - }, - }), + want := makeMetric(td, []pdata.Metric{ + internal.GaugeMetric("go_threads", k1v2, t1Ms, internal.DoublePoint(t1Ms, 19)), }) + + require.Equal(t, got, want, "Expecting values to be the same") */ } From 20bf87612627c6aae85c0b5f3507d65a60022672 Mon Sep 17 00:00:00 2001 From: Anthony J Mirabella Date: Mon, 22 Nov 2021 19:57:49 -0500 Subject: [PATCH 05/26] receiver/prometheus: fix tests Signed-off-by: Anthony J Mirabella --- .../metricstestutil/metricsutil_pdata.go | 4 +- receiver/prometheusreceiver/go.mod | 6 +- receiver/prometheusreceiver/go.sum | 377 +----------------- .../prometheusreceiver/internal/ocastore.go | 1 - .../internal/otlp_metricfamily.go | 6 +- .../internal/otlp_metricfamily_test.go | 17 +- .../internal/otlp_metrics_adjuster.go | 3 +- .../internal/otlp_metricsbuilder_test.go | 57 ++- .../internal/otlp_transaction.go | 26 +- .../internal/otlp_transaction_test.go | 13 +- .../metrics_receiver_helper_test.go | 11 + .../metrics_receiver_test.go | 26 +- 12 files changed, 115 insertions(+), 432 deletions(-) diff --git a/internal/coreinternal/metricstestutil/metricsutil_pdata.go b/internal/coreinternal/metricstestutil/metricsutil_pdata.go index 882446dcf1692..e65b5fd4ebc08 100644 --- a/internal/coreinternal/metricstestutil/metricsutil_pdata.go +++ b/internal/coreinternal/metricstestutil/metricsutil_pdata.go @@ -41,7 +41,7 @@ func DistPointPdata(ts pdata.Timestamp, bounds []float64, counts []uint64) *pdat func GaugeDistMetricPdata(name string, kvp []*KV, startTs pdata.Timestamp, points ...*pdata.HistogramDataPoint) *pdata.Metric { hMetric := CumulativeDistMetricPdata(name, kvp, startTs, points...) - hMetric.Histogram().SetAggregationTemporality(pdata.AggregationTemporalityDelta) + hMetric.Histogram().SetAggregationTemporality(pdata.MetricAggregationTemporalityDelta) return hMetric } @@ -50,7 +50,7 @@ func CumulativeDistMetricPdata(name string, kvp []*KV, startTs pdata.Timestamp, metric.SetName(name) metric.SetDataType(pdata.MetricDataTypeHistogram) histogram := metric.Histogram() - histogram.SetAggregationTemporality(pdata.AggregationTemporalityCumulative) + histogram.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) destPointL := histogram.DataPoints() // By default the AggregationTemporality is Cumulative until it'll be changed by the caller. diff --git a/receiver/prometheusreceiver/go.mod b/receiver/prometheusreceiver/go.mod index ebc742ce304df..90414ff7e94cf 100644 --- a/receiver/prometheusreceiver/go.mod +++ b/receiver/prometheusreceiver/go.mod @@ -3,7 +3,7 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/receiver/promet go 1.17 require ( - github.com/go-kit/kit v0.11.0 + github.com/go-kit/log v0.2.0 github.com/gogo/protobuf v1.3.2 github.com/golang/snappy v0.0.4 github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.39.0 @@ -36,7 +36,8 @@ require ( github.com/armon/go-metrics v0.3.9 // indirect github.com/aws/aws-sdk-go v1.40.56 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cenkalti/backoff/v4 v4.1.1 // indirect + github.com/cenkalti/backoff/v4 v4.1.2 // indirect + github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/containerd/containerd v1.4.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -63,7 +64,6 @@ require ( github.com/googleapis/gax-go/v2 v2.0.5 // indirect github.com/googleapis/gnostic v0.4.1 // indirect github.com/gophercloud/gophercloud v0.18.0 // indirect - github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/hashicorp/consul/api v1.8.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-hclog v0.12.2 // indirect diff --git a/receiver/prometheusreceiver/go.sum b/receiver/prometheusreceiver/go.sum index 7871dd42d4469..c871e7df6d121 100644 --- a/receiver/prometheusreceiver/go.sum +++ b/receiver/prometheusreceiver/go.sum @@ -1,4 +1,3 @@ -bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8= cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= @@ -48,20 +47,17 @@ collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= contrib.go.opencensus.io/exporter/prometheus v0.4.0 h1:0QfIkj9z/iVZgK31D9H9ohjjIDApI2GOPScCKwxedbs= contrib.go.opencensus.io/exporter/prometheus v0.4.0/go.mod h1:o7cosnyfuPVK0tB8q0QmaQNhGnptITnPQB+z1+qeFB0= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= -github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v41.3.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v55.2.0+incompatible h1:TL2/vJWJEPOrmv97nHcbvjXES0Ntlb9P95hqGA1J2dU= github.com/Azure/azure-sdk-for-go v55.2.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= -github.com/Azure/go-autorest v10.8.1+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= github.com/Azure/go-autorest/autorest v0.9.3/go.mod h1:GsRuLYvwzLjjjRoWEIyMUaYq8GNUx2nRB378IPt/1p0= github.com/Azure/go-autorest/autorest v0.10.0/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= github.com/Azure/go-autorest/autorest v0.10.1/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= -github.com/Azure/go-autorest/autorest v0.11.1/go.mod h1:JFgpikqFJ/MleTTxwepExTKnFUKKszPS8UavbQYUMuw= github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw= github.com/Azure/go-autorest/autorest v0.11.19 h1:7/IqD2fEYVha1EPeaiytVKhzmPV223pfkRIQUGOK2IE= github.com/Azure/go-autorest/autorest v0.11.19/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= @@ -70,7 +66,6 @@ github.com/Azure/go-autorest/autorest/adal v0.8.0/go.mod h1:Z6vX6WXXuyieHAXwMj0S github.com/Azure/go-autorest/autorest/adal v0.8.1/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= github.com/Azure/go-autorest/autorest/adal v0.8.3/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= -github.com/Azure/go-autorest/autorest/adal v0.9.0/go.mod h1:/c022QCutn2P7uY+/oQWWNcK9YU+MH96NgK+jErpbcg= github.com/Azure/go-autorest/autorest/adal v0.9.5/go.mod h1:B7KF7jKIeC9Mct5spmyCB/A8CG/sEz1vwIRGv/bbw7A= github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= github.com/Azure/go-autorest/autorest/adal v0.9.14 h1:G8hexQdV5D4khOXrWG2YuLCFKhWYmWD8bHYaXN5ophk= @@ -84,7 +79,6 @@ github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSY github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= -github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/mocks v0.4.1 h1:K0laFcLE6VLTOwNgSxaGbUcLPuGXlNkbVvq4cW4nIHk= github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/autorest/to v0.3.0/go.mod h1:MgwOyqaIuKdG4TL/2ywSsIWKAfJfgHDo8ObuUk3t5sA= @@ -109,25 +103,9 @@ github.com/HdrHistogram/hdrhistogram-go v1.0.1/go.mod h1:BWJ+nMSHY3L41Zj7CA3uXnl github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/sprig v2.16.0+incompatible/go.mod h1:y6hNFY5UBTIWBxnzTeuNhlNS5hqE0NB0E6fgfo2Br3o= -github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= -github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= -github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= -github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/go-winio v0.4.17-0.20210324224401-5516f17a5958/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= github.com/Microsoft/go-winio v0.4.17 h1:iT12IBVClFevaf8PuVyi3UmZOVh4OqnaLxDTW2O6j3w= github.com/Microsoft/go-winio v0.4.17/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= -github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= -github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg= -github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= -github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= -github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= -github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= -github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= -github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= -github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= -github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -136,7 +114,6 @@ github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbt github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/SAP/go-hdb v0.14.1/go.mod h1:7fdQLVC2lER3urZLjZCm0AuMQfApof92n3aylBPEkMo= -github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= @@ -152,7 +129,6 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15 h1:AUNCr9CiJuwrRYS3XieqF+Z9B9gNxo/eANAJCF2eiN4= github.com/alecthomas/units v0.0.0-20210208195552-ff826a37aa15/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= -github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -163,7 +139,6 @@ github.com/apache/arrow/go/arrow v0.0.0-20200923215132-ac86123a3f01/go.mod h1:QN github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= -github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.3.3/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= github.com/armon/go-metrics v0.3.9 h1:O2sNqxBdvq8Eq5xmzljcYzAORli6RWCvEym4cJf9m18= @@ -177,7 +152,6 @@ github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:o github.com/asaskevich/govalidator v0.0.0-20200428143746-21a406dcc535/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg= github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/aws/aws-lambda-go v1.13.3/go.mod h1:4UKl9IzQMoD+QF79YdCuzCwp8VbmG4VAQwij/eHl5CU= -github.com/aws/aws-sdk-go v1.15.11/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.29.16/go.mod h1:1KvfttTE3SPKMpo8g2c6jL3ZKfXtFvKscTgahTma5Xg= github.com/aws/aws-sdk-go v1.30.12/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= @@ -201,26 +175,15 @@ github.com/benbjohnson/clock v1.2.0 h1:9Re3G2TWxkE06LdMWMpcY6KV81GLXMGiYpPYUPkFA github.com/benbjohnson/clock v1.2.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/benbjohnson/immutable v0.2.1/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= github.com/benbjohnson/tmpl v1.0.0/go.mod h1:igT620JFIi44B6awvU9IsDhR77IXWtFigTLil/RPdps= -github.com/beorn7/perks v0.0.0-20160804104726-4c0e84591b9a/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= -github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= -github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= -github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bonitoo-io/go-sql-bigquery v0.3.4-1.4.0/go.mod h1:J4Y6YJm0qTWB9aFziB7cPeSyc6dOZFyJdteSeybVpXQ= -github.com/bshuster-repo/logrus-logstash-hook v0.4.1/go.mod h1:zsTqEiSzDgAa/8GZR7E1qaXrhYNDKBYy5/dWPTIflbk= -github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= -github.com/bugsnag/bugsnag-go v0.0.0-20141110184014-b1d153021fcd/go.mod h1:2oa8nejYd4cQ/b0hMIopN0lCRxU0bueqREvZLWFrtK8= -github.com/bugsnag/osext v0.0.0-20130617224835-0dd3f918b21b/go.mod h1:obH5gd0BsqsP2LwDJ9aOkm/6J86V6lyAXCoQWGw3K50= -github.com/bugsnag/panicwrap v0.0.0-20151223152923-e2c28503fcd0/go.mod h1:D/8v3kj0zr8ZAKg1AQ6crr+5VwKN5eIywRkfhyM/+dE= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/cactus/go-statsd-client/statsd v0.0.0-20191106001114-12b4e2b38748/go.mod h1:l/bIBLeOl9eX+wxJAzxS4TveKRtAqlyDpHjhkfO0MEI= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= @@ -228,7 +191,6 @@ github.com/cenkalti/backoff v0.0.0-20181003080854-62661b46c409/go.mod h1:90ReRw6 github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.1.0/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.1.2 h1:6Yo7N8UP2K6LWZnW94DLVSSrbobcWdVzAYOisuDPIFo= github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -237,19 +199,11 @@ github.com/census-instrumentation/opencensus-proto v0.3.0/go.mod h1:f6KPmirojxKA github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.0/go.mod h1:dgIUBU3pDso/gPgZ1osOZ0iQf77oPR28Tjxl5dIMyVM= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2 h1:YRXhKfTDauu4ajMg1TPgFO5jnlC2HCbmLXMcTG5cbYE= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/checkpoint-restore/go-criu/v4 v4.1.0/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw= -github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= -github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc= -github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= -github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= @@ -264,127 +218,27 @@ github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= -github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= -github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= -github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= -github.com/containerd/aufs v1.0.0/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= -github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E= -github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= -github.com/containerd/btrfs v1.0.0/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= -github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI= -github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= -github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= -github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= -github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= -github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= -github.com/containerd/cgroups v1.0.1/go.mod h1:0SJrPIenamHDcZhEcJMNBB85rHcUsw4f25ZfBiPYRkU= -github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= -github.com/containerd/console v1.0.1/go.mod h1:XUsP6YE/mKtz6bxc+I8UiKKTP04qjQL4qcS3XoQ5xkw= -github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= -github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.3 h1:ijQT13JedHSHrQGWFcGEwzcNKrAGIiZ+jSD5QQG07SY= github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= -github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= -github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= -github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoTJseu1FGOKuoA4nNb2s= -github.com/containerd/containerd v1.5.8 h1:NmkCC1/QxyZFBny8JogwLpOy2f+VEbO/f6bV2Mqtwuw= -github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s= -github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= -github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= -github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= -github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= -github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= -github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= -github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= -github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= -github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= -github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= -github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= -github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= -github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= -github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= -github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= -github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= -github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= -github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= -github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= -github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= -github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= -github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJrXQb0Dpc4ms= -github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= -github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= -github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= -github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= -github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= -github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= -github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= -github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= -github.com/containerd/ttrpc v1.1.0/go.mod h1:XX4ZTnoOId4HklF4edwc4DcqskFZuvXB1Evzy5KFQpQ= -github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= -github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= -github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= -github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= -github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw= -github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y= -github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= -github.com/containerd/zfs v0.0.0-20210324211415-d5c4544f0433/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= -github.com/containerd/zfs v1.0.0/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= -github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= -github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= -github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= -github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= -github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= -github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= -github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= -github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= -github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= -github.com/coreos/go-iptables v0.5.0/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= -github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= -github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= -github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= -github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4= -github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ= -github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s= -github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8= -github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v0.0.0-20161028175848-04cdfd42973b/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU= -github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0= -github.com/dgrijalva/jwt-go v0.0.0-20170104182250-a601269ab70c/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgrijalva/jwt-go/v4 v4.0.0-preview1/go.mod h1:+hnT3ywWDTAFrW5aE+u2Sa/wT555ZqwoCS+pk3p6ry4= github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= -github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dgryski/go-sip13 v0.0.0-20200911182023-62edffca9245/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/digitalocean/godo v1.62.0 h1:7Gw2KFsWkxl36qJa0s50tgXaE0Cgm51JdRP+MFQvNnM= @@ -392,22 +246,15 @@ github.com/digitalocean/godo v1.62.0/go.mod h1:p7dOjjtSBqCTUksqtA5Fd3uaKs9kyTq2x github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/dnaeon/go-vcr v1.0.1 h1:r8L/HqC0Hje5AXMu1ooW8oyQyOFv4GxqpL0nRP7SLLY= github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyGc8n1E= -github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY= -github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/docker v20.10.7+incompatible h1:Z6O9Nhsjv+ayUEeI1IojKbYcsGdgYSNqxe1s2MYzUhQ= github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= -github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= -github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= -github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= -github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= @@ -420,7 +267,6 @@ github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaB github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= -github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -448,13 +294,10 @@ github.com/foxcpp/go-mockdns v0.0.0-20201212160233-ede2f9158d15/go.mod h1:tPg4cp github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.4.0/go.mod h1:36zfPVQyHxymz4cH7wlDmVwDrJuljRB60qkgn7rorfQ= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= -github.com/garyburd/redigo v0.0.0-20150301180006-535138d7bcd7/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= github.com/gdamore/tcell v1.3.0/go.mod h1:Hjvr+Ofd+gLglo7RYKxxnzCBmev3BzsS67MebKS4zMM= github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= @@ -469,7 +312,6 @@ github.com/go-chi/chi v4.1.0+incompatible/go.mod h1:eB3wogJHnLi3x/kFX2A+IbTBlXxm github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= -github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= @@ -483,7 +325,6 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= -github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v0.4.0 h1:K7/B1jt6fIBQVd4Owv2MqGQClcgf0R266+7C/QjRcLc= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-ole/go-ole v1.2.5/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= @@ -615,22 +456,15 @@ github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWe github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= -github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= -github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= -github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= -github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= -github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= -github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.2.2-0.20190730201129-28a6bbf47e48/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= @@ -639,7 +473,6 @@ github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGw github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -721,7 +554,6 @@ github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm4 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -737,55 +569,43 @@ github.com/gophercloud/gophercloud v0.18.0 h1:V6hcuMPmjXg+js9flU8T3RIHDCjV7F5CG5 github.com/gophercloud/gophercloud v0.18.0/go.mod h1:wRtmUelyIIv3CSSDI47aUwbs075O6i+LY+pXsKCBsb4= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= -github.com/gorilla/handlers v0.0.0-20150720190736-60c7bfde3e33/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/OAirnOIQ= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= -github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= -github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= -github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.14.4/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/api v1.4.0/go.mod h1:xc8u05kyMa3Wjr9eEAsIAo3dg8+LywT5E/Cl7cNS5nU= +github.com/hashicorp/consul/api v1.8.1 h1:BOEQaMWoGMhmQ29fC26bi0qb7/rId9JzZP2V0Xmx7m8= github.com/hashicorp/consul/api v1.8.1/go.mod h1:sDjTOq0yUyv5G4h+BqSea7Fn6BU+XbolEz1952UB+mk= -github.com/hashicorp/consul/api v1.10.1 h1:MwZJp86nlnL+6+W1Zly4JUuVn9YHhMggBirMpHGD7kw= -github.com/hashicorp/consul/api v1.10.1/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/consul/sdk v0.3.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/consul/sdk v0.4.0/go.mod h1:fY08Y9z5SvJqevyZNy6WWPXiG3KwBPAvlcdx16zZ0fM= +github.com/hashicorp/consul/sdk v0.7.0 h1:H6R9d008jDcHPQPAqPNuydAshJ4v5/8URdFnUvK/+sc= github.com/hashicorp/consul/sdk v0.7.0/go.mod h1:fY08Y9z5SvJqevyZNy6WWPXiG3KwBPAvlcdx16zZ0fM= -github.com/hashicorp/consul/sdk v0.8.0 h1:OJtKBtEjboEZvG6AOUdh4Z1Zbyu0WcxQ0qatRrZHTVU= -github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= -github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= -github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= -github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.0.0-20180709165350-ff2cf002a8dd/go.mod h1:9bjs9uLqI8l75knNv3lV1kA55veR+WUPSiKIWcQHudI= github.com/hashicorp/go-hclog v0.8.0/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v0.12.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v0.12.2 h1:F1fdYblUEsxKiailtkhCCG2g4bipEgaHiDc8vffNpD4= github.com/hashicorp/go-hclog v0.12.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v0.16.2 h1:K4ev2ib4LdQETX5cSZBG0DVLk1jwGqSPXBjdah3veNs= -github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.2.0 h1:l6UW37iCXwZkZoAbEYnptSHVE/cQ5bOTPYG5W3vf9+8= github.com/hashicorp/go-immutable-radix v1.2.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= -github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= -github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= -github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= @@ -838,9 +658,6 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.4/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= -github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= -github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= @@ -861,11 +678,8 @@ github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bS github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= github.com/influxdata/tdigest v0.0.2-0.20210216194612-fc98d27c9e8b/go.mod h1:Z0kXnxzbTC2qrx4NaIzYkE1k66+6oEDQTvL95hQFh5Y= github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= -github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= -github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= -github.com/jmespath/go-jmespath v0.0.0-20160803190731-bd40a432e4c7/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -903,8 +717,6 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= @@ -940,7 +752,6 @@ github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= -github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -953,7 +764,6 @@ github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7 github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= -github.com/marstr/guid v1.1.0/go.mod h1:74gB1z2wpxxInTG6yaqA7KrtM0NZ+RbrcqDvYHefzho= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -972,7 +782,6 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.8/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -985,9 +794,7 @@ github.com/miekg/dns v1.1.29/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7 github.com/miekg/dns v1.1.42/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/miekg/dns v1.1.43 h1:JKfpVSCB84vrAmHzyrsxB5NAr5kLoMXZArPSw7Qlgyg= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= -github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mileusna/useragent v0.0.0-20190129205925-3e331f0949a5/go.mod h1:JWhYAp2EXqUtsxTKdeGlY8Wp44M7VxThC9FEoNGi2IE= -github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db/go.mod h1:l0dey0ia/Uv7NcFFVbCLtqEBQbrT4OCwCSKTEv6enCw= @@ -1012,16 +819,10 @@ github.com/mitchellh/mapstructure v1.4.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/mapstructure v1.4.2 h1:6h7AQ0yhTcIsmFmnAwQls75jp2Gzs4iB8W7pjMO+rqo= github.com/mitchellh/mapstructure v1.4.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= -github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= -github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= -github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= -github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -1036,10 +837,8 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mostynb/go-grpc-compression v1.1.15 h1:9pLWmZldgo3vstd3yGyNgpCzY5gvhCrCj3PyvnvlDiY= github.com/mostynb/go-grpc-compression v1.1.15/go.mod h1:OTK+ha9cKfSY0Pb3ESCzvGhzStJrudBxXPzuC3PaA5A= -github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -1051,58 +850,29 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/ncw/swift v1.0.47/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= -github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/oklog v0.3.2/go.mod h1:FCV+B7mhrz4o+ueLpx+KqkyXRGMWOYEvfiXtdGtbWGs= github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= -github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= -github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM= github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= -github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= -github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= -github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= -github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= -github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing-contrib/go-stdlib v0.0.0-20190519235532-cf7a6c988dc9/go.mod h1:PLldrQSroqzH70Xl+1DQcGnefIbqsKR7UDaiux3zV+w= github.com/opentracing-contrib/go-stdlib v1.0.0/go.mod h1:qtI1ogk+2JhVPIXVc6q+NHziSmy2W5GbdQZFUHADCBU= @@ -1121,10 +891,8 @@ github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0Mw github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= -github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ= github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= @@ -1137,7 +905,6 @@ github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi github.com/pierrec/lz4/v3 v3.3.4/go.mod h1:280XNCGS8jAcG++AHdd6SeWnzyJ1w9oow2vbORyey8Q= github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -1149,15 +916,11 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= -github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/alertmanager v0.20.0/go.mod h1:9g2i48FAyZW6BtbsnvHtMHQXl2aVtrORKwKVCQ+nbrg= github.com/prometheus/alertmanager v0.22.2/go.mod h1:rYinOWxFuCnNssc3iOjn2oMTlhLaPcUuqV5yk5JKUAE= -github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= -github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.2.1/go.mod h1:XMU6Z2MjaRKVu/dC1qupJI9SiNkDYzz3xecMgSW/F+U= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= @@ -1166,7 +929,6 @@ github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= github.com/prometheus/client_golang v1.11.0 h1:HNkLOAEQMIDv/K+04rukrLx6ch7msSRwf3/SASFAGtQ= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_model v0.0.0-20171117100541-99fa1f4be8e5/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -1174,10 +936,7 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= -github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= @@ -1192,18 +951,13 @@ github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+ github.com/prometheus/common v0.32.1 h1:hWIdL3N2HoUx3B8j3YN9mWor0qhY/NlEKZEaXxuIRh4= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/exporter-toolkit v0.5.1/go.mod h1:OCkM4805mmisBhLmVFw858QYi3v0wKdY6/UxrT0pZVg= -github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= -github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= -github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3 h1:4jVXhlkAyzOScmCkXBTOLRLTz8EeU+eyjrwB/EPq0VU= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= @@ -1212,7 +966,6 @@ github.com/prometheus/prometheus v1.8.2-0.20210621150501-ff58416a0b02 h1:waKRn/b github.com/prometheus/prometheus v1.8.2-0.20210621150501-ff58416a0b02/go.mod h1:fC6ROpjS/2o+MQTO7X8NSZLhLBSNlDzxaeDMqQm+TUM= github.com/prometheus/statsd_exporter v0.21.0 h1:hA05Q5RFeIjgwKIYEdFd59xu5Wwaznf33yKI+pyX6T8= github.com/prometheus/statsd_exporter v0.21.0/go.mod h1:rbT83sZq2V+p73lHhPZfMc3MLCHmSHelCh9hSGYNLTQ= -github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= github.com/rhnvrm/simples3 v0.6.1/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA= @@ -1233,18 +986,15 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= -github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sanity-io/litter v1.2.0/go.mod h1:JF6pZUFgu2Q0sBZ+HSV35P8TVPI1TTzEwyu9FXAw2W4= github.com/satori/go.uuid v0.0.0-20160603004225-b111a074d5ef/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/satori/go.uuid v1.2.1-0.20181028125025-b2ce2384e17b/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210223165440-c65ae3540d44 h1:3egqo0Vut6daANFm7tOXdNAa8v5/uLU+sgCJrc88Meo= github.com/scaleway/scaleway-sdk-go v1.0.0-beta.7.0.20210223165440-c65ae3540d44/go.mod h1:CJJ5VAbozOl0yEw7nHB9+7BXTJbIn6h7W+f6Gau5IP8= github.com/schollz/progressbar/v2 v2.13.2/go.mod h1:6YZjqdthH6SCZKv2rqGryrxPtfmRB/DWZxSMfCXPyD8= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= -github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/segmentio/kafka-go v0.1.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/segmentio/kafka-go v0.2.0/go.mod h1:X6itGqS9L4jDletMsxZ7Dz+JFWxM6JHfPOCvTvk+EJo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= @@ -1254,8 +1004,6 @@ github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= github.com/shurcooL/vfsgen v0.0.0-20200824052919-0d455de96546/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= -github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= -github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -1265,45 +1013,34 @@ github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/snowflakedb/gosnowflake v1.3.4/go.mod h1:NsRq2QeiMUuoNUJhp5Q6xGC4uBrsS9g6LwZVEkTWgsE= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= -github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.4.1 h1:s0hze+J0196ZfEMTs80N7UlFt0BDuQ7Q+JDnHiMWKdA= github.com/spf13/cast v1.4.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= -github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.2.1 h1:+KmjbUw1hriSNMF55oPrkZcb27aECyrj8V2ytv7kWDw= github.com/spf13/cobra v1.2.1/go.mod h1:ExllRjgxM/piMAM+3tAZvg8fsklGAf3tPfi+i8t68Nk= -github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= -github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= -github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= -github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= -github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -1313,10 +1050,6 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= -github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= -github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= github.com/tidwall/gjson v1.10.2 h1:APbLGOM0rrEkd8WBw9C24nllro4ajFuJu0Sc9hRz8Bo= github.com/tidwall/gjson v1.10.2/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -1335,7 +1068,6 @@ github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ github.com/tklauser/numcpus v0.3.0 h1:ILuRUQBtssgnxw0XXIjKUC56fgnOrFoQQ/4+DeU2biQ= github.com/tklauser/numcpus v0.3.0/go.mod h1:yFGUr7TUHQRAhyqBcEg0Ge34zDBAsIvJJcyE6boqnA8= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/uber-go/tally v3.3.15+incompatible/go.mod h1:YDTIBxdXyOU/sCWilKB4bgyufu1cEi0jdVnRdxvjnmU= github.com/uber/athenadriver v1.1.4/go.mod h1:tQjho4NzXw55LGfSZEcETuYydpY1vtmixUabHkC1K/E= @@ -1343,50 +1075,29 @@ github.com/uber/jaeger-client-go v2.23.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMW github.com/uber/jaeger-client-go v2.29.1+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk= github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= github.com/uber/jaeger-lib v2.4.1+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U= -github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= -github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= -github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= -github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= -github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= -github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= -github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= -github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= -github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= -github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= -github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= -github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= -github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= -github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= -go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= -go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= @@ -1400,7 +1111,6 @@ go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4S go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= go.mongodb.org/mongo-driver v1.4.6/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= go.mongodb.org/mongo-driver v1.5.1/go.mod h1:gRXCHX4Jo7J0IJ1oDQyUxF7jfy19UfxniMS4xxMmUqw= -go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -1447,7 +1157,6 @@ go.opentelemetry.io/otel/trace v1.2.0 h1:Ys3iqbqZhcf28hHzrm5WAquMkDHNZTUkw7KHbuN go.opentelemetry.io/otel/trace v1.2.0/go.mod h1:N5FLswTubnxKxOJHM7XZC074qpeEdLy3CgAVsdMucK0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -1474,10 +1183,8 @@ go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.19.1 h1:ue41HOKd1vGURxrmeKIgELGb3jPW9DMUDGtsinblHwI= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= -golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180505025534-4ec37c66abab/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -1499,12 +1206,10 @@ golang.org/x/crypto v0.0.0-20200220183623-bac4c82f6975/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200422194213-44a606286825/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201208171446-5f87f3452ae9/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= -golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210915214749-c084706c2272 h1:3erb+vDS8lU1sxfDHF4/hhWyaXnhIaO+7RgL4fDZORA= golang.org/x/crypto v0.0.0-20210915214749-c084706c2272/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1551,7 +1256,6 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1564,10 +1268,8 @@ golang.org/x/net v0.0.0-20190320064053-1272bf9dcd53/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -1593,7 +1295,6 @@ golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= -golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= @@ -1660,21 +1361,14 @@ golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1682,25 +1376,19 @@ golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191010194322-b09406accb47/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191128015809-6d18c012aee9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200217220822-9197077df867/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1711,22 +1399,13 @@ golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201117170446-d9b008d0a637/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1737,13 +1416,11 @@ golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210309074719-68d13333faf2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1775,7 +1452,6 @@ golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac h1:7zkz7BUtwNFFqcowJ+RIgu2MaV/MapERkDIy+mwPyjs= @@ -1871,7 +1547,6 @@ gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= -google.golang.org/api v0.0.0-20160322025152-9bf6e6e569ff/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= google.golang.org/api v0.3.1/go.mod h1:6wY9I6uQWHQ8EM57III9mq/AjF+i8G65rmVagqKMtkk= google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= @@ -1907,14 +1582,12 @@ google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCID google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190404172233-64821d5d2107/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190716160619-c506a9f90610/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= @@ -1927,7 +1600,6 @@ google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200108215221-bd8f9a0ef82f/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= -google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= @@ -1948,7 +1620,6 @@ google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= @@ -1962,7 +1633,6 @@ google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxH google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4 h1:ysnBoUyeL/H6RCvNRhWHjKoDEmguI+mPU+qHgK8qv/w= google.golang.org/genproto v0.0.0-20210917145530-b395a37504d4/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= -google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -2013,11 +1683,9 @@ google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp0 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -2029,17 +1697,13 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/fsnotify/fsnotify.v1 v1.4.7 h1:XNNYLJHt73EyYiCZi6+xjupS9CpvmiDgjPTAjrBlQbo= gopkg.in/fsnotify/fsnotify.v1 v1.4.7/go.mod h1:Fyux9zXlo4rWoMSIzpn9fDAYjalPqJ/K1qJ27s+7ltE= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= -gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo= gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= gopkg.in/go-playground/validator.v9 v9.29.1/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= -gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= @@ -2058,8 +1722,6 @@ gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= -gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= @@ -2072,33 +1734,14 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.17.5/go.mod h1:0zV5/ungglgy2Rlm3QK8fbxkXVs+BSJWpJP/+8gUVLY= -k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= -k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= -k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= k8s.io/api v0.21.1 h1:94bbZ5NTjdINJEdzOkpS4vdPhkb1VFpTYC9zh43f75c= k8s.io/api v0.21.1/go.mod h1:FstGROTmsSHBarKc8bylzXih8BLNYTiS3TZcsoEDg2s= k8s.io/apimachinery v0.17.5/go.mod h1:ioIo1G/a+uONV7Tv+ZmCbMG1/a3kVw5YcDdncd8ugQ0= -k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= -k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= k8s.io/apimachinery v0.21.1 h1:Q6XuHGlj2xc+hlMCvqyYfbv3H7SRGn2c8NycxJquDVs= k8s.io/apimachinery v0.21.1/go.mod h1:jbreFvJo3ov9rj7eWT7+sYiRx+qZuCYXwWT1bcDswPY= -k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= -k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= -k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= k8s.io/client-go v0.17.5/go.mod h1:S8uZpBpjJJdEH/fEyxcqg7Rn0P5jH+ilkgBHjriSmNo= -k8s.io/client-go v0.20.1/go.mod h1:/zcHdt1TeWSd5HoUe6elJmHSQ6uLLgp4bIJHVEuy+/Y= -k8s.io/client-go v0.20.4/go.mod h1:LiMv25ND1gLUdBeYxBIwKpkSC5IsozMMmOOeSJboP+k= -k8s.io/client-go v0.20.6/go.mod h1:nNQMnOvEUEsOzRRFIIkdmYOjAZrC8bgq0ExboWSU1I0= k8s.io/client-go v0.21.1 h1:bhblWYLZKUu+pm50plvQF8WpY6TXdRRtcS/K9WauOj4= k8s.io/client-go v0.21.1/go.mod h1:/kEw4RgW+3xnBGzvp9IWxKSNA+lXn3A7AuH3gdOAzLs= -k8s.io/component-base v0.20.1/go.mod h1:guxkoJnNoh8LNrbtiQOlyp2Y2XFCZQmrcg2n/DeYNLk= -k8s.io/component-base v0.20.4/go.mod h1:t4p9EdiagbVCJKrQ1RsA5/V4rFQNDfRlevJajlGwgjI= -k8s.io/component-base v0.20.6/go.mod h1:6f1MPBAeI+mvuts3sIdtpjljHWBQ2cIy38oBIWMYnrM= -k8s.io/cri-api v0.17.3/go.mod h1:X1sbHmuXhwaHs9xxYffLqJogVsnI+f6cPRcgPel7ywM= -k8s.io/cri-api v0.20.1/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= -k8s.io/cri-api v0.20.4/go.mod h1:2JRbKt+BFLTjtrILYVqQK5jqhI+XNdF6UiGMgczeBCI= -k8s.io/cri-api v0.20.6/go.mod h1:ew44AjNXwyn1s0U4xCKGodU7J1HzBeZ1MpGrpa5r8Yc= k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= k8s.io/klog v0.0.0-20181102134211-b9b56d5dfc92/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= @@ -2106,15 +1749,12 @@ k8s.io/klog v0.3.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= k8s.io/klog v1.0.0 h1:Pt+yjF5aB1xDSVbau4VsWe+dQNzA0qv1LlXdC2dF6Q8= k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I= k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= -k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= k8s.io/klog/v2 v2.8.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/klog/v2 v2.9.0 h1:D7HV+n1V57XeZ0m6tdRkfknthUaM06VFbWldOFh8kzM= k8s.io/klog/v2 v2.9.0/go.mod h1:hy9LJ/NvuK+iVyP4Ehqva4HxZG/oXyIS3n3Jmire4Ec= k8s.io/kube-openapi v0.0.0-20200316234421-82d701f24f9d/go.mod h1:F+5wygcW0wmRTnM3cOgIqGivxkwSWIWT5YdsDbeAOaU= -k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7 h1:vEx13qjvaZ4yfObSSXW7BrMc/KQBBT/Jyee8XtLf4x0= k8s.io/kube-openapi v0.0.0-20210305001622-591a79e4bda7/go.mod h1:wXW5VT87nVfh/iLV8FpR2uDvrFyomxbtb1KivDbvPTE= -k8s.io/kubernetes v1.13.0/go.mod h1:ocZa8+6APFNC2tX1DZASIbocyYT5jHzqFVsY5aoB7Jk= k8s.io/utils v0.0.0-20191114184206-e782cd3c129f/go.mod h1:sZAwmy6armz5eXlNoLmJcl4F1QuKu7sr+mFQ0byX7Ew= k8s.io/utils v0.0.0-20200414100711-2df71ebbae66/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20201110183641-67b214c5f920 h1:CbnUZsM497iRC5QMVkHwyl8s2tB3g7yaSHkYPkpgelw= @@ -2123,11 +1763,8 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.14/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= -sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.15/go.mod h1:LEScyzhFmoF5pso/YSeBstl57mOzx9xlU9n85RGrDQg= sigs.k8s.io/structured-merge-diff/v2 v2.0.1/go.mod h1:Wb7vfKAodbKgf6tn1Kl0VvGj7mRH6DGaRcixXEJXTsE= sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= -sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/structured-merge-diff/v4 v4.1.0 h1:C4r9BgJ98vrKnnVCjwCSXcWjWe0NKcUQkmzDXZXGwH8= sigs.k8s.io/structured-merge-diff/v4 v4.1.0/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= diff --git a/receiver/prometheusreceiver/internal/ocastore.go b/receiver/prometheusreceiver/internal/ocastore.go index 6688ded8c7949..8802a7532e4d1 100644 --- a/receiver/prometheusreceiver/internal/ocastore.go +++ b/receiver/prometheusreceiver/internal/ocastore.go @@ -23,7 +23,6 @@ import ( "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/scrape" "github.com/prometheus/prometheus/storage" - "go.uber.org/zap" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/config" diff --git a/receiver/prometheusreceiver/internal/otlp_metricfamily.go b/receiver/prometheusreceiver/internal/otlp_metricfamily.go index 47977b1f19fbf..0fbe843e53605 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricfamily.go +++ b/receiver/prometheusreceiver/internal/otlp_metricfamily.go @@ -217,7 +217,7 @@ func (mg *metricGroupPdata) toSummaryPoint(orderedLabelKeys []string, dest *pdat for _, p := range mg.complexValue { quantile := quantileValues.AppendEmpty() quantile.SetValue(p.value) - quantile.SetQuantile(p.boundary * 100) + quantile.SetQuantile(p.boundary) } // Based on the summary description from https://prometheus.io/docs/concepts/metric_types/#summary @@ -336,7 +336,7 @@ func (mf *metricFamilyPdata) ToMetricPdata(metrics *pdata.MetricSlice) (int, int switch mf.mtype { case pdata.MetricDataTypeHistogram: histogram := metric.Histogram() - histogram.SetAggregationTemporality(pdata.AggregationTemporalityCumulative) + histogram.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) hdpL := histogram.DataPoints() for _, mg := range mf.getGroups() { if !mg.toDistributionPoint(mf.labelKeysOrdered, &hdpL) { @@ -357,7 +357,7 @@ func (mf *metricFamilyPdata) ToMetricPdata(metrics *pdata.MetricSlice) (int, int case pdata.MetricDataTypeSum: sum := metric.Sum() - sum.SetAggregationTemporality(pdata.AggregationTemporalityCumulative) + sum.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) sdpL := sum.DataPoints() for _, mg := range mf.getGroups() { if !mg.toNumberDataPoint(mf.labelKeysOrdered, &sdpL) { diff --git a/receiver/prometheusreceiver/internal/otlp_metricfamily_test.go b/receiver/prometheusreceiver/internal/otlp_metricfamily_test.go index b2a4db2b32a2b..f8bfb70f712d5 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricfamily_test.go +++ b/receiver/prometheusreceiver/internal/otlp_metricfamily_test.go @@ -21,6 +21,7 @@ import ( "github.com/prometheus/prometheus/pkg/textparse" "github.com/prometheus/prometheus/scrape" "github.com/stretchr/testify/require" + "go.uber.org/zap" "go.opentelemetry.io/collector/model/pdata" ) @@ -92,7 +93,7 @@ func TestMetricGroupData_toDistributionUnitTest(t *testing.T) { { name: "histogram with startTimestamp of 11", metricName: "histogram", - intervalStartTimeMs: 1717, + intervalStartTimeMs: 11, labels: labels.Labels{{Name: "a", Value: "A"}, {Name: "le", Value: "0.75"}, {Name: "b", Value: "B"}}, scrapes: []*scrape{ {at: 11, value: 10, metric: "histogram_count"}, @@ -118,7 +119,7 @@ func TestMetricGroupData_toDistributionUnitTest(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - mp := newMetricFamilyPdata(tt.metricName, mc, testLogger, tt.intervalStartTimeMs).(*metricFamilyPdata) + mp := newMetricFamilyPdata(tt.metricName, mc, zap.NewNop(), tt.intervalStartTimeMs).(*metricFamilyPdata) for _, tv := range tt.scrapes { require.NoError(t, mp.Add(tv.metric, tt.labels.Copy(), tv.at, tv.value)) } @@ -216,16 +217,16 @@ func TestMetricGroupData_toSummaryUnitTest(t *testing.T) { qn0.SetQuantile(0) qn0.SetValue(8) qn50 := qtL.AppendEmpty() - qn50.SetQuantile(50) + qn50.SetQuantile(.5) qn50.SetValue(27) qn75 := qtL.AppendEmpty() - qn75.SetQuantile(75) + qn75.SetQuantile(.75) qn75.SetValue(33.7) qn90 := qtL.AppendEmpty() - qn90.SetQuantile(90) + qn90.SetQuantile(.9) qn90.SetValue(56) qn99 := qtL.AppendEmpty() - qn99.SetQuantile(99) + qn99.SetQuantile(.99) qn99.SetValue(82) point.SetTimestamp(14 * 1e6) // the time in milliseconds -> nanoseconds. point.SetStartTimestamp(10 * 1e5) @@ -240,7 +241,7 @@ func TestMetricGroupData_toSummaryUnitTest(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - mp := newMetricFamilyPdata(tt.name, mc, testLogger, 1).(*metricFamilyPdata) + mp := newMetricFamilyPdata(tt.name, mc, zap.NewNop(), 1).(*metricFamilyPdata) for _, lbs := range tt.labelsScrapes { for _, scrape := range lbs.scrapes { require.NoError(t, mp.Add(scrape.metric, lbs.labels.Copy(), scrape.at, scrape.value)) @@ -322,7 +323,7 @@ func TestMetricGroupData_toNumberDataUnitTest(t *testing.T) { for _, tt := range tests { tt := tt t.Run(tt.name, func(t *testing.T) { - mp := newMetricFamilyPdata(tt.metricKind, mc, testLogger, tt.intervalStartTimestampMs).(*metricFamilyPdata) + mp := newMetricFamilyPdata(tt.metricKind, mc, zap.NewNop(), tt.intervalStartTimestampMs).(*metricFamilyPdata) for _, tv := range tt.scrapes { require.NoError(t, mp.Add(tv.metric, tt.labels.Copy(), tv.at, tv.value)) } diff --git a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go index e10154d1e2da1..a754c47a3cea1 100644 --- a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go +++ b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go @@ -270,6 +270,7 @@ func (ma *MetricsAdjusterPdata) adjustMetricGauge(current *pdata.Metric) (resets // initial || reset timeseries. tsi.initial = current resets++ + continue } initialPoints := tsi.initial.Gauge().DataPoints() previousPoints := previous.Gauge().DataPoints() @@ -299,7 +300,7 @@ func (ma *MetricsAdjusterPdata) adjustMetricGauge(current *pdata.Metric) (resets func (ma *MetricsAdjusterPdata) adjustMetricHistogram(current *pdata.Metric) (resets int) { histogram := current.Histogram() - if histogram.AggregationTemporality() != pdata.AggregationTemporalityCumulative { + if histogram.AggregationTemporality() != pdata.MetricAggregationTemporalityCumulative { // Only dealing with CumulativeDistributions. return 0 } diff --git a/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go b/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go index 9fcb42ad105a8..863c45afcd9f3 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go +++ b/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go @@ -24,6 +24,7 @@ import ( "github.com/prometheus/prometheus/scrape" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/zap" "go.opentelemetry.io/collector/model/pdata" ) @@ -92,7 +93,7 @@ func runBuilderStartTimeTests(t *testing.T, tests []buildTestDataPdata, mc := newMockMetadataCache(testMetadata) st := startTs for _, page := range tt.inputs { - b := newMetricBuilderPdata(mc, true, startTimeMetricRegex, testLogger, 0) + b := newMetricBuilderPdata(mc, true, startTimeMetricRegex, zap.NewNop(), 0) b.startTime = defaultBuilderStartTime // set to a non-zero value for _, pt := range page.pts { // set ts for testing @@ -461,6 +462,7 @@ func Test_OTLPMetricBuilder_counters(t *testing.T) { m0.SetName("counter_test") m0.SetDataType(pdata.MetricDataTypeSum) sum := m0.Sum() + sum.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) pt0 := sum.DataPoints().AppendEmpty() pt0.SetDoubleVal(100.0) pt0.SetStartTimestamp(startTsNanos) @@ -486,6 +488,7 @@ func Test_OTLPMetricBuilder_counters(t *testing.T) { m0.SetName("counter_test") m0.SetDataType(pdata.MetricDataTypeSum) sum := m0.Sum() + sum.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) pt0 := sum.DataPoints().AppendEmpty() pt0.SetDoubleVal(150.0) pt0.SetStartTimestamp(startTsNanos) @@ -518,6 +521,7 @@ func Test_OTLPMetricBuilder_counters(t *testing.T) { m0.SetName("counter_test") m0.SetDataType(pdata.MetricDataTypeSum) sum0 := m0.Sum() + sum0.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) pt0 := sum0.DataPoints().AppendEmpty() pt0.SetDoubleVal(150.0) pt0.SetStartTimestamp(startTsNanos) @@ -534,6 +538,7 @@ func Test_OTLPMetricBuilder_counters(t *testing.T) { m1.SetName("counter_test2") m1.SetDataType(pdata.MetricDataTypeSum) sum1 := m1.Sum() + sum1.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) pt2 := sum1.DataPoints().AppendEmpty() pt2.SetDoubleVal(100.0) pt2.SetStartTimestamp(startTsNanos) @@ -558,6 +563,7 @@ func Test_OTLPMetricBuilder_counters(t *testing.T) { m0.SetName("poor_name_count") m0.SetDataType(pdata.MetricDataTypeSum) sum := m0.Sum() + sum.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) pt0 := sum.DataPoints().AppendEmpty() pt0.SetDoubleVal(100.0) pt0.SetStartTimestamp(startTsNanos) @@ -580,7 +586,7 @@ func runBuilderTestsPdata(t *testing.T, tests []buildTestDataPdata) { mc := newMockMetadataCache(testMetadata) st := startTs for i, page := range tt.inputs { - b := newMetricBuilderPdata(mc, true, "", testLogger, startTs) + b := newMetricBuilderPdata(mc, true, "", zap.NewNop(), startTs) b.startTime = defaultBuilderStartTime // set to a non-zero value b.intervalStartTimeMs = startTs for _, pt := range page.pts { @@ -590,13 +596,30 @@ func runBuilderTestsPdata(t *testing.T, tests []buildTestDataPdata) { } metrics, _, _, err := b.Build() assert.NoError(t, err) - assert.EqualValues(t, wants[i], metrics) + assertEquivalentMetrics(t, wants[i], metrics) st += interval } }) } } +func assertEquivalentMetrics(t *testing.T, want, got *pdata.MetricSlice) { + if !assert.Equal(t, want.Len(), got.Len()) { + return + } + wmap := map[string]pdata.Metric{} + gmap := map[string]pdata.Metric{} + + for i := 0; i < want.Len(); i++ { + wi := want.At(i) + wmap[wi.Name()] = wi + gi := got.At(i) + gmap[gi.Name()] = gi + } + + assert.EqualValues(t, wmap, gmap) +} + var ( startTsNanos = pdata.Timestamp(startTs * 1e6) startTsPlusIntervalNanos = pdata.Timestamp((startTs + interval) * 1e6) @@ -849,6 +872,7 @@ func Test_OTLPMetricBuilder_histogram(t *testing.T) { m0.SetName("hist_test") m0.SetDataType(pdata.MetricDataTypeHistogram) hist0 := m0.Histogram() + hist0.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) pt0 := hist0.DataPoints().AppendEmpty() pt0.SetCount(10) pt0.SetSum(99) @@ -885,6 +909,7 @@ func Test_OTLPMetricBuilder_histogram(t *testing.T) { m0.SetName("hist_test") m0.SetDataType(pdata.MetricDataTypeHistogram) hist0 := m0.Histogram() + hist0.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) pt0 := hist0.DataPoints().AppendEmpty() pt0.SetCount(10) pt0.SetSum(99) @@ -937,6 +962,7 @@ func Test_OTLPMetricBuilder_histogram(t *testing.T) { m0.SetName("hist_test") m0.SetDataType(pdata.MetricDataTypeHistogram) hist0 := m0.Histogram() + hist0.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) pt0 := hist0.DataPoints().AppendEmpty() pt0.SetCount(10) pt0.SetSum(99) @@ -961,6 +987,7 @@ func Test_OTLPMetricBuilder_histogram(t *testing.T) { m1.SetName("hist_test2") m1.SetDataType(pdata.MetricDataTypeHistogram) hist1 := m1.Histogram() + hist1.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) pt2 := hist1.DataPoints().AppendEmpty() pt2.SetCount(3) pt2.SetSum(50) @@ -991,6 +1018,7 @@ func Test_OTLPMetricBuilder_histogram(t *testing.T) { m0.SetName("hist_test") m0.SetDataType(pdata.MetricDataTypeHistogram) hist0 := m0.Histogram() + hist0.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) pt0 := hist0.DataPoints().AppendEmpty() pt0.SetCount(10) pt0.SetSum(99) @@ -1021,6 +1049,7 @@ func Test_OTLPMetricBuilder_histogram(t *testing.T) { m0.SetName("hist_test") m0.SetDataType(pdata.MetricDataTypeHistogram) hist0 := m0.Histogram() + hist0.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) pt0 := hist0.DataPoints().AppendEmpty() pt0.SetCount(3) pt0.SetSum(100) @@ -1050,6 +1079,7 @@ func Test_OTLPMetricBuilder_histogram(t *testing.T) { m0.SetName("hist_test") m0.SetDataType(pdata.MetricDataTypeHistogram) hist0 := m0.Histogram() + hist0.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) pt0 := hist0.DataPoints().AppendEmpty() pt0.SetCount(3) pt0.SetSum(100) @@ -1079,6 +1109,7 @@ func Test_OTLPMetricBuilder_histogram(t *testing.T) { m0.SetName("hist_test") m0.SetDataType(pdata.MetricDataTypeHistogram) hist0 := m0.Histogram() + hist0.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) pt0 := hist0.DataPoints().AppendEmpty() pt0.SetCount(3) pt0.SetSum(0) @@ -1187,13 +1218,13 @@ func Test_metricBuilder_summary(t *testing.T) { pt0.Attributes().InsertString("foo", "bar") qvL := pt0.QuantileValues() q50 := qvL.AppendEmpty() - q50.SetQuantile(50) + q50.SetQuantile(.50) q50.SetValue(1.0) q75 := qvL.AppendEmpty() - q75.SetQuantile(75) + q75.SetQuantile(.75) q75.SetValue(2.0) q100 := qvL.AppendEmpty() - q100.SetQuantile(100) + q100.SetQuantile(1) q100.SetValue(5.0) return []*pdata.MetricSlice{&mL0} @@ -1252,13 +1283,13 @@ func Test_metricBuilder_summary(t *testing.T) { pt0.Attributes().InsertString("foo", "bar") qvL := pt0.QuantileValues() q50 := qvL.AppendEmpty() - q50.SetQuantile(50) + q50.SetQuantile(.50) q50.SetValue(1.0) q75 := qvL.AppendEmpty() - q75.SetQuantile(75) + q75.SetQuantile(.75) q75.SetValue(2.0) q100 := qvL.AppendEmpty() - q100.SetQuantile(100) + q100.SetQuantile(1) q100.SetValue(5.0) return []*pdata.MetricSlice{&mL0} @@ -1272,7 +1303,7 @@ func Test_metricBuilder_summary(t *testing.T) { // Ensure that we reject duplicate label keys. See https://github.com/open-telemetry/wg-prometheus/issues/44. func TestMetricBuilderDuplicateLabelKeysAreRejected(t *testing.T) { mc := newMockMetadataCache(testMetadata) - mb := newMetricBuilderPdata(mc, true, "", testLogger, 0) + mb := newMetricBuilderPdata(mc, true, "", zap.NewNop(), 0) dupLabels := labels.Labels{ {Name: "__name__", Value: "test"}, @@ -1292,7 +1323,7 @@ func TestMetricBuilderDuplicateLabelKeysAreRejected(t *testing.T) { func Test_metricBuilder_baddata(t *testing.T) { t.Run("empty-metric-name", func(t *testing.T) { mc := newMockMetadataCache(testMetadata) - b := newMetricBuilderPdata(mc, true, "", testLogger, 0) + b := newMetricBuilderPdata(mc, true, "", zap.NewNop(), 0) b.startTime = 1.0 // set to a non-zero value if err := b.AddDataPoint(labels.FromStrings("a", "b"), startTs, 123); err != errMetricNameNotFound { t.Error("expecting errMetricNameNotFound error, but get nil") @@ -1306,7 +1337,7 @@ func Test_metricBuilder_baddata(t *testing.T) { t.Run("histogram-datapoint-no-bucket-label", func(t *testing.T) { mc := newMockMetadataCache(testMetadata) - b := newMetricBuilderPdata(mc, true, "", testLogger, 0) + b := newMetricBuilderPdata(mc, true, "", zap.NewNop(), 0) b.startTime = 1.0 // set to a non-zero value if err := b.AddDataPoint(createLabels("hist_test", "k", "v"), startTs, 123); err != errEmptyBoundaryLabel { t.Error("expecting errEmptyBoundaryLabel error, but get nil") @@ -1315,7 +1346,7 @@ func Test_metricBuilder_baddata(t *testing.T) { t.Run("summary-datapoint-no-quantile-label", func(t *testing.T) { mc := newMockMetadataCache(testMetadata) - b := newMetricBuilderPdata(mc, true, "", testLogger, 0) + b := newMetricBuilderPdata(mc, true, "", zap.NewNop(), 0) b.startTime = 1.0 // set to a non-zero value if err := b.AddDataPoint(createLabels("summary_test", "k", "v"), startTs, 123); err != errEmptyBoundaryLabel { t.Error("expecting errEmptyBoundaryLabel error, but get nil") diff --git a/receiver/prometheusreceiver/internal/otlp_transaction.go b/receiver/prometheusreceiver/internal/otlp_transaction.go index 8d3fcd780774a..52228932772fd 100644 --- a/receiver/prometheusreceiver/internal/otlp_transaction.go +++ b/receiver/prometheusreceiver/internal/otlp_transaction.go @@ -22,6 +22,7 @@ import ( "go.uber.org/zap" + "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/config" "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/model/pdata" @@ -74,7 +75,7 @@ type txConfig struct { ms *metadataService sink consumer.Metrics externalLabels labels.Labels - logger *zap.Logger + settings component.ReceiverCreateSettings } func newTransactionPdata(ctx context.Context, txc *txConfig) *transactionPdata { @@ -89,8 +90,8 @@ func newTransactionPdata(ctx context.Context, txc *txConfig) *transactionPdata { receiverID: txc.receiverID, metadataService: txc.ms, externalLabels: txc.externalLabels, - logger: txc.logger, - obsrecv: obsreport.NewReceiver(obsreport.ReceiverSettings{ReceiverID: txc.receiverID, Transport: transport}), + logger: txc.settings.Logger, + obsrecv: obsreport.NewReceiver(obsreport.ReceiverSettings{ReceiverID: txc.receiverID, Transport: transport, ReceiverCreateSettings: txc.settings}), } } @@ -107,6 +108,7 @@ func (t *transactionPdata) Append(ref uint64, labels labels.Labels, atMs int64, } if t.isNew { + t.startTimeMs = atMs if err := t.initTransaction(labels); err != nil { return 0, err } @@ -195,23 +197,23 @@ func adjustStartTimestampPdata(startTime float64, metricsL *pdata.MetricSlice) { case pdata.MetricDataTypeSum: dataPoints := metric.Sum().DataPoints() - for i := 0; i < dataPoints.Len(); i++ { - dataPoint := dataPoints.At(i) - dataPoint.SetStartTimestamp(startTimeTs) + for j := 0; j < dataPoints.Len(); j++ { + dp := dataPoints.At(j) + dp.SetStartTimestamp(startTimeTs) } case pdata.MetricDataTypeSummary: dataPoints := metric.Summary().DataPoints() - for i := 0; i < dataPoints.Len(); i++ { - dataPoint := dataPoints.At(i) - dataPoint.SetStartTimestamp(startTimeTs) + for j := 0; j < dataPoints.Len(); j++ { + dp := dataPoints.At(j) + dp.SetStartTimestamp(startTimeTs) } case pdata.MetricDataTypeHistogram: dataPoints := metric.Histogram().DataPoints() - for i := 0; i < dataPoints.Len(); i++ { - dataPoint := dataPoints.At(i) - dataPoint.SetStartTimestamp(startTimeTs) + for j := 0; j < dataPoints.Len(); j++ { + dp := dataPoints.At(j) + dp.SetStartTimestamp(startTimeTs) } default: diff --git a/receiver/prometheusreceiver/internal/otlp_transaction_test.go b/receiver/prometheusreceiver/internal/otlp_transaction_test.go index d0ede9f7193b2..49cf250f1f15b 100644 --- a/receiver/prometheusreceiver/internal/otlp_transaction_test.go +++ b/receiver/prometheusreceiver/internal/otlp_transaction_test.go @@ -24,6 +24,7 @@ import ( "github.com/prometheus/prometheus/scrape" "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component/componenttest" "go.opentelemetry.io/collector/config" "go.opentelemetry.io/collector/consumer/consumertest" ) @@ -62,7 +63,7 @@ func Test_transaction(t *testing.T) { t.Run("Commit Without Adding", func(t *testing.T) { nomc := consumertest.NewNop() - tr := newTransactionPdata(context.Background(), &txConfig{nil, true, "", rID, ms, nomc, nil, testLogger}) + tr := newTransactionPdata(context.Background(), &txConfig{nil, true, "", rID, ms, nomc, nil, componenttest.NewNopReceiverCreateSettings()}) if got := tr.Commit(); got != nil { t.Errorf("expecting nil from Commit() but got err %v", got) } @@ -70,7 +71,7 @@ func Test_transaction(t *testing.T) { t.Run("Rollback does nothing", func(t *testing.T) { nomc := consumertest.NewNop() - tr := newTransactionPdata(context.Background(), &txConfig{nil, true, "", rID, ms, nomc, nil, testLogger}) + tr := newTransactionPdata(context.Background(), &txConfig{nil, true, "", rID, ms, nomc, nil, componenttest.NewNopReceiverCreateSettings()}) if got := tr.Rollback(); got != nil { t.Errorf("expecting nil from Rollback() but got err %v", got) } @@ -79,7 +80,7 @@ func Test_transaction(t *testing.T) { badLabels := labels.Labels([]labels.Label{{Name: "foo", Value: "bar"}}) t.Run("Add One No Target", func(t *testing.T) { nomc := consumertest.NewNop() - tr := newTransactionPdata(context.Background(), &txConfig{nil, true, "", rID, ms, nomc, nil, testLogger}) + tr := newTransactionPdata(context.Background(), &txConfig{nil, true, "", rID, ms, nomc, nil, componenttest.NewNopReceiverCreateSettings()}) if _, got := tr.Append(0, badLabels, time.Now().Unix()*1000, 1.0); got == nil { t.Errorf("expecting error from Add() but got nil") } @@ -91,7 +92,7 @@ func Test_transaction(t *testing.T) { {Name: "foo", Value: "bar"}}) t.Run("Add One Job not found", func(t *testing.T) { nomc := consumertest.NewNop() - tr := newTransactionPdata(context.Background(), &txConfig{nil, true, "", rID, ms, nomc, nil, testLogger}) + tr := newTransactionPdata(context.Background(), &txConfig{nil, true, "", rID, ms, nomc, nil, componenttest.NewNopReceiverCreateSettings()}) if _, got := tr.Append(0, jobNotFoundLb, time.Now().Unix()*1000, 1.0); got == nil { t.Errorf("expecting error from Add() but got nil") } @@ -102,7 +103,7 @@ func Test_transaction(t *testing.T) { {Name: "__name__", Value: "foo"}}) t.Run("Add One Good", func(t *testing.T) { sink := new(consumertest.MetricsSink) - tr := newTransactionPdata(context.Background(), &txConfig{nil, true, "", rID, ms, sink, nil, testLogger}) + tr := newTransactionPdata(context.Background(), &txConfig{nil, true, "", rID, ms, sink, nil, componenttest.NewNopReceiverCreateSettings()}) if _, got := tr.Append(0, goodLabels, time.Now().Unix()*1000, 1.0); got != nil { t.Errorf("expecting error == nil from Add() but got: %v\n", got) } @@ -123,7 +124,7 @@ func Test_transaction(t *testing.T) { t.Run("Error when start time is zero", func(t *testing.T) { sink := new(consumertest.MetricsSink) - tr := newTransactionPdata(context.Background(), &txConfig{nil, true, "", rID, ms, sink, nil, testLogger}) + tr := newTransactionPdata(context.Background(), &txConfig{nil, true, "", rID, ms, sink, nil, componenttest.NewNopReceiverCreateSettings()}) if _, got := tr.Append(0, goodLabels, time.Now().Unix()*1000, 1.0); got != nil { t.Errorf("expecting error == nil from Add() but got: %v\n", got) } diff --git a/receiver/prometheusreceiver/metrics_receiver_helper_test.go b/receiver/prometheusreceiver/metrics_receiver_helper_test.go index 141d4e31a46db..444eef9627bfd 100644 --- a/receiver/prometheusreceiver/metrics_receiver_helper_test.go +++ b/receiver/prometheusreceiver/metrics_receiver_helper_test.go @@ -538,3 +538,14 @@ func splitMetricsByTarget(metrics []pdata.Metrics) map[string][]*pdata.ResourceM } return pResults } + +func getTS(ms pdata.MetricSlice) pdata.Timestamp { + for i := 0; i < ms.Len(); i++ { + m := ms.At(i) + switch m.DataType() { + case pdata.MetricDataTypeGauge: + return m.Gauge().DataPoints().At(0).Timestamp() + } + } + return 0 +} diff --git a/receiver/prometheusreceiver/metrics_receiver_test.go b/receiver/prometheusreceiver/metrics_receiver_test.go index 3cef9ad2084a3..00eb90ddf7d17 100644 --- a/receiver/prometheusreceiver/metrics_receiver_test.go +++ b/receiver/prometheusreceiver/metrics_receiver_test.go @@ -95,7 +95,7 @@ func verifyTarget1(t *testing.T, td *testData, resourceMetrics []*pdata.Resource wantAttributes := td.attributes metrics1 := m1.InstrumentationLibraryMetrics().At(0).Metrics() - ts1 := metrics1.At(0).Gauge().DataPoints().At(0).Timestamp() + ts1 := getTS(metrics1) e1 := []testExpectation{ assertMetricPresent("go_threads", compareMetricType(pdata.MetricDataTypeGauge), @@ -157,7 +157,7 @@ func verifyTarget1(t *testing.T, td *testData, resourceMetrics []*pdata.Resource assert.Equal(t, 9, metricsCount(m2)) metricsScrape2 := m2.InstrumentationLibraryMetrics().At(0).Metrics() - ts2 := metricsScrape2.At(0).Gauge().DataPoints().At(0).Timestamp() + ts2 := getTS(metricsScrape2) e2 := []testExpectation{ assertMetricPresent("go_threads", compareMetricType(pdata.MetricDataTypeGauge), @@ -289,7 +289,7 @@ func verifyTarget2(t *testing.T, td *testData, resourceMetrics []*pdata.Resource wantAttributes := td.attributes metrics1 := m1.InstrumentationLibraryMetrics().At(0).Metrics() - ts1 := metrics1.At(0).Gauge().DataPoints().At(0).Timestamp() + ts1 := getTS(metrics1) e1 := []testExpectation{ assertMetricPresent("go_threads", compareMetricType(pdata.MetricDataTypeGauge), @@ -329,7 +329,7 @@ func verifyTarget2(t *testing.T, td *testData, resourceMetrics []*pdata.Resource assert.Equal(t, 7, metricsCount(m2)) metricsScrape2 := m2.InstrumentationLibraryMetrics().At(0).Metrics() - ts2 := metricsScrape2.At(0).Gauge().DataPoints().At(0).Timestamp() + ts2 := getTS(metricsScrape2) e2 := []testExpectation{ assertMetricPresent("go_threads", compareMetricType(pdata.MetricDataTypeGauge), @@ -377,7 +377,7 @@ func verifyTarget2(t *testing.T, td *testData, resourceMetrics []*pdata.Resource assert.Equal(t, 7, metricsCount(m3)) metricsScrape3 := m3.InstrumentationLibraryMetrics().At(0).Metrics() - ts3 := metricsScrape3.At(0).Gauge().DataPoints().At(0).Timestamp() + ts3 := getTS(metricsScrape3) e3 := []testExpectation{ assertMetricPresent("go_threads", compareMetricType(pdata.MetricDataTypeGauge), @@ -425,7 +425,7 @@ func verifyTarget2(t *testing.T, td *testData, resourceMetrics []*pdata.Resource assert.Equal(t, 7, metricsCount(m4)) metricsScrape4 := m4.InstrumentationLibraryMetrics().At(0).Metrics() - ts4 := metricsScrape4.At(0).Gauge().DataPoints().At(0).Timestamp() + ts4 := getTS(metricsScrape4) e4 := []testExpectation{ assertMetricPresent("go_threads", compareMetricType(pdata.MetricDataTypeGauge), @@ -473,7 +473,7 @@ func verifyTarget2(t *testing.T, td *testData, resourceMetrics []*pdata.Resource assert.Equal(t, 7, metricsCount(m5)) metricsScrape5 := m5.InstrumentationLibraryMetrics().At(0).Metrics() - ts5 := metricsScrape5.At(0).Gauge().DataPoints().At(0).Timestamp() + ts5 := getTS(metricsScrape5) e5 := []testExpectation{ assertMetricPresent("go_threads", compareMetricType(pdata.MetricDataTypeGauge), @@ -599,7 +599,7 @@ func verifyTarget3(t *testing.T, td *testData, resourceMetrics []*pdata.Resource wantAttributes := td.attributes metrics1 := m1.InstrumentationLibraryMetrics().At(0).Metrics() - ts1 := metrics1.At(0).Gauge().DataPoints().At(0).Timestamp() + ts1 := getTS(metrics1) e1 := []testExpectation{ assertMetricPresent("go_threads", compareMetricType(pdata.MetricDataTypeGauge), @@ -651,7 +651,7 @@ func verifyTarget3(t *testing.T, td *testData, resourceMetrics []*pdata.Resource assert.Equal(t, 8, metricsCount(m2)) metricsScrape2 := m2.InstrumentationLibraryMetrics().At(0).Metrics() - ts2 := metricsScrape2.At(0).Gauge().DataPoints().At(0).Timestamp() + ts2 := getTS(metricsScrape2) e2 := []testExpectation{ assertMetricPresent("go_threads", compareMetricType(pdata.MetricDataTypeGauge), @@ -781,26 +781,26 @@ func verifyStartTimeMetricPage(t *testing.T, td *testData, result []*pdata.Resou case pdata.MetricDataTypeGauge: timestamp = nil for j := 0; j < metrics[i].Gauge().DataPoints().Len(); j++ { - time := timestamppb.New(metrics[i].Gauge().DataPoints().At(j).StartTimestamp().AsTime()) + time := metrics[i].Gauge().DataPoints().At(j).StartTimestamp() assert.Equal(t, timestamp.AsTime(), time.AsTime()) numTimeseries++ } case pdata.MetricDataTypeSum: for j := 0; j < metrics[i].Sum().DataPoints().Len(); j++ { - assert.Equal(t, timestamp.AsTime(), timestamppb.New(metrics[i].Sum().DataPoints().At(j).StartTimestamp().AsTime()).AsTime()) + assert.Equal(t, timestamp.AsTime(), metrics[i].Sum().DataPoints().At(j).StartTimestamp().AsTime()) numTimeseries++ } case pdata.MetricDataTypeHistogram: for j := 0; j < metrics[i].Histogram().DataPoints().Len(); j++ { - assert.Equal(t, timestamp.AsTime(), timestamppb.New(metrics[i].Histogram().DataPoints().At(j).StartTimestamp().AsTime()).AsTime()) + assert.Equal(t, timestamp.AsTime(), metrics[i].Histogram().DataPoints().At(j).StartTimestamp().AsTime()) numTimeseries++ } case pdata.MetricDataTypeSummary: for j := 0; j < metrics[i].Summary().DataPoints().Len(); j++ { - assert.Equal(t, timestamp.AsTime(), timestamppb.New(metrics[i].Summary().DataPoints().At(j).StartTimestamp().AsTime()).AsTime()) + assert.Equal(t, timestamp.AsTime(), metrics[i].Summary().DataPoints().At(j).StartTimestamp().AsTime()) numTimeseries++ } } From 0874f0f5c1494548712f02a253b13a5b32cfe8bd Mon Sep 17 00:00:00 2001 From: Anthony J Mirabella Date: Mon, 22 Nov 2021 20:04:38 -0500 Subject: [PATCH 06/26] receiver/prometheus: buffer metrics in builder to avoid issues with out-of-order exposition Signed-off-by: Anthony J Mirabella --- .../internal/otlp_metricsbuilder.go | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go b/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go index f177bf8e0c333..7e1c14a6aecdd 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go +++ b/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go @@ -184,6 +184,7 @@ func convToPdataMetricType(metricType textparse.MetricType) pdata.MetricDataType type metricBuilderPdata struct { metrics pdata.MetricSlice + families map[string]MetricFamilyPdata currentMf MetricFamilyPdata hasData bool hasInternalMetric bool @@ -207,6 +208,7 @@ func newMetricBuilderPdata(mc MetadataCache, useStartTimeMetric bool, startTimeM } return &metricBuilderPdata{ metrics: pdata.NewMetricSlice(), + families: map[string]MetricFamilyPdata{}, mc: mc, logger: logger, numTimeseries: 0, @@ -273,16 +275,18 @@ func (b *metricBuilderPdata) AddDataPoint(ls labels.Labels, t int64, v float64) b.hasData = true - if b.currentMf != nil && !b.currentMf.IsSameFamily(metricName) { - nTs, nDts := b.currentMf.ToMetricPdata(&b.metrics) - b.numTimeseries += nTs - b.droppedTimeseries += nDts - b.currentMf = newMetricFamilyPdata(metricName, b.mc, b.logger, b.intervalStartTimeMs) - } else if b.currentMf == nil { - b.currentMf = newMetricFamilyPdata(metricName, b.mc, b.logger, b.intervalStartTimeMs) + familyName := normalizeMetricName(metricName) + curMF, ok := b.families[familyName] + if !ok { + if mf, ok := b.families[metricName]; ok { + curMF = mf + } else { + curMF = newMetricFamilyPdata(metricName, b.mc, b.logger, b.intervalStartTimeMs) + b.families[familyName] = curMF + } } - return b.currentMf.Add(metricName, ls, t, v) + return curMF.Add(metricName, ls, t, v) } // Build an pdata.MetricSlice based on all added data complexValue. @@ -296,11 +300,10 @@ func (b *metricBuilderPdata) Build() (*pdata.MetricSlice, int, int, error) { return nil, 0, 0, errNoDataToBuild } - if b.currentMf != nil { - ts, dts := b.currentMf.ToMetricPdata(&b.metrics) + for _, mf := range b.families { + ts, dts := mf.ToMetricPdata(&b.metrics) b.numTimeseries += ts b.droppedTimeseries += dts - b.currentMf = nil } return &b.metrics, b.numTimeseries, b.droppedTimeseries, nil From 1c85958c91e7ec21582cf8c902be3dc4722d6e59 Mon Sep 17 00:00:00 2001 From: Anthony J Mirabella Date: Tue, 23 Nov 2021 12:29:48 -0500 Subject: [PATCH 07/26] receiver/prometheus: restore prom->OC->pdata pipeline Signed-off-by: Anthony J Mirabella --- receiver/prometheusreceiver/go.mod | 4 +- receiver/prometheusreceiver/go.sum | 3 + .../internal/metricfamily.go | 422 +++++ .../internal/metrics_adjuster.go | 317 ++++ .../internal/metrics_adjuster_test.go | 436 +++++ .../internal/metricsbuilder.go | 328 ++++ .../internal/metricsbuilder_test.go | 1501 +++++++++++++++++ .../internal/otlp_metricfamily.go | 48 +- .../internal/otlp_metrics_adjuster_test.go | 421 ++--- .../internal/otlp_metricsbuilder.go | 101 -- .../internal/otlp_metricsbuilder_test.go | 198 +-- .../internal/otlp_transaction.go | 20 +- .../internal/otlp_transaction_test.go | 2 +- .../internal/transaction.go | 260 +++ .../internal/transaction_test.go | 154 ++ 15 files changed, 3652 insertions(+), 563 deletions(-) create mode 100644 receiver/prometheusreceiver/internal/metricfamily.go create mode 100644 receiver/prometheusreceiver/internal/metrics_adjuster.go create mode 100644 receiver/prometheusreceiver/internal/metrics_adjuster_test.go create mode 100644 receiver/prometheusreceiver/internal/metricsbuilder.go create mode 100644 receiver/prometheusreceiver/internal/metricsbuilder_test.go create mode 100644 receiver/prometheusreceiver/internal/transaction.go create mode 100644 receiver/prometheusreceiver/internal/transaction_test.go diff --git a/receiver/prometheusreceiver/go.mod b/receiver/prometheusreceiver/go.mod index b17ed383faa56..bd3cd1494d05f 100644 --- a/receiver/prometheusreceiver/go.mod +++ b/receiver/prometheusreceiver/go.mod @@ -3,11 +3,13 @@ module github.com/open-telemetry/opentelemetry-collector-contrib/receiver/promet go 1.17 require ( + github.com/census-instrumentation/opencensus-proto v0.3.0 github.com/go-kit/log v0.2.0 github.com/gogo/protobuf v1.3.2 github.com/golang/snappy v0.0.4 github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prometheusremotewriteexporter v0.39.0 github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal v0.39.0 + github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus v0.39.0 github.com/prometheus/common v0.32.1 github.com/prometheus/prometheus v1.8.2-0.20210621150501-ff58416a0b02 github.com/stretchr/testify v1.7.0 @@ -37,7 +39,6 @@ require ( github.com/aws/aws-sdk-go v1.40.56 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cenkalti/backoff/v4 v4.1.2 // indirect - github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/containerd/containerd v1.4.3 // indirect github.com/davecgh/go-spew v1.1.1 // indirect @@ -64,6 +65,7 @@ require ( github.com/googleapis/gax-go/v2 v2.0.5 // indirect github.com/googleapis/gnostic v0.4.1 // indirect github.com/gophercloud/gophercloud v0.18.0 // indirect + github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/hashicorp/consul/api v1.8.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.1 // indirect github.com/hashicorp/go-hclog v0.12.2 // indirect diff --git a/receiver/prometheusreceiver/go.sum b/receiver/prometheusreceiver/go.sum index bae6a83919f8e..f3901d7c974ff 100644 --- a/receiver/prometheusreceiver/go.sum +++ b/receiver/prometheusreceiver/go.sum @@ -580,6 +580,7 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.14.4/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0= +github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= @@ -868,6 +869,8 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus v0.39.0 h1:678eZCNsUS6rXWLfeeDPAnNmS61qMXvoQeYH64BZyos= +github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus v0.39.0/go.mod h1:6NQp433EU5FmtkGavG5FuwFOnD4bAUhSDsK8gnSmIdM= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= diff --git a/receiver/prometheusreceiver/internal/metricfamily.go b/receiver/prometheusreceiver/internal/metricfamily.go new file mode 100644 index 0000000000000..bcb04ddf06068 --- /dev/null +++ b/receiver/prometheusreceiver/internal/metricfamily.go @@ -0,0 +1,422 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "fmt" + "sort" + "strings" + + metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/pkg/textparse" + "github.com/prometheus/prometheus/scrape" + "go.uber.org/zap" + "google.golang.org/protobuf/types/known/timestamppb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +// MetricFamily is unit which is corresponding to the metrics items which shared the same TYPE/UNIT/... metadata from +// a single scrape. +type MetricFamily interface { + Add(metricName string, ls labels.Labels, t int64, v float64) error + IsSameFamily(metricName string) bool + ToMetric() (*metricspb.Metric, int, int) +} + +type metricFamily struct { + name string + mtype metricspb.MetricDescriptor_Type + mc MetadataCache + droppedTimeseries int + labelKeys map[string]bool + labelKeysOrdered []string + metadata *scrape.MetricMetadata + groupOrders map[string]int + groups map[string]*metricGroup + intervalStartTimeMs int64 +} + +func newMetricFamily(metricName string, mc MetadataCache, logger *zap.Logger, intervalStartTimeMs int64) MetricFamily { + familyName := normalizeMetricName(metricName) + + // lookup metadata based on familyName + metadata, ok := mc.Metadata(familyName) + if !ok && metricName != familyName { + // use the original metricName as metricFamily + familyName = metricName + // perform a 2nd lookup with the original metric name. it can happen if there's a metric which is not histogram + // or summary, but ends with one of those _count/_sum suffixes + metadata, ok = mc.Metadata(metricName) + // still not found, this can happen when metric has no TYPE HINT + if !ok { + metadata.Metric = familyName + metadata.Type = textparse.MetricTypeUnknown + } + } else if !ok && isInternalMetric(metricName) { + metadata = defineInternalMetric(metricName, metadata, logger) + } else if !ok { + // Prometheus sends metrics without a type hint as gauges. + // MetricTypeUnknown is converted to a gauge in convToOCAMetricType() + metadata.Type = textparse.MetricTypeUnknown + } + ocaMetricType := convToOCAMetricType(metadata.Type) + + // If a counter has a _total suffix but metadata is stored without it, keep _total suffix as the name otherwise + // the metric sent won't have the suffix + if ocaMetricType == metricspb.MetricDescriptor_CUMULATIVE_DOUBLE && strings.HasSuffix(metricName, metricSuffixTotal) { + familyName = metricName + } else if ocaMetricType == metricspb.MetricDescriptor_UNSPECIFIED { + logger.Debug(fmt.Sprintf("Invalid metric : %s %+v", metricName, metadata)) + } + + return &metricFamily{ + name: familyName, + mtype: ocaMetricType, + mc: mc, + droppedTimeseries: 0, + labelKeys: make(map[string]bool), + labelKeysOrdered: make([]string, 0), + metadata: &metadata, + groupOrders: make(map[string]int), + groups: make(map[string]*metricGroup), + intervalStartTimeMs: intervalStartTimeMs, + } +} + +// Define manually the metadata of prometheus scrapper internal metrics +func defineInternalMetric(metricName string, metadata scrape.MetricMetadata, logger *zap.Logger) scrape.MetricMetadata { + if metadata.Metric != "" && metadata.Type != "" && metadata.Help != "" { + logger.Debug("Internal metric seems already fully defined") + return metadata + } + metadata.Metric = metricName + + switch metricName { + case scrapeUpMetricName: + metadata.Type = textparse.MetricTypeGauge + metadata.Help = "The scraping was successful" + case "scrape_duration_seconds": + metadata.Unit = "seconds" + metadata.Type = textparse.MetricTypeGauge + metadata.Help = "Duration of the scrape" + case "scrape_samples_scraped": + metadata.Type = textparse.MetricTypeGauge + metadata.Help = "The number of samples the target exposed" + case "scrape_series_added": + metadata.Type = textparse.MetricTypeGauge + metadata.Help = "The approximate number of new series in this scrape" + case "scrape_samples_post_metric_relabeling": + metadata.Type = textparse.MetricTypeGauge + metadata.Help = "The number of samples remaining after metric relabeling was applied" + } + return metadata +} + +func (mf *metricFamily) IsSameFamily(metricName string) bool { + // trim known suffix if necessary + familyName := normalizeMetricName(metricName) + return mf.name == familyName || familyName != metricName && mf.name == metricName +} + +// updateLabelKeys is used to store all the label keys of a same metric family in observed order. since prometheus +// receiver removes any label with empty value before feeding it to an appender, in order to figure out all the labels +// from the same metric family we will need to keep track of what labels have ever been observed. +func (mf *metricFamily) updateLabelKeys(ls labels.Labels) { + for _, l := range ls { + if isUsefulLabel(mf.mtype, l.Name) { + if _, ok := mf.labelKeys[l.Name]; !ok { + mf.labelKeys[l.Name] = true + // use insertion sort to maintain order + i := sort.SearchStrings(mf.labelKeysOrdered, l.Name) + mf.labelKeysOrdered = append(mf.labelKeysOrdered, "") + copy(mf.labelKeysOrdered[i+1:], mf.labelKeysOrdered[i:]) + mf.labelKeysOrdered[i] = l.Name + } + } + } +} + +func (mf *metricFamily) isCumulativeType() bool { + return mf.mtype == metricspb.MetricDescriptor_CUMULATIVE_DOUBLE || + mf.mtype == metricspb.MetricDescriptor_CUMULATIVE_INT64 || + mf.mtype == metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION || + mf.mtype == metricspb.MetricDescriptor_SUMMARY +} + +func (mf *metricFamily) getGroupKey(ls labels.Labels) string { + mf.updateLabelKeys(ls) + return dpgSignature(mf.labelKeysOrdered, ls) +} + +// getGroups to return groups in insertion order +func (mf *metricFamily) getGroups() []*metricGroup { + groups := make([]*metricGroup, len(mf.groupOrders)) + for k, v := range mf.groupOrders { + groups[v] = mf.groups[k] + } + + return groups +} + +func (mf *metricFamily) loadMetricGroupOrCreate(groupKey string, ls labels.Labels, ts int64) *metricGroup { + mg, ok := mf.groups[groupKey] + if !ok { + mg = &metricGroup{ + family: mf, + ts: ts, + ls: ls, + complexValue: make([]*dataPoint, 0), + intervalStartTimeMs: mf.intervalStartTimeMs, + } + mf.groups[groupKey] = mg + // maintaining data insertion order is helpful to generate stable/reproducible metric output + mf.groupOrders[groupKey] = len(mf.groupOrders) + } + return mg +} + +func (mf *metricFamily) getLabelKeys() []*metricspb.LabelKey { + lks := make([]*metricspb.LabelKey, len(mf.labelKeysOrdered)) + for i, k := range mf.labelKeysOrdered { + lks[i] = &metricspb.LabelKey{Key: k} + } + return lks +} + +func (mf *metricFamily) Add(metricName string, ls labels.Labels, t int64, v float64) error { + groupKey := mf.getGroupKey(ls) + mg := mf.loadMetricGroupOrCreate(groupKey, ls, t) + switch mf.mtype { + case metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION: + fallthrough + case metricspb.MetricDescriptor_SUMMARY: + switch { + case strings.HasSuffix(metricName, metricsSuffixSum): + // always use the timestamp from sum (count is ok too), because the startTs from quantiles won't be reliable + // in cases like remote server restart + mg.ts = t + mg.sum = v + mg.hasSum = true + case strings.HasSuffix(metricName, metricsSuffixCount): + mg.count = v + mg.hasCount = true + default: + boundary, err := getBoundary(mf.mtype, ls) + if err != nil { + mf.droppedTimeseries++ + return err + } + mg.complexValue = append(mg.complexValue, &dataPoint{value: v, boundary: boundary}) + } + default: + mg.value = v + } + + return nil +} + +func (mf *metricFamily) ToMetric() (*metricspb.Metric, int, int) { + timeseries := make([]*metricspb.TimeSeries, 0, len(mf.groups)) + switch mf.mtype { + // not supported currently + // case metricspb.MetricDescriptor_GAUGE_DISTRIBUTION: + // return nil + case metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION: + for _, mg := range mf.getGroups() { + tss := mg.toDistributionTimeSeries(mf.labelKeysOrdered) + if tss != nil { + timeseries = append(timeseries, tss) + } else { + mf.droppedTimeseries++ + } + } + case metricspb.MetricDescriptor_SUMMARY: + for _, mg := range mf.getGroups() { + tss := mg.toSummaryTimeSeries(mf.labelKeysOrdered) + if tss != nil { + timeseries = append(timeseries, tss) + } else { + mf.droppedTimeseries++ + } + } + default: + for _, mg := range mf.getGroups() { + tss := mg.toDoubleValueTimeSeries(mf.labelKeysOrdered) + if tss != nil { + timeseries = append(timeseries, tss) + } else { + mf.droppedTimeseries++ + } + } + } + + // note: the total number of timeseries is the length of timeseries plus the number of dropped timeseries. + numTimeseries := len(timeseries) + if numTimeseries != 0 { + return &metricspb.Metric{ + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: mf.name, + Description: mf.metadata.Help, + Unit: heuristicalMetricAndKnownUnits(mf.name, mf.metadata.Unit), + Type: mf.mtype, + LabelKeys: mf.getLabelKeys(), + }, + Timeseries: timeseries, + }, + numTimeseries + mf.droppedTimeseries, + mf.droppedTimeseries + } + return nil, mf.droppedTimeseries, mf.droppedTimeseries +} + +type dataPoint struct { + value float64 + boundary float64 +} + +// metricGroup, represents a single metric of a metric family. for example a histogram metric is usually represent by +// a couple data complexValue (buckets and count/sum), a group of a metric family always share a same set of tags. for +// simple types like counter and gauge, each data point is a group of itself +type metricGroup struct { + family *metricFamily + ts int64 + ls labels.Labels + count float64 + hasCount bool + sum float64 + hasSum bool + value float64 + complexValue []*dataPoint + intervalStartTimeMs int64 +} + +func (mg *metricGroup) sortPoints() { + sort.Slice(mg.complexValue, func(i, j int) bool { + return mg.complexValue[i].boundary < mg.complexValue[j].boundary + }) +} + +func (mg *metricGroup) toDistributionTimeSeries(orderedLabelKeys []string) *metricspb.TimeSeries { + if !(mg.hasCount) || len(mg.complexValue) == 0 { + return nil + } + mg.sortPoints() + // for OCAgent Proto, the bounds won't include +inf + bounds := make([]float64, len(mg.complexValue)-1) + buckets := make([]*metricspb.DistributionValue_Bucket, len(mg.complexValue)) + + for i := 0; i < len(mg.complexValue); i++ { + if i != len(mg.complexValue)-1 { + // not need to add +inf as bound to oc proto + bounds[i] = mg.complexValue[i].boundary + } + adjustedCount := mg.complexValue[i].value + if i != 0 { + adjustedCount -= mg.complexValue[i-1].value + } + buckets[i] = &metricspb.DistributionValue_Bucket{Count: int64(adjustedCount)} + } + + dv := &metricspb.DistributionValue{ + BucketOptions: &metricspb.DistributionValue_BucketOptions{ + Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ + Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ + Bounds: bounds, + }, + }, + }, + Count: int64(mg.count), + Sum: mg.sum, + Buckets: buckets, + // SumOfSquaredDeviation: // there's no way to compute this value from prometheus data + } + + return &metricspb.TimeSeries{ + StartTimestamp: timestampFromMs(mg.ts), + LabelValues: populateLabelValues(orderedLabelKeys, mg.ls), + Points: []*metricspb.Point{ + { + Timestamp: timestampFromMs(mg.ts), + Value: &metricspb.Point_DistributionValue{DistributionValue: dv}, + }, + }, + } +} + +func (mg *metricGroup) toSummaryTimeSeries(orderedLabelKeys []string) *metricspb.TimeSeries { + // expecting count to be provided, however, in the following two cases, they can be missed. + // 1. data is corrupted + // 2. ignored by startValue evaluation + if !(mg.hasCount) { + return nil + } + mg.sortPoints() + percentiles := make([]*metricspb.SummaryValue_Snapshot_ValueAtPercentile, len(mg.complexValue)) + for i, p := range mg.complexValue { + percentiles[i] = + &metricspb.SummaryValue_Snapshot_ValueAtPercentile{Percentile: p.boundary * 100, Value: p.value} + } + + // allow percentiles to be nil when no data provided from prometheus + var snapshot *metricspb.SummaryValue_Snapshot + if len(percentiles) != 0 { + snapshot = &metricspb.SummaryValue_Snapshot{ + PercentileValues: percentiles, + } + } + + // Based on the summary description from https://prometheus.io/docs/concepts/metric_types/#summary + // the quantiles are calculated over a sliding time window, however, the count is the total count of + // observations and the corresponding sum is a sum of all observed values, thus the sum and count used + // at the global level of the metricspb.SummaryValue + + summaryValue := &metricspb.SummaryValue{ + Sum: &wrapperspb.DoubleValue{Value: mg.sum}, + Count: &wrapperspb.Int64Value{Value: int64(mg.count)}, + Snapshot: snapshot, + } + return &metricspb.TimeSeries{ + StartTimestamp: timestampFromMs(mg.ts), + LabelValues: populateLabelValues(orderedLabelKeys, mg.ls), + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(mg.ts), Value: &metricspb.Point_SummaryValue{SummaryValue: summaryValue}}, + }, + } +} + +func (mg *metricGroup) toDoubleValueTimeSeries(orderedLabelKeys []string) *metricspb.TimeSeries { + var startTs *timestamppb.Timestamp + // gauge/undefined types has no start time + if mg.family.isCumulativeType() { + startTs = timestampFromMs(mg.intervalStartTimeMs) + } + + return &metricspb.TimeSeries{ + StartTimestamp: startTs, + Points: []*metricspb.Point{{Timestamp: timestampFromMs(mg.ts), Value: &metricspb.Point_DoubleValue{DoubleValue: mg.value}}}, + LabelValues: populateLabelValues(orderedLabelKeys, mg.ls), + } +} + +func populateLabelValues(orderedKeys []string, ls labels.Labels) []*metricspb.LabelValue { + lvs := make([]*metricspb.LabelValue, len(orderedKeys)) + lmap := ls.Map() + for i, k := range orderedKeys { + value := lmap[k] + lvs[i] = &metricspb.LabelValue{Value: value, HasValue: value != ""} + } + return lvs +} diff --git a/receiver/prometheusreceiver/internal/metrics_adjuster.go b/receiver/prometheusreceiver/internal/metrics_adjuster.go new file mode 100644 index 0000000000000..14d69f7e0f612 --- /dev/null +++ b/receiver/prometheusreceiver/internal/metrics_adjuster.go @@ -0,0 +1,317 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "fmt" + "strings" + "sync" + "time" + + metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" + "go.uber.org/zap" +) + +// Notes on garbage collection (gc): +// +// Job-level gc: +// The Prometheus receiver will likely execute in a long running service whose lifetime may exceed +// the lifetimes of many of the jobs that it is collecting from. In order to keep the JobsMap from +// leaking memory for entries of no-longer existing jobs, the JobsMap needs to remove entries that +// haven't been accessed for a long period of time. +// +// Timeseries-level gc: +// Some jobs that the Prometheus receiver is collecting from may export timeseries based on metrics +// from other jobs (e.g. cAdvisor). In order to keep the timeseriesMap from leaking memory for entries +// of no-longer existing jobs, the timeseriesMap for each job needs to remove entries that haven't +// been accessed for a long period of time. +// +// The gc strategy uses a standard mark-and-sweep approach - each time a timeseriesMap is accessed, +// it is marked. Similarly, each time a timeseriesinfo is accessed, it is also marked. +// +// At the end of each JobsMap.get(), if the last time the JobsMap was gc'd exceeds the 'gcInterval', +// the JobsMap is locked and any timeseriesMaps that are unmarked are removed from the JobsMap +// otherwise the timeseriesMap is gc'd +// +// The gc for the timeseriesMap is straightforward - the map is locked and, for each timeseriesinfo +// in the map, if it has not been marked, it is removed otherwise it is unmarked. +// +// Alternative Strategies +// 1. If the job-level gc doesn't run often enough, or runs too often, a separate go routine can +// be spawned at JobMap creation time that gc's at periodic intervals. This approach potentially +// adds more contention and latency to each scrape so the current approach is used. Note that +// the go routine will need to be cancelled upon Shutdown(). +// 2. If the gc of each timeseriesMap during the gc of the JobsMap causes too much contention, +// the gc of timeseriesMaps can be moved to the end of MetricsAdjuster().AdjustMetrics(). This +// approach requires adding 'lastGC' Time and (potentially) a gcInterval duration to +// timeseriesMap so the current approach is used instead. + +// timeseriesinfo contains the information necessary to adjust from the initial point and to detect +// resets. +type timeseriesinfo struct { + mark bool + initial *metricspb.TimeSeries + previous *metricspb.TimeSeries +} + +// timeseriesMap maps from a timeseries instance (metric * label values) to the timeseries info for +// the instance. +type timeseriesMap struct { + sync.RWMutex + mark bool + tsiMap map[string]*timeseriesinfo +} + +// Get the timeseriesinfo for the timeseries associated with the metric and label values. +func (tsm *timeseriesMap) get( + metric *metricspb.Metric, values []*metricspb.LabelValue) *timeseriesinfo { + name := metric.GetMetricDescriptor().GetName() + sig := getTimeseriesSignature(name, values) + tsi, ok := tsm.tsiMap[sig] + if !ok { + tsi = ×eriesinfo{} + tsm.tsiMap[sig] = tsi + } + tsm.mark = true + tsi.mark = true + return tsi +} + +// Remove timeseries that have aged out. +func (tsm *timeseriesMap) gc() { + tsm.Lock() + defer tsm.Unlock() + // this shouldn't happen under the current gc() strategy + if !tsm.mark { + return + } + for ts, tsi := range tsm.tsiMap { + if !tsi.mark { + delete(tsm.tsiMap, ts) + } else { + tsi.mark = false + } + } + tsm.mark = false +} + +func newTimeseriesMap() *timeseriesMap { + return ×eriesMap{mark: true, tsiMap: map[string]*timeseriesinfo{}} +} + +// Create a unique timeseries signature consisting of the metric name and label values. +func getTimeseriesSignature(name string, values []*metricspb.LabelValue) string { + labelValues := make([]string, 0, len(values)) + for _, label := range values { + if label.GetValue() != "" { + labelValues = append(labelValues, label.GetValue()) + } + } + return fmt.Sprintf("%s,%s", name, strings.Join(labelValues, ",")) +} + +// JobsMap maps from a job instance to a map of timeseries instances for the job. +type JobsMap struct { + sync.RWMutex + gcInterval time.Duration + lastGC time.Time + jobsMap map[string]*timeseriesMap +} + +// NewJobsMap creates a new (empty) JobsMap. +func NewJobsMap(gcInterval time.Duration) *JobsMap { + return &JobsMap{gcInterval: gcInterval, lastGC: time.Now(), jobsMap: make(map[string]*timeseriesMap)} +} + +// Remove jobs and timeseries that have aged out. +func (jm *JobsMap) gc() { + jm.Lock() + defer jm.Unlock() + // once the structure is locked, confirm that gc() is still necessary + if time.Since(jm.lastGC) > jm.gcInterval { + for sig, tsm := range jm.jobsMap { + tsm.RLock() + tsmNotMarked := !tsm.mark + tsm.RUnlock() + if tsmNotMarked { + delete(jm.jobsMap, sig) + } else { + tsm.gc() + } + } + jm.lastGC = time.Now() + } +} + +func (jm *JobsMap) maybeGC() { + // speculatively check if gc() is necessary, recheck once the structure is locked + jm.RLock() + defer jm.RUnlock() + if time.Since(jm.lastGC) > jm.gcInterval { + go jm.gc() + } +} + +func (jm *JobsMap) get(job, instance string) *timeseriesMap { + sig := job + ":" + instance + jm.RLock() + tsm, ok := jm.jobsMap[sig] + jm.RUnlock() + defer jm.maybeGC() + if ok { + return tsm + } + jm.Lock() + defer jm.Unlock() + tsm2, ok2 := jm.jobsMap[sig] + if ok2 { + return tsm2 + } + tsm2 = newTimeseriesMap() + jm.jobsMap[sig] = tsm2 + return tsm2 +} + +// MetricsAdjuster takes a map from a metric instance to the initial point in the metrics instance +// and provides AdjustMetrics, which takes a sequence of metrics and adjust their start times based on +// the initial points. +type MetricsAdjuster struct { + tsm *timeseriesMap + logger *zap.Logger +} + +// NewMetricsAdjuster is a constructor for MetricsAdjuster. +func NewMetricsAdjuster(tsm *timeseriesMap, logger *zap.Logger) *MetricsAdjuster { + return &MetricsAdjuster{ + tsm: tsm, + logger: logger, + } +} + +// AdjustMetrics takes a sequence of metrics and adjust their start times based on the initial and +// previous points in the timeseriesMap. +// Returns the total number of timeseries that had reset start times. +func (ma *MetricsAdjuster) AdjustMetrics(metrics []*metricspb.Metric) ([]*metricspb.Metric, int) { + var adjusted = make([]*metricspb.Metric, 0, len(metrics)) + resets := 0 + ma.tsm.Lock() + defer ma.tsm.Unlock() + for _, metric := range metrics { + d := ma.adjustMetric(metric) + resets += d + adjusted = append(adjusted, metric) + } + return adjusted, resets +} + +// Returns the number of timeseries with reset start times. +// +// Types of metrics returned supported by prometheus: +// - MetricDescriptor_GAUGE_DOUBLE +// - MetricDescriptor_GAUGE_DISTRIBUTION +// - MetricDescriptor_CUMULATIVE_DOUBLE +// - MetricDescriptor_CUMULATIVE_DISTRIBUTION +// - MetricDescriptor_SUMMARY +func (ma *MetricsAdjuster) adjustMetric(metric *metricspb.Metric) int { + switch metric.MetricDescriptor.Type { + case metricspb.MetricDescriptor_GAUGE_DOUBLE, metricspb.MetricDescriptor_GAUGE_DISTRIBUTION: + // gauges don't need to be adjusted so no additional processing is necessary + return 0 + default: + return ma.adjustMetricTimeseries(metric) + } +} + +// Returns the number of timeseries that had reset start times. +func (ma *MetricsAdjuster) adjustMetricTimeseries(metric *metricspb.Metric) int { + resets := 0 + filtered := make([]*metricspb.TimeSeries, 0, len(metric.GetTimeseries())) + for _, current := range metric.GetTimeseries() { + tsi := ma.tsm.get(metric, current.GetLabelValues()) + if tsi.initial == nil || !ma.adjustTimeseries(metric.MetricDescriptor.Type, current, tsi.initial, tsi.previous) { + // initial || reset timeseries + tsi.initial = current + resets++ + } + tsi.previous = current + filtered = append(filtered, current) + } + metric.Timeseries = filtered + return resets +} + +// Returns true if 'current' was adjusted and false if 'current' is an the initial occurrence or a +// reset of the timeseries. +func (ma *MetricsAdjuster) adjustTimeseries(metricType metricspb.MetricDescriptor_Type, + current, initial, previous *metricspb.TimeSeries) bool { + if !ma.adjustPoints( + metricType, current.GetPoints(), initial.GetPoints(), previous.GetPoints()) { + return false + } + current.StartTimestamp = initial.StartTimestamp + return true +} + +func (ma *MetricsAdjuster) adjustPoints(metricType metricspb.MetricDescriptor_Type, + current, initial, previous []*metricspb.Point) bool { + if len(current) != 1 || len(initial) != 1 || len(previous) != 1 { + ma.logger.Info("Adjusting Points, all lengths should be 1", + zap.Int("len(current)", len(current)), zap.Int("len(initial)", len(initial)), zap.Int("len(previous)", len(previous))) + return true + } + return ma.isReset(metricType, current[0], previous[0]) +} + +func (ma *MetricsAdjuster) isReset(metricType metricspb.MetricDescriptor_Type, + current, previous *metricspb.Point) bool { + switch metricType { + case metricspb.MetricDescriptor_CUMULATIVE_DOUBLE: + if current.GetDoubleValue() < previous.GetDoubleValue() { + // reset detected + return false + } + case metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION: + // note: sum of squared deviation not currently supported + currentDist := current.GetDistributionValue() + previousDist := previous.GetDistributionValue() + if currentDist == nil || previousDist == nil { + return false + } + if currentDist.Count < previousDist.Count || currentDist.Sum < previousDist.Sum { + // reset detected + return false + } + case metricspb.MetricDescriptor_SUMMARY: + currentSummary := current.GetSummaryValue() + previousSummary := previous.GetSummaryValue() + if currentSummary == nil || previousSummary == nil { + return false + } + if (currentSummary.Count != nil && + previousSummary.Count != nil && + currentSummary.Count.GetValue() < previousSummary.Count.GetValue()) || + + (currentSummary.Sum != nil && + previousSummary.Sum != nil && + currentSummary.Sum.GetValue() < previousSummary.Sum.GetValue()) { + // reset detected + return false + } + default: + // this shouldn't happen + ma.logger.Info("Adjust - skipping unexpected point", zap.String("type", metricType.String())) + } + return true +} diff --git a/receiver/prometheusreceiver/internal/metrics_adjuster_test.go b/receiver/prometheusreceiver/internal/metrics_adjuster_test.go new file mode 100644 index 0000000000000..712f5b3f930c7 --- /dev/null +++ b/receiver/prometheusreceiver/internal/metrics_adjuster_test.go @@ -0,0 +1,436 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "testing" + "time" + + metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + "github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/metricstestutil" +) + +func Test_gauge(t *testing.T) { + script := []*metricsAdjusterTest{{ + "Gauge: round 1 - gauge not adjusted", + []*metricspb.Metric{metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44)))}, + []*metricspb.Metric{metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44)))}, + 0, + }, { + "Gauge: round 2 - gauge not adjusted", + []*metricspb.Metric{metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.Double(t2Ms, 66)))}, + []*metricspb.Metric{metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.Double(t2Ms, 66)))}, + 0, + }, { + "Gauge: round 3 - value less than previous value - gauge is not adjusted", + []*metricspb.Metric{metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t3Ms, 55)))}, + []*metricspb.Metric{metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t3Ms, 55)))}, + 0, + }} + runScript(t, NewJobsMap(time.Minute).get("job", "0"), script) +} + +func Test_gaugeDistribution(t *testing.T) { + script := []*metricsAdjusterTest{{ + "GaugeDist: round 1 - gauge distribution not adjusted", + []*metricspb.Metric{metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7})))}, + []*metricspb.Metric{metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7})))}, + 0, + }, { + "GaugeDist: round 2 - gauge distribution not adjusted", + []*metricspb.Metric{metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.DistPt(t2Ms, bounds0, []int64{6, 5, 8, 11})))}, + []*metricspb.Metric{metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.DistPt(t2Ms, bounds0, []int64{6, 5, 8, 11})))}, + 0, + }, { + "GaugeDist: round 3 - count/sum less than previous - gauge distribution not adjusted", + []*metricspb.Metric{metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.DistPt(t3Ms, bounds0, []int64{2, 0, 1, 5})))}, + []*metricspb.Metric{metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.DistPt(t3Ms, bounds0, []int64{2, 0, 1, 5})))}, + 0, + }} + runScript(t, NewJobsMap(time.Minute).get("job", "0"), script) +} + +func Test_cumulative(t *testing.T) { + script := []*metricsAdjusterTest{{ + "Cumulative: round 1 - initial instance, start time is established", + []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44)))}, + []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44)))}, + 1, + }, { + "Cumulative: round 2 - instance adjusted based on round 1", + []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.Double(t2Ms, 66)))}, + []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t2Ms, 66)))}, + 0, + }, { + "Cumulative: round 3 - instance reset (value less than previous value), start time is reset", + []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t3Ms, 55)))}, + []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t3Ms, 55)))}, + 1, + }, { + "Cumulative: round 4 - instance adjusted based on round 3", + []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.Double(t4Ms, 72)))}, + []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t4Ms, 72)))}, + 0, + }} + runScript(t, NewJobsMap(time.Minute).get("job", "0"), script) +} + +func Test_cumulativeDistribution(t *testing.T) { + script := []*metricsAdjusterTest{{ + "CumulativeDist: round 1 - initial instance, start time is established", + []*metricspb.Metric{metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7})))}, + []*metricspb.Metric{metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7})))}, + 1, + }, { + "CumulativeDist: round 2 - instance adjusted based on round 1", + []*metricspb.Metric{metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.DistPt(t2Ms, bounds0, []int64{6, 3, 4, 8})))}, + []*metricspb.Metric{metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t2Ms, bounds0, []int64{6, 3, 4, 8})))}, + 0, + }, { + "CumulativeDist: round 3 - instance reset (value less than previous value), start time is reset", + []*metricspb.Metric{metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.DistPt(t3Ms, bounds0, []int64{5, 3, 2, 7})))}, + []*metricspb.Metric{metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.DistPt(t3Ms, bounds0, []int64{5, 3, 2, 7})))}, + 1, + }, { + "CumulativeDist: round 4 - instance adjusted based on round 3", + []*metricspb.Metric{metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.DistPt(t4Ms, bounds0, []int64{7, 4, 2, 12})))}, + []*metricspb.Metric{metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.DistPt(t4Ms, bounds0, []int64{7, 4, 2, 12})))}, + 0, + }} + runScript(t, NewJobsMap(time.Minute).get("job", "0"), script) +} + +func Test_summary_no_count(t *testing.T) { + script := []*metricsAdjusterTest{{ + "Summary No Count: round 1 - initial instance, start time is established", + []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.SummPt(t1Ms, 10, 40, percent0, []float64{1, 5, 8})))}, + []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.SummPt(t1Ms, 10, 40, percent0, []float64{1, 5, 8})))}, + 1, + }, { + "Summary No Count: round 2 - instance adjusted based on round 1", + []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.SummPt(t2Ms, 15, 70, percent0, []float64{7, 44, 9})))}, + []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.SummPt(t2Ms, 15, 70, percent0, []float64{7, 44, 9})))}, + 0, + }, { + "Summary No Count: round 3 - instance reset (count less than previous), start time is reset", + []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.SummPt(t3Ms, 12, 66, percent0, []float64{3, 22, 5})))}, + []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.SummPt(t3Ms, 12, 66, percent0, []float64{3, 22, 5})))}, + 1, + }, { + "Summary No Count: round 4 - instance adjusted based on round 3", + []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.SummPt(t4Ms, 14, 96, percent0, []float64{9, 47, 8})))}, + []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.SummPt(t4Ms, 14, 96, percent0, []float64{9, 47, 8})))}, + 0, + }} + + for _, test := range script { + test.metrics[0].GetTimeseries()[0].Points[0].GetSummaryValue().Count = nil + test.adjusted[0].GetTimeseries()[0].Points[0].GetSummaryValue().Count = nil + } + + runScript(t, NewJobsMap(time.Minute).get("job", "0"), script) +} + +func Test_summary(t *testing.T) { + script := []*metricsAdjusterTest{{ + "Summary: round 1 - initial instance, start time is established", + []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.SummPt(t1Ms, 10, 40, percent0, []float64{1, 5, 8})))}, + []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.SummPt(t1Ms, 10, 40, percent0, []float64{1, 5, 8})))}, + 1, + }, { + "Summary: round 2 - instance adjusted based on round 1", + []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.SummPt(t2Ms, 15, 70, percent0, []float64{7, 44, 9})))}, + []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.SummPt(t2Ms, 15, 70, percent0, []float64{7, 44, 9})))}, + 0, + }, { + "Summary: round 3 - instance reset (count less than previous), start time is reset", + []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.SummPt(t3Ms, 12, 66, percent0, []float64{3, 22, 5})))}, + []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.SummPt(t3Ms, 12, 66, percent0, []float64{3, 22, 5})))}, + 1, + }, { + "Summary: round 4 - instance adjusted based on round 3", + []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.SummPt(t4Ms, 14, 96, percent0, []float64{9, 47, 8})))}, + []*metricspb.Metric{metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.SummPt(t4Ms, 14, 96, percent0, []float64{9, 47, 8})))}, + 0, + }} + runScript(t, NewJobsMap(time.Minute).get("job", "0"), script) +} + +func Test_multiMetrics(t *testing.T) { + script := []*metricsAdjusterTest{{ + "MultiMetrics: round 1 - combined round 1 of individual metrics", + []*metricspb.Metric{ + metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44))), + metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7}))), + metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44))), + metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7}))), + metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.SummPt(t1Ms, 10, 40, percent0, []float64{1, 5, 8}))), + }, + []*metricspb.Metric{ + metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44))), + metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7}))), + metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44))), + metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7}))), + metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.SummPt(t1Ms, 10, 40, percent0, []float64{1, 5, 8}))), + }, + 3, + }, { + "MultiMetrics: round 2 - combined round 2 of individual metrics", + []*metricspb.Metric{ + metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.Double(t2Ms, 66))), + metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.DistPt(t2Ms, bounds0, []int64{6, 5, 8, 11}))), + metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.Double(t2Ms, 66))), + metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.DistPt(t2Ms, bounds0, []int64{6, 3, 4, 8}))), + metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.SummPt(t2Ms, 15, 70, percent0, []float64{7, 44, 9}))), + }, + []*metricspb.Metric{ + metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.Double(t2Ms, 66))), + metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.DistPt(t2Ms, bounds0, []int64{6, 5, 8, 11}))), + metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t2Ms, 66))), + metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t2Ms, bounds0, []int64{6, 3, 4, 8}))), + metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.SummPt(t2Ms, 15, 70, percent0, []float64{7, 44, 9}))), + }, + 0, + }, { + "MultiMetrics: round 3 - combined round 3 of individual metrics", + []*metricspb.Metric{ + metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t3Ms, 55))), + metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.DistPt(t3Ms, bounds0, []int64{2, 0, 1, 5}))), + metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t3Ms, 55))), + metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.DistPt(t3Ms, bounds0, []int64{5, 3, 2, 7}))), + metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.SummPt(t3Ms, 12, 66, percent0, []float64{3, 22, 5}))), + }, + []*metricspb.Metric{ + metricstestutil.Gauge(g1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t3Ms, 55))), + metricstestutil.GaugeDist(gd1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.DistPt(t3Ms, bounds0, []int64{2, 0, 1, 5}))), + metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t3Ms, 55))), + metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.DistPt(t3Ms, bounds0, []int64{5, 3, 2, 7}))), + metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.SummPt(t3Ms, 12, 66, percent0, []float64{3, 22, 5}))), + }, + 3, + }, { + "MultiMetrics: round 4 - combined round 4 of individual metrics", + []*metricspb.Metric{ + metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.Double(t4Ms, 72))), + metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.DistPt(t4Ms, bounds0, []int64{7, 4, 2, 12}))), + metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.SummPt(t4Ms, 14, 96, percent0, []float64{9, 47, 8}))), + }, + []*metricspb.Metric{ + metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t4Ms, 72))), + metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.DistPt(t4Ms, bounds0, []int64{7, 4, 2, 12}))), + metricstestutil.Summary(s1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.SummPt(t4Ms, 14, 96, percent0, []float64{9, 47, 8}))), + }, + 0, + }} + runScript(t, NewJobsMap(time.Minute).get("job", "0"), script) +} + +func Test_multiTimeseries(t *testing.T) { + script := []*metricsAdjusterTest{{ + "MultiTimeseries: round 1 - initial first instance, start time is established", + []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44)))}, + []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44)))}, + 1, + }, { + "MultiTimeseries: round 2 - first instance adjusted based on round 1, initial second instance", + []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.Double(t2Ms, 66)), metricstestutil.Timeseries(t2Ms, v10v20, metricstestutil.Double(t2Ms, 20)))}, + []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t2Ms, 66)), metricstestutil.Timeseries(t2Ms, v10v20, metricstestutil.Double(t2Ms, 20)))}, + 1, + }, { + "MultiTimeseries: round 3 - first instance adjusted based on round 1, second based on round 2", + []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t3Ms, 88)), metricstestutil.Timeseries(t3Ms, v10v20, metricstestutil.Double(t3Ms, 49)))}, + []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t3Ms, 88)), metricstestutil.Timeseries(t2Ms, v10v20, metricstestutil.Double(t3Ms, 49)))}, + 0, + }, { + "MultiTimeseries: round 4 - first instance reset, second instance adjusted based on round 2, initial third instance", + []*metricspb.Metric{ + metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.Double(t4Ms, 87)), metricstestutil.Timeseries(t4Ms, v10v20, metricstestutil.Double(t4Ms, 57)), metricstestutil.Timeseries(t4Ms, v100v200, metricstestutil.Double(t4Ms, 10)))}, + []*metricspb.Metric{ + metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.Double(t4Ms, 87)), metricstestutil.Timeseries(t2Ms, v10v20, metricstestutil.Double(t4Ms, 57)), metricstestutil.Timeseries(t4Ms, v100v200, metricstestutil.Double(t4Ms, 10)))}, + 2, + }, { + "MultiTimeseries: round 5 - first instance adjusted based on round 4, second on round 2, third on round 4", + []*metricspb.Metric{ + metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t5Ms, v1v2, metricstestutil.Double(t5Ms, 90)), metricstestutil.Timeseries(t5Ms, v10v20, metricstestutil.Double(t5Ms, 65)), metricstestutil.Timeseries(t5Ms, v100v200, metricstestutil.Double(t5Ms, 22)))}, + []*metricspb.Metric{ + metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.Double(t5Ms, 90)), metricstestutil.Timeseries(t2Ms, v10v20, metricstestutil.Double(t5Ms, 65)), metricstestutil.Timeseries(t4Ms, v100v200, metricstestutil.Double(t5Ms, 22)))}, + 0, + }} + runScript(t, NewJobsMap(time.Minute).get("job", "0"), script) +} + +func Test_emptyLabels(t *testing.T) { + script := []*metricsAdjusterTest{{ + "EmptyLabels: round 1 - initial instance, implicitly empty labels, start time is established", + []*metricspb.Metric{metricstestutil.Cumulative(c1, []string{}, metricstestutil.Timeseries(t1Ms, []string{}, metricstestutil.Double(t1Ms, 44)))}, + []*metricspb.Metric{metricstestutil.Cumulative(c1, []string{}, metricstestutil.Timeseries(t1Ms, []string{}, metricstestutil.Double(t1Ms, 44)))}, + 1, + }, { + "EmptyLabels: round 2 - instance adjusted based on round 1", + []*metricspb.Metric{metricstestutil.Cumulative(c1, []string{}, metricstestutil.Timeseries(t2Ms, []string{}, metricstestutil.Double(t2Ms, 66)))}, + []*metricspb.Metric{metricstestutil.Cumulative(c1, []string{}, metricstestutil.Timeseries(t1Ms, []string{}, metricstestutil.Double(t2Ms, 66)))}, + 0, + }, { + "EmptyLabels: round 3 - one explicitly empty label, instance adjusted based on round 1", + []*metricspb.Metric{metricstestutil.Cumulative(c1, k1, metricstestutil.Timeseries(t3Ms, []string{""}, metricstestutil.Double(t3Ms, 77)))}, + []*metricspb.Metric{metricstestutil.Cumulative(c1, k1, metricstestutil.Timeseries(t1Ms, []string{""}, metricstestutil.Double(t3Ms, 77)))}, + 0, + }, { + "EmptyLabels: round 4 - three explicitly empty labels, instance adjusted based on round 1", + []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2k3, metricstestutil.Timeseries(t3Ms, []string{"", "", ""}, metricstestutil.Double(t3Ms, 88)))}, + []*metricspb.Metric{metricstestutil.Cumulative(c1, k1k2k3, metricstestutil.Timeseries(t1Ms, []string{"", "", ""}, metricstestutil.Double(t3Ms, 88)))}, + 0, + }} + runScript(t, NewJobsMap(time.Minute).get("job", "0"), script) +} + +func Test_tsGC(t *testing.T) { + script1 := []*metricsAdjusterTest{{ + "TsGC: round 1 - initial instances, start time is established", + []*metricspb.Metric{ + metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44)), metricstestutil.Timeseries(t1Ms, v10v20, metricstestutil.Double(t1Ms, 20))), + metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7})), metricstestutil.Timeseries(t1Ms, v10v20, metricstestutil.DistPt(t1Ms, bounds0, []int64{40, 20, 30, 70}))), + }, + []*metricspb.Metric{ + metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44)), metricstestutil.Timeseries(t1Ms, v10v20, metricstestutil.Double(t1Ms, 20))), + metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7})), metricstestutil.Timeseries(t1Ms, v10v20, metricstestutil.DistPt(t1Ms, bounds0, []int64{40, 20, 30, 70}))), + }, + 4, + }} + + script2 := []*metricsAdjusterTest{{ + "TsGC: round 2 - metrics first timeseries adjusted based on round 2, second timeseries not updated", + []*metricspb.Metric{ + metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.Double(t2Ms, 88))), + metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t2Ms, v1v2, metricstestutil.DistPt(t2Ms, bounds0, []int64{8, 7, 9, 14}))), + }, + []*metricspb.Metric{ + metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t2Ms, 88))), + metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t2Ms, bounds0, []int64{8, 7, 9, 14}))), + }, + 0, + }} + + script3 := []*metricsAdjusterTest{{ + "TsGC: round 3 - metrics first timeseries adjusted based on round 2, second timeseries empty due to timeseries gc()", + []*metricspb.Metric{ + metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.Double(t3Ms, 99)), metricstestutil.Timeseries(t3Ms, v10v20, metricstestutil.Double(t3Ms, 80))), + metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t3Ms, v1v2, metricstestutil.DistPt(t3Ms, bounds0, []int64{9, 8, 10, 15})), metricstestutil.Timeseries(t3Ms, v10v20, metricstestutil.DistPt(t3Ms, bounds0, []int64{55, 66, 33, 77}))), + }, + []*metricspb.Metric{ + metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t3Ms, 99)), metricstestutil.Timeseries(t3Ms, v10v20, metricstestutil.Double(t3Ms, 80))), + metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t3Ms, bounds0, []int64{9, 8, 10, 15})), metricstestutil.Timeseries(t3Ms, v10v20, metricstestutil.DistPt(t3Ms, bounds0, []int64{55, 66, 33, 77}))), + }, + 2, + }} + + jobsMap := NewJobsMap(time.Minute) + + // run round 1 + runScript(t, jobsMap.get("job", "0"), script1) + // gc the tsmap, unmarking all entries + jobsMap.get("job", "0").gc() + // run round 2 - update metrics first timeseries only + runScript(t, jobsMap.get("job", "0"), script2) + // gc the tsmap, collecting umarked entries + jobsMap.get("job", "0").gc() + // run round 3 - verify that metrics second timeseries have been gc'd + runScript(t, jobsMap.get("job", "0"), script3) +} + +func Test_jobGC(t *testing.T) { + job1Script1 := []*metricsAdjusterTest{{ + "JobGC: job 1, round 1 - initial instances, adjusted should be empty", + []*metricspb.Metric{ + metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44)), metricstestutil.Timeseries(t1Ms, v10v20, metricstestutil.Double(t1Ms, 20))), + metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7})), metricstestutil.Timeseries(t1Ms, v10v20, metricstestutil.DistPt(t1Ms, bounds0, []int64{40, 20, 30, 70}))), + }, + []*metricspb.Metric{ + metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.Double(t1Ms, 44)), metricstestutil.Timeseries(t1Ms, v10v20, metricstestutil.Double(t1Ms, 20))), + metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t1Ms, v1v2, metricstestutil.DistPt(t1Ms, bounds0, []int64{4, 2, 3, 7})), metricstestutil.Timeseries(t1Ms, v10v20, metricstestutil.DistPt(t1Ms, bounds0, []int64{40, 20, 30, 70}))), + }, + 4, + }} + + job2Script1 := []*metricsAdjusterTest{{ + "JobGC: job2, round 1 - no metrics adjusted, just trigger gc", + []*metricspb.Metric{}, + []*metricspb.Metric{}, + 0, + }} + + job1Script2 := []*metricsAdjusterTest{{ + "JobGC: job 1, round 2 - metrics timeseries empty due to job-level gc", + []*metricspb.Metric{ + metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.Double(t4Ms, 99)), metricstestutil.Timeseries(t4Ms, v10v20, metricstestutil.Double(t4Ms, 80))), + metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.DistPt(t4Ms, bounds0, []int64{9, 8, 10, 15})), metricstestutil.Timeseries(t4Ms, v10v20, metricstestutil.DistPt(t4Ms, bounds0, []int64{55, 66, 33, 77}))), + }, + []*metricspb.Metric{ + metricstestutil.Cumulative(c1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.Double(t4Ms, 99)), metricstestutil.Timeseries(t4Ms, v10v20, metricstestutil.Double(t4Ms, 80))), + metricstestutil.CumulativeDist(cd1, k1k2, metricstestutil.Timeseries(t4Ms, v1v2, metricstestutil.DistPt(t4Ms, bounds0, []int64{9, 8, 10, 15})), metricstestutil.Timeseries(t4Ms, v10v20, metricstestutil.DistPt(t4Ms, bounds0, []int64{55, 66, 33, 77}))), + }, + 4, + }} + + gcInterval := 10 * time.Millisecond + jobsMap := NewJobsMap(gcInterval) + + // run job 1, round 1 - all entries marked + runScript(t, jobsMap.get("job", "0"), job1Script1) + // sleep longer than gcInterval to enable job gc in the next run + time.Sleep(2 * gcInterval) + // run job 2, round1 - trigger job gc, unmarking all entries + runScript(t, jobsMap.get("job", "1"), job2Script1) + // sleep longer than gcInterval to enable job gc in the next run + time.Sleep(2 * gcInterval) + // re-run job 2, round1 - trigger job gc, removing unmarked entries + runScript(t, jobsMap.get("job", "1"), job2Script1) + // ensure that at least one jobsMap.gc() completed + jobsMap.gc() + // run job 1, round 2 - verify that all job 1 timeseries have been gc'd + runScript(t, jobsMap.get("job", "0"), job1Script2) +} + +var ( + t1Ms = time.Unix(0, 1000000) + t2Ms = time.Unix(0, 2000000) + t3Ms = time.Unix(0, 3000000) + t4Ms = time.Unix(0, 5000000) + t5Ms = time.Unix(0, 5000000) +) + +type metricsAdjusterTest struct { + description string + metrics []*metricspb.Metric + adjusted []*metricspb.Metric + resets int +} + +func runScript(t *testing.T, tsm *timeseriesMap, script []*metricsAdjusterTest) { + l := zap.NewNop() + t.Cleanup(func() { require.NoError(t, l.Sync()) }) // flushes buffer, if any + ma := NewMetricsAdjuster(tsm, l) + + for _, test := range script { + expectedResets := test.resets + adjusted, resets := ma.AdjustMetrics(test.metrics) + assert.EqualValuesf(t, test.adjusted, adjusted, "Test: %v - expected: %v, actual: %v", test.description, test.adjusted, adjusted) + assert.Equalf(t, expectedResets, resets, "Test: %v", test.description) + } +} diff --git a/receiver/prometheusreceiver/internal/metricsbuilder.go b/receiver/prometheusreceiver/internal/metricsbuilder.go new file mode 100644 index 0000000000000..e2c7fddf5ac63 --- /dev/null +++ b/receiver/prometheusreceiver/internal/metricsbuilder.go @@ -0,0 +1,328 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "errors" + "fmt" + "regexp" + "sort" + "strconv" + "strings" + + metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/pkg/textparse" + "go.uber.org/zap" + "google.golang.org/protobuf/types/known/timestamppb" +) + +const ( + metricsSuffixCount = "_count" + metricsSuffixBucket = "_bucket" + metricsSuffixSum = "_sum" + metricSuffixTotal = "_total" + startTimeMetricName = "process_start_time_seconds" + scrapeUpMetricName = "up" +) + +var ( + trimmableSuffixes = []string{metricsSuffixBucket, metricsSuffixCount, metricsSuffixSum, metricSuffixTotal} + errNoDataToBuild = errors.New("there's no data to build") + errNoBoundaryLabel = errors.New("given metricType has no BucketLabel or QuantileLabel") + errEmptyBoundaryLabel = errors.New("BucketLabel or QuantileLabel is empty") +) + +type metricBuilder struct { + hasData bool + hasInternalMetric bool + mc MetadataCache + metrics []*metricspb.Metric + numTimeseries int + droppedTimeseries int + useStartTimeMetric bool + startTimeMetricRegex *regexp.Regexp + startTime float64 + intervalStartTimeMs int64 + logger *zap.Logger + currentMf MetricFamily +} + +// newMetricBuilder creates a MetricBuilder which is allowed to feed all the datapoints from a single prometheus +// scraped page by calling its AddDataPoint function, and turn them into an opencensus data.MetricsData object +// by calling its Build function +func newMetricBuilder(mc MetadataCache, useStartTimeMetric bool, startTimeMetricRegex string, logger *zap.Logger, intervalStartTimeMs int64) *metricBuilder { + var regex *regexp.Regexp + if startTimeMetricRegex != "" { + regex, _ = regexp.Compile(startTimeMetricRegex) + } + return &metricBuilder{ + mc: mc, + metrics: make([]*metricspb.Metric, 0), + logger: logger, + numTimeseries: 0, + droppedTimeseries: 0, + useStartTimeMetric: useStartTimeMetric, + startTimeMetricRegex: regex, + intervalStartTimeMs: intervalStartTimeMs, + } +} + +func (b *metricBuilder) matchStartTimeMetric(metricName string) bool { + if b.startTimeMetricRegex != nil { + return b.startTimeMetricRegex.MatchString(metricName) + } + + return metricName == startTimeMetricName +} + +// AddDataPoint is for feeding prometheus data complexValue in its processing order +func (b *metricBuilder) AddDataPoint(ls labels.Labels, t int64, v float64) error { + // Any datapoint with duplicate labels MUST be rejected per: + // * https://github.com/open-telemetry/wg-prometheus/issues/44 + // * https://github.com/open-telemetry/opentelemetry-collector/issues/3407 + // as Prometheus rejects such too as of version 2.16.0, released on 2020-02-13. + seen := make(map[string]bool) + dupLabels := make([]string, 0, len(ls)) + for _, label := range ls { + if _, ok := seen[label.Name]; ok { + dupLabels = append(dupLabels, label.Name) + } + seen[label.Name] = true + } + if len(dupLabels) != 0 { + sort.Strings(dupLabels) + return fmt.Errorf("invalid sample: non-unique label names: %q", dupLabels) + } + + metricName := ls.Get(model.MetricNameLabel) + switch { + case metricName == "": + b.numTimeseries++ + b.droppedTimeseries++ + return errMetricNameNotFound + case isInternalMetric(metricName): + b.hasInternalMetric = true + lm := ls.Map() + // See https://www.prometheus.io/docs/concepts/jobs_instances/#automatically-generated-labels-and-time-series + // up: 1 if the instance is healthy, i.e. reachable, or 0 if the scrape failed. + if metricName == scrapeUpMetricName && v != 1.0 { + if v == 0.0 { + b.logger.Warn("Failed to scrape Prometheus endpoint", + zap.Int64("scrape_timestamp", t), + zap.String("target_labels", fmt.Sprintf("%v", lm))) + } else { + b.logger.Warn("The 'up' metric contains invalid value", + zap.Float64("value", v), + zap.Int64("scrape_timestamp", t), + zap.String("target_labels", fmt.Sprintf("%v", lm))) + } + } + case b.useStartTimeMetric && b.matchStartTimeMetric(metricName): + b.startTime = v + } + + b.hasData = true + + if b.currentMf != nil && !b.currentMf.IsSameFamily(metricName) { + m, ts, dts := b.currentMf.ToMetric() + b.numTimeseries += ts + b.droppedTimeseries += dts + if m != nil { + b.metrics = append(b.metrics, m) + } + b.currentMf = newMetricFamily(metricName, b.mc, b.logger, b.intervalStartTimeMs) + } else if b.currentMf == nil { + b.currentMf = newMetricFamily(metricName, b.mc, b.logger, b.intervalStartTimeMs) + } + + return b.currentMf.Add(metricName, ls, t, v) +} + +// Build an opencensus data.MetricsData based on all added data complexValue. +// The only error returned by this function is errNoDataToBuild. +func (b *metricBuilder) Build() ([]*metricspb.Metric, int, int, error) { + if !b.hasData { + if b.hasInternalMetric { + return make([]*metricspb.Metric, 0), 0, 0, nil + } + return nil, 0, 0, errNoDataToBuild + } + + if b.currentMf != nil { + m, ts, dts := b.currentMf.ToMetric() + b.numTimeseries += ts + b.droppedTimeseries += dts + if m != nil { + b.metrics = append(b.metrics, m) + } + b.currentMf = nil + } + + return b.metrics, b.numTimeseries, b.droppedTimeseries, nil +} + +// TODO: move the following helper functions to a proper place, as they are not called directly in this go file + +func isUsefulLabel(mType metricspb.MetricDescriptor_Type, labelKey string) bool { + switch labelKey { + case model.MetricNameLabel, model.InstanceLabel, model.SchemeLabel, model.MetricsPathLabel, model.JobLabel: + return false + case model.BucketLabel: + return mType != metricspb.MetricDescriptor_GAUGE_DISTRIBUTION && + mType != metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION + case model.QuantileLabel: + return mType != metricspb.MetricDescriptor_SUMMARY + } + return true +} + +// dpgSignature is used to create a key for data complexValue belong to a same group of a metric family +func dpgSignature(orderedKnownLabelKeys []string, ls labels.Labels) string { + size := 0 + for _, k := range orderedKnownLabelKeys { + v := ls.Get(k) + if v == "" { + continue + } + // 2 enclosing quotes + 1 equality sign = 3 extra chars. + // Note: if any character in the label value requires escaping, + // we'll need more space than that, which will lead to some + // extra allocation. + size += 3 + len(k) + len(v) + } + sign := make([]byte, 0, size) + for _, k := range orderedKnownLabelKeys { + v := ls.Get(k) + if v == "" { + continue + } + sign = strconv.AppendQuote(sign, k+"="+v) + } + return string(sign) +} + +func normalizeMetricName(name string) string { + for _, s := range trimmableSuffixes { + if strings.HasSuffix(name, s) && name != s { + return strings.TrimSuffix(name, s) + } + } + return name +} + +func getBoundary(metricType metricspb.MetricDescriptor_Type, labels labels.Labels) (float64, error) { + labelName := "" + switch metricType { + case metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, + metricspb.MetricDescriptor_GAUGE_DISTRIBUTION: + labelName = model.BucketLabel + case metricspb.MetricDescriptor_SUMMARY: + labelName = model.QuantileLabel + default: + return 0, errNoBoundaryLabel + } + + v := labels.Get(labelName) + if v == "" { + return 0, errEmptyBoundaryLabel + } + + return strconv.ParseFloat(v, 64) +} + +func convToOCAMetricType(metricType textparse.MetricType) metricspb.MetricDescriptor_Type { + switch metricType { + case textparse.MetricTypeCounter: + // always use float64, as it's the internal data type used in prometheus + return metricspb.MetricDescriptor_CUMULATIVE_DOUBLE + // textparse.MetricTypeUnknown is converted to gauge by default to fix Prometheus untyped metrics from being dropped + case textparse.MetricTypeGauge, textparse.MetricTypeUnknown: + return metricspb.MetricDescriptor_GAUGE_DOUBLE + case textparse.MetricTypeHistogram: + return metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION + // dropping support for gaugehistogram for now until we have an official spec of its implementation + // a draft can be found in: https://docs.google.com/document/d/1KwV0mAXwwbvvifBvDKH_LU1YjyXE_wxCkHNoCGq1GX0/edit#heading=h.1cvzqd4ksd23 + // case textparse.MetricTypeGaugeHistogram: + // return metricspb.MetricDescriptor_GAUGE_DISTRIBUTION + case textparse.MetricTypeSummary: + return metricspb.MetricDescriptor_SUMMARY + default: + // including: textparse.MetricTypeInfo, textparse.MetricTypeStateset + return metricspb.MetricDescriptor_UNSPECIFIED + } +} + +/* + code borrowed from the original promreceiver +*/ + +func heuristicalMetricAndKnownUnits(metricName, parsedUnit string) string { + if parsedUnit != "" { + return parsedUnit + } + lastUnderscoreIndex := strings.LastIndex(metricName, "_") + if lastUnderscoreIndex <= 0 || lastUnderscoreIndex >= len(metricName)-1 { + return "" + } + + unit := "" + + supposedUnit := metricName[lastUnderscoreIndex+1:] + switch strings.ToLower(supposedUnit) { + case "millisecond", "milliseconds", "ms": + unit = "ms" + case "second", "seconds", "s": + unit = "s" + case "microsecond", "microseconds", "us": + unit = "us" + case "nanosecond", "nanoseconds", "ns": + unit = "ns" + case "byte", "bytes", "by": + unit = "By" + case "bit", "bits": + unit = "Bi" + case "kilogram", "kilograms", "kg": + unit = "kg" + case "gram", "grams", "g": + unit = "g" + case "meter", "meters", "metre", "metres", "m": + unit = "m" + case "kilometer", "kilometers", "kilometre", "kilometres", "km": + unit = "km" + case "milimeter", "milimeters", "milimetre", "milimetres", "mm": + unit = "mm" + case "nanogram", "ng", "nanograms": + unit = "ng" + } + + return unit +} + +func timestampFromMs(timeAtMs int64) *timestamppb.Timestamp { + secs, ns := timeAtMs/1e3, (timeAtMs%1e3)*1e6 + return ×tamppb.Timestamp{ + Seconds: secs, + Nanos: int32(ns), + } +} + +func isInternalMetric(metricName string) bool { + if metricName == scrapeUpMetricName || strings.HasPrefix(metricName, "scrape_") { + return true + } + return false +} diff --git a/receiver/prometheusreceiver/internal/metricsbuilder_test.go b/receiver/prometheusreceiver/internal/metricsbuilder_test.go new file mode 100644 index 0000000000000..46e6cbcd268ec --- /dev/null +++ b/receiver/prometheusreceiver/internal/metricsbuilder_test.go @@ -0,0 +1,1501 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "reflect" + "runtime" + "testing" + + metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/pkg/textparse" + "github.com/prometheus/prometheus/scrape" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +const startTs = int64(1555366610000) +const interval = int64(15 * 1000) +const defaultBuilderStartTime = float64(1.0) + +var testMetadata = map[string]scrape.MetricMetadata{ + "counter_test": {Metric: "counter_test", Type: textparse.MetricTypeCounter, Help: "", Unit: ""}, + "counter_test2": {Metric: "counter_test2", Type: textparse.MetricTypeCounter, Help: "", Unit: ""}, + "gauge_test": {Metric: "gauge_test", Type: textparse.MetricTypeGauge, Help: "", Unit: ""}, + "gauge_test2": {Metric: "gauge_test2", Type: textparse.MetricTypeGauge, Help: "", Unit: ""}, + "hist_test": {Metric: "hist_test", Type: textparse.MetricTypeHistogram, Help: "", Unit: ""}, + "hist_test2": {Metric: "hist_test2", Type: textparse.MetricTypeHistogram, Help: "", Unit: ""}, + "ghist_test": {Metric: "ghist_test", Type: textparse.MetricTypeGaugeHistogram, Help: "", Unit: ""}, + "summary_test": {Metric: "summary_test", Type: textparse.MetricTypeSummary, Help: "", Unit: ""}, + "summary_test2": {Metric: "summary_test2", Type: textparse.MetricTypeSummary, Help: "", Unit: ""}, + "unknown_test": {Metric: "unknown_test", Type: textparse.MetricTypeUnknown, Help: "", Unit: ""}, + "poor_name_count": {Metric: "poor_name_count", Type: textparse.MetricTypeCounter, Help: "", Unit: ""}, + "up": {Metric: "up", Type: textparse.MetricTypeCounter, Help: "", Unit: ""}, + "scrape_foo": {Metric: "scrape_foo", Type: textparse.MetricTypeCounter, Help: "", Unit: ""}, + "example_process_start_time_seconds": {Metric: "example_process_start_time_seconds", + Type: textparse.MetricTypeGauge, Help: "", Unit: ""}, + "process_start_time_seconds": {Metric: "process_start_time_seconds", + Type: textparse.MetricTypeGauge, Help: "", Unit: ""}, + "badprocess_start_time_seconds": {Metric: "badprocess_start_time_seconds", + Type: textparse.MetricTypeGauge, Help: "", Unit: ""}, +} + +type testDataPoint struct { + lb labels.Labels + t int64 + v float64 +} + +type testScrapedPage struct { + pts []*testDataPoint +} + +type buildTestData struct { + name string + inputs []*testScrapedPage + wants [][]*metricspb.Metric +} + +func createLabels(mFamily string, tagPairs ...string) labels.Labels { + lm := make(map[string]string) + lm[model.MetricNameLabel] = mFamily + if len(tagPairs)%2 != 0 { + panic("tag pairs is not even") + } + + for i := 0; i < len(tagPairs); i += 2 { + lm[tagPairs[i]] = tagPairs[i+1] + } + + return labels.FromMap(lm) +} + +func createDataPoint(mname string, value float64, tagPairs ...string) *testDataPoint { + return &testDataPoint{ + lb: createLabels(mname, tagPairs...), + v: value, + } +} + +func runBuilderTests(t *testing.T, tests []buildTestData) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.EqualValues(t, len(tt.wants), len(tt.inputs)) + mc := newMockMetadataCache(testMetadata) + st := startTs + for i, page := range tt.inputs { + b := newMetricBuilder(mc, true, "", testTelemetry.ToReceiverCreateSettings().Logger, startTs) + b.startTime = defaultBuilderStartTime // set to a non-zero value + for _, pt := range page.pts { + // set ts for testing + pt.t = st + assert.NoError(t, b.AddDataPoint(pt.lb, pt.t, pt.v)) + } + metrics, _, _, err := b.Build() + assert.NoError(t, err) + assert.EqualValues(t, tt.wants[i], metrics) + st += interval + } + }) + } +} + +func runBuilderStartTimeTests(t *testing.T, tests []buildTestData, + startTimeMetricRegex string, expectedBuilderStartTime float64) { + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mc := newMockMetadataCache(testMetadata) + st := startTs + for _, page := range tt.inputs { + b := newMetricBuilder(mc, true, startTimeMetricRegex, + testTelemetry.Logger, 0) + b.startTime = defaultBuilderStartTime // set to a non-zero value + for _, pt := range page.pts { + // set ts for testing + pt.t = st + assert.NoError(t, b.AddDataPoint(pt.lb, pt.t, pt.v)) + } + _, _, _, err := b.Build() + assert.NoError(t, err) + assert.EqualValues(t, b.startTime, expectedBuilderStartTime) + st += interval + } + }) + } +} + +func Test_startTimeMetricMatch(t *testing.T) { + matchBuilderStartTime := 123.456 + matchTests := []buildTestData{ + { + name: "prefix_match", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("example_process_start_time_seconds", + matchBuilderStartTime, "foo", "bar"), + }, + }, + }, + }, + { + name: "match", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("process_start_time_seconds", + matchBuilderStartTime, "foo", "bar"), + }, + }, + }, + }, + } + nomatchTests := []buildTestData{ + { + name: "nomatch1", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("_process_start_time_seconds", + matchBuilderStartTime, "foo", "bar"), + }, + }, + }, + }, + { + name: "nomatch2", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("subprocess_start_time_seconds", + matchBuilderStartTime, "foo", "bar"), + }, + }, + }, + }, + } + + runBuilderStartTimeTests(t, matchTests, "^(.+_)*process_start_time_seconds$", matchBuilderStartTime) + runBuilderStartTimeTests(t, nomatchTests, "^(.+_)*process_start_time_seconds$", defaultBuilderStartTime) +} + +func Test_metricBuilder_counters(t *testing.T) { + tests := []buildTestData{ + { + name: "single-item", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("counter_test", 100, "foo", "bar"), + }, + }, + }, + wants: [][]*metricspb.Metric{ + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "counter_test", + Type: metricspb.MetricDescriptor_CUMULATIVE_DOUBLE, + LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: timestampFromMs(startTs), + LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 100.0}}, + }, + }, + }, + }, + }, + }, + }, + // Some counters such as "python_gc_collections_total" have metadata key as "python_gc_collections" but still need + // to be converted using full metric name as "python_gc_collections_total" to match Prometheus functionality + { + name: "counter-with-metadata-without-total-suffix", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("counter_test_total", 100, "foo", "bar"), + }, + }, + }, + wants: [][]*metricspb.Metric{ + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "counter_test_total", + Type: metricspb.MetricDescriptor_CUMULATIVE_DOUBLE, + LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: timestampFromMs(startTs), + LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 100.0}}, + }, + }, + }, + }, + }, + }, + }, + { + name: "two-items", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("counter_test", 150, "foo", "bar"), + createDataPoint("counter_test", 25, "foo", "other"), + }, + }, + }, + wants: [][]*metricspb.Metric{ + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "counter_test", + Type: metricspb.MetricDescriptor_CUMULATIVE_DOUBLE, + LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: timestampFromMs(startTs), + LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 150.0}}, + }, + }, + { + StartTimestamp: timestampFromMs(startTs), + LabelValues: []*metricspb.LabelValue{{Value: "other", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 25.0}}, + }, + }, + }, + }, + }, + }, + }, + { + name: "two-metrics", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("counter_test", 150, "foo", "bar"), + createDataPoint("counter_test", 25, "foo", "other"), + createDataPoint("counter_test2", 100, "foo", "bar"), + }, + }, + }, + wants: [][]*metricspb.Metric{ + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "counter_test", + Type: metricspb.MetricDescriptor_CUMULATIVE_DOUBLE, + LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: timestampFromMs(startTs), + LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 150.0}}, + }, + }, + { + StartTimestamp: timestampFromMs(startTs), + LabelValues: []*metricspb.LabelValue{{Value: "other", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 25.0}}, + }, + }, + }, + }, + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "counter_test2", + Type: metricspb.MetricDescriptor_CUMULATIVE_DOUBLE, + LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: timestampFromMs(startTs), + LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 100.0}}, + }, + }, + }, + }, + }, + }, + }, + { + name: "metrics-with-poor-names", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("poor_name_count", 100, "foo", "bar"), + }, + }, + }, + wants: [][]*metricspb.Metric{ + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "poor_name_count", + Type: metricspb.MetricDescriptor_CUMULATIVE_DOUBLE, + LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: timestampFromMs(startTs), + LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 100.0}}, + }, + }, + }, + }, + }, + }, + }, + } + + runBuilderTests(t, tests) +} + +func Test_metricBuilder_gauges(t *testing.T) { + tests := []buildTestData{ + { + name: "one-gauge", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("gauge_test", 100, "foo", "bar"), + }, + }, + { + pts: []*testDataPoint{ + createDataPoint("gauge_test", 90, "foo", "bar"), + }, + }, + }, + wants: [][]*metricspb.Metric{ + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "gauge_test", + Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, + LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 100.0}}, + }, + }, + }, + }, + }, + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "gauge_test", + Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, + LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs + interval), Value: &metricspb.Point_DoubleValue{DoubleValue: 90.0}}, + }, + }, + }, + }, + }, + }, + }, + { + name: "gauge-with-different-tags", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("gauge_test", 100, "foo", "bar"), + createDataPoint("gauge_test", 200, "bar", "foo"), + }, + }, + }, + wants: [][]*metricspb.Metric{ + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "gauge_test", + Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, + LabelKeys: []*metricspb.LabelKey{{Key: "bar"}, {Key: "foo"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + LabelValues: []*metricspb.LabelValue{{Value: "", HasValue: false}, {Value: "bar", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 100.0}}, + }, + }, + { + LabelValues: []*metricspb.LabelValue{{Value: "foo", HasValue: true}, {Value: "", HasValue: false}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 200.0}}, + }, + }, + }, + }, + }, + }, + }, + { + // TODO: A decision need to be made. If we want to have the behavior which can generate different tag key + // sets because metrics come and go + name: "gauge-comes-and-go-with-different-tagset", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("gauge_test", 100, "foo", "bar"), + createDataPoint("gauge_test", 200, "bar", "foo"), + }, + }, + { + pts: []*testDataPoint{ + createDataPoint("gauge_test", 20, "foo", "bar"), + }, + }, + }, + wants: [][]*metricspb.Metric{ + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "gauge_test", + Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, + LabelKeys: []*metricspb.LabelKey{{Key: "bar"}, {Key: "foo"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + LabelValues: []*metricspb.LabelValue{{Value: "", HasValue: false}, {Value: "bar", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 100.0}}, + }, + }, + { + LabelValues: []*metricspb.LabelValue{{Value: "foo", HasValue: true}, {Value: "", HasValue: false}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 200.0}}, + }, + }, + }, + }, + }, + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "gauge_test", + Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, + LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs + interval), Value: &metricspb.Point_DoubleValue{DoubleValue: 20.0}}, + }, + }, + }, + }, + }, + }, + }, + } + + runBuilderTests(t, tests) +} + +func Test_metricBuilder_untype(t *testing.T) { + tests := []buildTestData{ + { + name: "one-unknown", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("unknown_test", 100, "foo", "bar"), + }, + }, + }, + wants: [][]*metricspb.Metric{ + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "unknown_test", + Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, + LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 100.0}}, + }, + }, + }, + }, + }, + }, + }, + { + name: "no-type-hint", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("something_not_exists", 100, "foo", "bar"), + createDataPoint("theother_not_exists", 200, "foo", "bar"), + createDataPoint("theother_not_exists", 300, "bar", "foo"), + }, + }, + }, + wants: [][]*metricspb.Metric{ + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "something_not_exists", + Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, + LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 100.0}}, + }, + }, + }, + }, + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "theother_not_exists", + Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, + LabelKeys: []*metricspb.LabelKey{{Key: "bar"}, {Key: "foo"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + LabelValues: []*metricspb.LabelValue{{Value: "", HasValue: false}, {Value: "bar", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 200.0}}, + }, + }, + { + LabelValues: []*metricspb.LabelValue{{Value: "foo", HasValue: true}, {Value: "", HasValue: false}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 300.0}}, + }, + }, + }, + }, + }, + }, + }, + { + name: "untype-metric-poor-names", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("some_count", 100, "foo", "bar"), + }, + }, + }, + wants: [][]*metricspb.Metric{ + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "some_count", + Type: metricspb.MetricDescriptor_GAUGE_DOUBLE, + LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DoubleValue{DoubleValue: 100.0}}, + }, + }, + }, + }, + }, + }, + }, + } + + runBuilderTests(t, tests) +} + +func Test_metricBuilder_histogram(t *testing.T) { + tests := []buildTestData{ + { + name: "single item", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("hist_test", 1, "foo", "bar", "le", "10"), + createDataPoint("hist_test", 2, "foo", "bar", "le", "20"), + createDataPoint("hist_test", 10, "foo", "bar", "le", "+inf"), + createDataPoint("hist_test_sum", 99, "foo", "bar"), + createDataPoint("hist_test_count", 10, "foo", "bar"), + }, + }, + }, + wants: [][]*metricspb.Metric{ + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "hist_test", + Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, + LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: timestampFromMs(startTs), + LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DistributionValue{ + DistributionValue: &metricspb.DistributionValue{ + BucketOptions: &metricspb.DistributionValue_BucketOptions{ + Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ + Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ + Bounds: []float64{10, 20}, + }, + }, + }, + Count: 10, + Sum: 99.0, + Buckets: []*metricspb.DistributionValue_Bucket{{Count: 1}, {Count: 1}, {Count: 8}}, + }}}, + }, + }, + }, + }, + }, + }, + }, + { + name: "multi-groups", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("hist_test", 1, "foo", "bar", "le", "10"), + createDataPoint("hist_test", 2, "foo", "bar", "le", "20"), + createDataPoint("hist_test", 10, "foo", "bar", "le", "+inf"), + createDataPoint("hist_test_sum", 99, "foo", "bar"), + createDataPoint("hist_test_count", 10, "foo", "bar"), + createDataPoint("hist_test", 1, "key2", "v2", "le", "10"), + createDataPoint("hist_test", 2, "key2", "v2", "le", "20"), + createDataPoint("hist_test", 3, "key2", "v2", "le", "+inf"), + createDataPoint("hist_test_sum", 50, "key2", "v2"), + createDataPoint("hist_test_count", 3, "key2", "v2"), + }, + }, + }, + wants: [][]*metricspb.Metric{ + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "hist_test", + Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, + LabelKeys: []*metricspb.LabelKey{{Key: "foo"}, {Key: "key2"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: timestampFromMs(startTs), + LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}, {Value: "", HasValue: false}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DistributionValue{ + DistributionValue: &metricspb.DistributionValue{ + BucketOptions: &metricspb.DistributionValue_BucketOptions{ + Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ + Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ + Bounds: []float64{10, 20}, + }, + }, + }, + Count: 10, + Sum: 99.0, + Buckets: []*metricspb.DistributionValue_Bucket{{Count: 1}, {Count: 1}, {Count: 8}}, + }}}, + }, + }, + { + StartTimestamp: timestampFromMs(startTs), + LabelValues: []*metricspb.LabelValue{{Value: "", HasValue: false}, {Value: "v2", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DistributionValue{ + DistributionValue: &metricspb.DistributionValue{ + BucketOptions: &metricspb.DistributionValue_BucketOptions{ + Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ + Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ + Bounds: []float64{10, 20}, + }, + }, + }, + Count: 3, + Sum: 50.0, + Buckets: []*metricspb.DistributionValue_Bucket{{Count: 1}, {Count: 1}, {Count: 1}}, + }}}, + }, + }, + }, + }, + }, + }, + }, + { + name: "multi-groups-and-families", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("hist_test", 1, "foo", "bar", "le", "10"), + createDataPoint("hist_test", 2, "foo", "bar", "le", "20"), + createDataPoint("hist_test", 10, "foo", "bar", "le", "+inf"), + createDataPoint("hist_test_sum", 99, "foo", "bar"), + createDataPoint("hist_test_count", 10, "foo", "bar"), + createDataPoint("hist_test", 1, "key2", "v2", "le", "10"), + createDataPoint("hist_test", 2, "key2", "v2", "le", "20"), + createDataPoint("hist_test", 3, "key2", "v2", "le", "+inf"), + createDataPoint("hist_test_sum", 50, "key2", "v2"), + createDataPoint("hist_test_count", 3, "key2", "v2"), + createDataPoint("hist_test2", 1, "le", "10"), + createDataPoint("hist_test2", 2, "le", "20"), + createDataPoint("hist_test2", 3, "le", "+inf"), + createDataPoint("hist_test2_sum", 50), + createDataPoint("hist_test2_count", 3), + }, + }, + }, + wants: [][]*metricspb.Metric{ + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "hist_test", + Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, + LabelKeys: []*metricspb.LabelKey{{Key: "foo"}, {Key: "key2"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: timestampFromMs(startTs), + LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}, {Value: "", HasValue: false}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DistributionValue{ + DistributionValue: &metricspb.DistributionValue{ + BucketOptions: &metricspb.DistributionValue_BucketOptions{ + Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ + Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ + Bounds: []float64{10, 20}, + }, + }, + }, + Count: 10, + Sum: 99.0, + Buckets: []*metricspb.DistributionValue_Bucket{{Count: 1}, {Count: 1}, {Count: 8}}, + }}}, + }, + }, + { + StartTimestamp: timestampFromMs(startTs), + LabelValues: []*metricspb.LabelValue{{Value: "", HasValue: false}, {Value: "v2", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DistributionValue{ + DistributionValue: &metricspb.DistributionValue{ + BucketOptions: &metricspb.DistributionValue_BucketOptions{ + Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ + Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ + Bounds: []float64{10, 20}, + }, + }, + }, + Count: 3, + Sum: 50.0, + Buckets: []*metricspb.DistributionValue_Bucket{{Count: 1}, {Count: 1}, {Count: 1}}, + }}}, + }, + }, + }, + }, + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "hist_test2", + Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, + LabelKeys: []*metricspb.LabelKey{}}, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: timestampFromMs(startTs), + LabelValues: []*metricspb.LabelValue{}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DistributionValue{ + DistributionValue: &metricspb.DistributionValue{ + BucketOptions: &metricspb.DistributionValue_BucketOptions{ + Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ + Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ + Bounds: []float64{10, 20}, + }, + }, + }, + Count: 3, + Sum: 50.0, + Buckets: []*metricspb.DistributionValue_Bucket{{Count: 1}, {Count: 1}, {Count: 1}}, + }}}, + }, + }, + }, + }, + }, + }, + }, + { + name: "unordered-buckets", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("hist_test", 10, "foo", "bar", "le", "+inf"), + createDataPoint("hist_test", 1, "foo", "bar", "le", "10"), + createDataPoint("hist_test", 2, "foo", "bar", "le", "20"), + createDataPoint("hist_test_sum", 99, "foo", "bar"), + createDataPoint("hist_test_count", 10, "foo", "bar"), + }, + }, + }, + wants: [][]*metricspb.Metric{ + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "hist_test", + Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, + LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: timestampFromMs(startTs), + LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DistributionValue{ + DistributionValue: &metricspb.DistributionValue{ + BucketOptions: &metricspb.DistributionValue_BucketOptions{ + Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ + Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ + Bounds: []float64{10, 20}, + }, + }, + }, + Count: 10, + Sum: 99.0, + Buckets: []*metricspb.DistributionValue_Bucket{{Count: 1}, {Count: 1}, {Count: 8}}, + }}}, + }, + }, + }, + }, + }, + }, + }, + { + // this won't likely happen in real env, as prometheus wont generate histogram with less than 3 buckets + name: "only-one-bucket", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("hist_test", 3, "le", "+inf"), + createDataPoint("hist_test_count", 3), + createDataPoint("hist_test_sum", 100), + }, + }, + }, + wants: [][]*metricspb.Metric{ + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "hist_test", + Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, + LabelKeys: []*metricspb.LabelKey{}}, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: timestampFromMs(startTs), + LabelValues: []*metricspb.LabelValue{}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DistributionValue{ + DistributionValue: &metricspb.DistributionValue{ + BucketOptions: &metricspb.DistributionValue_BucketOptions{ + Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ + Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ + Bounds: []float64{}, + }, + }, + }, + Count: 3, + Sum: 100, + Buckets: []*metricspb.DistributionValue_Bucket{{Count: 3}}, + }}}, + }, + }, + }, + }, + }, + }, + }, + { + // this won't likely happen in real env, as prometheus wont generate histogram with less than 3 buckets + name: "only-one-bucket-noninf", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("hist_test", 3, "le", "20"), + createDataPoint("hist_test_count", 3), + createDataPoint("hist_test_sum", 100), + }, + }, + }, + wants: [][]*metricspb.Metric{ + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "hist_test", + Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, + LabelKeys: []*metricspb.LabelKey{}}, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: timestampFromMs(startTs), + LabelValues: []*metricspb.LabelValue{}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DistributionValue{ + DistributionValue: &metricspb.DistributionValue{ + BucketOptions: &metricspb.DistributionValue_BucketOptions{ + Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ + Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ + Bounds: []float64{}, + }, + }, + }, + Count: 3, + Sum: 100, + Buckets: []*metricspb.DistributionValue_Bucket{{Count: 3}}, + }}}, + }, + }, + }, + }, + }, + }, + }, + { + name: "no-sum", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("hist_test", 1, "foo", "bar", "le", "10"), + createDataPoint("hist_test", 2, "foo", "bar", "le", "20"), + createDataPoint("hist_test", 3, "foo", "bar", "le", "+inf"), + createDataPoint("hist_test_count", 3, "foo", "bar"), + }, + }, + }, + wants: [][]*metricspb.Metric{ + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "hist_test", + Type: metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, + LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: timestampFromMs(startTs), + LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_DistributionValue{ + DistributionValue: &metricspb.DistributionValue{ + BucketOptions: &metricspb.DistributionValue_BucketOptions{ + Type: &metricspb.DistributionValue_BucketOptions_Explicit_{ + Explicit: &metricspb.DistributionValue_BucketOptions_Explicit{ + Bounds: []float64{10, 20}, + }, + }, + }, + Count: 3, + Sum: 0, + Buckets: []*metricspb.DistributionValue_Bucket{{Count: 1}, {Count: 1}, {Count: 1}}, + }}}, + }, + }, + }, + }, + }, + }, + }, + { + name: "corrupted-no-buckets", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("hist_test_sum", 99), + createDataPoint("hist_test_count", 10), + }, + }, + }, + wants: [][]*metricspb.Metric{ + {}, + }, + }, + { + name: "corrupted-no-count", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("hist_test", 1, "foo", "bar", "le", "10"), + createDataPoint("hist_test", 2, "foo", "bar", "le", "20"), + createDataPoint("hist_test", 3, "foo", "bar", "le", "+inf"), + createDataPoint("hist_test_sum", 99, "foo", "bar"), + }, + }, + }, + wants: [][]*metricspb.Metric{ + {}, + }, + }, + } + + runBuilderTests(t, tests) +} + +func Test_metricBuilder_summary(t *testing.T) { + tests := []buildTestData{ + { + name: "no-sum-and-count", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("summary_test", 5, "foo", "bar", "quantile", "1"), + }, + }, + }, + wants: [][]*metricspb.Metric{ + {}, + }, + }, + { + name: "no-count", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("summary_test", 1, "foo", "bar", "quantile", "0.5"), + createDataPoint("summary_test", 2, "foo", "bar", "quantile", "0.75"), + createDataPoint("summary_test", 5, "foo", "bar", "quantile", "1"), + createDataPoint("summary_test_sum", 500, "foo", "bar"), + }, + }, + }, + wants: [][]*metricspb.Metric{ + {}, + }, + }, + { + name: "no-sum", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("summary_test", 1, "foo", "bar", "quantile", "0.5"), + createDataPoint("summary_test", 2, "foo", "bar", "quantile", "0.75"), + createDataPoint("summary_test", 5, "foo", "bar", "quantile", "1"), + createDataPoint("summary_test_count", 500, "foo", "bar"), + }, + }, + }, + wants: [][]*metricspb.Metric{ + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "summary_test", + Type: metricspb.MetricDescriptor_SUMMARY, + LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: timestampFromMs(startTs), + LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_SummaryValue{ + SummaryValue: &metricspb.SummaryValue{ + Sum: &wrapperspb.DoubleValue{Value: 0.0}, + Count: &wrapperspb.Int64Value{Value: 500}, + Snapshot: &metricspb.SummaryValue_Snapshot{ + PercentileValues: []*metricspb.SummaryValue_Snapshot_ValueAtPercentile{ + {Percentile: 50.0, Value: 1}, + {Percentile: 75.0, Value: 2}, + {Percentile: 100.0, Value: 5}, + }, + }}}}, + }, + }, + }, + }, + }, + }, + }, + { + name: "empty-quantiles", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("summary_test_sum", 100, "foo", "bar"), + createDataPoint("summary_test_count", 500, "foo", "bar"), + }, + }, + }, + wants: [][]*metricspb.Metric{ + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "summary_test", + Type: metricspb.MetricDescriptor_SUMMARY, + LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: timestampFromMs(startTs), + LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, + Points: []*metricspb.Point{ + { + Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_SummaryValue{ + SummaryValue: &metricspb.SummaryValue{ + Sum: &wrapperspb.DoubleValue{Value: 100.0}, + Count: &wrapperspb.Int64Value{Value: 500}, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "regular-summary", + inputs: []*testScrapedPage{ + { + pts: []*testDataPoint{ + createDataPoint("summary_test", 1, "foo", "bar", "quantile", "0.5"), + createDataPoint("summary_test", 2, "foo", "bar", "quantile", "0.75"), + createDataPoint("summary_test", 5, "foo", "bar", "quantile", "1"), + createDataPoint("summary_test_sum", 100, "foo", "bar"), + createDataPoint("summary_test_count", 500, "foo", "bar"), + }, + }, + }, + wants: [][]*metricspb.Metric{ + { + { + MetricDescriptor: &metricspb.MetricDescriptor{ + Name: "summary_test", + Type: metricspb.MetricDescriptor_SUMMARY, + LabelKeys: []*metricspb.LabelKey{{Key: "foo"}}}, + Timeseries: []*metricspb.TimeSeries{ + { + StartTimestamp: timestampFromMs(startTs), + LabelValues: []*metricspb.LabelValue{{Value: "bar", HasValue: true}}, + Points: []*metricspb.Point{ + {Timestamp: timestampFromMs(startTs), Value: &metricspb.Point_SummaryValue{ + SummaryValue: &metricspb.SummaryValue{ + Sum: &wrapperspb.DoubleValue{Value: 100.0}, + Count: &wrapperspb.Int64Value{Value: 500}, + Snapshot: &metricspb.SummaryValue_Snapshot{ + PercentileValues: []*metricspb.SummaryValue_Snapshot_ValueAtPercentile{ + {Percentile: 50.0, Value: 1}, + {Percentile: 75.0, Value: 2}, + {Percentile: 100.0, Value: 5}, + }, + }}}}, + }, + }, + }, + }, + }, + }, + }, + } + + runBuilderTests(t, tests) +} + +func Test_metricBuilder_baddata(t *testing.T) { + t.Run("empty-metric-name", func(t *testing.T) { + mc := newMockMetadataCache(testMetadata) + b := newMetricBuilder(mc, true, "", testTelemetry.Logger, 0) + b.startTime = 1.0 // set to a non-zero value + if err := b.AddDataPoint(labels.FromStrings("a", "b"), startTs, 123); err != errMetricNameNotFound { + t.Error("expecting errMetricNameNotFound error, but get nil") + return + } + + if _, _, _, err := b.Build(); err != errNoDataToBuild { + t.Error("expecting errNoDataToBuild error, but get nil") + } + }) + + t.Run("histogram-datapoint-no-bucket-label", func(t *testing.T) { + mc := newMockMetadataCache(testMetadata) + b := newMetricBuilder(mc, true, "", testTelemetry.Logger, 0) + b.startTime = 1.0 // set to a non-zero value + if err := b.AddDataPoint(createLabels("hist_test", "k", "v"), startTs, 123); err != errEmptyBoundaryLabel { + t.Error("expecting errEmptyBoundaryLabel error, but get nil") + } + }) + + t.Run("summary-datapoint-no-quantile-label", func(t *testing.T) { + mc := newMockMetadataCache(testMetadata) + b := newMetricBuilder(mc, true, "", testTelemetry.Logger, 0) + b.startTime = 1.0 // set to a non-zero value + if err := b.AddDataPoint(createLabels("summary_test", "k", "v"), startTs, 123); err != errEmptyBoundaryLabel { + t.Error("expecting errEmptyBoundaryLabel error, but get nil") + } + }) + +} + +func Test_isUsefulLabel(t *testing.T) { + type args struct { + mType metricspb.MetricDescriptor_Type + labelKey string + } + tests := []struct { + name string + args args + want bool + }{ + {"metricName", args{metricspb.MetricDescriptor_GAUGE_DOUBLE, model.MetricNameLabel}, false}, + {"instance", args{metricspb.MetricDescriptor_GAUGE_DOUBLE, model.InstanceLabel}, false}, + {"scheme", args{metricspb.MetricDescriptor_GAUGE_DOUBLE, model.SchemeLabel}, false}, + {"metricPath", args{metricspb.MetricDescriptor_GAUGE_DOUBLE, model.MetricsPathLabel}, false}, + {"job", args{metricspb.MetricDescriptor_GAUGE_DOUBLE, model.JobLabel}, false}, + {"bucket", args{metricspb.MetricDescriptor_GAUGE_DOUBLE, model.BucketLabel}, true}, + {"bucketForGaugeDistribution", args{metricspb.MetricDescriptor_GAUGE_DISTRIBUTION, model.BucketLabel}, false}, + {"bucketForCumulativeDistribution", args{metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, model.BucketLabel}, false}, + {"Quantile", args{metricspb.MetricDescriptor_GAUGE_DOUBLE, model.QuantileLabel}, true}, + {"QuantileForSummay", args{metricspb.MetricDescriptor_SUMMARY, model.QuantileLabel}, false}, + {"other", args{metricspb.MetricDescriptor_GAUGE_DOUBLE, "other"}, true}, + {"empty", args{metricspb.MetricDescriptor_GAUGE_DOUBLE, ""}, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := isUsefulLabel(tt.args.mType, tt.args.labelKey); got != tt.want { + t.Errorf("isUsefulLabel() = %v, want %v", got, tt.want) + } + }) + } +} + +func Benchmark_dpgSignature(b *testing.B) { + knownLabelKeys := []string{"a", "b"} + ls := labels.FromStrings("a", "va", "b", "vb", "x", "xa") + b.ReportAllocs() + for i := 0; i < b.N; i++ { + runtime.KeepAlive(dpgSignature(knownLabelKeys, ls)) + } +} + +func Test_dpgSignature(t *testing.T) { + knownLabelKeys := []string{"a", "b"} + + tests := []struct { + name string + ls labels.Labels + want string + }{ + {"1st label", labels.FromStrings("a", "va"), `"a=va"`}, + {"2nd label", labels.FromStrings("b", "vb"), `"b=vb"`}, + {"two labels", labels.FromStrings("a", "va", "b", "vb"), `"a=va""b=vb"`}, + {"extra label", labels.FromStrings("a", "va", "b", "vb", "x", "xa"), `"a=va""b=vb"`}, + {"different order", labels.FromStrings("b", "vb", "a", "va"), `"a=va""b=vb"`}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := dpgSignature(knownLabelKeys, tt.ls); got != tt.want { + t.Errorf("dpgSignature() = %q, want %q", got, tt.want) + } + }) + } + + // this is important for caching start values, as new metrics with new tag of a same group can come up in a 2nd run, + // however, its order within the group is not predictable. we need to have a way to generate a stable key even if + // the total number of keys changes in between different scrape runs + t.Run("knownLabelKeys updated", func(t *testing.T) { + ls := labels.FromStrings("a", "va") + want := dpgSignature(knownLabelKeys, ls) + got := dpgSignature(append(knownLabelKeys, "c"), ls) + if got != want { + t.Errorf("dpgSignature() = %v, want %v", got, want) + } + }) +} + +func Test_normalizeMetricName(t *testing.T) { + tests := []struct { + name string + mname string + want string + }{ + {"normal", "normal", "normal"}, + {"count", "foo_count", "foo"}, + {"bucket", "foo_bucket", "foo"}, + {"sum", "foo_sum", "foo"}, + {"total", "foo_total", "foo"}, + {"no_prefix", "_sum", "_sum"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := normalizeMetricName(tt.mname); got != tt.want { + t.Errorf("normalizeMetricName() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_getBoundary(t *testing.T) { + ls := labels.FromStrings("le", "100.0", "foo", "bar", "quantile", "0.5") + ls2 := labels.FromStrings("foo", "bar") + ls3 := labels.FromStrings("le", "xyz", "foo", "bar", "quantile", "0.5") + type args struct { + metricType metricspb.MetricDescriptor_Type + labels labels.Labels + } + tests := []struct { + name string + args args + want float64 + wantErr bool + }{ + {"histogram", args{metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION, ls}, 100.0, false}, + {"gaugehistogram", args{metricspb.MetricDescriptor_GAUGE_DISTRIBUTION, ls}, 100.0, false}, + {"gaugehistogram_no_label", args{metricspb.MetricDescriptor_GAUGE_DISTRIBUTION, ls2}, 0, true}, + {"gaugehistogram_bad_value", args{metricspb.MetricDescriptor_GAUGE_DISTRIBUTION, ls3}, 0, true}, + {"summary", args{metricspb.MetricDescriptor_SUMMARY, ls}, 0.5, false}, + {"otherType", args{metricspb.MetricDescriptor_GAUGE_DOUBLE, ls}, 0, true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getBoundary(tt.args.metricType, tt.args.labels) + if (err != nil) != tt.wantErr { + t.Errorf("getBoundary() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("getBoundary() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_convToOCAMetricType(t *testing.T) { + tests := []struct { + name string + metricType textparse.MetricType + want metricspb.MetricDescriptor_Type + }{ + {"counter", textparse.MetricTypeCounter, metricspb.MetricDescriptor_CUMULATIVE_DOUBLE}, + {"gauge", textparse.MetricTypeGauge, metricspb.MetricDescriptor_GAUGE_DOUBLE}, + {"histogram", textparse.MetricTypeHistogram, metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION}, + {"guageHistogram", textparse.MetricTypeGaugeHistogram, metricspb.MetricDescriptor_UNSPECIFIED}, + {"summary", textparse.MetricTypeSummary, metricspb.MetricDescriptor_SUMMARY}, + {"info", textparse.MetricTypeInfo, metricspb.MetricDescriptor_UNSPECIFIED}, + {"stateset", textparse.MetricTypeStateset, metricspb.MetricDescriptor_UNSPECIFIED}, + {"unknown", textparse.MetricTypeUnknown, metricspb.MetricDescriptor_GAUGE_DOUBLE}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := convToOCAMetricType(tt.metricType); !reflect.DeepEqual(got, tt.want) { + t.Errorf("convToOCAMetricType() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_heuristicalMetricAndKnownUnits(t *testing.T) { + tests := []struct { + metricName string + parsedUnit string + want string + }{ + {"test", "ms", "ms"}, + {"millisecond", "", ""}, + {"test_millisecond", "", "ms"}, + {"test_milliseconds", "", "ms"}, + {"test_ms", "", "ms"}, + {"test_second", "", "s"}, + {"test_seconds", "", "s"}, + {"test_s", "", "s"}, + {"test_microsecond", "", "us"}, + {"test_microseconds", "", "us"}, + {"test_us", "", "us"}, + {"test_nanosecond", "", "ns"}, + {"test_nanoseconds", "", "ns"}, + {"test_ns", "", "ns"}, + {"test_byte", "", "By"}, + {"test_bytes", "", "By"}, + {"test_by", "", "By"}, + {"test_bit", "", "Bi"}, + {"test_bits", "", "Bi"}, + {"test_kilogram", "", "kg"}, + {"test_kilograms", "", "kg"}, + {"test_kg", "", "kg"}, + {"test_gram", "", "g"}, + {"test_grams", "", "g"}, + {"test_g", "", "g"}, + {"test_nanogram", "", "ng"}, + {"test_nanograms", "", "ng"}, + {"test_ng", "", "ng"}, + {"test_meter", "", "m"}, + {"test_meters", "", "m"}, + {"test_metre", "", "m"}, + {"test_metres", "", "m"}, + {"test_m", "", "m"}, + {"test_kilometer", "", "km"}, + {"test_kilometers", "", "km"}, + {"test_kilometre", "", "km"}, + {"test_kilometres", "", "km"}, + {"test_km", "", "km"}, + {"test_milimeter", "", "mm"}, + {"test_milimeters", "", "mm"}, + {"test_milimetre", "", "mm"}, + {"test_milimetres", "", "mm"}, + {"test_mm", "", "mm"}, + } + for _, tt := range tests { + t.Run(tt.metricName, func(t *testing.T) { + if got := heuristicalMetricAndKnownUnits(tt.metricName, tt.parsedUnit); got != tt.want { + t.Errorf("heuristicalMetricAndKnownUnits() = %v, want %v", got, tt.want) + } + }) + } +} + +// Ensure that we reject duplicate label keys. See https://github.com/open-telemetry/wg-prometheus/issues/44. +func TestMetricBuilderDuplicateLabelKeysAreRejected(t *testing.T) { + mc := newMockMetadataCache(testMetadata) + mb := newMetricBuilder(mc, true, "", testTelemetry.Logger, 0) + + dupLabels := labels.Labels{ + {Name: "__name__", Value: "test"}, + {Name: "a", Value: "1"}, + {Name: "a", Value: "1"}, + {Name: "z", Value: "9"}, + {Name: "z", Value: "1"}, + {Name: "instance", Value: "0.0.0.0:8855"}, + {Name: "job", Value: "test"}, + } + + err := mb.AddDataPoint(dupLabels, 1917, 1.0) + require.NotNil(t, err) + require.Contains(t, err.Error(), `invalid sample: non-unique label names: ["a" "z"]`) +} diff --git a/receiver/prometheusreceiver/internal/otlp_metricfamily.go b/receiver/prometheusreceiver/internal/otlp_metricfamily.go index 0fbe843e53605..c96db6e88414b 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricfamily.go +++ b/receiver/prometheusreceiver/internal/otlp_metricfamily.go @@ -28,11 +28,6 @@ import ( "go.opentelemetry.io/collector/model/pdata" ) -type dataPoint struct { - value float64 - boundary float64 -} - // MetricFamilyPdata is unit which is corresponding to the metrics items which shared the same TYPE/UNIT/... metadata from // a single scrape. type MetricFamilyPdata interface { @@ -187,9 +182,9 @@ func (mg *metricGroupPdata) toDistributionPoint(orderedLabelKeys []string, dest point.SetSum(mg.sum) point.SetBucketCounts(bucketCounts) // The timestamp MUST be in retrieved from milliseconds and converted to nanoseconds. - tsNanos := timestampFromMs(mg.ts) + tsNanos := pdataTimestampFromMs(mg.ts) if mg.family.isCumulativeTypePdata() { - point.SetStartTimestamp(timestampFromMs(mg.intervalStartTimeMs)) + point.SetStartTimestamp(pdataTimestampFromMs(mg.intervalStartTimeMs)) } point.SetTimestamp(tsNanos) populateAttributesPdata(orderedLabelKeys, mg.ls, point.Attributes()) @@ -197,7 +192,7 @@ func (mg *metricGroupPdata) toDistributionPoint(orderedLabelKeys []string, dest return true } -func timestampFromMs(timeAtMs int64) pdata.Timestamp { +func pdataTimestampFromMs(timeAtMs int64) pdata.Timestamp { secs, ns := timeAtMs/1e3, (timeAtMs%1e3)*1e6 return pdata.NewTimestampFromTime(time.Unix(secs, ns)) } @@ -225,10 +220,10 @@ func (mg *metricGroupPdata) toSummaryPoint(orderedLabelKeys []string, dest *pdat // observations and the corresponding sum is a sum of all observed values, thus the sum and count used // at the global level of the metricspb.SummaryValue // The timestamp MUST be in retrieved from milliseconds and converted to nanoseconds. - tsNanos := timestampFromMs(mg.ts) + tsNanos := pdataTimestampFromMs(mg.ts) point.SetTimestamp(tsNanos) if mg.family.isCumulativeTypePdata() { - point.SetStartTimestamp(timestampFromMs(mg.intervalStartTimeMs)) + point.SetStartTimestamp(pdataTimestampFromMs(mg.intervalStartTimeMs)) } point.SetSum(mg.sum) point.SetCount(uint64(mg.count)) @@ -239,10 +234,10 @@ func (mg *metricGroupPdata) toSummaryPoint(orderedLabelKeys []string, dest *pdat func (mg *metricGroupPdata) toNumberDataPoint(orderedLabelKeys []string, dest *pdata.NumberDataPointSlice) bool { var startTsNanos pdata.Timestamp - tsNanos := timestampFromMs(mg.ts) + tsNanos := pdataTimestampFromMs(mg.ts) // gauge/undefined types have no start time. if mg.family.isCumulativeTypePdata() { - startTsNanos = timestampFromMs(mg.intervalStartTimeMs) + startTsNanos = pdataTimestampFromMs(mg.intervalStartTimeMs) } point := dest.AppendEmpty() @@ -387,32 +382,3 @@ func (mf *metricFamilyPdata) ToMetricPdata(metrics *pdata.MetricSlice) (int, int // note: the total number of points is the number of points+droppedTimeseries. return pointCount + mf.droppedTimeseries, mf.droppedTimeseries } - -// Define manually the metadata of prometheus scrapper internal metrics -func defineInternalMetric(metricName string, metadata scrape.MetricMetadata, logger *zap.Logger) scrape.MetricMetadata { - if metadata.Metric != "" && metadata.Type != "" && metadata.Help != "" { - logger.Debug("Internal metric seems already fully defined") - return metadata - } - metadata.Metric = metricName - - switch metricName { - case scrapeUpMetricName: - metadata.Type = textparse.MetricTypeGauge - metadata.Help = "The scraping was successful" - case "scrape_duration_seconds": - metadata.Unit = "seconds" - metadata.Type = textparse.MetricTypeGauge - metadata.Help = "Duration of the scrape" - case "scrape_samples_scraped": - metadata.Type = textparse.MetricTypeGauge - metadata.Help = "The number of samples the target exposed" - case "scrape_series_added": - metadata.Type = textparse.MetricTypeGauge - metadata.Help = "The approximate number of new series in this scrape" - case "scrape_samples_post_metric_relabeling": - metadata.Type = textparse.MetricTypeGauge - metadata.Help = "The number of samples remaining after metric relabeling was applied" - } - return metadata -} diff --git a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go index d46887cc12fa8..12172210c90d9 100644 --- a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go +++ b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go @@ -28,15 +28,16 @@ import ( ) var ( - t1Ms = pdata.Timestamp(time.Unix(0, 1000000).UnixNano()) - t2Ms = pdata.Timestamp(time.Unix(0, 2000000).UnixNano()) - t3Ms = pdata.Timestamp(time.Unix(0, 3000000).UnixNano()) - t4Ms = pdata.Timestamp(time.Unix(0, 5000000).UnixNano()) - t5Ms = pdata.Timestamp(time.Unix(0, 5000000).UnixNano()) + pdt1Ms = pdata.Timestamp(time.Unix(0, 1000000).UnixNano()) + pdt2Ms = pdata.Timestamp(time.Unix(0, 2000000).UnixNano()) + pdt3Ms = pdata.Timestamp(time.Unix(0, 3000000).UnixNano()) + pdt4Ms = pdata.Timestamp(time.Unix(0, 5000000).UnixNano()) + pdt5Ms = pdata.Timestamp(time.Unix(0, 5000000).UnixNano()) bounds0 = []float64{1, 2, 4} percent0 = []float64{10, 50, 90} + g1 = "gauge1" gd1 = "gaugedist1" c1 = "cumulative1" cd1 = "cumulativedist1" @@ -49,8 +50,8 @@ var ( v100v200 = []string{"v100", "v200"} ) -func Test_gauge(t *testing.T) { - script := []*metricsAdjusterTest{ +func Test_gauge_pdata(t *testing.T) { + script := []*metricsAdjusterTestPdata{ { "Gauge: round 1 - gauge not adjusted", func() *pdata.MetricSlice { @@ -60,9 +61,9 @@ func Test_gauge(t *testing.T) { m0.SetName("gauge1") g0 := m0.Gauge() pt0 := g0.DataPoints().AppendEmpty() - pt0.SetStartTimestamp(t1Ms) + pt0.SetStartTimestamp(pdt1Ms) pt0.Attributes().InsertString("v1", "v2") - pt0.SetTimestamp(t1Ms) + pt0.SetTimestamp(pdt1Ms) pt0.SetDoubleVal(44) return &mL }(), @@ -73,9 +74,9 @@ func Test_gauge(t *testing.T) { m0.SetName("gauge1") g0 := m0.Gauge() pt0 := g0.DataPoints().AppendEmpty() - pt0.SetStartTimestamp(t1Ms) + pt0.SetStartTimestamp(pdt1Ms) pt0.Attributes().InsertString("v1", "v2") - pt0.SetTimestamp(t1Ms) + pt0.SetTimestamp(pdt1Ms) pt0.SetDoubleVal(44) return &mL }(), @@ -90,10 +91,10 @@ func Test_gauge(t *testing.T) { m0.SetName("gauge1") g0 := m0.Gauge() pt0 := g0.DataPoints().AppendEmpty() - pt0.SetStartTimestamp(t2Ms) + pt0.SetStartTimestamp(pdt2Ms) pt0.Attributes().InsertString("k1", "v1") pt0.Attributes().InsertString("k2", "v2") - pt0.SetTimestamp(t2Ms) + pt0.SetTimestamp(pdt2Ms) pt0.SetDoubleVal(66) return &mL @@ -105,10 +106,10 @@ func Test_gauge(t *testing.T) { m0.SetName("gauge1") g0 := m0.Gauge() pt0 := g0.DataPoints().AppendEmpty() - pt0.SetStartTimestamp(t2Ms) + pt0.SetStartTimestamp(pdt2Ms) pt0.Attributes().InsertString("k1", "v1") pt0.Attributes().InsertString("k2", "v2") - pt0.SetTimestamp(t2Ms) + pt0.SetTimestamp(pdt2Ms) pt0.SetDoubleVal(66) return &mL }(), @@ -123,10 +124,10 @@ func Test_gauge(t *testing.T) { m0.SetName("gauge1") g0 := m0.Gauge() pt0 := g0.DataPoints().AppendEmpty() - pt0.SetStartTimestamp(t3Ms) + pt0.SetStartTimestamp(pdt3Ms) pt0.Attributes().InsertString("k1", "v1") pt0.Attributes().InsertString("k2", "v2") - pt0.SetTimestamp(t3Ms) + pt0.SetTimestamp(pdt3Ms) pt0.SetDoubleVal(55) return &mL @@ -138,10 +139,10 @@ func Test_gauge(t *testing.T) { m0.SetName("gauge1") g0 := m0.Gauge() pt0 := g0.DataPoints().AppendEmpty() - pt0.SetStartTimestamp(t3Ms) + pt0.SetStartTimestamp(pdt3Ms) pt0.Attributes().InsertString("k1", "v1") pt0.Attributes().InsertString("k2", "v2") - pt0.SetTimestamp(t3Ms) + pt0.SetTimestamp(pdt3Ms) pt0.SetDoubleVal(55) return &mL @@ -149,11 +150,11 @@ func Test_gauge(t *testing.T) { 0, }, } - runScript(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) + runScriptPdata(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) } -func Test_cumulative(t *testing.T) { - script := []*metricsAdjusterTest{ +func Test_cumulative_pdata(t *testing.T) { + script := []*metricsAdjusterTestPdata{ { "Cumulative: round 1 - initial instance, start time is established", func() *pdata.MetricSlice { @@ -163,10 +164,10 @@ func Test_cumulative(t *testing.T) { m0.SetName("cumulative1") g0 := m0.Sum() pt0 := g0.DataPoints().AppendEmpty() - pt0.SetStartTimestamp(t1Ms) + pt0.SetStartTimestamp(pdt1Ms) pt0.Attributes().InsertString("k1", "v1") pt0.Attributes().InsertString("k2", "v2") - pt0.SetTimestamp(t1Ms) + pt0.SetTimestamp(pdt1Ms) pt0.SetDoubleVal(44) return &mL @@ -178,10 +179,10 @@ func Test_cumulative(t *testing.T) { m0.SetName("cumulative1") g0 := m0.Sum() pt0 := g0.DataPoints().AppendEmpty() - pt0.SetStartTimestamp(t1Ms) + pt0.SetStartTimestamp(pdt1Ms) pt0.Attributes().InsertString("k1", "v1") pt0.Attributes().InsertString("k2", "v2") - pt0.SetTimestamp(t1Ms) + pt0.SetTimestamp(pdt1Ms) pt0.SetDoubleVal(44) return &mL @@ -197,10 +198,10 @@ func Test_cumulative(t *testing.T) { m0.SetName("cumulative1") g0 := m0.Sum() pt0 := g0.DataPoints().AppendEmpty() - pt0.SetStartTimestamp(t2Ms) + pt0.SetStartTimestamp(pdt2Ms) pt0.Attributes().InsertString("k1", "v1") pt0.Attributes().InsertString("k2", "v2") - pt0.SetTimestamp(t2Ms) + pt0.SetTimestamp(pdt2Ms) pt0.SetDoubleVal(66) return &mL @@ -212,10 +213,10 @@ func Test_cumulative(t *testing.T) { m0.SetName("cumulative1") g0 := m0.Sum() pt0 := g0.DataPoints().AppendEmpty() - pt0.SetStartTimestamp(t1Ms) + pt0.SetStartTimestamp(pdt1Ms) pt0.Attributes().InsertString("k1", "v1") pt0.Attributes().InsertString("k2", "v2") - pt0.SetTimestamp(t2Ms) + pt0.SetTimestamp(pdt2Ms) pt0.SetDoubleVal(66) return &mL @@ -231,10 +232,10 @@ func Test_cumulative(t *testing.T) { m0.SetName("cumulative1") g0 := m0.Sum() pt0 := g0.DataPoints().AppendEmpty() - pt0.SetStartTimestamp(t3Ms) + pt0.SetStartTimestamp(pdt3Ms) pt0.Attributes().InsertString("k1", "v1") pt0.Attributes().InsertString("k2", "v2") - pt0.SetTimestamp(t3Ms) + pt0.SetTimestamp(pdt3Ms) pt0.SetDoubleVal(55) return &mL @@ -246,10 +247,10 @@ func Test_cumulative(t *testing.T) { m0.SetName("cumulative1") g0 := m0.Sum() pt0 := g0.DataPoints().AppendEmpty() - pt0.SetStartTimestamp(t3Ms) + pt0.SetStartTimestamp(pdt3Ms) pt0.Attributes().InsertString("k1", "v1") pt0.Attributes().InsertString("k2", "v2") - pt0.SetTimestamp(t3Ms) + pt0.SetTimestamp(pdt3Ms) pt0.SetDoubleVal(55) return &mL @@ -265,10 +266,10 @@ func Test_cumulative(t *testing.T) { m0.SetName("cumulative1") g0 := m0.Sum() pt0 := g0.DataPoints().AppendEmpty() - pt0.SetStartTimestamp(t4Ms) + pt0.SetStartTimestamp(pdt4Ms) pt0.Attributes().InsertString("k1", "v1") pt0.Attributes().InsertString("k2", "v2") - pt0.SetTimestamp(t4Ms) + pt0.SetTimestamp(pdt4Ms) pt0.SetDoubleVal(72) return &mL @@ -280,10 +281,10 @@ func Test_cumulative(t *testing.T) { m0.SetName("cumulative1") g0 := m0.Sum() pt0 := g0.DataPoints().AppendEmpty() - pt0.SetStartTimestamp(t3Ms) + pt0.SetStartTimestamp(pdt3Ms) pt0.Attributes().InsertString("k1", "v1") pt0.Attributes().InsertString("k2", "v2") - pt0.SetTimestamp(t4Ms) + pt0.SetTimestamp(pdt4Ms) pt0.SetDoubleVal(72) return &mL @@ -291,7 +292,7 @@ func Test_cumulative(t *testing.T) { 0, }, } - runScript(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) + runScriptPdata(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) } func populateHistogram(hdp *pdata.HistogramDataPoint, timestamp pdata.Timestamp, bounds []float64, counts []uint64) { @@ -310,28 +311,28 @@ func populateHistogram(hdp *pdata.HistogramDataPoint, timestamp pdata.Timestamp, hdp.SetExplicitBounds(bounds) } -func Test_gaugeDistribution(t *testing.T) { - script := []*metricsAdjusterTest{ +func Test_gaugeDistribution_pdata(t *testing.T) { + script := []*metricsAdjusterTestPdata{ { "GaugeDist: round 1 - gauge distribution not adjusted", - metricSlice(gaugeDistMetric(gd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7}))), - metricSlice(gaugeDistMetric(gd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7}))), + metricSlice(gaugeDistMetric(gd1, k1v1k2v2, pdt1Ms, distPoint(pdt1Ms, bounds0, []uint64{4, 2, 3, 7}))), + metricSlice(gaugeDistMetric(gd1, k1v1k2v2, pdt1Ms, distPoint(pdt1Ms, bounds0, []uint64{4, 2, 3, 7}))), 0, }, { "GaugeDist: round 2 - gauge distribution not adjusted", - metricSlice(gaugeDistMetric(gd1, k1v1k2v2, t2Ms, distPoint(t2Ms, bounds0, []uint64{6, 5, 8, 11}))), - metricSlice(gaugeDistMetric(gd1, k1v1k2v2, t2Ms, distPoint(t2Ms, bounds0, []uint64{6, 5, 8, 11}))), + metricSlice(gaugeDistMetric(gd1, k1v1k2v2, pdt2Ms, distPoint(pdt2Ms, bounds0, []uint64{6, 5, 8, 11}))), + metricSlice(gaugeDistMetric(gd1, k1v1k2v2, pdt2Ms, distPoint(pdt2Ms, bounds0, []uint64{6, 5, 8, 11}))), 0, }, { "GaugeDist: round 3 - count/sum less than previous - gauge distribution not adjusted", - metricSlice(gaugeDistMetric(gd1, k1v1k2v2, t3Ms, distPoint(t3Ms, bounds0, []uint64{2, 0, 1, 5}))), - metricSlice(gaugeDistMetric(gd1, k1v1k2v2, t3Ms, distPoint(t3Ms, bounds0, []uint64{2, 0, 1, 5}))), + metricSlice(gaugeDistMetric(gd1, k1v1k2v2, pdt3Ms, distPoint(pdt3Ms, bounds0, []uint64{2, 0, 1, 5}))), + metricSlice(gaugeDistMetric(gd1, k1v1k2v2, pdt3Ms, distPoint(pdt3Ms, bounds0, []uint64{2, 0, 1, 5}))), 0, }, } - runScript(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) + runScriptPdata(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) } func populateSummary(sdp *pdata.SummaryDataPoint, timestamp pdata.Timestamp, count uint64, sum float64, quantilePercents, quantileValues []float64) { @@ -346,8 +347,8 @@ func populateSummary(sdp *pdata.SummaryDataPoint, timestamp pdata.Timestamp, cou sdp.SetSum(sum) } -func Test_summary_no_count(t *testing.T) { - script := []*metricsAdjusterTest{ +func Test_summary_no_count_pdata(t *testing.T) { + script := []*metricsAdjusterTestPdata{ { "Summary No Count: round 1 - initial instance, start time is established", func() *pdata.MetricSlice { @@ -358,7 +359,7 @@ func Test_summary_no_count(t *testing.T) { s0 := m0.Summary() pt0 := s0.DataPoints().AppendEmpty() pt0.Attributes().InsertString("v1", "v2") - populateSummary(&pt0, t1Ms, 10, 40, percent0, []float64{1, 5, 8}) + populateSummary(&pt0, pdt1Ms, 10, 40, percent0, []float64{1, 5, 8}) return &mL }(), func() *pdata.MetricSlice { @@ -369,7 +370,7 @@ func Test_summary_no_count(t *testing.T) { s0 := m0.Summary() pt0 := s0.DataPoints().AppendEmpty() pt0.Attributes().InsertString("v1", "v2") - populateSummary(&pt0, t1Ms, 10, 40, percent0, []float64{1, 5, 8}) + populateSummary(&pt0, pdt1Ms, 10, 40, percent0, []float64{1, 5, 8}) return &mL }(), 1, @@ -384,7 +385,7 @@ func Test_summary_no_count(t *testing.T) { s0 := m0.Summary() pt0 := s0.DataPoints().AppendEmpty() pt0.Attributes().InsertString("v1", "v2") - populateSummary(&pt0, t2Ms, 15, 70, percent0, []float64{7, 44, 9}) + populateSummary(&pt0, pdt2Ms, 15, 70, percent0, []float64{7, 44, 9}) return &mL }(), func() *pdata.MetricSlice { @@ -395,7 +396,7 @@ func Test_summary_no_count(t *testing.T) { s0 := m0.Summary() pt0 := s0.DataPoints().AppendEmpty() pt0.Attributes().InsertString("v1", "v2") - populateSummary(&pt0, t2Ms, 15, 70, percent0, []float64{7, 44, 9}) + populateSummary(&pt0, pdt2Ms, 15, 70, percent0, []float64{7, 44, 9}) return &mL }(), 0, @@ -410,7 +411,7 @@ func Test_summary_no_count(t *testing.T) { s0 := m0.Summary() pt0 := s0.DataPoints().AppendEmpty() pt0.Attributes().InsertString("v1", "v2") - populateSummary(&pt0, t3Ms, 12, 66, percent0, []float64{3, 22, 5}) + populateSummary(&pt0, pdt3Ms, 12, 66, percent0, []float64{3, 22, 5}) return &mL }(), func() *pdata.MetricSlice { @@ -421,7 +422,7 @@ func Test_summary_no_count(t *testing.T) { s0 := m0.Summary() pt0 := s0.DataPoints().AppendEmpty() pt0.Attributes().InsertString("v1", "v2") - populateSummary(&pt0, t3Ms, 12, 66, percent0, []float64{3, 22, 5}) + populateSummary(&pt0, pdt3Ms, 12, 66, percent0, []float64{3, 22, 5}) return &mL }(), 1, @@ -436,8 +437,8 @@ func Test_summary_no_count(t *testing.T) { s0 := m0.Summary() pt0 := s0.DataPoints().AppendEmpty() pt0.Attributes().InsertString("v1", "v2") - populateSummary(&pt0, t4Ms, 14, 96, percent0, []float64{9, 47, 8}) - pt0.SetStartTimestamp(t4Ms) + populateSummary(&pt0, pdt4Ms, 14, 96, percent0, []float64{9, 47, 8}) + pt0.SetStartTimestamp(pdt4Ms) return &mL }(), func() *pdata.MetricSlice { @@ -448,61 +449,61 @@ func Test_summary_no_count(t *testing.T) { s0 := m0.Summary() pt0 := s0.DataPoints().AppendEmpty() pt0.Attributes().InsertString("v1", "v2") - populateSummary(&pt0, t4Ms, 14, 96, percent0, []float64{9, 47, 8}) + populateSummary(&pt0, pdt4Ms, 14, 96, percent0, []float64{9, 47, 8}) return &mL }(), 0, }, } - runScript(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) + runScriptPdata(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) } -func Test_summary(t *testing.T) { - script := []*metricsAdjusterTest{ +func Test_summary_pdata(t *testing.T) { + script := []*metricsAdjusterTestPdata{ { "Summary: round 1 - initial instance, start time is established", metricSlice( - summaryMetric(s1, k1v1k2v2, t1Ms, summaryPoint(t1Ms, 10, 40, percent0, []float64{1, 5, 8})), + summaryMetric(s1, k1v1k2v2, pdt1Ms, summaryPoint(pdt1Ms, 10, 40, percent0, []float64{1, 5, 8})), ), metricSlice( - summaryMetric(s1, k1v1k2v2, t1Ms, summaryPoint(t1Ms, 10, 40, percent0, []float64{1, 5, 8})), + summaryMetric(s1, k1v1k2v2, pdt1Ms, summaryPoint(pdt1Ms, 10, 40, percent0, []float64{1, 5, 8})), ), 1, }, { "Summary: round 2 - instance adjusted based on round 1", metricSlice( - summaryMetric(s1, k1v1k2v2, t2Ms, summaryPoint(t2Ms, 15, 70, percent0, []float64{7, 44, 9})), + summaryMetric(s1, k1v1k2v2, pdt2Ms, summaryPoint(pdt2Ms, 15, 70, percent0, []float64{7, 44, 9})), ), metricSlice( - summaryMetric(s1, k1v1k2v2, t1Ms, summaryPoint(t2Ms, 15, 70, percent0, []float64{7, 44, 9})), + summaryMetric(s1, k1v1k2v2, pdt1Ms, summaryPoint(pdt2Ms, 15, 70, percent0, []float64{7, 44, 9})), ), 0, }, { "Summary: round 3 - instance reset (count less than previous), start time is reset", metricSlice( - summaryMetric(s1, k1v1k2v2, t3Ms, summaryPoint(t3Ms, 12, 66, percent0, []float64{3, 22, 5})), + summaryMetric(s1, k1v1k2v2, pdt3Ms, summaryPoint(pdt3Ms, 12, 66, percent0, []float64{3, 22, 5})), ), metricSlice( - summaryMetric(s1, k1v1k2v2, t3Ms, summaryPoint(t3Ms, 12, 66, percent0, []float64{3, 22, 5})), + summaryMetric(s1, k1v1k2v2, pdt3Ms, summaryPoint(pdt3Ms, 12, 66, percent0, []float64{3, 22, 5})), ), 1, }, { "Summary: round 4 - instance adjusted based on round 3", metricSlice( - summaryMetric(s1, k1v1k2v2, t4Ms, summaryPoint(t4Ms, 14, 96, percent0, []float64{9, 47, 8})), + summaryMetric(s1, k1v1k2v2, pdt4Ms, summaryPoint(pdt4Ms, 14, 96, percent0, []float64{9, 47, 8})), ), metricSlice( - summaryMetric(s1, k1v1k2v2, t3Ms, summaryPoint(t4Ms, 14, 96, percent0, []float64{9, 47, 8})), + summaryMetric(s1, k1v1k2v2, pdt3Ms, summaryPoint(pdt4Ms, 14, 96, percent0, []float64{9, 47, 8})), ), 0, }, } - runScript(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) + runScriptPdata(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) } type kv = metricstestutil.KV @@ -544,163 +545,163 @@ var ( } ) -func Test_cumulativeDistribution(t *testing.T) { - script := []*metricsAdjusterTest{ +func Test_cumulativeDistribution_pdata(t *testing.T) { + script := []*metricsAdjusterTestPdata{ { "CumulativeDist: round 1 - initial instance, start time is established", - metricSlice(histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7}))), - metricSlice(histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7}))), + metricSlice(histogramMetric(cd1, k1v1k2v2, pdt1Ms, distPoint(pdt1Ms, bounds0, []uint64{4, 2, 3, 7}))), + metricSlice(histogramMetric(cd1, k1v1k2v2, pdt1Ms, distPoint(pdt1Ms, bounds0, []uint64{4, 2, 3, 7}))), 1, }, { "CumulativeDist: round 2 - instance adjusted based on round 1", - metricSlice(histogramMetric(cd1, k1v1k2v2, t2Ms, distPoint(t2Ms, bounds0, []uint64{6, 3, 4, 8}))), - metricSlice(histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t2Ms, bounds0, []uint64{6, 3, 4, 8}))), + metricSlice(histogramMetric(cd1, k1v1k2v2, pdt2Ms, distPoint(pdt2Ms, bounds0, []uint64{6, 3, 4, 8}))), + metricSlice(histogramMetric(cd1, k1v1k2v2, pdt1Ms, distPoint(pdt2Ms, bounds0, []uint64{6, 3, 4, 8}))), 0, }, { "CumulativeDist: round 3 - instance reset (value less than previous value), start time is reset", - metricSlice(histogramMetric(cd1, k1v1k2v2, t3Ms, distPoint(t3Ms, bounds0, []uint64{5, 3, 2, 7}))), - metricSlice(histogramMetric(cd1, k1v1k2v2, t3Ms, distPoint(t3Ms, bounds0, []uint64{5, 3, 2, 7}))), + metricSlice(histogramMetric(cd1, k1v1k2v2, pdt3Ms, distPoint(pdt3Ms, bounds0, []uint64{5, 3, 2, 7}))), + metricSlice(histogramMetric(cd1, k1v1k2v2, pdt3Ms, distPoint(pdt3Ms, bounds0, []uint64{5, 3, 2, 7}))), 1, }, { "CumulativeDist: round 4 - instance adjusted based on round 3", - metricSlice(histogramMetric(cd1, k1v1k2v2, t4Ms, distPoint(t4Ms, bounds0, []uint64{7, 4, 2, 12}))), - metricSlice(histogramMetric(cd1, k1v1k2v2, t3Ms, distPoint(t4Ms, bounds0, []uint64{7, 4, 2, 12}))), + metricSlice(histogramMetric(cd1, k1v1k2v2, pdt4Ms, distPoint(pdt4Ms, bounds0, []uint64{7, 4, 2, 12}))), + metricSlice(histogramMetric(cd1, k1v1k2v2, pdt3Ms, distPoint(pdt4Ms, bounds0, []uint64{7, 4, 2, 12}))), 0, }, } - runScript(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) + runScriptPdata(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) } -func Test_multiMetrics(t *testing.T) { +func Test_multiMetrics_pdata(t *testing.T) { g1 := "gauge1" - script := []*metricsAdjusterTest{ + script := []*metricsAdjusterTestPdata{ { "MultiMetrics: round 1 - combined round 1 of individual metrics", metricSlice( - gaugeMetric(g1, k1v1k2v2, t1Ms, doublePoint(t1Ms, 44)), - gaugeDistMetric(gd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7})), - sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t1Ms, 44)), - histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7})), - summaryMetric(s1, k1v1k2v2, t1Ms, summaryPoint(t1Ms, 10, 40, percent0, []float64{1, 5, 8})), + gaugeMetric(g1, k1v1k2v2, pdt1Ms, doublePoint(pdt1Ms, 44)), + gaugeDistMetric(gd1, k1v1k2v2, pdt1Ms, distPoint(pdt1Ms, bounds0, []uint64{4, 2, 3, 7})), + sumMetric(c1, k1v1k2v2, pdt1Ms, doublePoint(pdt1Ms, 44)), + histogramMetric(cd1, k1v1k2v2, pdt1Ms, distPoint(pdt1Ms, bounds0, []uint64{4, 2, 3, 7})), + summaryMetric(s1, k1v1k2v2, pdt1Ms, summaryPoint(pdt1Ms, 10, 40, percent0, []float64{1, 5, 8})), ), metricSlice( - gaugeMetric(g1, k1v1k2v2, t1Ms, doublePoint(t1Ms, 44)), - gaugeDistMetric(gd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7})), - sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t1Ms, 44)), - histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7})), - summaryMetric(s1, k1v1k2v2, t1Ms, summaryPoint(t1Ms, 10, 40, percent0, []float64{1, 5, 8})), + gaugeMetric(g1, k1v1k2v2, pdt1Ms, doublePoint(pdt1Ms, 44)), + gaugeDistMetric(gd1, k1v1k2v2, pdt1Ms, distPoint(pdt1Ms, bounds0, []uint64{4, 2, 3, 7})), + sumMetric(c1, k1v1k2v2, pdt1Ms, doublePoint(pdt1Ms, 44)), + histogramMetric(cd1, k1v1k2v2, pdt1Ms, distPoint(pdt1Ms, bounds0, []uint64{4, 2, 3, 7})), + summaryMetric(s1, k1v1k2v2, pdt1Ms, summaryPoint(pdt1Ms, 10, 40, percent0, []float64{1, 5, 8})), ), 3, }, { "MultiMetrics: round 2 - combined round 2 of individual metrics", metricSlice( - gaugeMetric(g1, k1v1k2v2, t2Ms, doublePoint(t2Ms, 66)), - gaugeDistMetric(gd1, k1v1k2v2, t2Ms, distPoint(t2Ms, bounds0, []uint64{6, 5, 8, 11})), - sumMetric(c1, k1v1k2v2, t2Ms, doublePoint(t2Ms, 66)), - histogramMetric(cd1, k1v1k2v2, t2Ms, distPoint(t2Ms, bounds0, []uint64{6, 3, 4, 8})), - summaryMetric(s1, k1v1k2v2, t2Ms, summaryPoint(t2Ms, 15, 70, percent0, []float64{7, 44, 9})), + gaugeMetric(g1, k1v1k2v2, pdt2Ms, doublePoint(pdt2Ms, 66)), + gaugeDistMetric(gd1, k1v1k2v2, pdt2Ms, distPoint(pdt2Ms, bounds0, []uint64{6, 5, 8, 11})), + sumMetric(c1, k1v1k2v2, pdt2Ms, doublePoint(pdt2Ms, 66)), + histogramMetric(cd1, k1v1k2v2, pdt2Ms, distPoint(pdt2Ms, bounds0, []uint64{6, 3, 4, 8})), + summaryMetric(s1, k1v1k2v2, pdt2Ms, summaryPoint(pdt2Ms, 15, 70, percent0, []float64{7, 44, 9})), ), metricSlice( - gaugeMetric(g1, k1v1k2v2, t2Ms, doublePoint(t2Ms, 66)), - gaugeDistMetric(gd1, k1v1k2v2, t2Ms, distPoint(t2Ms, bounds0, []uint64{6, 5, 8, 11})), - sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t2Ms, 66)), - histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t2Ms, bounds0, []uint64{6, 3, 4, 8})), - summaryMetric(s1, k1v1k2v2, t1Ms, summaryPoint(t2Ms, 15, 70, percent0, []float64{7, 44, 9})), + gaugeMetric(g1, k1v1k2v2, pdt2Ms, doublePoint(pdt2Ms, 66)), + gaugeDistMetric(gd1, k1v1k2v2, pdt2Ms, distPoint(pdt2Ms, bounds0, []uint64{6, 5, 8, 11})), + sumMetric(c1, k1v1k2v2, pdt1Ms, doublePoint(pdt2Ms, 66)), + histogramMetric(cd1, k1v1k2v2, pdt1Ms, distPoint(pdt2Ms, bounds0, []uint64{6, 3, 4, 8})), + summaryMetric(s1, k1v1k2v2, pdt1Ms, summaryPoint(pdt2Ms, 15, 70, percent0, []float64{7, 44, 9})), ), 0, }, { "MultiMetrics: round 3 - combined round 3 of individual metrics", metricSlice( - gaugeMetric(g1, k1v1k2v2, t3Ms, doublePoint(t3Ms, 55)), - gaugeDistMetric(gd1, k1v1k2v2, t3Ms, distPoint(t3Ms, bounds0, []uint64{2, 0, 1, 5})), - sumMetric(c1, k1v1k2v2, t3Ms, doublePoint(t3Ms, 55)), - histogramMetric(cd1, k1v1k2v2, t3Ms, distPoint(t3Ms, bounds0, []uint64{5, 3, 2, 7})), - summaryMetric(s1, k1v1k2v2, t3Ms, summaryPoint(t3Ms, 12, 66, percent0, []float64{3, 22, 5})), + gaugeMetric(g1, k1v1k2v2, pdt3Ms, doublePoint(pdt3Ms, 55)), + gaugeDistMetric(gd1, k1v1k2v2, pdt3Ms, distPoint(pdt3Ms, bounds0, []uint64{2, 0, 1, 5})), + sumMetric(c1, k1v1k2v2, pdt3Ms, doublePoint(pdt3Ms, 55)), + histogramMetric(cd1, k1v1k2v2, pdt3Ms, distPoint(pdt3Ms, bounds0, []uint64{5, 3, 2, 7})), + summaryMetric(s1, k1v1k2v2, pdt3Ms, summaryPoint(pdt3Ms, 12, 66, percent0, []float64{3, 22, 5})), ), metricSlice( - gaugeMetric(g1, k1v1k2v2, t3Ms, doublePoint(t3Ms, 55)), - gaugeDistMetric(gd1, k1v1k2v2, t3Ms, distPoint(t3Ms, bounds0, []uint64{2, 0, 1, 5})), - sumMetric(c1, k1v1k2v2, t3Ms, doublePoint(t3Ms, 55)), - histogramMetric(cd1, k1v1k2v2, t3Ms, distPoint(t3Ms, bounds0, []uint64{5, 3, 2, 7})), - summaryMetric(s1, k1v1k2v2, t3Ms, summaryPoint(t3Ms, 12, 66, percent0, []float64{3, 22, 5})), + gaugeMetric(g1, k1v1k2v2, pdt3Ms, doublePoint(pdt3Ms, 55)), + gaugeDistMetric(gd1, k1v1k2v2, pdt3Ms, distPoint(pdt3Ms, bounds0, []uint64{2, 0, 1, 5})), + sumMetric(c1, k1v1k2v2, pdt3Ms, doublePoint(pdt3Ms, 55)), + histogramMetric(cd1, k1v1k2v2, pdt3Ms, distPoint(pdt3Ms, bounds0, []uint64{5, 3, 2, 7})), + summaryMetric(s1, k1v1k2v2, pdt3Ms, summaryPoint(pdt3Ms, 12, 66, percent0, []float64{3, 22, 5})), ), 3, }, { "MultiMetrics: round 4 - combined round 4 of individual metrics", metricSlice( - sumMetric(c1, k1v1k2v2, t4Ms, doublePoint(t4Ms, 72)), - histogramMetric(cd1, k1v1k2v2, t4Ms, distPoint(t4Ms, bounds0, []uint64{7, 4, 2, 12})), - summaryMetric(s1, k1v1k2v2, t4Ms, summaryPoint(t4Ms, 14, 96, percent0, []float64{9, 47, 8})), + sumMetric(c1, k1v1k2v2, pdt4Ms, doublePoint(pdt4Ms, 72)), + histogramMetric(cd1, k1v1k2v2, pdt4Ms, distPoint(pdt4Ms, bounds0, []uint64{7, 4, 2, 12})), + summaryMetric(s1, k1v1k2v2, pdt4Ms, summaryPoint(pdt4Ms, 14, 96, percent0, []float64{9, 47, 8})), ), metricSlice( - sumMetric(c1, k1v1k2v2, t3Ms, doublePoint(t4Ms, 72)), - histogramMetric(cd1, k1v1k2v2, t3Ms, distPoint(t4Ms, bounds0, []uint64{7, 4, 2, 12})), - summaryMetric(s1, k1v1k2v2, t3Ms, summaryPoint(t4Ms, 14, 96, percent0, []float64{9, 47, 8})), + sumMetric(c1, k1v1k2v2, pdt3Ms, doublePoint(pdt4Ms, 72)), + histogramMetric(cd1, k1v1k2v2, pdt3Ms, distPoint(pdt4Ms, bounds0, []uint64{7, 4, 2, 12})), + summaryMetric(s1, k1v1k2v2, pdt3Ms, summaryPoint(pdt4Ms, 14, 96, percent0, []float64{9, 47, 8})), ), 0, }, } - runScript(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) + runScriptPdata(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) } -func Test_multiTimeseries(t *testing.T) { - script := []*metricsAdjusterTest{ +func Test_multiTimeseries_pdata(t *testing.T) { + script := []*metricsAdjusterTestPdata{ { "MultiTimeseries: round 1 - initial first instance, start time is established", - metricSlice(sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t1Ms, 44))), - metricSlice(sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t1Ms, 44))), + metricSlice(sumMetric(c1, k1v1k2v2, pdt1Ms, doublePoint(pdt1Ms, 44))), + metricSlice(sumMetric(c1, k1v1k2v2, pdt1Ms, doublePoint(pdt1Ms, 44))), 1, }, { "MultiTimeseries: round 2 - first instance adjusted based on round 1, initial second instance", metricSlice( - sumMetric(c1, k1v1k2v2, t2Ms, doublePoint(t2Ms, 66)), - sumMetric(c1, k1v10k2v20, t2Ms, doublePoint(t2Ms, 20.0)), + sumMetric(c1, k1v1k2v2, pdt2Ms, doublePoint(pdt2Ms, 66)), + sumMetric(c1, k1v10k2v20, pdt2Ms, doublePoint(pdt2Ms, 20.0)), ), metricSlice( - sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t2Ms, 66)), - sumMetric(c1, k1v10k2v20, t2Ms, doublePoint(t2Ms, 20.0)), + sumMetric(c1, k1v1k2v2, pdt1Ms, doublePoint(pdt2Ms, 66)), + sumMetric(c1, k1v10k2v20, pdt2Ms, doublePoint(pdt2Ms, 20.0)), ), 1, }, { "MultiTimeseries: round 3 - first instance adjusted based on round 1, second based on round 2", metricSlice( - sumMetric(c1, k1v1k2v2, t3Ms, doublePoint(t3Ms, 88.0)), - sumMetric(c1, k1v10k2v20, t3Ms, doublePoint(t3Ms, 49.0)), + sumMetric(c1, k1v1k2v2, pdt3Ms, doublePoint(pdt3Ms, 88.0)), + sumMetric(c1, k1v10k2v20, pdt3Ms, doublePoint(pdt3Ms, 49.0)), ), metricSlice( - sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t3Ms, 88.0)), - sumMetric(c1, k1v10k2v20, t2Ms, doublePoint(t3Ms, 49.0)), + sumMetric(c1, k1v1k2v2, pdt1Ms, doublePoint(pdt3Ms, 88.0)), + sumMetric(c1, k1v10k2v20, pdt2Ms, doublePoint(pdt3Ms, 49.0)), ), 0, }, { "MultiTimeseries: round 4 - first instance reset, second instance adjusted based on round 2, initial third instance", metricSlice( - sumMetric(c1, k1v1k2v2, t4Ms, doublePoint(t4Ms, 87.0)), - sumMetric(c1, k1v10k2v20, t4Ms, doublePoint(t4Ms, 57.0)), - sumMetric(c1, k1v100k2v200, t4Ms, doublePoint(t4Ms, 10.0)), + sumMetric(c1, k1v1k2v2, pdt4Ms, doublePoint(pdt4Ms, 87.0)), + sumMetric(c1, k1v10k2v20, pdt4Ms, doublePoint(pdt4Ms, 57.0)), + sumMetric(c1, k1v100k2v200, pdt4Ms, doublePoint(pdt4Ms, 10.0)), ), metricSlice( - sumMetric(c1, k1v1k2v2, t4Ms, doublePoint(t4Ms, 87.0)), - sumMetric(c1, k1v10k2v20, t2Ms, doublePoint(t4Ms, 57.0)), - sumMetric(c1, k1v100k2v200, t4Ms, doublePoint(t4Ms, 10.0)), + sumMetric(c1, k1v1k2v2, pdt4Ms, doublePoint(pdt4Ms, 87.0)), + sumMetric(c1, k1v10k2v20, pdt2Ms, doublePoint(pdt4Ms, 57.0)), + sumMetric(c1, k1v100k2v200, pdt4Ms, doublePoint(pdt4Ms, 10.0)), ), 2, }, { "MultiTimeseries: round 5 - first instance adjusted based on round 4, second on round 2, third on round 4", metricSlice( - sumMetric(c1, k1v1k2v2, t5Ms, doublePoint(t5Ms, 90.0)), - sumMetric(c1, k1v10k2v20, t5Ms, doublePoint(t5Ms, 65.0)), - sumMetric(c1, k1v100k2v200, t5Ms, doublePoint(t5Ms, 22.0)), + sumMetric(c1, k1v1k2v2, pdt5Ms, doublePoint(pdt5Ms, 90.0)), + sumMetric(c1, k1v10k2v20, pdt5Ms, doublePoint(pdt5Ms, 65.0)), + sumMetric(c1, k1v100k2v200, pdt5Ms, doublePoint(pdt5Ms, 22.0)), ), metricSlice( - sumMetric(c1, k1v1k2v2, t4Ms, doublePoint(t5Ms, 90.0)), - sumMetric(c1, k1v10k2v20, t2Ms, doublePoint(t5Ms, 65.0)), - sumMetric(c1, k1v100k2v200, t4Ms, doublePoint(t5Ms, 22.0)), + sumMetric(c1, k1v1k2v2, pdt4Ms, doublePoint(pdt5Ms, 90.0)), + sumMetric(c1, k1v10k2v20, pdt2Ms, doublePoint(pdt5Ms, 65.0)), + sumMetric(c1, k1v100k2v200, pdt4Ms, doublePoint(pdt5Ms, 22.0)), ), 0, }, } - runScript(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) + runScriptPdata(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) } var ( @@ -709,82 +710,82 @@ var ( k1vEmptyk2vEmptyk3vEmpty = []*kv{{"k1", ""}, {"k2", ""}, {"k3", ""}} ) -func Test_emptyLabels(t *testing.T) { - script := []*metricsAdjusterTest{ +func Test_emptyLabels_pdata(t *testing.T) { + script := []*metricsAdjusterTestPdata{ { "EmptyLabels: round 1 - initial instance, implicitly empty labels, start time is established", - metricSlice(sumMetric(c1, emptyLabels, t1Ms, doublePoint(t1Ms, 44))), - metricSlice(sumMetric(c1, emptyLabels, t1Ms, doublePoint(t1Ms, 44))), + metricSlice(sumMetric(c1, emptyLabels, pdt1Ms, doublePoint(pdt1Ms, 44))), + metricSlice(sumMetric(c1, emptyLabels, pdt1Ms, doublePoint(pdt1Ms, 44))), 1, }, { "EmptyLabels: round 2 - instance adjusted based on round 1", - metricSlice(sumMetric(c1, emptyLabels, t2Ms, doublePoint(t2Ms, 66))), - metricSlice(sumMetric(c1, emptyLabels, t1Ms, doublePoint(t2Ms, 66))), + metricSlice(sumMetric(c1, emptyLabels, pdt2Ms, doublePoint(pdt2Ms, 66))), + metricSlice(sumMetric(c1, emptyLabels, pdt1Ms, doublePoint(pdt2Ms, 66))), 0, }, { "EmptyLabels: round 3 - one explicitly empty label, instance adjusted based on round 1", - metricSlice(sumMetric(c1, k1vEmpty, t3Ms, doublePoint(t3Ms, 77))), - metricSlice(sumMetric(c1, k1vEmpty, t1Ms, doublePoint(t3Ms, 77))), + metricSlice(sumMetric(c1, k1vEmpty, pdt3Ms, doublePoint(pdt3Ms, 77))), + metricSlice(sumMetric(c1, k1vEmpty, pdt1Ms, doublePoint(pdt3Ms, 77))), 0, }, { "EmptyLabels: round 4 - three explicitly empty labels, instance adjusted based on round 1", - metricSlice(sumMetric(c1, k1vEmptyk2vEmptyk3vEmpty, t3Ms, doublePoint(t3Ms, 88))), - metricSlice(sumMetric(c1, k1vEmptyk2vEmptyk3vEmpty, t1Ms, doublePoint(t3Ms, 88))), + metricSlice(sumMetric(c1, k1vEmptyk2vEmptyk3vEmpty, pdt3Ms, doublePoint(pdt3Ms, 88))), + metricSlice(sumMetric(c1, k1vEmptyk2vEmptyk3vEmpty, pdt1Ms, doublePoint(pdt3Ms, 88))), 0, }, } - runScript(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) + runScriptPdata(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) } -func Test_tsGC(t *testing.T) { - script1 := []*metricsAdjusterTest{ +func Test_tsGC_pdata(t *testing.T) { + script1 := []*metricsAdjusterTestPdata{ { "TsGC: round 1 - initial instances, start time is established", metricSlice( - sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t1Ms, 44)), - sumMetric(c1, k1v10k2v20, t1Ms, doublePoint(t1Ms, 20)), - histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7})), - histogramMetric(cd1, k1v10k2v20, t1Ms, distPoint(t1Ms, bounds0, []uint64{40, 20, 30, 70})), + sumMetric(c1, k1v1k2v2, pdt1Ms, doublePoint(pdt1Ms, 44)), + sumMetric(c1, k1v10k2v20, pdt1Ms, doublePoint(pdt1Ms, 20)), + histogramMetric(cd1, k1v1k2v2, pdt1Ms, distPoint(pdt1Ms, bounds0, []uint64{4, 2, 3, 7})), + histogramMetric(cd1, k1v10k2v20, pdt1Ms, distPoint(pdt1Ms, bounds0, []uint64{40, 20, 30, 70})), ), metricSlice( - sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t1Ms, 44)), - sumMetric(c1, k1v10k2v20, t1Ms, doublePoint(t1Ms, 20)), - histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7})), - histogramMetric(cd1, k1v10k2v20, t1Ms, distPoint(t1Ms, bounds0, []uint64{40, 20, 30, 70})), + sumMetric(c1, k1v1k2v2, pdt1Ms, doublePoint(pdt1Ms, 44)), + sumMetric(c1, k1v10k2v20, pdt1Ms, doublePoint(pdt1Ms, 20)), + histogramMetric(cd1, k1v1k2v2, pdt1Ms, distPoint(pdt1Ms, bounds0, []uint64{4, 2, 3, 7})), + histogramMetric(cd1, k1v10k2v20, pdt1Ms, distPoint(pdt1Ms, bounds0, []uint64{40, 20, 30, 70})), ), 4, }, } - script2 := []*metricsAdjusterTest{ + script2 := []*metricsAdjusterTestPdata{ { "TsGC: round 2 - metrics first timeseries adjusted based on round 2, second timeseries not updated", metricSlice( - sumMetric(c1, k1v1k2v2, t2Ms, doublePoint(t2Ms, 88)), - histogramMetric(cd1, k1v1k2v2, t2Ms, distPoint(t2Ms, bounds0, []uint64{8, 7, 9, 14})), + sumMetric(c1, k1v1k2v2, pdt2Ms, doublePoint(pdt2Ms, 88)), + histogramMetric(cd1, k1v1k2v2, pdt2Ms, distPoint(pdt2Ms, bounds0, []uint64{8, 7, 9, 14})), ), metricSlice( - sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t2Ms, 88)), - histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t2Ms, bounds0, []uint64{8, 7, 9, 14})), + sumMetric(c1, k1v1k2v2, pdt1Ms, doublePoint(pdt2Ms, 88)), + histogramMetric(cd1, k1v1k2v2, pdt1Ms, distPoint(pdt2Ms, bounds0, []uint64{8, 7, 9, 14})), ), 0, }, } - script3 := []*metricsAdjusterTest{ + script3 := []*metricsAdjusterTestPdata{ { "TsGC: round 3 - metrics first timeseries adjusted based on round 2, second timeseries empty due to timeseries gc()", metricSlice( - sumMetric(c1, k1v1k2v2, t3Ms, doublePoint(t3Ms, 99)), - sumMetric(c1, k1v10k2v20, t3Ms, doublePoint(t3Ms, 80)), - histogramMetric(cd1, k1v1k2v2, t3Ms, distPoint(t3Ms, bounds0, []uint64{9, 8, 10, 15})), - histogramMetric(cd1, k1v10k2v20, t3Ms, distPoint(t3Ms, bounds0, []uint64{55, 66, 33, 77})), + sumMetric(c1, k1v1k2v2, pdt3Ms, doublePoint(pdt3Ms, 99)), + sumMetric(c1, k1v10k2v20, pdt3Ms, doublePoint(pdt3Ms, 80)), + histogramMetric(cd1, k1v1k2v2, pdt3Ms, distPoint(pdt3Ms, bounds0, []uint64{9, 8, 10, 15})), + histogramMetric(cd1, k1v10k2v20, pdt3Ms, distPoint(pdt3Ms, bounds0, []uint64{55, 66, 33, 77})), ), metricSlice( - sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t3Ms, 99)), - sumMetric(c1, k1v10k2v20, t3Ms, doublePoint(t3Ms, 80)), - histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t3Ms, bounds0, []uint64{9, 8, 10, 15})), - histogramMetric(cd1, k1v10k2v20, t3Ms, distPoint(t3Ms, bounds0, []uint64{55, 66, 33, 77})), + sumMetric(c1, k1v1k2v2, pdt1Ms, doublePoint(pdt3Ms, 99)), + sumMetric(c1, k1v10k2v20, pdt3Ms, doublePoint(pdt3Ms, 80)), + histogramMetric(cd1, k1v1k2v2, pdt1Ms, distPoint(pdt3Ms, bounds0, []uint64{9, 8, 10, 15})), + histogramMetric(cd1, k1v10k2v20, pdt3Ms, distPoint(pdt3Ms, bounds0, []uint64{55, 66, 33, 77})), ), 2, }, @@ -793,39 +794,39 @@ func Test_tsGC(t *testing.T) { jobsMap := NewJobsMapPdata(time.Minute) // run round 1 - runScript(t, jobsMap.get("job", "0"), script1) + runScriptPdata(t, jobsMap.get("job", "0"), script1) // gc the tsmap, unmarking all entries jobsMap.get("job", "0").gc() // run round 2 - update metrics first timeseries only - runScript(t, jobsMap.get("job", "0"), script2) + runScriptPdata(t, jobsMap.get("job", "0"), script2) // gc the tsmap, collecting umarked entries jobsMap.get("job", "0").gc() // run round 3 - verify that metrics second timeseries have been gc'd - runScript(t, jobsMap.get("job", "0"), script3) + runScriptPdata(t, jobsMap.get("job", "0"), script3) } -func Test_jobGC(t *testing.T) { - job1Script1 := []*metricsAdjusterTest{ +func Test_jobGC_pdata(t *testing.T) { + job1Script1 := []*metricsAdjusterTestPdata{ { "JobGC: job 1, round 1 - initial instances, adjusted should be empty", metricSlice( - sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t1Ms, 44)), - sumMetric(c1, k1v10k2v20, t1Ms, doublePoint(t1Ms, 20)), - histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7})), - histogramMetric(cd1, k1v10k2v20, t1Ms, distPoint(t1Ms, bounds0, []uint64{40, 20, 30, 70})), + sumMetric(c1, k1v1k2v2, pdt1Ms, doublePoint(pdt1Ms, 44)), + sumMetric(c1, k1v10k2v20, pdt1Ms, doublePoint(pdt1Ms, 20)), + histogramMetric(cd1, k1v1k2v2, pdt1Ms, distPoint(pdt1Ms, bounds0, []uint64{4, 2, 3, 7})), + histogramMetric(cd1, k1v10k2v20, pdt1Ms, distPoint(pdt1Ms, bounds0, []uint64{40, 20, 30, 70})), ), metricSlice( - sumMetric(c1, k1v1k2v2, t1Ms, doublePoint(t1Ms, 44)), - sumMetric(c1, k1v10k2v20, t1Ms, doublePoint(t1Ms, 20)), - histogramMetric(cd1, k1v1k2v2, t1Ms, distPoint(t1Ms, bounds0, []uint64{4, 2, 3, 7})), - histogramMetric(cd1, k1v10k2v20, t1Ms, distPoint(t1Ms, bounds0, []uint64{40, 20, 30, 70})), + sumMetric(c1, k1v1k2v2, pdt1Ms, doublePoint(pdt1Ms, 44)), + sumMetric(c1, k1v10k2v20, pdt1Ms, doublePoint(pdt1Ms, 20)), + histogramMetric(cd1, k1v1k2v2, pdt1Ms, distPoint(pdt1Ms, bounds0, []uint64{4, 2, 3, 7})), + histogramMetric(cd1, k1v10k2v20, pdt1Ms, distPoint(pdt1Ms, bounds0, []uint64{40, 20, 30, 70})), ), 4, }, } emptyMetricSlice := func() *pdata.MetricSlice { ms := pdata.NewMetricSlice(); return &ms } - job2Script1 := []*metricsAdjusterTest{ + job2Script1 := []*metricsAdjusterTestPdata{ { "JobGC: job2, round 1 - no metrics adjusted, just trigger gc", emptyMetricSlice(), @@ -834,20 +835,20 @@ func Test_jobGC(t *testing.T) { }, } - job1Script2 := []*metricsAdjusterTest{ + job1Script2 := []*metricsAdjusterTestPdata{ { "JobGC: job 1, round 2 - metrics timeseries empty due to job-level gc", metricSlice( - sumMetric(c1, k1v1k2v2, t4Ms, doublePoint(t4Ms, 99)), - sumMetric(c1, k1v10k2v20, t4Ms, doublePoint(t4Ms, 80)), - histogramMetric(cd1, k1v1k2v2, t4Ms, distPoint(t4Ms, bounds0, []uint64{9, 8, 10, 15})), - histogramMetric(cd1, k1v10k2v20, t4Ms, distPoint(t4Ms, bounds0, []uint64{55, 66, 33, 77})), + sumMetric(c1, k1v1k2v2, pdt4Ms, doublePoint(pdt4Ms, 99)), + sumMetric(c1, k1v10k2v20, pdt4Ms, doublePoint(pdt4Ms, 80)), + histogramMetric(cd1, k1v1k2v2, pdt4Ms, distPoint(pdt4Ms, bounds0, []uint64{9, 8, 10, 15})), + histogramMetric(cd1, k1v10k2v20, pdt4Ms, distPoint(pdt4Ms, bounds0, []uint64{55, 66, 33, 77})), ), metricSlice( - sumMetric(c1, k1v1k2v2, t4Ms, doublePoint(t4Ms, 99)), - sumMetric(c1, k1v10k2v20, t4Ms, doublePoint(t4Ms, 80)), - histogramMetric(cd1, k1v1k2v2, t4Ms, distPoint(t4Ms, bounds0, []uint64{9, 8, 10, 15})), - histogramMetric(cd1, k1v10k2v20, t4Ms, distPoint(t4Ms, bounds0, []uint64{55, 66, 33, 77})), + sumMetric(c1, k1v1k2v2, pdt4Ms, doublePoint(pdt4Ms, 99)), + sumMetric(c1, k1v10k2v20, pdt4Ms, doublePoint(pdt4Ms, 80)), + histogramMetric(cd1, k1v1k2v2, pdt4Ms, distPoint(pdt4Ms, bounds0, []uint64{9, 8, 10, 15})), + histogramMetric(cd1, k1v10k2v20, pdt4Ms, distPoint(pdt4Ms, bounds0, []uint64{55, 66, 33, 77})), ), 4, }, @@ -857,30 +858,30 @@ func Test_jobGC(t *testing.T) { jobsMap := NewJobsMapPdata(gcInterval) // run job 1, round 1 - all entries marked - runScript(t, jobsMap.get("job", "0"), job1Script1) + runScriptPdata(t, jobsMap.get("job", "0"), job1Script1) // sleep longer than gcInterval to enable job gc in the next run time.Sleep(2 * gcInterval) // run job 2, round1 - trigger job gc, unmarking all entries - runScript(t, jobsMap.get("job", "1"), job2Script1) + runScriptPdata(t, jobsMap.get("job", "1"), job2Script1) // sleep longer than gcInterval to enable job gc in the next run time.Sleep(2 * gcInterval) // re-run job 2, round1 - trigger job gc, removing unmarked entries - runScript(t, jobsMap.get("job", "1"), job2Script1) + runScriptPdata(t, jobsMap.get("job", "1"), job2Script1) // ensure that at least one jobsMap.gc() completed jobsMap.gc() // run job 1, round 2 - verify that all job 1 timeseries have been gc'd - runScript(t, jobsMap.get("job", "0"), job1Script2) + runScriptPdata(t, jobsMap.get("job", "0"), job1Script2) return } -type metricsAdjusterTest struct { +type metricsAdjusterTestPdata struct { description string metrics *pdata.MetricSlice adjusted *pdata.MetricSlice resets int } -func runScript(t *testing.T, tsm *timeseriesMapPdata, script []*metricsAdjusterTest) { +func runScriptPdata(t *testing.T, tsm *timeseriesMapPdata, script []*metricsAdjusterTestPdata) { l := zap.NewNop() t.Cleanup(func() { require.NoError(t, l.Sync()) }) // flushes buffer, if any ma := NewMetricsAdjusterPdata(tsm, l) diff --git a/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go b/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go index 7e1c14a6aecdd..34af03cc1fb91 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go +++ b/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go @@ -15,12 +15,10 @@ package internal import ( - "errors" "fmt" "regexp" "sort" "strconv" - "strings" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/pkg/labels" @@ -30,105 +28,6 @@ import ( "go.opentelemetry.io/collector/model/pdata" ) -const ( - metricsSuffixCount = "_count" - metricsSuffixBucket = "_bucket" - metricsSuffixSum = "_sum" - metricSuffixTotal = "_total" - startTimeMetricName = "process_start_time_seconds" - scrapeUpMetricName = "up" -) - -var ( - trimmableSuffixes = []string{metricsSuffixBucket, metricsSuffixCount, metricsSuffixSum, metricSuffixTotal} - errNoDataToBuild = errors.New("there's no data to build") - errNoBoundaryLabel = errors.New("given metricType has no BucketLabel or QuantileLabel") - errEmptyBoundaryLabel = errors.New("BucketLabel or QuantileLabel is empty") -) - -// dpgSignature is used to create a key for data complexValue belong to a same group of a metric family -func dpgSignature(orderedKnownLabelKeys []string, ls labels.Labels) string { - size := 0 - for _, k := range orderedKnownLabelKeys { - v := ls.Get(k) - if v == "" { - continue - } - // 2 enclosing quotes + 1 equality sign = 3 extra chars. - // Note: if any character in the label value requires escaping, - // we'll need more space than that, which will lead to some - // extra allocation. - size += 3 + len(k) + len(v) - } - sign := make([]byte, 0, size) - for _, k := range orderedKnownLabelKeys { - v := ls.Get(k) - if v == "" { - continue - } - sign = strconv.AppendQuote(sign, k+"="+v) - } - return string(sign) -} - -func normalizeMetricName(name string) string { - for _, s := range trimmableSuffixes { - if strings.HasSuffix(name, s) && name != s { - return strings.TrimSuffix(name, s) - } - } - return name -} - -// Code borrowed from the original promreceiver -func heuristicalMetricAndKnownUnits(metricName, parsedUnit string) string { - if parsedUnit != "" { - return parsedUnit - } - lastUnderscoreIndex := strings.LastIndex(metricName, "_") - if lastUnderscoreIndex <= 0 || lastUnderscoreIndex >= len(metricName)-1 { - return "" - } - - unit := "" - - supposedUnit := metricName[lastUnderscoreIndex+1:] - switch strings.ToLower(supposedUnit) { - case "millisecond", "milliseconds", "ms": - unit = "ms" - case "second", "seconds", "s": - unit = "s" - case "microsecond", "microseconds", "us": - unit = "us" - case "nanosecond", "nanoseconds", "ns": - unit = "ns" - case "byte", "bytes", "by": - unit = "By" - case "bit", "bits": - unit = "Bi" - case "kilogram", "kilograms", "kg": - unit = "kg" - case "gram", "grams", "g": - unit = "g" - case "meter", "meters", "metre", "metres", "m": - unit = "m" - case "kilometer", "kilometers", "kilometre", "kilometres", "km": - unit = "km" - case "milimeter", "milimeters", "milimetre", "milimetres", "mm": - unit = "mm" - case "nanogram", "ng", "nanograms": - unit = "ng" - } - - return unit -} - -func isInternalMetric(metricName string) bool { - if metricName == scrapeUpMetricName || strings.HasPrefix(metricName, "scrape_") { - return true - } - return false -} func isUsefulLabelPdata(mType pdata.MetricDataType, labelKey string) bool { switch labelKey { case model.MetricNameLabel, model.InstanceLabel, model.SchemeLabel, model.MetricsPathLabel, model.JobLabel: diff --git a/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go b/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go index 863c45afcd9f3..124d3c278ead6 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go +++ b/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go @@ -15,13 +15,11 @@ package internal import ( - "runtime" "testing" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/textparse" - "github.com/prometheus/prometheus/scrape" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "go.uber.org/zap" @@ -29,64 +27,7 @@ import ( "go.opentelemetry.io/collector/model/pdata" ) -const startTs = int64(1555366610000) -const interval = int64(15 * 1000) -const defaultBuilderStartTime = float64(1.0) - -var testMetadata = map[string]scrape.MetricMetadata{ - "counter_test": {Metric: "counter_test", Type: textparse.MetricTypeCounter, Help: "", Unit: ""}, - "counter_test2": {Metric: "counter_test2", Type: textparse.MetricTypeCounter, Help: "", Unit: ""}, - "gauge_test": {Metric: "gauge_test", Type: textparse.MetricTypeGauge, Help: "", Unit: ""}, - "gauge_test2": {Metric: "gauge_test2", Type: textparse.MetricTypeGauge, Help: "", Unit: ""}, - "hist_test": {Metric: "hist_test", Type: textparse.MetricTypeHistogram, Help: "", Unit: ""}, - "hist_test2": {Metric: "hist_test2", Type: textparse.MetricTypeHistogram, Help: "", Unit: ""}, - "ghist_test": {Metric: "ghist_test", Type: textparse.MetricTypeGaugeHistogram, Help: "", Unit: ""}, - "summary_test": {Metric: "summary_test", Type: textparse.MetricTypeSummary, Help: "", Unit: ""}, - "summary_test2": {Metric: "summary_test2", Type: textparse.MetricTypeSummary, Help: "", Unit: ""}, - "unknown_test": {Metric: "unknown_test", Type: textparse.MetricTypeUnknown, Help: "", Unit: ""}, - "poor_name_count": {Metric: "poor_name_count", Type: textparse.MetricTypeCounter, Help: "", Unit: ""}, - "up": {Metric: "up", Type: textparse.MetricTypeCounter, Help: "", Unit: ""}, - "scrape_foo": {Metric: "scrape_foo", Type: textparse.MetricTypeCounter, Help: "", Unit: ""}, - "example_process_start_time_seconds": {Metric: "example_process_start_time_seconds", - Type: textparse.MetricTypeGauge, Help: "", Unit: ""}, - "process_start_time_seconds": {Metric: "process_start_time_seconds", - Type: textparse.MetricTypeGauge, Help: "", Unit: ""}, - "badprocess_start_time_seconds": {Metric: "badprocess_start_time_seconds", - Type: textparse.MetricTypeGauge, Help: "", Unit: ""}, -} - -type testDataPoint struct { - lb labels.Labels - t int64 - v float64 -} - -type testScrapedPage struct { - pts []*testDataPoint -} - -func createLabels(mFamily string, tagPairs ...string) labels.Labels { - lm := make(map[string]string) - lm[model.MetricNameLabel] = mFamily - if len(tagPairs)%2 != 0 { - panic("tag pairs is not even") - } - - for i := 0; i < len(tagPairs); i += 2 { - lm[tagPairs[i]] = tagPairs[i+1] - } - - return labels.FromMap(lm) -} - -func createDataPoint(mname string, value float64, tagPairs ...string) *testDataPoint { - return &testDataPoint{ - lb: createLabels(mname, tagPairs...), - v: value, - } -} - -func runBuilderStartTimeTests(t *testing.T, tests []buildTestDataPdata, +func runBuilderStartTimeTestsPdata(t *testing.T, tests []buildTestDataPdata, startTimeMetricRegex string, expectedBuilderStartTime float64) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -109,7 +50,7 @@ func runBuilderStartTimeTests(t *testing.T, tests []buildTestDataPdata, } } -func Test_startTimeMetricMatch(t *testing.T) { +func Test_startTimeMetricMatch_pdata(t *testing.T) { matchBuilderStartTime := 123.456 matchTests := []buildTestDataPdata{ { @@ -160,67 +101,8 @@ func Test_startTimeMetricMatch(t *testing.T) { }, } - runBuilderStartTimeTests(t, matchTests, "^(.+_)*process_start_time_seconds$", matchBuilderStartTime) - runBuilderStartTimeTests(t, nomatchTests, "^(.+_)*process_start_time_seconds$", defaultBuilderStartTime) -} - -func Test_heuristicalMetricAndKnownUnits(t *testing.T) { - tests := []struct { - metricName string - parsedUnit string - want string - }{ - {"test", "ms", "ms"}, - {"millisecond", "", ""}, - {"test_millisecond", "", "ms"}, - {"test_milliseconds", "", "ms"}, - {"test_ms", "", "ms"}, - {"test_second", "", "s"}, - {"test_seconds", "", "s"}, - {"test_s", "", "s"}, - {"test_microsecond", "", "us"}, - {"test_microseconds", "", "us"}, - {"test_us", "", "us"}, - {"test_nanosecond", "", "ns"}, - {"test_nanoseconds", "", "ns"}, - {"test_ns", "", "ns"}, - {"test_byte", "", "By"}, - {"test_bytes", "", "By"}, - {"test_by", "", "By"}, - {"test_bit", "", "Bi"}, - {"test_bits", "", "Bi"}, - {"test_kilogram", "", "kg"}, - {"test_kilograms", "", "kg"}, - {"test_kg", "", "kg"}, - {"test_gram", "", "g"}, - {"test_grams", "", "g"}, - {"test_g", "", "g"}, - {"test_nanogram", "", "ng"}, - {"test_nanograms", "", "ng"}, - {"test_ng", "", "ng"}, - {"test_meter", "", "m"}, - {"test_meters", "", "m"}, - {"test_metre", "", "m"}, - {"test_metres", "", "m"}, - {"test_m", "", "m"}, - {"test_kilometer", "", "km"}, - {"test_kilometers", "", "km"}, - {"test_kilometre", "", "km"}, - {"test_kilometres", "", "km"}, - {"test_km", "", "km"}, - {"test_milimeter", "", "mm"}, - {"test_milimeters", "", "mm"}, - {"test_milimetre", "", "mm"}, - {"test_milimetres", "", "mm"}, - {"test_mm", "", "mm"}, - } - for _, tt := range tests { - t.Run(tt.metricName, func(t *testing.T) { - if got := heuristicalMetricAndKnownUnits(tt.metricName, tt.parsedUnit); got != tt.want { - t.Errorf("heuristicalMetricAndKnownUnits() = %v, want %v", got, tt.want) - } - }) - } + runBuilderStartTimeTestsPdata(t, matchTests, "^(.+_)*process_start_time_seconds$", matchBuilderStartTime) + runBuilderStartTimeTestsPdata(t, nomatchTests, "^(.+_)*process_start_time_seconds$", defaultBuilderStartTime) } func TestGetBoundaryPdata(t *testing.T) { @@ -1159,7 +1041,7 @@ func Test_OTLPMetricBuilder_histogram(t *testing.T) { runBuilderTestsPdata(t, tests) } -func Test_metricBuilder_summary(t *testing.T) { +func Test_OTLPMetricBuilder_summary(t *testing.T) { tests := []buildTestDataPdata{ { name: "no-sum-and-count", @@ -1301,7 +1183,7 @@ func Test_metricBuilder_summary(t *testing.T) { } // Ensure that we reject duplicate label keys. See https://github.com/open-telemetry/wg-prometheus/issues/44. -func TestMetricBuilderDuplicateLabelKeysAreRejected(t *testing.T) { +func TestOTLPMetricBuilderDuplicateLabelKeysAreRejected(t *testing.T) { mc := newMockMetadataCache(testMetadata) mb := newMetricBuilderPdata(mc, true, "", zap.NewNop(), 0) @@ -1320,7 +1202,7 @@ func TestMetricBuilderDuplicateLabelKeysAreRejected(t *testing.T) { require.Contains(t, err.Error(), `invalid sample: non-unique label names: ["a" "z"]`) } -func Test_metricBuilder_baddata(t *testing.T) { +func Test_OTLPMetricBuilder_baddata(t *testing.T) { t.Run("empty-metric-name", func(t *testing.T) { mc := newMockMetadataCache(testMetadata) b := newMetricBuilderPdata(mc, true, "", zap.NewNop(), 0) @@ -1353,69 +1235,3 @@ func Test_metricBuilder_baddata(t *testing.T) { } }) } - -func Benchmark_dpgSignature(b *testing.B) { - knownLabelKeys := []string{"a", "b"} - labels := labels.FromStrings("a", "va", "b", "vb", "x", "xa") - b.ReportAllocs() - for i := 0; i < b.N; i++ { - runtime.KeepAlive(dpgSignature(knownLabelKeys, labels)) - } -} - -func Test_dpgSignature(t *testing.T) { - knownLabelKeys := []string{"a", "b"} - - tests := []struct { - name string - ls labels.Labels - want string - }{ - {"1st label", labels.FromStrings("a", "va"), `"a=va"`}, - {"2nd label", labels.FromStrings("b", "vb"), `"b=vb"`}, - {"two labels", labels.FromStrings("a", "va", "b", "vb"), `"a=va""b=vb"`}, - {"extra label", labels.FromStrings("a", "va", "b", "vb", "x", "xa"), `"a=va""b=vb"`}, - {"different order", labels.FromStrings("b", "vb", "a", "va"), `"a=va""b=vb"`}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := dpgSignature(knownLabelKeys, tt.ls); got != tt.want { - t.Errorf("dpgSignature() = %q, want %q", got, tt.want) - } - }) - } - - // this is important for caching start values, as new metrics with new tag of a same group can come up in a 2nd run, - // however, its order within the group is not predictable. we need to have a way to generate a stable key even if - // the total number of keys changes in between different scrape runs - t.Run("knownLabelKeys updated", func(t *testing.T) { - ls := labels.FromStrings("a", "va") - want := dpgSignature(knownLabelKeys, ls) - got := dpgSignature(append(knownLabelKeys, "c"), ls) - if got != want { - t.Errorf("dpgSignature() = %v, want %v", got, want) - } - }) -} - -func Test_normalizeMetricName(t *testing.T) { - tests := []struct { - name string - mname string - want string - }{ - {"normal", "normal", "normal"}, - {"count", "foo_count", "foo"}, - {"bucket", "foo_bucket", "foo"}, - {"sum", "foo_sum", "foo"}, - {"total", "foo_total", "foo"}, - {"no_prefix", "_sum", "_sum"}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := normalizeMetricName(tt.mname); got != tt.want { - t.Errorf("normalizeMetricName() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/receiver/prometheusreceiver/internal/otlp_transaction.go b/receiver/prometheusreceiver/internal/otlp_transaction.go index 52228932772fd..c8f786ced3207 100644 --- a/receiver/prometheusreceiver/internal/otlp_transaction.go +++ b/receiver/prometheusreceiver/internal/otlp_transaction.go @@ -16,7 +16,6 @@ package internal import ( "context" - "errors" "sync/atomic" "time" @@ -33,21 +32,6 @@ import ( "github.com/prometheus/prometheus/pkg/labels" ) -const ( - portAttr = "port" - schemeAttr = "scheme" - jobAttr = "job" - instanceAttr = "instance" - - transport = "http" - dataformat = "prometheus" -) - -var errMetricNameNotFound = errors.New("metricName not found from labels") -var errTransactionAborted = errors.New("transaction aborted") -var errNoJobInstance = errors.New("job or instance cannot be found from labels") -var errNoStartTimeMetrics = errors.New("process_start_time_seconds metric is missing") - type transactionPdata struct { id int64 isNew bool @@ -181,14 +165,14 @@ func (t *transactionPdata) Rollback() error { return nil } -func timestampFromFloat64(ts float64) pdata.Timestamp { +func pdataTimestampFromFloat64(ts float64) pdata.Timestamp { secs := int64(ts) nanos := int64((ts - float64(secs)) * 1e9) return pdata.NewTimestampFromTime(time.Unix(secs, nanos)) } func adjustStartTimestampPdata(startTime float64, metricsL *pdata.MetricSlice) { - startTimeTs := timestampFromFloat64(startTime) + startTimeTs := pdataTimestampFromFloat64(startTime) for i := 0; i < metricsL.Len(); i++ { metric := metricsL.At(i) switch metric.DataType() { diff --git a/receiver/prometheusreceiver/internal/otlp_transaction_test.go b/receiver/prometheusreceiver/internal/otlp_transaction_test.go index 49cf250f1f15b..91afde7c6b8a1 100644 --- a/receiver/prometheusreceiver/internal/otlp_transaction_test.go +++ b/receiver/prometheusreceiver/internal/otlp_transaction_test.go @@ -29,7 +29,7 @@ import ( "go.opentelemetry.io/collector/consumer/consumertest" ) -func Test_transaction(t *testing.T) { +func Test_transaction_pdata(t *testing.T) { // discoveredLabels contain labels prior to any processing discoveredLabels := labels.New( labels.Label{ diff --git a/receiver/prometheusreceiver/internal/transaction.go b/receiver/prometheusreceiver/internal/transaction.go new file mode 100644 index 0000000000000..11351f98ea9d1 --- /dev/null +++ b/receiver/prometheusreceiver/internal/transaction.go @@ -0,0 +1,260 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "context" + "errors" + "net" + "sync/atomic" + + commonpb "github.com/census-instrumentation/opencensus-proto/gen-go/agent/common/v1" + metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" + resourcepb "github.com/census-instrumentation/opencensus-proto/gen-go/resource/v1" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/exemplar" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/storage" + "go.opentelemetry.io/collector/component" + "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/consumer" + "go.opentelemetry.io/collector/obsreport" + "go.uber.org/zap" + "google.golang.org/protobuf/types/known/timestamppb" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus" +) + +const ( + portAttr = "port" + schemeAttr = "scheme" + jobAttr = "job" + instanceAttr = "instance" + + transport = "http" + dataformat = "prometheus" +) + +var errMetricNameNotFound = errors.New("metricName not found from labels") +var errTransactionAborted = errors.New("transaction aborted") +var errNoJobInstance = errors.New("job or instance cannot be found from labels") +var errNoStartTimeMetrics = errors.New("process_start_time_seconds metric is missing") + +// A transaction is corresponding to an individual scrape operation or stale report. +// That said, whenever prometheus receiver scrapped a target metric endpoint a page of raw metrics is returned, +// a transaction, which acts as appender, is created to process this page of data, the scrapeLoop will call the Add or +// AddFast method to insert metrics data points, when finished either Commit, which means success, is called and data +// will be flush to the downstream consumer, or Rollback, which means discard all the data, is called and all data +// points are discarded. +type transaction struct { + id int64 + ctx context.Context + isNew bool + sink consumer.Metrics + job string + instance string + jobsMap *JobsMap + useStartTimeMetric bool + startTimeMetricRegex string + ms *metadataService + node *commonpb.Node + resource *resourcepb.Resource + metricBuilder *metricBuilder + externalLabels labels.Labels + logger *zap.Logger + obsrecv *obsreport.Receiver + startTimeMs int64 +} + +func newTransaction( + ctx context.Context, + jobsMap *JobsMap, + useStartTimeMetric bool, + startTimeMetricRegex string, + receiverID config.ComponentID, + ms *metadataService, + sink consumer.Metrics, + externalLabels labels.Labels, + set component.ReceiverCreateSettings) *transaction { + return &transaction{ + id: atomic.AddInt64(&idSeq, 1), + ctx: ctx, + isNew: true, + sink: sink, + jobsMap: jobsMap, + useStartTimeMetric: useStartTimeMetric, + startTimeMetricRegex: startTimeMetricRegex, + ms: ms, + externalLabels: externalLabels, + logger: set.Logger, + obsrecv: obsreport.NewReceiver(obsreport.ReceiverSettings{ + ReceiverID: receiverID, + Transport: transport, + ReceiverCreateSettings: set, + }), + startTimeMs: -1, + } +} + +// ensure *transaction has implemented the storage.Appender interface +var _ storage.Appender = (*transaction)(nil) + +// Append always returns 0 to disable label caching. +func (tr *transaction) Append(ref uint64, ls labels.Labels, t int64, v float64) (uint64, error) { + if tr.startTimeMs < 0 { + tr.startTimeMs = t + } + + select { + case <-tr.ctx.Done(): + return 0, errTransactionAborted + default: + } + if len(tr.externalLabels) > 0 { + // TODO(jbd): Improve the allocs. + ls = append(ls, tr.externalLabels...) + } + if tr.isNew { + if err := tr.initTransaction(ls); err != nil { + return 0, err + } + } + return 0, tr.metricBuilder.AddDataPoint(ls, t, v) +} + +func (tr *transaction) AppendExemplar(ref uint64, l labels.Labels, e exemplar.Exemplar) (uint64, error) { + return 0, nil +} + +// AddFast always returns error since caching is not supported by Add() function. +func (tr *transaction) AddFast(_ uint64, _ int64, _ float64) error { + return storage.ErrNotFound +} + +func (tr *transaction) initTransaction(ls labels.Labels) error { + job, instance := ls.Get(model.JobLabel), ls.Get(model.InstanceLabel) + if job == "" || instance == "" { + return errNoJobInstance + } + // discover the binding target when this method is called for the first time during a transaction + mc, err := tr.ms.Get(job, instance) + if err != nil { + return err + } + if tr.jobsMap != nil { + tr.job = job + tr.instance = instance + } + tr.node, tr.resource = createNodeAndResource(job, instance, mc.SharedLabels().Get(model.SchemeLabel)) + tr.metricBuilder = newMetricBuilder(mc, tr.useStartTimeMetric, tr.startTimeMetricRegex, tr.logger, tr.startTimeMs) + tr.isNew = false + return nil +} + +// Commit submits metrics data to consumers. +func (tr *transaction) Commit() error { + if tr.isNew { + // In a situation like not able to connect to the remote server, scrapeloop will still commit even if it had + // never added any data points, that the transaction has not been initialized. + return nil + } + + tr.startTimeMs = -1 + + ctx := tr.obsrecv.StartMetricsOp(tr.ctx) + metrics, _, _, err := tr.metricBuilder.Build() + if err != nil { + // Only error by Build() is errNoDataToBuild, with numReceivedPoints set to zero. + tr.obsrecv.EndMetricsOp(ctx, dataformat, 0, err) + return err + } + + if tr.useStartTimeMetric { + // startTime is mandatory in this case, but may be zero when the + // process_start_time_seconds metric is missing from the target endpoint. + if tr.metricBuilder.startTime == 0.0 { + // Since we are unable to adjust metrics properly, we will drop them + // and return an error. + err = errNoStartTimeMetrics + tr.obsrecv.EndMetricsOp(ctx, dataformat, 0, err) + return err + } + + adjustStartTimestamp(tr.metricBuilder.startTime, metrics) + } else { + // AdjustMetrics - jobsMap has to be non-nil in this case. + // Note: metrics could be empty after adjustment, which needs to be checked before passing it on to ConsumeMetrics() + metrics, _ = NewMetricsAdjuster(tr.jobsMap.get(tr.job, tr.instance), tr.logger).AdjustMetrics(metrics) + } + + numPoints := 0 + if len(metrics) > 0 { + md := opencensus.OCToMetrics(tr.node, tr.resource, metrics) + numPoints = md.DataPointCount() + err = tr.sink.ConsumeMetrics(ctx, md) + } + tr.obsrecv.EndMetricsOp(ctx, dataformat, numPoints, err) + return err +} + +func (tr *transaction) Rollback() error { + tr.startTimeMs = -1 + return nil +} + +func adjustStartTimestamp(startTime float64, metrics []*metricspb.Metric) { + startTimeTs := timestampFromFloat64(startTime) + for _, metric := range metrics { + switch metric.GetMetricDescriptor().GetType() { + case metricspb.MetricDescriptor_GAUGE_DOUBLE, metricspb.MetricDescriptor_GAUGE_DISTRIBUTION: + continue + default: + for _, ts := range metric.GetTimeseries() { + ts.StartTimestamp = startTimeTs + } + } + } +} + +func timestampFromFloat64(ts float64) *timestamppb.Timestamp { + secs := int64(ts) + nanos := int64((ts - float64(secs)) * 1e9) + return ×tamppb.Timestamp{ + Seconds: secs, + Nanos: int32(nanos), + } +} + +func createNodeAndResource(job, instance, scheme string) (*commonpb.Node, *resourcepb.Resource) { + host, port, err := net.SplitHostPort(instance) + if err != nil { + host = instance + } + node := &commonpb.Node{ + ServiceInfo: &commonpb.ServiceInfo{Name: job}, + Identifier: &commonpb.ProcessIdentifier{ + HostName: host, + }, + } + resource := &resourcepb.Resource{ + Labels: map[string]string{ + jobAttr: job, + instanceAttr: instance, + portAttr: port, + schemeAttr: scheme, + }, + } + return node, resource +} diff --git a/receiver/prometheusreceiver/internal/transaction_test.go b/receiver/prometheusreceiver/internal/transaction_test.go new file mode 100644 index 0000000000000..a67b774bc51c6 --- /dev/null +++ b/receiver/prometheusreceiver/internal/transaction_test.go @@ -0,0 +1,154 @@ +// Copyright The OpenTelemetry Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package internal + +import ( + "context" + "testing" + "time" + + agentmetricspb "github.com/census-instrumentation/opencensus-proto/gen-go/agent/metrics/v1" + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/labels" + "github.com/prometheus/prometheus/scrape" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/consumer/consumertest" + "google.golang.org/protobuf/proto" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus" +) + +func Test_transaction(t *testing.T) { + // discoveredLabels contain labels prior to any processing + discoveredLabels := labels.New( + labels.Label{ + Name: model.AddressLabel, + Value: "address:8080", + }, + labels.Label{ + Name: model.MetricNameLabel, + Value: "foo", + }, + labels.Label{ + Name: model.SchemeLabel, + Value: "http", + }, + ) + // processedLabels contain label values after processing (e.g. relabeling) + processedLabels := labels.New( + labels.Label{ + Name: model.InstanceLabel, + Value: "localhost:8080", + }, + ) + + ms := &metadataService{ + sm: &mockScrapeManager{targets: map[string][]*scrape.Target{ + "test": {scrape.NewTarget(processedLabels, discoveredLabels, nil)}, + }}, + } + + rID := config.NewComponentID("prometheus") + + t.Run("Commit Without Adding", func(t *testing.T) { + nomc := consumertest.NewNop() + tr := newTransaction(context.Background(), nil, true, "", rID, ms, nomc, nil, testTelemetry.ToReceiverCreateSettings()) + if got := tr.Commit(); got != nil { + t.Errorf("expecting nil from Commit() but got err %v", got) + } + }) + + t.Run("Rollback dose nothing", func(t *testing.T) { + nomc := consumertest.NewNop() + tr := newTransaction(context.Background(), nil, true, "", rID, ms, nomc, nil, testTelemetry.ToReceiverCreateSettings()) + if got := tr.Rollback(); got != nil { + t.Errorf("expecting nil from Rollback() but got err %v", got) + } + }) + + badLabels := labels.Labels([]labels.Label{{Name: "foo", Value: "bar"}}) + t.Run("Add One No Target", func(t *testing.T) { + nomc := consumertest.NewNop() + tr := newTransaction(context.Background(), nil, true, "", rID, ms, nomc, nil, testTelemetry.ToReceiverCreateSettings()) + if _, got := tr.Append(0, badLabels, time.Now().Unix()*1000, 1.0); got == nil { + t.Errorf("expecting error from Add() but got nil") + } + }) + + jobNotFoundLb := labels.Labels([]labels.Label{ + {Name: "instance", Value: "localhost:8080"}, + {Name: "job", Value: "test2"}, + {Name: "foo", Value: "bar"}}) + t.Run("Add One Job not found", func(t *testing.T) { + nomc := consumertest.NewNop() + tr := newTransaction(context.Background(), nil, true, "", rID, ms, nomc, nil, testTelemetry.ToReceiverCreateSettings()) + if _, got := tr.Append(0, jobNotFoundLb, time.Now().Unix()*1000, 1.0); got == nil { + t.Errorf("expecting error from Add() but got nil") + } + }) + + goodLabels := labels.Labels([]labels.Label{{Name: "instance", Value: "localhost:8080"}, + {Name: "job", Value: "test"}, + {Name: "__name__", Value: "foo"}}) + t.Run("Add One Good", func(t *testing.T) { + sink := new(consumertest.MetricsSink) + tr := newTransaction(context.Background(), nil, true, "", rID, ms, sink, nil, testTelemetry.ToReceiverCreateSettings()) + if _, got := tr.Append(0, goodLabels, time.Now().Unix()*1000, 1.0); got != nil { + t.Errorf("expecting error == nil from Add() but got: %v\n", got) + } + tr.metricBuilder.startTime = 1.0 // set to a non-zero value + if got := tr.Commit(); got != nil { + t.Errorf("expecting nil from Commit() but got err %v", got) + } + expectedNode, expectedResource := createNodeAndResource("test", "localhost:8080", "http") + mds := sink.AllMetrics() + if len(mds) != 1 { + t.Fatalf("wanted one batch, got %v\n", sink.AllMetrics()) + } + var ocmds []*agentmetricspb.ExportMetricsServiceRequest + rms := mds[0].ResourceMetrics() + for i := 0; i < rms.Len(); i++ { + ocmd := &agentmetricspb.ExportMetricsServiceRequest{} + ocmd.Node, ocmd.Resource, ocmd.Metrics = opencensus.ResourceMetricsToOC(rms.At(i)) + ocmds = append(ocmds, ocmd) + } + require.Len(t, ocmds, 1) + if !proto.Equal(ocmds[0].Node, expectedNode) { + t.Errorf("generated node %v and expected node %v is different\n", ocmds[0].Node, expectedNode) + } + if !proto.Equal(ocmds[0].Resource, expectedResource) { + t.Errorf("generated resource %v and expected resource %v is different\n", ocmds[0].Resource, expectedResource) + } + + // TODO: re-enable this when handle unspecified OC type + // assert.Len(t, ocmds[0].Metrics, 1) + }) + + t.Run("Error when start time is zero", func(t *testing.T) { + sink := new(consumertest.MetricsSink) + tr := newTransaction(context.Background(), nil, true, "", rID, ms, sink, nil, testTelemetry.ToReceiverCreateSettings()) + if _, got := tr.Append(0, goodLabels, time.Now().Unix()*1000, 1.0); got != nil { + t.Errorf("expecting error == nil from Add() but got: %v\n", got) + } + tr.metricBuilder.startTime = 0 // zero value means the start time metric is missing + got := tr.Commit() + if got == nil { + t.Error("expecting error from Commit() but got nil") + } else if got.Error() != errNoStartTimeMetrics.Error() { + t.Errorf("expected error %q but got %q", errNoStartTimeMetrics, got) + } + }) +} From 65fd778ca5de2794ba9682d3b2bffaa5babf64ef Mon Sep 17 00:00:00 2001 From: Anthony J Mirabella Date: Tue, 23 Nov 2021 13:47:45 -0500 Subject: [PATCH 08/26] receiver/prometheus: add feature gate to control whether to use OpenCensus or pdata directly Signed-off-by: Anthony J Mirabella --- receiver/prometheusreceiver/config.go | 12 ++++ receiver/prometheusreceiver/factory.go | 2 + .../prometheusreceiver/internal/ocastore.go | 59 +++++++++++++------ .../internal/ocastore_test.go | 2 +- .../prometheusreceiver/metrics_receiver.go | 7 +-- 5 files changed, 58 insertions(+), 24 deletions(-) diff --git a/receiver/prometheusreceiver/config.go b/receiver/prometheusreceiver/config.go index 67117fa9cebd7..88e067c6baa25 100644 --- a/receiver/prometheusreceiver/config.go +++ b/receiver/prometheusreceiver/config.go @@ -31,6 +31,7 @@ import ( "github.com/prometheus/prometheus/discovery/kubernetes" "github.com/prometheus/prometheus/discovery/targetgroup" "go.opentelemetry.io/collector/config" + "go.opentelemetry.io/collector/service/featuregate" "gopkg.in/yaml.v2" ) @@ -39,6 +40,16 @@ const ( prometheusConfigKey = "config" ) +var pdataPipelineGate = featuregate.Gate{ + ID: "receiver.prometheus.OTLPDirect", + Enabled: false, + Description: "Controls whether to use a new translation directly from Prometheus timeseries to pdata, without an intermediate representation as OpenCensus data.", +} + +func init() { + featuregate.Register(pdataPipelineGate) +} + // Config defines configuration for Prometheus receiver. type Config struct { config.ReceiverSettings `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct @@ -47,6 +58,7 @@ type Config struct { BufferCount int `mapstructure:"buffer_count"` UseStartTimeMetric bool `mapstructure:"use_start_time_metric"` StartTimeMetricRegex string `mapstructure:"start_time_metric_regex"` + pdataDirect bool // ConfigPlaceholder is just an entry to make the configuration pass a check // that requires that all keys present in the config actually exist on the diff --git a/receiver/prometheusreceiver/factory.go b/receiver/prometheusreceiver/factory.go index 0115cc69706ac..7c869f90557a4 100644 --- a/receiver/prometheusreceiver/factory.go +++ b/receiver/prometheusreceiver/factory.go @@ -23,6 +23,7 @@ import ( "go.opentelemetry.io/collector/config" "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/receiver/receiverhelper" + "go.opentelemetry.io/collector/service/featuregate" ) // This file implements config for Prometheus receiver. @@ -44,6 +45,7 @@ func NewFactory() component.ReceiverFactory { func createDefaultConfig() config.Receiver { return &Config{ ReceiverSettings: config.NewReceiverSettings(config.NewComponentID(typeStr)), + pdataDirect: featuregate.IsEnabled(pdataPipelineGate.ID), } } diff --git a/receiver/prometheusreceiver/internal/ocastore.go b/receiver/prometheusreceiver/internal/ocastore.go index 8802a7532e4d1..81aa9f6f5380a 100644 --- a/receiver/prometheusreceiver/internal/ocastore.go +++ b/receiver/prometheusreceiver/internal/ocastore.go @@ -18,6 +18,7 @@ import ( "context" "errors" "sync/atomic" + "time" "github.com/prometheus/prometheus/pkg/exemplar" "github.com/prometheus/prometheus/pkg/labels" @@ -45,11 +46,13 @@ type OcaStore struct { running int32 // access atomically sink consumer.Metrics mc *metadataService - jobsMap *JobsMapPdata + jobsMapPdata *JobsMapPdata + jobsMapOC *JobsMap useStartTimeMetric bool startTimeMetricRegex string receiverID config.ComponentID externalLabels labels.Labels + pdataDirect bool settings component.ReceiverCreateSettings } @@ -59,21 +62,29 @@ func NewOcaStore( ctx context.Context, sink consumer.Metrics, set component.ReceiverCreateSettings, - jobsMap *JobsMapPdata, useStartTimeMetric bool, startTimeMetricRegex string, receiverID config.ComponentID, - externalLabels labels.Labels) *OcaStore { + externalLabels labels.Labels, + pdataDirect bool) *OcaStore { + var jobsMapPdata *JobsMapPdata + var jobsMapOC *JobsMap + if !useStartTimeMetric { + jobsMapPdata = NewJobsMapPdata(2 * time.Minute) + jobsMapOC = NewJobsMap(2 * time.Minute) + } return &OcaStore{ running: runningStateInit, ctx: ctx, sink: sink, settings: set, - jobsMap: jobsMap, + jobsMapPdata: jobsMapPdata, + jobsMapOC: jobsMapOC, useStartTimeMetric: useStartTimeMetric, startTimeMetricRegex: startTimeMetricRegex, receiverID: receiverID, externalLabels: externalLabels, + pdataDirect: pdataDirect, } } @@ -88,19 +99,33 @@ func (o *OcaStore) SetScrapeManager(scrapeManager *scrape.Manager) { func (o *OcaStore) Appender(context.Context) storage.Appender { state := atomic.LoadInt32(&o.running) if state == runningStateReady { - return newTransactionPdata( - o.ctx, - &txConfig{ - jobsMap: o.jobsMap, - useStartTimeMetric: o.useStartTimeMetric, - startTimeMetricRegex: o.startTimeMetricRegex, - receiverID: o.receiverID, - ms: o.mc, - sink: o.sink, - externalLabels: o.externalLabels, - settings: o.settings, - }, - ) + if o.pdataDirect { + return newTransactionPdata( + o.ctx, + &txConfig{ + jobsMap: o.jobsMapPdata, + useStartTimeMetric: o.useStartTimeMetric, + startTimeMetricRegex: o.startTimeMetricRegex, + receiverID: o.receiverID, + ms: o.mc, + sink: o.sink, + externalLabels: o.externalLabels, + settings: o.settings, + }, + ) + } else { + return newTransaction( + o.ctx, + o.jobsMapOC, + o.useStartTimeMetric, + o.startTimeMetricRegex, + o.receiverID, + o.mc, + o.sink, + o.externalLabels, + o.settings, + ) + } } else if state == runningStateInit { panic("ScrapeManager is not set") } diff --git a/receiver/prometheusreceiver/internal/ocastore_test.go b/receiver/prometheusreceiver/internal/ocastore_test.go index bf0adeb5a7e52..a9a6c46e7d270 100644 --- a/receiver/prometheusreceiver/internal/ocastore_test.go +++ b/receiver/prometheusreceiver/internal/ocastore_test.go @@ -26,7 +26,7 @@ import ( ) func TestOcaStore(t *testing.T) { - o := NewOcaStore(context.Background(), nil, testTelemetry.ToReceiverCreateSettings(), nil, false, "", config.NewComponentID("prometheus"), nil) + o := NewOcaStore(context.Background(), nil, testTelemetry.ToReceiverCreateSettings(), false, "", config.NewComponentID("prometheus"), nil, false) o.SetScrapeManager(&scrape.Manager{}) app := o.Appender(context.Background()) diff --git a/receiver/prometheusreceiver/metrics_receiver.go b/receiver/prometheusreceiver/metrics_receiver.go index a5b576ce7557a..9886eb7a81c8c 100644 --- a/receiver/prometheusreceiver/metrics_receiver.go +++ b/receiver/prometheusreceiver/metrics_receiver.go @@ -16,7 +16,6 @@ package prometheusreceiver import ( "context" - "time" "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/scrape" @@ -72,21 +71,17 @@ func (r *pReceiver) Start(_ context.Context, host component.Host) error { } }() - var jobsMap *internal.JobsMapPdata - if !r.cfg.UseStartTimeMetric { - jobsMap = internal.NewJobsMapPdata(2 * time.Minute) - } // Per component.Component Start instructions, for async operations we should not use the // incoming context, it may get cancelled. r.ocaStore = internal.NewOcaStore( context.Background(), r.consumer, r.settings, - jobsMap, r.cfg.UseStartTimeMetric, r.cfg.StartTimeMetricRegex, r.cfg.ID(), r.cfg.PrometheusConfig.GlobalConfig.ExternalLabels, + r.cfg.pdataDirect, ) r.scrapeManager = scrape.NewManager(logger, r.ocaStore) r.ocaStore.SetScrapeManager(r.scrapeManager) From 6770a1c46567a279b4a4974fd3e86c00f731f592 Mon Sep 17 00:00:00 2001 From: Anthony J Mirabella Date: Tue, 23 Nov 2021 13:53:16 -0500 Subject: [PATCH 09/26] receiver/prometheus: localize metrics test helpers Signed-off-by: Anthony J Mirabella --- .../internal}/metricsutil_pdata.go | 22 +++++++++---------- .../internal/otlp_metrics_adjuster_test.go | 20 +++++++---------- 2 files changed, 19 insertions(+), 23 deletions(-) rename {internal/coreinternal/metricstestutil => receiver/prometheusreceiver/internal}/metricsutil_pdata.go (85%) diff --git a/internal/coreinternal/metricstestutil/metricsutil_pdata.go b/receiver/prometheusreceiver/internal/metricsutil_pdata.go similarity index 85% rename from internal/coreinternal/metricstestutil/metricsutil_pdata.go rename to receiver/prometheusreceiver/internal/metricsutil_pdata.go index e65b5fd4ebc08..b1fa57c9aba9b 100644 --- a/internal/coreinternal/metricstestutil/metricsutil_pdata.go +++ b/receiver/prometheusreceiver/internal/metricsutil_pdata.go @@ -12,15 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -package metricstestutil +package internal import "go.opentelemetry.io/collector/model/pdata" -type KV struct { +type kv struct { Key, Value string } -func DistPointPdata(ts pdata.Timestamp, bounds []float64, counts []uint64) *pdata.HistogramDataPoint { +func distPointPdata(ts pdata.Timestamp, bounds []float64, counts []uint64) *pdata.HistogramDataPoint { hdp := pdata.NewHistogramDataPoint() hdp.SetExplicitBounds(bounds) hdp.SetBucketCounts(counts) @@ -39,13 +39,13 @@ func DistPointPdata(ts pdata.Timestamp, bounds []float64, counts []uint64) *pdat return &hdp } -func GaugeDistMetricPdata(name string, kvp []*KV, startTs pdata.Timestamp, points ...*pdata.HistogramDataPoint) *pdata.Metric { - hMetric := CumulativeDistMetricPdata(name, kvp, startTs, points...) +func gaugeDistMetricPdata(name string, kvp []*kv, startTs pdata.Timestamp, points ...*pdata.HistogramDataPoint) *pdata.Metric { + hMetric := cumulativeDistMetricPdata(name, kvp, startTs, points...) hMetric.Histogram().SetAggregationTemporality(pdata.MetricAggregationTemporalityDelta) return hMetric } -func CumulativeDistMetricPdata(name string, kvp []*KV, startTs pdata.Timestamp, points ...*pdata.HistogramDataPoint) *pdata.Metric { +func cumulativeDistMetricPdata(name string, kvp []*kv, startTs pdata.Timestamp, points ...*pdata.HistogramDataPoint) *pdata.Metric { metric := pdata.NewMetric() metric.SetName(name) metric.SetDataType(pdata.MetricDataTypeHistogram) @@ -66,7 +66,7 @@ func CumulativeDistMetricPdata(name string, kvp []*KV, startTs pdata.Timestamp, return &metric } -func DoublePointPdata(ts pdata.Timestamp, value float64) *pdata.NumberDataPoint { +func doublePointPdata(ts pdata.Timestamp, value float64) *pdata.NumberDataPoint { ndp := pdata.NewNumberDataPoint() ndp.SetTimestamp(ts) ndp.SetDoubleVal(value) @@ -74,7 +74,7 @@ func DoublePointPdata(ts pdata.Timestamp, value float64) *pdata.NumberDataPoint return &ndp } -func GaugeMetricPdata(name string, kvp []*KV, startTs pdata.Timestamp, points ...*pdata.NumberDataPoint) *pdata.Metric { +func gaugeMetricPdata(name string, kvp []*kv, startTs pdata.Timestamp, points ...*pdata.NumberDataPoint) *pdata.Metric { metric := pdata.NewMetric() metric.SetName(name) metric.SetDataType(pdata.MetricDataTypeGauge) @@ -92,7 +92,7 @@ func GaugeMetricPdata(name string, kvp []*KV, startTs pdata.Timestamp, points .. return &metric } -func SummaryPointPdata(ts pdata.Timestamp, count uint64, sum float64, quantiles, values []float64) *pdata.SummaryDataPoint { +func summaryPointPdata(ts pdata.Timestamp, count uint64, sum float64, quantiles, values []float64) *pdata.SummaryDataPoint { sdp := pdata.NewSummaryDataPoint() sdp.SetTimestamp(ts) sdp.SetCount(count) @@ -106,7 +106,7 @@ func SummaryPointPdata(ts pdata.Timestamp, count uint64, sum float64, quantiles, return &sdp } -func SummaryMetricPdata(name string, kvp []*KV, startTs pdata.Timestamp, points ...*pdata.SummaryDataPoint) *pdata.Metric { +func summaryMetricPdata(name string, kvp []*kv, startTs pdata.Timestamp, points ...*pdata.SummaryDataPoint) *pdata.Metric { metric := pdata.NewMetric() metric.SetName(name) metric.SetDataType(pdata.MetricDataTypeSummary) @@ -124,7 +124,7 @@ func SummaryMetricPdata(name string, kvp []*KV, startTs pdata.Timestamp, points return &metric } -func SumMetricPdata(name string, kvp []*KV, startTs pdata.Timestamp, points ...*pdata.NumberDataPoint) *pdata.Metric { +func sumMetricPdata(name string, kvp []*kv, startTs pdata.Timestamp, points ...*pdata.NumberDataPoint) *pdata.Metric { metric := pdata.NewMetric() metric.SetName(name) metric.SetDataType(pdata.MetricDataTypeSum) diff --git a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go index 12172210c90d9..3079f919ddd57 100644 --- a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go +++ b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go @@ -23,8 +23,6 @@ import ( "go.uber.org/zap" "go.opentelemetry.io/collector/model/pdata" - - "github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal/metricstestutil" ) var ( @@ -506,17 +504,15 @@ func Test_summary_pdata(t *testing.T) { runScriptPdata(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) } -type kv = metricstestutil.KV - var ( - distPoint = metricstestutil.DistPointPdata - gaugeDistMetric = metricstestutil.GaugeDistMetricPdata - histogramMetric = metricstestutil.CumulativeDistMetricPdata - doublePoint = metricstestutil.DoublePointPdata - gaugeMetric = metricstestutil.GaugeMetricPdata - summaryPoint = metricstestutil.SummaryPointPdata - summaryMetric = metricstestutil.SummaryMetricPdata - sumMetric = metricstestutil.SumMetricPdata + distPoint = distPointPdata + gaugeDistMetric = gaugeDistMetricPdata + histogramMetric = cumulativeDistMetricPdata + doublePoint = doublePointPdata + gaugeMetric = gaugeMetricPdata + summaryPoint = summaryPointPdata + summaryMetric = summaryMetricPdata + sumMetric = sumMetricPdata ) func metricSlice(metrics ...*pdata.Metric) *pdata.MetricSlice { From 9208e41389f48dde343fc506c61b1fa48df26f2d Mon Sep 17 00:00:00 2001 From: Anthony J Mirabella Date: Tue, 23 Nov 2021 14:02:15 -0500 Subject: [PATCH 10/26] receiver/prometheus: store metrics in a map in metricbuilder to avoid issues with out-of-order exposition data Signed-off-by: Anthony J Mirabella --- .../internal/metricsbuilder.go | 27 +++++++++---------- .../internal/otlp_metricsbuilder.go | 1 - 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/receiver/prometheusreceiver/internal/metricsbuilder.go b/receiver/prometheusreceiver/internal/metricsbuilder.go index e2c7fddf5ac63..83f2cf6e58024 100644 --- a/receiver/prometheusreceiver/internal/metricsbuilder.go +++ b/receiver/prometheusreceiver/internal/metricsbuilder.go @@ -58,7 +58,7 @@ type metricBuilder struct { startTime float64 intervalStartTimeMs int64 logger *zap.Logger - currentMf MetricFamily + families map[string]MetricFamily } // newMetricBuilder creates a MetricBuilder which is allowed to feed all the datapoints from a single prometheus @@ -72,6 +72,7 @@ func newMetricBuilder(mc MetadataCache, useStartTimeMetric bool, startTimeMetric return &metricBuilder{ mc: mc, metrics: make([]*metricspb.Metric, 0), + families: map[string]MetricFamily{}, logger: logger, numTimeseries: 0, droppedTimeseries: 0, @@ -137,19 +138,18 @@ func (b *metricBuilder) AddDataPoint(ls labels.Labels, t int64, v float64) error b.hasData = true - if b.currentMf != nil && !b.currentMf.IsSameFamily(metricName) { - m, ts, dts := b.currentMf.ToMetric() - b.numTimeseries += ts - b.droppedTimeseries += dts - if m != nil { - b.metrics = append(b.metrics, m) + familyName := normalizeMetricName(metricName) + curMF, ok := b.families[familyName] + if !ok { + if mf, ok := b.families[metricName]; ok { + curMF = mf + } else { + curMF = newMetricFamily(metricName, b.mc, b.logger, b.intervalStartTimeMs) + b.families[familyName] = curMF } - b.currentMf = newMetricFamily(metricName, b.mc, b.logger, b.intervalStartTimeMs) - } else if b.currentMf == nil { - b.currentMf = newMetricFamily(metricName, b.mc, b.logger, b.intervalStartTimeMs) } - return b.currentMf.Add(metricName, ls, t, v) + return curMF.Add(metricName, ls, t, v) } // Build an opencensus data.MetricsData based on all added data complexValue. @@ -162,14 +162,13 @@ func (b *metricBuilder) Build() ([]*metricspb.Metric, int, int, error) { return nil, 0, 0, errNoDataToBuild } - if b.currentMf != nil { - m, ts, dts := b.currentMf.ToMetric() + for _, mf := range b.families { + m, ts, dts := mf.ToMetric() b.numTimeseries += ts b.droppedTimeseries += dts if m != nil { b.metrics = append(b.metrics, m) } - b.currentMf = nil } return b.metrics, b.numTimeseries, b.droppedTimeseries, nil diff --git a/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go b/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go index 34af03cc1fb91..5d34afc0d2dad 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go +++ b/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go @@ -84,7 +84,6 @@ func convToPdataMetricType(metricType textparse.MetricType) pdata.MetricDataType type metricBuilderPdata struct { metrics pdata.MetricSlice families map[string]MetricFamilyPdata - currentMf MetricFamilyPdata hasData bool hasInternalMetric bool mc MetadataCache From 8845e1867e820e10519de1ad4847fc64413be332 Mon Sep 17 00:00:00 2001 From: Anthony J Mirabella Date: Tue, 23 Nov 2021 15:09:53 -0500 Subject: [PATCH 11/26] receiver/prometheus: run component tests against both OC and pdata export paths Signed-off-by: Anthony J Mirabella --- .../metrics_receiver_helper_test.go | 146 ++++++++++-------- .../metrics_receiver_labels_test.go | 41 ++--- 2 files changed, 101 insertions(+), 86 deletions(-) diff --git a/receiver/prometheusreceiver/metrics_receiver_helper_test.go b/receiver/prometheusreceiver/metrics_receiver_helper_test.go index 444eef9627bfd..e7ebabca77daa 100644 --- a/receiver/prometheusreceiver/metrics_receiver_helper_test.go +++ b/receiver/prometheusreceiver/metrics_receiver_helper_test.go @@ -442,73 +442,95 @@ func compareSummary(count uint64, sum float64, quantiles [][]float64) summaryPoi } func testComponent(t *testing.T, targets []*testData, useStartTimeMetric bool, startTimeMetricRegex string) { - // 1. setup mock server - mp, cfg, err := setupMockPrometheus(targets...) - require.Nilf(t, err, "Failed to create Prometheus config: %v", err) - defer mp.Close() - - cms := new(consumertest.MetricsSink) - rcvr := newPrometheusReceiver(componenttest.NewNopReceiverCreateSettings(), &Config{ - ReceiverSettings: config.NewReceiverSettings(config.NewComponentID(typeStr)), - PrometheusConfig: cfg, - UseStartTimeMetric: useStartTimeMetric, - StartTimeMetricRegex: startTimeMetricRegex}, cms) - - require.NoError(t, rcvr.Start(context.Background(), componenttest.NewNopHost()), "Failed to invoke Start: %v", err) - t.Cleanup(func() { - // verify state after shutdown is called - assert.Lenf(t, flattenTargets(rcvr.scrapeManager.TargetsAll()), len(targets), "expected %v targets to be running", len(targets)) - require.NoError(t, rcvr.Shutdown(context.Background())) - assert.Len(t, flattenTargets(rcvr.scrapeManager.TargetsAll()), 0, "expected scrape manager to have no targets") - }) - - // wait for all provided data to be scraped - mp.wg.Wait() - metrics := cms.AllMetrics() - - // split and store results by target name - pResults := splitMetricsByTarget(metrics) - lres, lep := len(pResults), len(mp.endpoints) - assert.Equalf(t, lep, lres, "want %d targets, but got %v\n", lep, lres) - - // loop to validate outputs for each targets - for _, target := range targets { - t.Run(target.name, func(t *testing.T) { - validScrapes := getValidScrapes(t, pResults[target.name]) - target.validateFunc(t, target, validScrapes) + for _, pdataDirect := range []bool{false, true} { + pipelineType := "OpenCensus" + if pdataDirect { + pipelineType = "pdata" + } + t.Run(pipelineType, func(t *testing.T) { // 1. setup mock server + mp, cfg, err := setupMockPrometheus(targets...) + require.Nilf(t, err, "Failed to create Prometheus config: %v", err) + defer mp.Close() + + cms := new(consumertest.MetricsSink) + rcvr := newPrometheusReceiver(componenttest.NewNopReceiverCreateSettings(), &Config{ + ReceiverSettings: config.NewReceiverSettings(config.NewComponentID(typeStr)), + PrometheusConfig: cfg, + UseStartTimeMetric: useStartTimeMetric, + StartTimeMetricRegex: startTimeMetricRegex, + pdataDirect: pdataDirect, + }, cms) + + require.NoError(t, rcvr.Start(context.Background(), componenttest.NewNopHost()), "Failed to invoke Start: %v", err) + t.Cleanup(func() { + // verify state after shutdown is called + assert.Lenf(t, flattenTargets(rcvr.scrapeManager.TargetsAll()), len(targets), "expected %v targets to be running", len(targets)) + require.NoError(t, rcvr.Shutdown(context.Background())) + assert.Len(t, flattenTargets(rcvr.scrapeManager.TargetsAll()), 0, "expected scrape manager to have no targets") + }) + + // wait for all provided data to be scraped + mp.wg.Wait() + metrics := cms.AllMetrics() + + // split and store results by target name + pResults := splitMetricsByTarget(metrics) + lres, lep := len(pResults), len(mp.endpoints) + assert.Equalf(t, lep, lres, "want %d targets, but got %v\n", lep, lres) + + // loop to validate outputs for each targets + for _, target := range targets { + t.Run(target.name, func(t *testing.T) { + validScrapes := getValidScrapes(t, pResults[target.name]) + target.validateFunc(t, target, validScrapes) + }) + } }) } } // starts prometheus receiver with custom config, retrieves metrics from MetricsSink -func testComponentCustomConfig(t *testing.T, targets []*testData, mp *mockPrometheus, cfg *promcfg.Config) { - ctx := context.Background() - defer mp.Close() - - cms := new(consumertest.MetricsSink) - receiver := newPrometheusReceiver(componenttest.NewNopReceiverCreateSettings(), &Config{ - ReceiverSettings: config.NewReceiverSettings(config.NewComponentID(typeStr)), - PrometheusConfig: cfg}, cms) - - require.NoError(t, receiver.Start(ctx, componenttest.NewNopHost())) - - // verify state after shutdown is called - t.Cleanup(func() { require.NoError(t, receiver.Shutdown(ctx)) }) - - // wait for all provided data to be scraped - mp.wg.Wait() - metrics := cms.AllMetrics() - - // split and store results by target name - pResults := splitMetricsByTarget(metrics) - lres, lep := len(pResults), len(mp.endpoints) - assert.Equalf(t, lep, lres, "want %d targets, but got %v\n", lep, lres) - - // loop to validate outputs for each targets - for _, target := range targets { - t.Run(target.name, func(t *testing.T) { - validScrapes := getValidScrapes(t, pResults[target.name]) - target.validateFunc(t, target, validScrapes) +func testComponentCustomConfig(t *testing.T, targets []*testData, cfgMut func(*promcfg.Config)) { + for _, pdataDirect := range []bool{false, true} { + pipelineType := "OpenCensus" + if pdataDirect { + pipelineType = "pdata" + } + t.Run(pipelineType, func(t *testing.T) { + ctx := context.Background() + mp, cfg, err := setupMockPrometheus(targets...) + cfgMut(cfg) + require.Nilf(t, err, "Failed to create Prometheus config: %v", err) + defer mp.Close() + + cms := new(consumertest.MetricsSink) + receiver := newPrometheusReceiver(componenttest.NewNopReceiverCreateSettings(), &Config{ + ReceiverSettings: config.NewReceiverSettings(config.NewComponentID(typeStr)), + PrometheusConfig: cfg, + pdataDirect: pdataDirect, + }, cms) + + require.NoError(t, receiver.Start(ctx, componenttest.NewNopHost())) + + // verify state after shutdown is called + t.Cleanup(func() { require.NoError(t, receiver.Shutdown(ctx)) }) + + // wait for all provided data to be scraped + mp.wg.Wait() + metrics := cms.AllMetrics() + + // split and store results by target name + pResults := splitMetricsByTarget(metrics) + lres, lep := len(pResults), len(mp.endpoints) + assert.Equalf(t, lep, lres, "want %d targets, but got %v\n", lep, lres) + + // loop to validate outputs for each targets + for _, target := range targets { + t.Run(target.name, func(t *testing.T) { + validScrapes := getValidScrapes(t, pResults[target.name]) + target.validateFunc(t, target, validScrapes) + }) + } }) } } diff --git a/receiver/prometheusreceiver/metrics_receiver_labels_test.go b/receiver/prometheusreceiver/metrics_receiver_labels_test.go index 27511448e47cd..5c588e5738d4b 100644 --- a/receiver/prometheusreceiver/metrics_receiver_labels_test.go +++ b/receiver/prometheusreceiver/metrics_receiver_labels_test.go @@ -17,6 +17,7 @@ package prometheusreceiver import ( "testing" + promcfg "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/pkg/labels" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/model/pdata" @@ -38,11 +39,9 @@ func TestExternalLabels(t *testing.T) { }, } - mp, cfg, err := setupMockPrometheus(targets...) - cfg.GlobalConfig.ExternalLabels = labels.FromStrings("key", "value") - require.Nilf(t, err, "Failed to create Prometheus config: %v", err) - - testComponentCustomConfig(t, targets, mp, cfg) + testComponentCustomConfig(t, targets, func(cfg *promcfg.Config) { + cfg.GlobalConfig.ExternalLabels = labels.FromStrings("key", "value") + }) } func verifyExternalLabels(t *testing.T, td *testData, rms []*pdata.ResourceMetrics) { @@ -130,15 +129,12 @@ func TestLabelLimitConfig(t *testing.T) { }, } - mp, cfg, err := setupMockPrometheus(targets...) - require.Nilf(t, err, "Failed to create Prometheus config: %v", err) - - // set label limit in scrape_config - for _, scrapeCfg := range cfg.ScrapeConfigs { - scrapeCfg.LabelLimit = 5 - } - - testComponentCustomConfig(t, targets, mp, cfg) + testComponentCustomConfig(t, targets, func(cfg *promcfg.Config) { + // set label limit in scrape_config + for _, scrapeCfg := range cfg.ScrapeConfigs { + scrapeCfg.LabelLimit = 5 + } + }) } const targetLabelNameLimit1 = ` @@ -157,7 +153,7 @@ func verifyLabelLengthTarget1(t *testing.T, td *testData, rms []*pdata.ResourceM want := td.attributes metrics1 := rms[0].InstrumentationLibraryMetrics().At(0).Metrics() - ts1 := metrics1.At(0).Gauge().DataPoints().At(0).Timestamp() + ts1 := getTS(metrics1) doCompare(t, "scrape-labelNameLimit", want, rms[0], []testExpectation{ assertMetricPresent("test_counter0", @@ -216,13 +212,10 @@ func TestLabelNameLimitConfig(t *testing.T) { }, } - mp, cfg, err := setupMockPrometheus(targets...) - require.Nilf(t, err, "Failed to create Prometheus config: %v", err) - - // set label name limit in scrape_config - for _, scrapeCfg := range cfg.ScrapeConfigs { - scrapeCfg.LabelNameLengthLimit = 20 - } - - testComponentCustomConfig(t, targets, mp, cfg) + testComponentCustomConfig(t, targets, func(cfg *promcfg.Config) { + // set label name limit in scrape_config + for _, scrapeCfg := range cfg.ScrapeConfigs { + scrapeCfg.LabelNameLengthLimit = 20 + } + }) } From e71027501477fbd81522d93dcbc23d53b625586a Mon Sep 17 00:00:00 2001 From: Anthony J Mirabella Date: Tue, 23 Nov 2021 15:56:55 -0500 Subject: [PATCH 12/26] receiver/prometheus: ensure OC metrics builder tests are deterministic Signed-off-by: Anthony J Mirabella --- .../internal/metricsbuilder_test.go | 19 ++++++++++++++++++- .../internal/otlp_metricsbuilder_test.go | 4 ++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/receiver/prometheusreceiver/internal/metricsbuilder_test.go b/receiver/prometheusreceiver/internal/metricsbuilder_test.go index 46e6cbcd268ec..da8ddfee2875b 100644 --- a/receiver/prometheusreceiver/internal/metricsbuilder_test.go +++ b/receiver/prometheusreceiver/internal/metricsbuilder_test.go @@ -108,13 +108,30 @@ func runBuilderTests(t *testing.T, tests []buildTestData) { } metrics, _, _, err := b.Build() assert.NoError(t, err) - assert.EqualValues(t, tt.wants[i], metrics) + assertEquivalentMetrics(t, tt.wants[i], metrics) st += interval } }) } } +func assertEquivalentMetrics(t *testing.T, want, got []*metricspb.Metric) { + if !assert.Equal(t, len(want), len(got)) { + return + } + wmap := map[string]*metricspb.Metric{} + gmap := map[string]*metricspb.Metric{} + + for i := 0; i < len(want); i++ { + wi := want[i] + wmap[wi.GetMetricDescriptor().GetName()] = wi + gi := got[i] + gmap[gi.GetMetricDescriptor().GetName()] = gi + } + + assert.EqualValues(t, wmap, gmap) +} + func runBuilderStartTimeTests(t *testing.T, tests []buildTestData, startTimeMetricRegex string, expectedBuilderStartTime float64) { for _, tt := range tests { diff --git a/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go b/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go index 124d3c278ead6..9eb5ef5145d2e 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go +++ b/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go @@ -478,14 +478,14 @@ func runBuilderTestsPdata(t *testing.T, tests []buildTestDataPdata) { } metrics, _, _, err := b.Build() assert.NoError(t, err) - assertEquivalentMetrics(t, wants[i], metrics) + assertEquivalentMetricsPdata(t, wants[i], metrics) st += interval } }) } } -func assertEquivalentMetrics(t *testing.T, want, got *pdata.MetricSlice) { +func assertEquivalentMetricsPdata(t *testing.T, want, got *pdata.MetricSlice) { if !assert.Equal(t, want.Len(), got.Len()) { return } From 5a56ab7cd4731482f06d957f963ede611c7a46dc Mon Sep 17 00:00:00 2001 From: Anthony J Mirabella Date: Tue, 23 Nov 2021 16:20:28 -0500 Subject: [PATCH 13/26] receiver/prometheus: restore replace directive to ensure use of local copy of opencensus translator during development Signed-off-by: Anthony J Mirabella --- receiver/prometheusreceiver/go.mod | 2 ++ receiver/prometheusreceiver/go.sum | 2 -- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/receiver/prometheusreceiver/go.mod b/receiver/prometheusreceiver/go.mod index bd3cd1494d05f..4645af24584f3 100644 --- a/receiver/prometheusreceiver/go.mod +++ b/receiver/prometheusreceiver/go.mod @@ -158,3 +158,5 @@ replace github.com/open-telemetry/opentelemetry-collector-contrib/exporter/prome replace github.com/open-telemetry/opentelemetry-collector-contrib/internal/coreinternal => ../../internal/coreinternal replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/resourcetotelemetry => ../../pkg/resourcetotelemetry + +replace github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus => ../../pkg/translator/opencensus diff --git a/receiver/prometheusreceiver/go.sum b/receiver/prometheusreceiver/go.sum index f3901d7c974ff..06efb1e6514dd 100644 --- a/receiver/prometheusreceiver/go.sum +++ b/receiver/prometheusreceiver/go.sum @@ -869,8 +869,6 @@ github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus v0.39.0 h1:678eZCNsUS6rXWLfeeDPAnNmS61qMXvoQeYH64BZyos= -github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus v0.39.0/go.mod h1:6NQp433EU5FmtkGavG5FuwFOnD4bAUhSDsK8gnSmIdM= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= From f862b9454fe48017c78b3fe617ddac4b46aa56ed Mon Sep 17 00:00:00 2001 From: Anthony J Mirabella Date: Tue, 23 Nov 2021 16:34:16 -0500 Subject: [PATCH 14/26] receiver/prometheus: fix lint issues Signed-off-by: Anthony J Mirabella --- .../prometheusreceiver/internal/ocastore.go | 23 +++++++++---------- .../internal/otlp_metrics_adjuster_test.go | 17 -------------- 2 files changed, 11 insertions(+), 29 deletions(-) diff --git a/receiver/prometheusreceiver/internal/ocastore.go b/receiver/prometheusreceiver/internal/ocastore.go index 81aa9f6f5380a..0e402017bac0c 100644 --- a/receiver/prometheusreceiver/internal/ocastore.go +++ b/receiver/prometheusreceiver/internal/ocastore.go @@ -113,19 +113,18 @@ func (o *OcaStore) Appender(context.Context) storage.Appender { settings: o.settings, }, ) - } else { - return newTransaction( - o.ctx, - o.jobsMapOC, - o.useStartTimeMetric, - o.startTimeMetricRegex, - o.receiverID, - o.mc, - o.sink, - o.externalLabels, - o.settings, - ) } + return newTransaction( + o.ctx, + o.jobsMapOC, + o.useStartTimeMetric, + o.startTimeMetricRegex, + o.receiverID, + o.mc, + o.sink, + o.externalLabels, + o.settings, + ) } else if state == runningStateInit { panic("ScrapeManager is not set") } diff --git a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go index 3079f919ddd57..2a11650a4246a 100644 --- a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go +++ b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go @@ -293,22 +293,6 @@ func Test_cumulative_pdata(t *testing.T) { runScriptPdata(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) } -func populateHistogram(hdp *pdata.HistogramDataPoint, timestamp pdata.Timestamp, bounds []float64, counts []uint64) { - count := uint64(0) - sum := float64(0) - for i, counti := range counts { - if i > 0 { - sum += float64(counti) * bounds[i-1] - } - count += counti - } - hdp.SetBucketCounts(counts) - hdp.SetSum(sum) - hdp.SetCount(count) - hdp.SetTimestamp(timestamp) - hdp.SetExplicitBounds(bounds) -} - func Test_gaugeDistribution_pdata(t *testing.T) { script := []*metricsAdjusterTestPdata{ { @@ -867,7 +851,6 @@ func Test_jobGC_pdata(t *testing.T) { jobsMap.gc() // run job 1, round 2 - verify that all job 1 timeseries have been gc'd runScriptPdata(t, jobsMap.get("job", "0"), job1Script2) - return } type metricsAdjusterTestPdata struct { From 598f4a2be09cc20d5bc540426aaff5ffd2271c51 Mon Sep 17 00:00:00 2001 From: Anthony J Mirabella Date: Tue, 23 Nov 2021 18:57:48 -0500 Subject: [PATCH 15/26] receiver/prometheus: fix lint issues Signed-off-by: Anthony J Mirabella --- receiver/prometheusreceiver/internal/ocastore.go | 1 - .../prometheusreceiver/internal/otlp_metricfamily.go | 3 +-- .../internal/otlp_metricfamily_test.go | 3 +-- .../internal/otlp_metrics_adjuster.go | 1 - .../internal/otlp_metrics_adjuster_test.go | 3 +-- .../prometheusreceiver/internal/otlp_metricsbuilder.go | 3 +-- .../internal/otlp_metricsbuilder_test.go | 3 +-- .../prometheusreceiver/internal/otlp_transaction.go | 10 ++++------ .../internal/otlp_transaction_test.go | 1 - .../prometheusreceiver/internal/prom_to_otlp_test.go | 1 - receiver/prometheusreceiver/metrics_receiver.go | 3 +-- 11 files changed, 10 insertions(+), 22 deletions(-) diff --git a/receiver/prometheusreceiver/internal/ocastore.go b/receiver/prometheusreceiver/internal/ocastore.go index 0e402017bac0c..4455e23570e73 100644 --- a/receiver/prometheusreceiver/internal/ocastore.go +++ b/receiver/prometheusreceiver/internal/ocastore.go @@ -24,7 +24,6 @@ import ( "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/scrape" "github.com/prometheus/prometheus/storage" - "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/config" "go.opentelemetry.io/collector/consumer" diff --git a/receiver/prometheusreceiver/internal/otlp_metricfamily.go b/receiver/prometheusreceiver/internal/otlp_metricfamily.go index c96db6e88414b..58007f09ee177 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricfamily.go +++ b/receiver/prometheusreceiver/internal/otlp_metricfamily.go @@ -23,9 +23,8 @@ import ( "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/textparse" "github.com/prometheus/prometheus/scrape" - "go.uber.org/zap" - "go.opentelemetry.io/collector/model/pdata" + "go.uber.org/zap" ) // MetricFamilyPdata is unit which is corresponding to the metrics items which shared the same TYPE/UNIT/... metadata from diff --git a/receiver/prometheusreceiver/internal/otlp_metricfamily_test.go b/receiver/prometheusreceiver/internal/otlp_metricfamily_test.go index f8bfb70f712d5..1fb678e99963e 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricfamily_test.go +++ b/receiver/prometheusreceiver/internal/otlp_metricfamily_test.go @@ -21,9 +21,8 @@ import ( "github.com/prometheus/prometheus/pkg/textparse" "github.com/prometheus/prometheus/scrape" "github.com/stretchr/testify/require" - "go.uber.org/zap" - "go.opentelemetry.io/collector/model/pdata" + "go.uber.org/zap" ) type byLookupMetadataCache map[string]scrape.MetricMetadata diff --git a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go index a754c47a3cea1..5276b3710a2b1 100644 --- a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go +++ b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go @@ -21,7 +21,6 @@ import ( "time" "go.opentelemetry.io/collector/model/pdata" - "go.uber.org/zap" ) diff --git a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go index 2a11650a4246a..fc689c8ed0834 100644 --- a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go +++ b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go @@ -20,9 +20,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.uber.org/zap" - "go.opentelemetry.io/collector/model/pdata" + "go.uber.org/zap" ) var ( diff --git a/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go b/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go index 5d34afc0d2dad..c677f9c082ce7 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go +++ b/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go @@ -23,9 +23,8 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/pkg/textparse" - "go.uber.org/zap" - "go.opentelemetry.io/collector/model/pdata" + "go.uber.org/zap" ) func isUsefulLabelPdata(mType pdata.MetricDataType, labelKey string) bool { diff --git a/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go b/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go index 9eb5ef5145d2e..96f7f3d90e095 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go +++ b/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go @@ -22,9 +22,8 @@ import ( "github.com/prometheus/prometheus/pkg/textparse" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.uber.org/zap" - "go.opentelemetry.io/collector/model/pdata" + "go.uber.org/zap" ) func runBuilderStartTimeTestsPdata(t *testing.T, tests []buildTestDataPdata, diff --git a/receiver/prometheusreceiver/internal/otlp_transaction.go b/receiver/prometheusreceiver/internal/otlp_transaction.go index c8f786ced3207..b23691bdfff55 100644 --- a/receiver/prometheusreceiver/internal/otlp_transaction.go +++ b/receiver/prometheusreceiver/internal/otlp_transaction.go @@ -19,17 +19,15 @@ import ( "sync/atomic" "time" - "go.uber.org/zap" - + "github.com/prometheus/common/model" + "github.com/prometheus/prometheus/pkg/exemplar" + "github.com/prometheus/prometheus/pkg/labels" "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/config" "go.opentelemetry.io/collector/consumer" "go.opentelemetry.io/collector/model/pdata" "go.opentelemetry.io/collector/obsreport" - - "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/pkg/exemplar" - "github.com/prometheus/prometheus/pkg/labels" + "go.uber.org/zap" ) type transactionPdata struct { diff --git a/receiver/prometheusreceiver/internal/otlp_transaction_test.go b/receiver/prometheusreceiver/internal/otlp_transaction_test.go index 91afde7c6b8a1..07c03b07d8293 100644 --- a/receiver/prometheusreceiver/internal/otlp_transaction_test.go +++ b/receiver/prometheusreceiver/internal/otlp_transaction_test.go @@ -23,7 +23,6 @@ import ( "github.com/prometheus/prometheus/pkg/labels" "github.com/prometheus/prometheus/scrape" "github.com/stretchr/testify/require" - "go.opentelemetry.io/collector/component/componenttest" "go.opentelemetry.io/collector/config" "go.opentelemetry.io/collector/consumer/consumertest" diff --git a/receiver/prometheusreceiver/internal/prom_to_otlp_test.go b/receiver/prometheusreceiver/internal/prom_to_otlp_test.go index fe61e5ab486d5..6d6681bf83453 100644 --- a/receiver/prometheusreceiver/internal/prom_to_otlp_test.go +++ b/receiver/prometheusreceiver/internal/prom_to_otlp_test.go @@ -18,7 +18,6 @@ import ( "testing" "github.com/stretchr/testify/require" - "go.opentelemetry.io/collector/model/pdata" ) diff --git a/receiver/prometheusreceiver/metrics_receiver.go b/receiver/prometheusreceiver/metrics_receiver.go index 9886eb7a81c8c..49293788aeecb 100644 --- a/receiver/prometheusreceiver/metrics_receiver.go +++ b/receiver/prometheusreceiver/metrics_receiver.go @@ -19,10 +19,9 @@ import ( "github.com/prometheus/prometheus/discovery" "github.com/prometheus/prometheus/scrape" - "go.uber.org/zap" - "go.opentelemetry.io/collector/component" "go.opentelemetry.io/collector/consumer" + "go.uber.org/zap" "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/internal" ) From 838f7a5fb6ce94a52adb29bc67874f472958d949 Mon Sep 17 00:00:00 2001 From: Anthony J Mirabella Date: Tue, 23 Nov 2021 20:22:00 -0500 Subject: [PATCH 16/26] make porto Signed-off-by: Anthony J Mirabella --- extension/oidcauthextension/authdata.go | 2 +- processor/redactionprocessor/config.go | 2 +- processor/redactionprocessor/factory.go | 2 +- processor/redactionprocessor/processor.go | 2 +- receiver/cloudfoundryreceiver/converter.go | 2 +- receiver/cloudfoundryreceiver/stream.go | 2 +- receiver/cloudfoundryreceiver/uaa.go | 2 +- receiver/prometheusreceiver/internal/metricsutil_pdata.go | 2 +- receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go | 2 +- receiver/prometheusreceiver/internal/otlp_transaction.go | 2 +- 10 files changed, 10 insertions(+), 10 deletions(-) diff --git a/extension/oidcauthextension/authdata.go b/extension/oidcauthextension/authdata.go index 6f1d495734525..87808a892febe 100644 --- a/extension/oidcauthextension/authdata.go +++ b/extension/oidcauthextension/authdata.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package oidcauthextension +package oidcauthextension // import "github.com/open-telemetry/opentelemetry-collector-contrib/extension/oidcauthextension" import "go.opentelemetry.io/collector/client" diff --git a/processor/redactionprocessor/config.go b/processor/redactionprocessor/config.go index a5949b35c022f..b2ba6d903ba07 100644 --- a/processor/redactionprocessor/config.go +++ b/processor/redactionprocessor/config.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package redactionprocessor +package redactionprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/redactionprocessor" import ( "go.opentelemetry.io/collector/config" diff --git a/processor/redactionprocessor/factory.go b/processor/redactionprocessor/factory.go index d71ae070af502..5b3a0657cdac6 100644 --- a/processor/redactionprocessor/factory.go +++ b/processor/redactionprocessor/factory.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package redactionprocessor +package redactionprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/redactionprocessor" import ( "context" diff --git a/processor/redactionprocessor/processor.go b/processor/redactionprocessor/processor.go index ecfb61c3ab4f3..89469e3038808 100644 --- a/processor/redactionprocessor/processor.go +++ b/processor/redactionprocessor/processor.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package redactionprocessor +package redactionprocessor // import "github.com/open-telemetry/opentelemetry-collector-contrib/processor/redactionprocessor" import ( "context" diff --git a/receiver/cloudfoundryreceiver/converter.go b/receiver/cloudfoundryreceiver/converter.go index 3a7c83e3ae6da..8e3c0a4800c42 100644 --- a/receiver/cloudfoundryreceiver/converter.go +++ b/receiver/cloudfoundryreceiver/converter.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cloudfoundryreceiver +package cloudfoundryreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/cloudfoundryreceiver" import ( "time" diff --git a/receiver/cloudfoundryreceiver/stream.go b/receiver/cloudfoundryreceiver/stream.go index 8563cb4537afd..7760c3a9d45ad 100644 --- a/receiver/cloudfoundryreceiver/stream.go +++ b/receiver/cloudfoundryreceiver/stream.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cloudfoundryreceiver +package cloudfoundryreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/cloudfoundryreceiver" import ( "context" diff --git a/receiver/cloudfoundryreceiver/uaa.go b/receiver/cloudfoundryreceiver/uaa.go index f8b343f735272..c3c7645d5084f 100644 --- a/receiver/cloudfoundryreceiver/uaa.go +++ b/receiver/cloudfoundryreceiver/uaa.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package cloudfoundryreceiver +package cloudfoundryreceiver // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/cloudfoundryreceiver" import ( "fmt" diff --git a/receiver/prometheusreceiver/internal/metricsutil_pdata.go b/receiver/prometheusreceiver/internal/metricsutil_pdata.go index b1fa57c9aba9b..27d230a968356 100644 --- a/receiver/prometheusreceiver/internal/metricsutil_pdata.go +++ b/receiver/prometheusreceiver/internal/metricsutil_pdata.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package internal // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/internal" import "go.opentelemetry.io/collector/model/pdata" diff --git a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go index 5276b3710a2b1..deef4e0efc6dd 100644 --- a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go +++ b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package internal // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/internal" import ( "fmt" diff --git a/receiver/prometheusreceiver/internal/otlp_transaction.go b/receiver/prometheusreceiver/internal/otlp_transaction.go index b23691bdfff55..7a9f56c3af7e2 100644 --- a/receiver/prometheusreceiver/internal/otlp_transaction.go +++ b/receiver/prometheusreceiver/internal/otlp_transaction.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package internal +package internal // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/internal" import ( "context" From 7e04b44797bde14914e405f30b550c3f61a6d19d Mon Sep 17 00:00:00 2001 From: Anthony J Mirabella Date: Wed, 24 Nov 2021 11:59:31 -0500 Subject: [PATCH 17/26] receiver/prometheus: run e2e tests in parallel to avoid doubling test suite runtime Signed-off-by: Anthony J Mirabella --- .../metrics_receiver_helper_test.go | 29 +++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/receiver/prometheusreceiver/metrics_receiver_helper_test.go b/receiver/prometheusreceiver/metrics_receiver_helper_test.go index e7ebabca77daa..6e5853687b891 100644 --- a/receiver/prometheusreceiver/metrics_receiver_helper_test.go +++ b/receiver/prometheusreceiver/metrics_receiver_helper_test.go @@ -111,6 +111,10 @@ type testData struct { validateFunc func(t *testing.T, td *testData, result []*pdata.ResourceMetrics) } +func (t testData) clone() *testData { + return &testData{name: t.name, pages: t.pages, validateFunc: t.validateFunc} +} + // setupMockPrometheus to create a mocked prometheus based on targets, returning the server and a prometheus exporting // config func setupMockPrometheus(tds ...*testData) (*mockPrometheus, *promcfg.Config, error) { @@ -277,6 +281,7 @@ type testExpectation func(*testing.T, *pdata.ResourceMetrics) func doCompare(t *testing.T, name string, want pdata.AttributeMap, got *pdata.ResourceMetrics, expectations []testExpectation) { t.Run(name, func(t *testing.T) { + t.Parallel() assert.Equal(t, expectedScrapeMetricCount, countScrapeMetricsRM(got)) assert.Equal(t, want.Len(), got.Resource().Attributes().Len()) for k, v := range want.AsRaw() { @@ -448,7 +453,13 @@ func testComponent(t *testing.T, targets []*testData, useStartTimeMetric bool, s pipelineType = "pdata" } t.Run(pipelineType, func(t *testing.T) { // 1. setup mock server - mp, cfg, err := setupMockPrometheus(targets...) + // Clone the target data to allow the tests to run in parallel + tgts := make([]*testData, len(targets)) + for i, t := range targets { + tgts[i] = t.clone() + } + t.Parallel() + mp, cfg, err := setupMockPrometheus(tgts...) require.Nilf(t, err, "Failed to create Prometheus config: %v", err) defer mp.Close() @@ -464,7 +475,7 @@ func testComponent(t *testing.T, targets []*testData, useStartTimeMetric bool, s require.NoError(t, rcvr.Start(context.Background(), componenttest.NewNopHost()), "Failed to invoke Start: %v", err) t.Cleanup(func() { // verify state after shutdown is called - assert.Lenf(t, flattenTargets(rcvr.scrapeManager.TargetsAll()), len(targets), "expected %v targets to be running", len(targets)) + assert.Lenf(t, flattenTargets(rcvr.scrapeManager.TargetsAll()), len(tgts), "expected %v targets to be running", len(tgts)) require.NoError(t, rcvr.Shutdown(context.Background())) assert.Len(t, flattenTargets(rcvr.scrapeManager.TargetsAll()), 0, "expected scrape manager to have no targets") }) @@ -479,8 +490,9 @@ func testComponent(t *testing.T, targets []*testData, useStartTimeMetric bool, s assert.Equalf(t, lep, lres, "want %d targets, but got %v\n", lep, lres) // loop to validate outputs for each targets - for _, target := range targets { + for _, target := range tgts { t.Run(target.name, func(t *testing.T) { + t.Parallel() validScrapes := getValidScrapes(t, pResults[target.name]) target.validateFunc(t, target, validScrapes) }) @@ -497,8 +509,14 @@ func testComponentCustomConfig(t *testing.T, targets []*testData, cfgMut func(*p pipelineType = "pdata" } t.Run(pipelineType, func(t *testing.T) { + // Clone the target data to allow the tests to run in parallel + tgts := make([]*testData, len(targets)) + for i, t := range targets { + tgts[i] = t.clone() + } + t.Parallel() ctx := context.Background() - mp, cfg, err := setupMockPrometheus(targets...) + mp, cfg, err := setupMockPrometheus(tgts...) cfgMut(cfg) require.Nilf(t, err, "Failed to create Prometheus config: %v", err) defer mp.Close() @@ -525,8 +543,9 @@ func testComponentCustomConfig(t *testing.T, targets []*testData, cfgMut func(*p assert.Equalf(t, lep, lres, "want %d targets, but got %v\n", lep, lres) // loop to validate outputs for each targets - for _, target := range targets { + for _, target := range tgts { t.Run(target.name, func(t *testing.T) { + t.Parallel() validScrapes := getValidScrapes(t, pResults[target.name]) target.validateFunc(t, target, validScrapes) }) From d3ff42a7aa38fb781ae06fd092c5ed2e1b92aca7 Mon Sep 17 00:00:00 2001 From: Anthony J Mirabella Date: Wed, 24 Nov 2021 13:23:10 -0500 Subject: [PATCH 18/26] Revert "receiver/prometheus: run e2e tests in parallel to avoid doubling test suite runtime" This reverts commit 7e04b44797bde14914e405f30b550c3f61a6d19d. --- .../metrics_receiver_helper_test.go | 29 ++++--------------- 1 file changed, 5 insertions(+), 24 deletions(-) diff --git a/receiver/prometheusreceiver/metrics_receiver_helper_test.go b/receiver/prometheusreceiver/metrics_receiver_helper_test.go index 6e5853687b891..e7ebabca77daa 100644 --- a/receiver/prometheusreceiver/metrics_receiver_helper_test.go +++ b/receiver/prometheusreceiver/metrics_receiver_helper_test.go @@ -111,10 +111,6 @@ type testData struct { validateFunc func(t *testing.T, td *testData, result []*pdata.ResourceMetrics) } -func (t testData) clone() *testData { - return &testData{name: t.name, pages: t.pages, validateFunc: t.validateFunc} -} - // setupMockPrometheus to create a mocked prometheus based on targets, returning the server and a prometheus exporting // config func setupMockPrometheus(tds ...*testData) (*mockPrometheus, *promcfg.Config, error) { @@ -281,7 +277,6 @@ type testExpectation func(*testing.T, *pdata.ResourceMetrics) func doCompare(t *testing.T, name string, want pdata.AttributeMap, got *pdata.ResourceMetrics, expectations []testExpectation) { t.Run(name, func(t *testing.T) { - t.Parallel() assert.Equal(t, expectedScrapeMetricCount, countScrapeMetricsRM(got)) assert.Equal(t, want.Len(), got.Resource().Attributes().Len()) for k, v := range want.AsRaw() { @@ -453,13 +448,7 @@ func testComponent(t *testing.T, targets []*testData, useStartTimeMetric bool, s pipelineType = "pdata" } t.Run(pipelineType, func(t *testing.T) { // 1. setup mock server - // Clone the target data to allow the tests to run in parallel - tgts := make([]*testData, len(targets)) - for i, t := range targets { - tgts[i] = t.clone() - } - t.Parallel() - mp, cfg, err := setupMockPrometheus(tgts...) + mp, cfg, err := setupMockPrometheus(targets...) require.Nilf(t, err, "Failed to create Prometheus config: %v", err) defer mp.Close() @@ -475,7 +464,7 @@ func testComponent(t *testing.T, targets []*testData, useStartTimeMetric bool, s require.NoError(t, rcvr.Start(context.Background(), componenttest.NewNopHost()), "Failed to invoke Start: %v", err) t.Cleanup(func() { // verify state after shutdown is called - assert.Lenf(t, flattenTargets(rcvr.scrapeManager.TargetsAll()), len(tgts), "expected %v targets to be running", len(tgts)) + assert.Lenf(t, flattenTargets(rcvr.scrapeManager.TargetsAll()), len(targets), "expected %v targets to be running", len(targets)) require.NoError(t, rcvr.Shutdown(context.Background())) assert.Len(t, flattenTargets(rcvr.scrapeManager.TargetsAll()), 0, "expected scrape manager to have no targets") }) @@ -490,9 +479,8 @@ func testComponent(t *testing.T, targets []*testData, useStartTimeMetric bool, s assert.Equalf(t, lep, lres, "want %d targets, but got %v\n", lep, lres) // loop to validate outputs for each targets - for _, target := range tgts { + for _, target := range targets { t.Run(target.name, func(t *testing.T) { - t.Parallel() validScrapes := getValidScrapes(t, pResults[target.name]) target.validateFunc(t, target, validScrapes) }) @@ -509,14 +497,8 @@ func testComponentCustomConfig(t *testing.T, targets []*testData, cfgMut func(*p pipelineType = "pdata" } t.Run(pipelineType, func(t *testing.T) { - // Clone the target data to allow the tests to run in parallel - tgts := make([]*testData, len(targets)) - for i, t := range targets { - tgts[i] = t.clone() - } - t.Parallel() ctx := context.Background() - mp, cfg, err := setupMockPrometheus(tgts...) + mp, cfg, err := setupMockPrometheus(targets...) cfgMut(cfg) require.Nilf(t, err, "Failed to create Prometheus config: %v", err) defer mp.Close() @@ -543,9 +525,8 @@ func testComponentCustomConfig(t *testing.T, targets []*testData, cfgMut func(*p assert.Equalf(t, lep, lres, "want %d targets, but got %v\n", lep, lres) // loop to validate outputs for each targets - for _, target := range tgts { + for _, target := range targets { t.Run(target.name, func(t *testing.T) { - t.Parallel() validScrapes := getValidScrapes(t, pResults[target.name]) target.validateFunc(t, target, validScrapes) }) From 4fb78ab28e178916910ccf2de5e9d4f74dadf9ce Mon Sep 17 00:00:00 2001 From: Anthony J Mirabella Date: Wed, 24 Nov 2021 14:01:07 -0500 Subject: [PATCH 19/26] receiver/prometheus: address PR feedback Signed-off-by: Anthony J Mirabella --- receiver/prometheusreceiver/internal/metricfamily.go | 7 ------- .../prometheusreceiver/internal/metricsutil_pdata.go | 5 ++++- .../prometheusreceiver/internal/otlp_metricfamily.go | 8 +------- .../internal/otlp_metrics_adjuster.go | 8 ++++---- .../internal/otlp_metricsbuilder.go | 12 ++++++------ .../internal/otlp_metricsbuilder_test.go | 5 +++++ .../prometheusreceiver/internal/otlp_transaction.go | 8 ++++---- 7 files changed, 24 insertions(+), 29 deletions(-) diff --git a/receiver/prometheusreceiver/internal/metricfamily.go b/receiver/prometheusreceiver/internal/metricfamily.go index 06d4a24ced64f..97b4dd05c5413 100644 --- a/receiver/prometheusreceiver/internal/metricfamily.go +++ b/receiver/prometheusreceiver/internal/metricfamily.go @@ -32,7 +32,6 @@ import ( // a single scrape. type MetricFamily interface { Add(metricName string, ls labels.Labels, t int64, v float64) error - IsSameFamily(metricName string) bool ToMetric() (*metricspb.Metric, int, int) } @@ -125,12 +124,6 @@ func defineInternalMetric(metricName string, metadata scrape.MetricMetadata, log return metadata } -func (mf *metricFamily) IsSameFamily(metricName string) bool { - // trim known suffix if necessary - familyName := normalizeMetricName(metricName) - return mf.name == familyName || familyName != metricName && mf.name == metricName -} - // updateLabelKeys is used to store all the label keys of a same metric family in observed order. since prometheus // receiver removes any label with empty value before feeding it to an appender, in order to figure out all the labels // from the same metric family we will need to keep track of what labels have ever been observed. diff --git a/receiver/prometheusreceiver/internal/metricsutil_pdata.go b/receiver/prometheusreceiver/internal/metricsutil_pdata.go index 27d230a968356..a0f10562bf10b 100644 --- a/receiver/prometheusreceiver/internal/metricsutil_pdata.go +++ b/receiver/prometheusreceiver/internal/metricsutil_pdata.go @@ -128,8 +128,11 @@ func sumMetricPdata(name string, kvp []*kv, startTs pdata.Timestamp, points ...* metric := pdata.NewMetric() metric.SetName(name) metric.SetDataType(pdata.MetricDataTypeSum) + sum := metric.Sum() + sum.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) + sum.SetIsMonotonic(true) - destPointL := metric.Sum().DataPoints() + destPointL := sum.DataPoints() for _, point := range points { destPoint := destPointL.AppendEmpty() point.CopyTo(destPoint) diff --git a/receiver/prometheusreceiver/internal/otlp_metricfamily.go b/receiver/prometheusreceiver/internal/otlp_metricfamily.go index ff8daae6e45b5..18a53e812fdbc 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricfamily.go +++ b/receiver/prometheusreceiver/internal/otlp_metricfamily.go @@ -31,7 +31,6 @@ import ( // a single scrape. type MetricFamilyPdata interface { Add(metricName string, ls labels.Labels, t int64, v float64) error - IsSameFamily(metricName string) bool ToMetricPdata(metrics *pdata.MetricSlice) (int, int) } @@ -115,12 +114,6 @@ func newMetricFamilyPdata(metricName string, mc MetadataCache, logger *zap.Logge } } -func (mf *metricFamilyPdata) IsSameFamily(metricName string) bool { - // trim known suffix if necessary - familyName := normalizeMetricName(metricName) - return mf.name == familyName || familyName != metricName && mf.name == metricName -} - // updateLabelKeys is used to store all the label keys of a same metric family in observed order. since prometheus // receiver removes any label with empty value before feeding it to an appender, in order to figure out all the labels // from the same metric family we will need to keep track of what labels have ever been observed. @@ -352,6 +345,7 @@ func (mf *metricFamilyPdata) ToMetricPdata(metrics *pdata.MetricSlice) (int, int case pdata.MetricDataTypeSum: sum := metric.Sum() sum.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) + sum.SetIsMonotonic(true) sdpL := sum.DataPoints() for _, mg := range mf.getGroups() { if !mg.toNumberDataPoint(mf.labelKeysOrdered, &sdpL) { diff --git a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go index deef4e0efc6dd..c8cd19620c602 100644 --- a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go +++ b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go @@ -274,7 +274,7 @@ func (ma *MetricsAdjusterPdata) adjustMetricGauge(current *pdata.Metric) (resets initialPoints := tsi.initial.Gauge().DataPoints() previousPoints := previous.Gauge().DataPoints() if i >= initialPoints.Len() || i >= previousPoints.Len() { - ma.logger.Info("Adjusting Points, all lengths should be equal", + ma.logger.Debug("Adjusting Points, all lengths should be equal", zap.Int("len(current)", currentPoints.Len()), zap.Int("len(initial)", initialPoints.Len()), zap.Int("len(previous)", previousPoints.Len())) @@ -325,7 +325,7 @@ func (ma *MetricsAdjusterPdata) adjustMetricHistogram(current *pdata.Metric) (re initialPoints := tsi.initial.Histogram().DataPoints() previousPoints := previous.Histogram().DataPoints() if i >= initialPoints.Len() || i >= previousPoints.Len() { - ma.logger.Info("Adjusting Points, all lengths should be equal", + ma.logger.Debug("Adjusting Points, all lengths should be equal", zap.Int("len(current)", currentPoints.Len()), zap.Int("len(initial)", initialPoints.Len()), zap.Int("len(previous)", previousPoints.Len())) @@ -365,7 +365,7 @@ func (ma *MetricsAdjusterPdata) adjustMetricSum(current *pdata.Metric) (resets i initialPoints := tsi.initial.Sum().DataPoints() previousPoints := previous.Sum().DataPoints() if i >= initialPoints.Len() || i >= previousPoints.Len() { - ma.logger.Info("Adjusting Points, all lengths should be equal", + ma.logger.Debug("Adjusting Points, all lengths should be equal", zap.Int("len(current)", currentPoints.Len()), zap.Int("len(initial)", initialPoints.Len()), zap.Int("len(previous)", previousPoints.Len())) @@ -405,7 +405,7 @@ func (ma *MetricsAdjusterPdata) adjustMetricSummary(current *pdata.Metric) (rese initialPoints := tsi.initial.Summary().DataPoints() previousPoints := previous.Summary().DataPoints() if i >= initialPoints.Len() || i >= previousPoints.Len() { - ma.logger.Info("Adjusting Points, all lengths should be equal", + ma.logger.Debug("Adjusting Points, all lengths should be equal", zap.Int("len(current)", currentPoints.Len()), zap.Int("len(initial)", initialPoints.Len()), zap.Int("len(previous)", previousPoints.Len())) diff --git a/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go b/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go index 4e87fa71e8496..e5b1b50556e0b 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go +++ b/receiver/prometheusreceiver/internal/otlp_metricsbuilder.go @@ -60,22 +60,22 @@ func getBoundaryPdata(metricType pdata.MetricDataType, labels labels.Labels) (fl func convToPdataMetricType(metricType textparse.MetricType) pdata.MetricDataType { switch metricType { - case textparse.MetricTypeCounter: // metricspb.MetricDescriptor_CUMULATIVE_DOUBLE + case textparse.MetricTypeCounter: // always use float64, as it's the internal data type used in prometheus return pdata.MetricDataTypeSum - // textparse.MetricTypeUnknown is converted to gauge by default to fix Prometheus untyped metrics from being dropped - case textparse.MetricTypeGauge, textparse.MetricTypeUnknown: // metricspb.MetricDescriptor_GAUGE_DOUBLE + // textparse.MetricTypeUnknown is converted to gauge by default to prevent Prometheus untyped metrics from being dropped + case textparse.MetricTypeGauge, textparse.MetricTypeUnknown: return pdata.MetricDataTypeGauge - case textparse.MetricTypeHistogram: // metricspb.MetricDescriptor_CUMULATIVE_DISTRIBUTION + case textparse.MetricTypeHistogram: return pdata.MetricDataTypeHistogram // dropping support for gaugehistogram for now until we have an official spec of its implementation // a draft can be found in: https://docs.google.com/document/d/1KwV0mAXwwbvvifBvDKH_LU1YjyXE_wxCkHNoCGq1GX0/edit#heading=h.1cvzqd4ksd23 // case textparse.MetricTypeGaugeHistogram: - // return metricspb.MetricDescriptor_GAUGE_DISTRIBUTION + // return case textparse.MetricTypeSummary: return pdata.MetricDataTypeSummary default: - // including: textparse.MetricTypeInfo, textparse.MetricTypeStateset + // including: textparse.MetricTypeGaugeHistogram, textparse.MetricTypeInfo, textparse.MetricTypeStateset return pdata.MetricDataTypeNone } } diff --git a/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go b/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go index 96f7f3d90e095..602f2cde8895e 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go +++ b/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go @@ -344,6 +344,7 @@ func Test_OTLPMetricBuilder_counters(t *testing.T) { m0.SetDataType(pdata.MetricDataTypeSum) sum := m0.Sum() sum.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) + sum.SetIsMonotonic(true) pt0 := sum.DataPoints().AppendEmpty() pt0.SetDoubleVal(100.0) pt0.SetStartTimestamp(startTsNanos) @@ -370,6 +371,7 @@ func Test_OTLPMetricBuilder_counters(t *testing.T) { m0.SetDataType(pdata.MetricDataTypeSum) sum := m0.Sum() sum.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) + sum.SetIsMonotonic(true) pt0 := sum.DataPoints().AppendEmpty() pt0.SetDoubleVal(150.0) pt0.SetStartTimestamp(startTsNanos) @@ -403,6 +405,7 @@ func Test_OTLPMetricBuilder_counters(t *testing.T) { m0.SetDataType(pdata.MetricDataTypeSum) sum0 := m0.Sum() sum0.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) + sum0.SetIsMonotonic(true) pt0 := sum0.DataPoints().AppendEmpty() pt0.SetDoubleVal(150.0) pt0.SetStartTimestamp(startTsNanos) @@ -420,6 +423,7 @@ func Test_OTLPMetricBuilder_counters(t *testing.T) { m1.SetDataType(pdata.MetricDataTypeSum) sum1 := m1.Sum() sum1.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) + sum1.SetIsMonotonic(true) pt2 := sum1.DataPoints().AppendEmpty() pt2.SetDoubleVal(100.0) pt2.SetStartTimestamp(startTsNanos) @@ -445,6 +449,7 @@ func Test_OTLPMetricBuilder_counters(t *testing.T) { m0.SetDataType(pdata.MetricDataTypeSum) sum := m0.Sum() sum.SetAggregationTemporality(pdata.MetricAggregationTemporalityCumulative) + sum.SetIsMonotonic(true) pt0 := sum.DataPoints().AppendEmpty() pt0.SetDoubleVal(100.0) pt0.SetStartTimestamp(startTsNanos) diff --git a/receiver/prometheusreceiver/internal/otlp_transaction.go b/receiver/prometheusreceiver/internal/otlp_transaction.go index 7a9f56c3af7e2..f5492dd25013e 100644 --- a/receiver/prometheusreceiver/internal/otlp_transaction.go +++ b/receiver/prometheusreceiver/internal/otlp_transaction.go @@ -143,7 +143,7 @@ func (t *transactionPdata) Commit() error { return err } // Otherwise adjust the startTimestamp for all the metrics. - adjustStartTimestampPdata(t.metricBuilder.startTime, metricsL) + t.adjustStartTimestampPdata(metricsL) } else { // TODO: Derive numPoints in this case. _ = NewMetricsAdjusterPdata(t.jobsMap.get(t.job, t.instance), t.logger).AdjustMetrics(metricsL) @@ -169,8 +169,8 @@ func pdataTimestampFromFloat64(ts float64) pdata.Timestamp { return pdata.NewTimestampFromTime(time.Unix(secs, nanos)) } -func adjustStartTimestampPdata(startTime float64, metricsL *pdata.MetricSlice) { - startTimeTs := pdataTimestampFromFloat64(startTime) +func (t transactionPdata) adjustStartTimestampPdata(metricsL *pdata.MetricSlice) { + startTimeTs := pdataTimestampFromFloat64(t.metricBuilder.startTime) for i := 0; i < metricsL.Len(); i++ { metric := metricsL.At(i) switch metric.DataType() { @@ -199,7 +199,7 @@ func adjustStartTimestampPdata(startTime float64, metricsL *pdata.MetricSlice) { } default: - panic("Unknown type:: " + metric.DataType().String()) + t.logger.Warn("Unknown metric type", zap.String("type", metric.DataType().String())) } } } From 514e284243b8f97ba5aba24504b355cfc4753ac6 Mon Sep 17 00:00:00 2001 From: Anthony J Mirabella Date: Wed, 24 Nov 2021 19:00:12 -0500 Subject: [PATCH 20/26] receiver/prometheus: ensure test helper can extract timestamp from metrics of any type Signed-off-by: Anthony J Mirabella --- .../metrics_receiver_helper_test.go | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/receiver/prometheusreceiver/metrics_receiver_helper_test.go b/receiver/prometheusreceiver/metrics_receiver_helper_test.go index c117fc7d3c0d0..62e2fa044da33 100644 --- a/receiver/prometheusreceiver/metrics_receiver_helper_test.go +++ b/receiver/prometheusreceiver/metrics_receiver_helper_test.go @@ -578,12 +578,21 @@ func splitMetricsByTarget(metrics []pdata.Metrics) map[string][]*pdata.ResourceM } func getTS(ms pdata.MetricSlice) pdata.Timestamp { - for i := 0; i < ms.Len(); i++ { - m := ms.At(i) - switch m.DataType() { - case pdata.MetricDataTypeGauge: - return m.Gauge().DataPoints().At(0).Timestamp() - } + if ms.Len() == 0 { + return 0 + } + m := ms.At(0) + switch m.DataType() { + case pdata.MetricDataTypeGauge: + return m.Gauge().DataPoints().At(0).Timestamp() + case pdata.MetricDataTypeSum: + return m.Sum().DataPoints().At(0).Timestamp() + case pdata.MetricDataTypeHistogram: + return m.Histogram().DataPoints().At(0).Timestamp() + case pdata.MetricDataTypeSummary: + return m.Summary().DataPoints().At(0).Timestamp() + case pdata.MetricDataTypeExponentialHistogram: + return m.ExponentialHistogram().DataPoints().At(0).Timestamp() } return 0 } From ac4dbd98a0662102e48e01bc82e9329c2fb9b2bb Mon Sep 17 00:00:00 2001 From: Anthony J Mirabella Date: Wed, 24 Nov 2021 19:01:25 -0500 Subject: [PATCH 21/26] receiver/prometheus: tests should not be expecting empty label values Signed-off-by: Anthony J Mirabella --- .../internal/otlp_metricsbuilder_test.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go b/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go index 602f2cde8895e..da6596c5b9589 100644 --- a/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go +++ b/receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go @@ -573,7 +573,6 @@ func Test_OTLPMetricBuilder_gauges(t *testing.T) { pt0.SetDoubleVal(100.0) pt0.SetStartTimestamp(0) pt0.SetTimestamp(startTsNanos) - pt0.Attributes().InsertString("bar", "") pt0.Attributes().InsertString("foo", "bar") pt1 := gauge0.DataPoints().AppendEmpty() @@ -581,7 +580,6 @@ func Test_OTLPMetricBuilder_gauges(t *testing.T) { pt1.SetStartTimestamp(0) pt1.SetTimestamp(startTsNanos) pt1.Attributes().InsertString("bar", "foo") - pt1.Attributes().InsertString("foo", "") return []*pdata.MetricSlice{&mL0} }, @@ -613,7 +611,6 @@ func Test_OTLPMetricBuilder_gauges(t *testing.T) { pt0.SetDoubleVal(100.0) pt0.SetStartTimestamp(0) pt0.SetTimestamp(startTsNanos) - pt0.Attributes().InsertString("bar", "") pt0.Attributes().InsertString("foo", "bar") pt1 := gauge0.DataPoints().AppendEmpty() @@ -621,7 +618,6 @@ func Test_OTLPMetricBuilder_gauges(t *testing.T) { pt1.SetStartTimestamp(0) pt1.SetTimestamp(startTsNanos) pt1.Attributes().InsertString("bar", "foo") - pt1.Attributes().InsertString("foo", "") mL1 := pdata.NewMetricSlice() m1 := mL1.AppendEmpty() @@ -697,14 +693,12 @@ func Test_OTLPMetricBuilder_untype(t *testing.T) { pt1 := gauge1.DataPoints().AppendEmpty() pt1.SetDoubleVal(200.0) pt1.SetTimestamp(startTsNanos) - pt1.Attributes().InsertString("bar", "") pt1.Attributes().InsertString("foo", "bar") pt2 := gauge1.DataPoints().AppendEmpty() pt2.SetDoubleVal(300.0) pt2.SetTimestamp(startTsNanos) pt2.Attributes().InsertString("bar", "foo") - pt2.Attributes().InsertString("foo", "") return []*pdata.MetricSlice{&mL0} }, @@ -804,7 +798,6 @@ func Test_OTLPMetricBuilder_histogram(t *testing.T) { pt0.SetTimestamp(startTsNanos) pt0.SetStartTimestamp(startTsNanos) pt0.Attributes().InsertString("foo", "bar") - pt0.Attributes().InsertString("key2", "") pt1 := hist0.DataPoints().AppendEmpty() pt1.SetCount(3) @@ -813,7 +806,6 @@ func Test_OTLPMetricBuilder_histogram(t *testing.T) { pt1.SetBucketCounts([]uint64{1, 1, 1}) pt1.SetTimestamp(startTsNanos) pt1.SetStartTimestamp(startTsNanos) - pt1.Attributes().InsertString("foo", "") pt1.Attributes().InsertString("key2", "v2") return []*pdata.MetricSlice{&mL0} @@ -857,7 +849,6 @@ func Test_OTLPMetricBuilder_histogram(t *testing.T) { pt0.SetTimestamp(startTsNanos) pt0.SetStartTimestamp(startTsNanos) pt0.Attributes().InsertString("foo", "bar") - pt0.Attributes().InsertString("key2", "") pt1 := hist0.DataPoints().AppendEmpty() pt1.SetCount(3) @@ -866,7 +857,6 @@ func Test_OTLPMetricBuilder_histogram(t *testing.T) { pt1.SetBucketCounts([]uint64{1, 1, 1}) pt1.SetTimestamp(startTsNanos) pt1.SetStartTimestamp(startTsNanos) - pt1.Attributes().InsertString("foo", "") pt1.Attributes().InsertString("key2", "v2") m1 := mL0.AppendEmpty() From d0bd35d56484d6290c81b54a9d02c2e0fa8cd871 Mon Sep 17 00:00:00 2001 From: Anthony J Mirabella Date: Mon, 29 Nov 2021 15:52:05 -0500 Subject: [PATCH 22/26] make porto Signed-off-by: Anthony J Mirabella --- .../subprocessmanager/exec_command_other.go | 2 +- .../subprocessmanager/exec_command_win.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/receiver/prometheusexecreceiver/subprocessmanager/exec_command_other.go b/receiver/prometheusexecreceiver/subprocessmanager/exec_command_other.go index 32baa0069574d..69e37a6208601 100644 --- a/receiver/prometheusexecreceiver/subprocessmanager/exec_command_other.go +++ b/receiver/prometheusexecreceiver/subprocessmanager/exec_command_other.go @@ -14,7 +14,7 @@ //go:build !windows -package subprocessmanager +package subprocessmanager // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusexecreceiver/subprocessmanager" import ( "os/exec" diff --git a/receiver/prometheusexecreceiver/subprocessmanager/exec_command_win.go b/receiver/prometheusexecreceiver/subprocessmanager/exec_command_win.go index 1ec4d10a899a0..04d0720dbddd6 100644 --- a/receiver/prometheusexecreceiver/subprocessmanager/exec_command_win.go +++ b/receiver/prometheusexecreceiver/subprocessmanager/exec_command_win.go @@ -14,7 +14,7 @@ //go:build windows -package subprocessmanager +package subprocessmanager // import "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusexecreceiver/subprocessmanager" import ( "os" From 5f5284241bb12ecbc0d551be6456b49b5fd3cd3e Mon Sep 17 00:00:00 2001 From: Anthony J Mirabella Date: Tue, 30 Nov 2021 00:24:21 -0500 Subject: [PATCH 23/26] receiver/prometheus: remove gauge histogram pdata adjustment tests Signed-off-by: Anthony J Mirabella --- ...til_pdata.go => metricsutil_pdata_test.go} | 0 .../internal/otlp_metrics_adjuster_test.go | 31 ------------------- 2 files changed, 31 deletions(-) rename receiver/prometheusreceiver/internal/{metricsutil_pdata.go => metricsutil_pdata_test.go} (100%) diff --git a/receiver/prometheusreceiver/internal/metricsutil_pdata.go b/receiver/prometheusreceiver/internal/metricsutil_pdata_test.go similarity index 100% rename from receiver/prometheusreceiver/internal/metricsutil_pdata.go rename to receiver/prometheusreceiver/internal/metricsutil_pdata_test.go diff --git a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go index fc689c8ed0834..57dfb2d2a0424 100644 --- a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go +++ b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster_test.go @@ -292,30 +292,6 @@ func Test_cumulative_pdata(t *testing.T) { runScriptPdata(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) } -func Test_gaugeDistribution_pdata(t *testing.T) { - script := []*metricsAdjusterTestPdata{ - { - "GaugeDist: round 1 - gauge distribution not adjusted", - metricSlice(gaugeDistMetric(gd1, k1v1k2v2, pdt1Ms, distPoint(pdt1Ms, bounds0, []uint64{4, 2, 3, 7}))), - metricSlice(gaugeDistMetric(gd1, k1v1k2v2, pdt1Ms, distPoint(pdt1Ms, bounds0, []uint64{4, 2, 3, 7}))), - 0, - }, - { - "GaugeDist: round 2 - gauge distribution not adjusted", - metricSlice(gaugeDistMetric(gd1, k1v1k2v2, pdt2Ms, distPoint(pdt2Ms, bounds0, []uint64{6, 5, 8, 11}))), - metricSlice(gaugeDistMetric(gd1, k1v1k2v2, pdt2Ms, distPoint(pdt2Ms, bounds0, []uint64{6, 5, 8, 11}))), - 0, - }, - { - "GaugeDist: round 3 - count/sum less than previous - gauge distribution not adjusted", - metricSlice(gaugeDistMetric(gd1, k1v1k2v2, pdt3Ms, distPoint(pdt3Ms, bounds0, []uint64{2, 0, 1, 5}))), - metricSlice(gaugeDistMetric(gd1, k1v1k2v2, pdt3Ms, distPoint(pdt3Ms, bounds0, []uint64{2, 0, 1, 5}))), - 0, - }, - } - runScriptPdata(t, NewJobsMapPdata(time.Minute).get("job", "0"), script) -} - func populateSummary(sdp *pdata.SummaryDataPoint, timestamp pdata.Timestamp, count uint64, sum float64, quantilePercents, quantileValues []float64) { quantiles := sdp.QuantileValues() for i := range quantilePercents { @@ -489,7 +465,6 @@ func Test_summary_pdata(t *testing.T) { var ( distPoint = distPointPdata - gaugeDistMetric = gaugeDistMetricPdata histogramMetric = cumulativeDistMetricPdata doublePoint = doublePointPdata gaugeMetric = gaugeMetricPdata @@ -558,14 +533,12 @@ func Test_multiMetrics_pdata(t *testing.T) { "MultiMetrics: round 1 - combined round 1 of individual metrics", metricSlice( gaugeMetric(g1, k1v1k2v2, pdt1Ms, doublePoint(pdt1Ms, 44)), - gaugeDistMetric(gd1, k1v1k2v2, pdt1Ms, distPoint(pdt1Ms, bounds0, []uint64{4, 2, 3, 7})), sumMetric(c1, k1v1k2v2, pdt1Ms, doublePoint(pdt1Ms, 44)), histogramMetric(cd1, k1v1k2v2, pdt1Ms, distPoint(pdt1Ms, bounds0, []uint64{4, 2, 3, 7})), summaryMetric(s1, k1v1k2v2, pdt1Ms, summaryPoint(pdt1Ms, 10, 40, percent0, []float64{1, 5, 8})), ), metricSlice( gaugeMetric(g1, k1v1k2v2, pdt1Ms, doublePoint(pdt1Ms, 44)), - gaugeDistMetric(gd1, k1v1k2v2, pdt1Ms, distPoint(pdt1Ms, bounds0, []uint64{4, 2, 3, 7})), sumMetric(c1, k1v1k2v2, pdt1Ms, doublePoint(pdt1Ms, 44)), histogramMetric(cd1, k1v1k2v2, pdt1Ms, distPoint(pdt1Ms, bounds0, []uint64{4, 2, 3, 7})), summaryMetric(s1, k1v1k2v2, pdt1Ms, summaryPoint(pdt1Ms, 10, 40, percent0, []float64{1, 5, 8})), @@ -575,14 +548,12 @@ func Test_multiMetrics_pdata(t *testing.T) { "MultiMetrics: round 2 - combined round 2 of individual metrics", metricSlice( gaugeMetric(g1, k1v1k2v2, pdt2Ms, doublePoint(pdt2Ms, 66)), - gaugeDistMetric(gd1, k1v1k2v2, pdt2Ms, distPoint(pdt2Ms, bounds0, []uint64{6, 5, 8, 11})), sumMetric(c1, k1v1k2v2, pdt2Ms, doublePoint(pdt2Ms, 66)), histogramMetric(cd1, k1v1k2v2, pdt2Ms, distPoint(pdt2Ms, bounds0, []uint64{6, 3, 4, 8})), summaryMetric(s1, k1v1k2v2, pdt2Ms, summaryPoint(pdt2Ms, 15, 70, percent0, []float64{7, 44, 9})), ), metricSlice( gaugeMetric(g1, k1v1k2v2, pdt2Ms, doublePoint(pdt2Ms, 66)), - gaugeDistMetric(gd1, k1v1k2v2, pdt2Ms, distPoint(pdt2Ms, bounds0, []uint64{6, 5, 8, 11})), sumMetric(c1, k1v1k2v2, pdt1Ms, doublePoint(pdt2Ms, 66)), histogramMetric(cd1, k1v1k2v2, pdt1Ms, distPoint(pdt2Ms, bounds0, []uint64{6, 3, 4, 8})), summaryMetric(s1, k1v1k2v2, pdt1Ms, summaryPoint(pdt2Ms, 15, 70, percent0, []float64{7, 44, 9})), @@ -592,14 +563,12 @@ func Test_multiMetrics_pdata(t *testing.T) { "MultiMetrics: round 3 - combined round 3 of individual metrics", metricSlice( gaugeMetric(g1, k1v1k2v2, pdt3Ms, doublePoint(pdt3Ms, 55)), - gaugeDistMetric(gd1, k1v1k2v2, pdt3Ms, distPoint(pdt3Ms, bounds0, []uint64{2, 0, 1, 5})), sumMetric(c1, k1v1k2v2, pdt3Ms, doublePoint(pdt3Ms, 55)), histogramMetric(cd1, k1v1k2v2, pdt3Ms, distPoint(pdt3Ms, bounds0, []uint64{5, 3, 2, 7})), summaryMetric(s1, k1v1k2v2, pdt3Ms, summaryPoint(pdt3Ms, 12, 66, percent0, []float64{3, 22, 5})), ), metricSlice( gaugeMetric(g1, k1v1k2v2, pdt3Ms, doublePoint(pdt3Ms, 55)), - gaugeDistMetric(gd1, k1v1k2v2, pdt3Ms, distPoint(pdt3Ms, bounds0, []uint64{2, 0, 1, 5})), sumMetric(c1, k1v1k2v2, pdt3Ms, doublePoint(pdt3Ms, 55)), histogramMetric(cd1, k1v1k2v2, pdt3Ms, distPoint(pdt3Ms, bounds0, []uint64{5, 3, 2, 7})), summaryMetric(s1, k1v1k2v2, pdt3Ms, summaryPoint(pdt3Ms, 12, 66, percent0, []float64{3, 22, 5})), From 24060cfe99bfc43e5eea05345fcc942b9b77cd46 Mon Sep 17 00:00:00 2001 From: Anthony J Mirabella Date: Tue, 30 Nov 2021 12:11:44 -0500 Subject: [PATCH 24/26] receiver/prometheus: remove unused gauge histogram data generator helper Signed-off-by: Anthony J Mirabella --- .../prometheusreceiver/internal/metricsutil_pdata_test.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/receiver/prometheusreceiver/internal/metricsutil_pdata_test.go b/receiver/prometheusreceiver/internal/metricsutil_pdata_test.go index a0f10562bf10b..7fd3eb7fb7fa1 100644 --- a/receiver/prometheusreceiver/internal/metricsutil_pdata_test.go +++ b/receiver/prometheusreceiver/internal/metricsutil_pdata_test.go @@ -39,12 +39,6 @@ func distPointPdata(ts pdata.Timestamp, bounds []float64, counts []uint64) *pdat return &hdp } -func gaugeDistMetricPdata(name string, kvp []*kv, startTs pdata.Timestamp, points ...*pdata.HistogramDataPoint) *pdata.Metric { - hMetric := cumulativeDistMetricPdata(name, kvp, startTs, points...) - hMetric.Histogram().SetAggregationTemporality(pdata.MetricAggregationTemporalityDelta) - return hMetric -} - func cumulativeDistMetricPdata(name string, kvp []*kv, startTs pdata.Timestamp, points ...*pdata.HistogramDataPoint) *pdata.Metric { metric := pdata.NewMetric() metric.SetName(name) From 934413199a8b7b63cd2aad8db1edc858bb7172eb Mon Sep 17 00:00:00 2001 From: Anthony J Mirabella Date: Mon, 6 Dec 2021 18:13:14 -0500 Subject: [PATCH 25/26] Fix lint errors Signed-off-by: Anthony J Mirabella --- receiver/prometheusreceiver/internal/prom_to_otlp_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/receiver/prometheusreceiver/internal/prom_to_otlp_test.go b/receiver/prometheusreceiver/internal/prom_to_otlp_test.go index ec3a7e6b0fe6b..0fa776b92e85d 100644 --- a/receiver/prometheusreceiver/internal/prom_to_otlp_test.go +++ b/receiver/prometheusreceiver/internal/prom_to_otlp_test.go @@ -18,9 +18,10 @@ import ( "testing" metricspb "github.com/census-instrumentation/opencensus-proto/gen-go/metrics/v1" - "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus" "github.com/stretchr/testify/require" "go.opentelemetry.io/collector/model/pdata" + + "github.com/open-telemetry/opentelemetry-collector-contrib/pkg/translator/opencensus" ) // Parity test to ensure that createNodeAndResource produces identical results to createNodeAndResourcePdata. From 9ed90fe2baa0f61782f1bc15054b314d07884de6 Mon Sep 17 00:00:00 2001 From: Anthony J Mirabella Date: Tue, 7 Dec 2021 19:58:09 -0500 Subject: [PATCH 26/26] receiver/prometheus: Add comments detailing use of locks in metrics adjuster. Signed-off-by: Anthony J Mirabella --- .../internal/otlp_metrics_adjuster.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go index c8cd19620c602..3138f9728c3a5 100644 --- a/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go +++ b/receiver/prometheusreceiver/internal/otlp_metrics_adjuster.go @@ -70,12 +70,17 @@ type timeseriesinfoPdata struct { // the instance. type timeseriesMapPdata struct { sync.RWMutex + // The mutex is used to protect access to the member fields. It is acquired for the entirety of + // AdjustMetrics() and also acquired by gc(). + mark bool tsiMap map[string]*timeseriesinfoPdata } // Get the timeseriesinfo for the timeseries associated with the metric and label values. func (tsm *timeseriesMapPdata) get(metric *pdata.Metric, kv pdata.AttributeMap) *timeseriesinfoPdata { + // This should only be invoked be functions called (directly or indirectly) by AdjustMetrics(). + // The lock protecting tsm.tsiMap is acquired there. name := metric.Name() sig := getTimeseriesSignaturePdata(name, kv) if metric.DataType() == pdata.MetricDataTypeHistogram { @@ -133,6 +138,9 @@ func newTimeseriesMapPdata() *timeseriesMapPdata { // JobsMapPdata maps from a job instance to a map of timeseriesPdata instances for the job. type JobsMapPdata struct { sync.RWMutex + // The mutex is used to protect access to the member fields. It is acquired for most of + // get() and also acquired by gc(). + gcInterval time.Duration lastGC time.Time jobsMap map[string]*timeseriesMapPdata @@ -152,10 +160,12 @@ func (jm *JobsMapPdata) gc() { for sig, tsm := range jm.jobsMap { tsm.RLock() tsmNotMarked := !tsm.mark + // take a read lock here, no need to get a full lock as we have a lock on the JobsMapPdata tsm.RUnlock() if tsmNotMarked { delete(jm.jobsMap, sig) } else { + // a full lock will be obtained in here, if required. tsm.gc() } } @@ -174,6 +184,7 @@ func (jm *JobsMapPdata) maybeGC() { func (jm *JobsMapPdata) get(job, instance string) *timeseriesMapPdata { sig := job + ":" + instance + // a read locke is taken here as we will not need to modify jobsMap if the target timeseriesMap is available. jm.RLock() tsm, ok := jm.jobsMap[sig] jm.RUnlock() @@ -183,6 +194,8 @@ func (jm *JobsMapPdata) get(job, instance string) *timeseriesMapPdata { } jm.Lock() defer jm.Unlock() + // Now that we've got an exclusive lock, check once more to ensure an entry wasn't created in the interim + // and then create a new timeseriesMap if required. tsm2, ok2 := jm.jobsMap[sig] if ok2 { return tsm2 @@ -213,6 +226,8 @@ func NewMetricsAdjusterPdata(tsm *timeseriesMapPdata, logger *zap.Logger) *Metri // Returns the total number of timeseries that had reset start times. func (ma *MetricsAdjusterPdata) AdjustMetrics(metricL *pdata.MetricSlice) int { resets := 0 + // The lock on the relevant timeseriesMap is held throughout the adjustment process to ensure that + // nothing else can modify the data used for adjustment. ma.tsm.Lock() defer ma.tsm.Unlock() for i := 0; i < metricL.Len(); i++ {