diff --git a/.chloggen/prometheusreceiver-targetallocator-unset-config.yaml b/.chloggen/prometheusreceiver-targetallocator-unset-config.yaml new file mode 100644 index 0000000000000..a031533d2f20a --- /dev/null +++ b/.chloggen/prometheusreceiver-targetallocator-unset-config.yaml @@ -0,0 +1,27 @@ +# Use this changelog template to create an entry for release notes. + +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: bug_fix + +# The name of the component, or a single word describing the area of concern, (e.g. filelogreceiver) +component: prometheusreceiver + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Fix invalid metric name validation error in scrape start from target allocator. + +# Mandatory: One or more tracking issues related to the change. You can use the PR number here if no issue exists. +issues: [35459,40788] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: Prometheus made setting metric_name_validation_scheme, metric_name_escaping_scheme mandatory mandatory, use sane defaults. + +# If your change doesn't affect end users or the exported elements of any package, +# you should instead start your pull request title with [chore] or use the "Skip Changelog" label. +# Optional: The change log or logs in which this entry should be included. +# e.g. '[user]' or '[user, api]' +# Include 'user' if the change is relevant to end users. +# Include 'api' if there is a change to a library API. +# Default: '[user]' +change_logs: [] diff --git a/receiver/prometheusreceiver/metrics_receiver_target_allocator_test.go b/receiver/prometheusreceiver/metrics_receiver_target_allocator_test.go new file mode 100644 index 0000000000000..ae3f6deafa824 --- /dev/null +++ b/receiver/prometheusreceiver/metrics_receiver_target_allocator_test.go @@ -0,0 +1,174 @@ +// Copyright The OpenTelemetry Authors +// SPDX-License-Identifier: Apache-2.0 + +package prometheusreceiver + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + promTestUtil "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/prometheus/common/model" + "github.com/prometheus/common/promslog" + promConfig "github.com/prometheus/prometheus/config" + promHTTP "github.com/prometheus/prometheus/discovery/http" + promTG "github.com/prometheus/prometheus/discovery/targetgroup" + "github.com/stretchr/testify/require" + "go.opentelemetry.io/collector/component/componenttest" + "go.opentelemetry.io/collector/config/confighttp" + "go.opentelemetry.io/collector/consumer/consumertest" + "go.opentelemetry.io/collector/receiver/receivertest" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "gopkg.in/yaml.v3" + + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/internal/metadata" + "github.com/open-telemetry/opentelemetry-collector-contrib/receiver/prometheusreceiver/targetallocator" +) + +const exportedMetrics = ` +# HELP test_gauge0 This is my gauge +# TYPE test_gauge0 gauge +test_gauge0{label1="value1",label2="value2"} 10 +` + +func TestTargetAllocatorProvidesEmptyScrapeConfig(t *testing.T) { + // Make a prometheus exporter that can serve some metrics. + mockProm := newMockPrometheus(map[string][]mockPrometheusResponse{ + "/metrics": { + { + code: 200, + data: exportedMetrics, + }, + }, + }) + t.Cleanup(func() { mockProm.srv.Close() }) + + // Fake TargetAllocator to serve discovery and targets. + tas := newMockTargetAllocator(mockProm.srv.Listener.Addr().String()) + t.Cleanup(func() { tas.srv.Close() }) + + promSDConfig := &promHTTP.SDConfig{ + RefreshInterval: model.Duration(45 * time.Second), + URL: tas.srv.URL, + } + + pCfg, err := promConfig.Load("", promslog.NewNopLogger()) + require.NoError(t, err) + + config := &Config{ + PrometheusConfig: (*PromConfig)(pCfg), + StartTimeMetricRegex: "", + TargetAllocator: &targetallocator.Config{ + ClientConfig: confighttp.ClientConfig{ + Endpoint: tas.srv.URL, + }, + CollectorID: "1", + HTTPSDConfig: (*targetallocator.PromHTTPSDConfig)(promSDConfig), + Interval: 60 * time.Second, + }, + } + + cms := new(consumertest.MetricsSink) + settings := receivertest.NewNopSettings(metadata.Type) + logsOverWarn := atomic.Int64{} + settings.Logger, err = zap.NewDevelopment(zap.Hooks(func(logentry zapcore.Entry) error { + if logentry.Level >= zapcore.WarnLevel { + logsOverWarn.Add(1) + } + return nil + })) + require.NoError(t, err) + receiver, err := newPrometheusReceiver(settings, config, cms) + require.NoError(t, err, "Failed to create Prometheus receiver") + receiver.skipOffsetting = true + + require.NoError(t, receiver.Start(context.Background(), componenttest.NewNopHost()), "Failed to start Prometheus receiver") + t.Cleanup(func() { + require.NoError(t, receiver.Shutdown(context.Background())) + }) + + metricsCount := 0 + require.Eventually(t, func() bool { + metrics := cms.AllMetrics() + // Scrape was a success and we got metrics. + if len(metrics) > 0 { + metricsCount = len(metrics) + return true + } + // There was a log line above WARN level. + if logsOverWarn.Load() > 0 { + return true + } + return false + }, 30*time.Second, 100*time.Millisecond, "Failed to scrape the metrics via target allocator") + + require.Zero(t, logsOverWarn.Load(), "There are log messages over the WARN level, see logs") + + require.NoError(t, promTestUtil.GatherAndCompare(receiver.registry, bytes.NewBufferString(fmt.Sprintf(` + # TYPE prometheus_target_scrape_pools_failed_total counter + # HELP prometheus_target_scrape_pools_failed_total Total number of scrape pool creations that failed. + prometheus_target_scrape_pools_failed_total{receiver="%s"} 0 +`, receiver.settings.ID)), "prometheus_target_scrape_pools_failed_total"), "Prometheus scrape manager reports failed scrape pools") + + require.Positive(t, metricsCount, "No metrics were scraped even though successful") +} + +type mockTargetAllocator struct { + address string + srv *httptest.Server +} + +func newMockTargetAllocator(address string) *mockTargetAllocator { + s := &mockTargetAllocator{ + address: address, + } + srv := httptest.NewServer(s) + s.srv = srv + return s +} + +func (mp *mockTargetAllocator) ServeHTTP(rw http.ResponseWriter, req *http.Request) { + if strings.HasSuffix(req.URL.Path, "/scrape_configs") { + job := make(map[string]any) + job["job_name"] = "test" + // Do not set any fields in the scrape config to verify that we have sane defaults. + + result := make(map[string]any) + result["test"] = job + + data, err := yaml.Marshal(&result) + if err != nil { + return + } + + _, _ = rw.Write(data) + return + } + + response := []*promTG.Group{ + { + Targets: []model.LabelSet{ + { + model.AddressLabel: model.LabelValue(mp.address), + model.SchemeLabel: "http", + }, + }, + }, + } + + data, err := json.Marshal(&response) + if err != nil { + return + } + rw.Header().Set("Content-Type", "application/json") + _, _ = rw.Write(data) +} diff --git a/receiver/prometheusreceiver/targetallocator/manager.go b/receiver/prometheusreceiver/targetallocator/manager.go index 1ee8e3d1bcac2..c20fbf7f9b7e8 100644 --- a/receiver/prometheusreceiver/targetallocator/manager.go +++ b/receiver/prometheusreceiver/targetallocator/manager.go @@ -154,6 +154,21 @@ func (m *Manager) sync(compareHash uint64, httpClient *http.Client) (uint64, err scrapeConfig.ScrapeFallbackProtocol = promconfig.PrometheusText0_0_4 } + // TODO(krajorama): remove once + // https://github.com/prometheus/prometheus/issues/16750 is solved + // https://github.com/open-telemetry/opentelemetry-collector-contrib/issues/35459 + // is implemented and is default. + if m.promCfg.GlobalConfig.MetricNameValidationScheme == "" { + m.promCfg.GlobalConfig.MetricNameValidationScheme = promconfig.LegacyValidationConfig + } + + // Validate the scrape config and also fill in the defaults from the global config as needed. + err = scrapeConfig.Validate(m.promCfg.GlobalConfig) + if err != nil { + m.settings.Logger.Error("Failed to validate the scrape configuration", zap.Error(err)) + return 0, err + } + m.promCfg.ScrapeConfigs = append(m.promCfg.ScrapeConfigs, scrapeConfig) }