Add TLS config options for tempo remote_write

CHANGELOG.md

@@ -1,5 +1,7 @@
 # Main (unreleased)
 
+- [FEATURE] Add TLS config options for tempo `remote_write`s. (@mapno)
+
 # v0.16.0 (2021-06-17)
 
 - [FEATURE] (beta) A Grafana Agent Operator is now available. (@rfratto)

docs/configuration-reference.md

@@ -2042,9 +2042,22 @@ remote_write:
     # Controls whether or not TLS is required.  See https://godoc.org/google.golang.org/grpc#WithInsecure
     [ insecure: <boolean> | default = false ]
 
-    # Disable validation of the server certificate. Only used when insecure is set
-    # to false.
+    # Deprecated in favor of tls_config
+    # If both `insecure_skip_verify` and `tls_config.insecure_skip_verify` are used,
+    # the latter take precedence.
     [ insecure_skip_verify: <bool> | default = false ]
+
+    # Controls TLS settings of the exporter's client. See https://github.com/open-telemetry/opentelemetry-collector/blob/v0.21.0/config/configtls/README.md
+    # This should be used only if `insecure` is set to false
+    tls_config:
+      # Path to the CA cert. For a client this verifies the server certificate. If empty uses system root CA.
+      [ca_file: <string>]
+      # Path to the TLS cert to use for TLS required connections
+      [cert_file: <string>]
+      # Path to the TLS key to use for TLS required connections
+      [key_file: <string>]
+      # Disable validation of the server certificate.
+      [ insecure_skip_verify: <bool> | default = false ]
 
     # Sets the `Authorization` header on every trace push with the
     # configured username and password.

docs/migration-guide.md

@@ -3,6 +3,43 @@
 This is a guide detailing all breaking changes that have happened in prior
 releases and how to migrate to newer versions.
 
+# Unreleased
+
+## Tempo: Remote write TLS config
+
+Tempo `remote_write` now supports configuring TLS settings in the trace
+exporter's client. `insecure_skip_verify` is moved into this setting's block.
+
+Old config's with `insecure_skip_verify` outside `tls_config` will continue
+to work until it's fully deprecated.
+If both `insecure_skip_verify` and `tls_config.insecure_skip_verify` are used,
+the latter take precedence
+
+Example old config:
+
+```
+tempo:
+  configs:
+    - name: default
+      remote_write:
+        - endpoint: otel-collector:55680
+          insecure: true
+          insecure_skip_verify: true
+```
+
+Example new config:
+
+```
+tempo:
+  configs:
+    - name: default
+      remote_write:
+        - endpoint: otel-collector:55680
+          insecure: true
+          tls_config:
+            insecure_skip_verify: true
+```
+
 # v0.15.0
 
 ## Tempo: `automatic_logging` changes

pkg/tempo/config.go

@@ -178,10 +178,12 @@ var DefaultRemoteWriteConfig = RemoteWriteConfig{
 
 // RemoteWriteConfig controls the configuration of an exporter
 type RemoteWriteConfig struct {
-	Endpoint           string                 `yaml:"endpoint,omitempty"`
-	Compression        string                 `yaml:"compression,omitempty"`
-	Insecure           bool                   `yaml:"insecure,omitempty"`
+	Endpoint    string `yaml:"endpoint,omitempty"`
+	Compression string `yaml:"compression,omitempty"`
+	Insecure    bool   `yaml:"insecure,omitempty"`
+	// Deprecated
 	InsecureSkipVerify bool                   `yaml:"insecure_skip_verify,omitempty"`
+	TLSConfig          *prom_config.TLSConfig `yaml:"tls_config,omitempty"`
 	BasicAuth          *prom_config.BasicAuth `yaml:"basic_auth,omitempty"`
 	Headers            map[string]string      `yaml:"headers,omitempty"`
 	SendingQueue       map[string]interface{} `yaml:"sending_queue,omitempty"`    // https://github.com/open-telemetry/opentelemetry-collector/blob/7d7ae2eb34b5d387627875c498d7f43619f37ee3/exporter/exporterhelper/queued_retry.go#L30
@@ -246,44 +248,56 @@ type exporterConfig struct {
 }
 
 // exporter builds an OTel exporter from RemoteWriteConfig
-func exporter(remoteWriteConfig RemoteWriteConfig) (map[string]interface{}, error) {
-	if len(remoteWriteConfig.Endpoint) == 0 {
+func exporter(rwCfg RemoteWriteConfig) (map[string]interface{}, error) {
+	if len(rwCfg.Endpoint) == 0 {
 		return nil, errors.New("must have a configured a backend endpoint")
 	}
 
 	headers := map[string]string{}
-	if remoteWriteConfig.Headers != nil {
-		headers = remoteWriteConfig.Headers
+	if rwCfg.Headers != nil {
+		headers = rwCfg.Headers
 	}
 
-	if remoteWriteConfig.BasicAuth != nil {
-		password := string(remoteWriteConfig.BasicAuth.Password)
+	if rwCfg.BasicAuth != nil {
+		password := string(rwCfg.BasicAuth.Password)
 
-		if len(remoteWriteConfig.BasicAuth.PasswordFile) > 0 {
-			buff, err := ioutil.ReadFile(remoteWriteConfig.BasicAuth.PasswordFile)
+		if len(rwCfg.BasicAuth.PasswordFile) > 0 {
+			buff, err := ioutil.ReadFile(rwCfg.BasicAuth.PasswordFile)
 			if err != nil {
-				return nil, fmt.Errorf("unable to load password file %s: %w", remoteWriteConfig.BasicAuth.PasswordFile, err)
+				return nil, fmt.Errorf("unable to load password file %s: %w", rwCfg.BasicAuth.PasswordFile, err)
 			}
 			password = string(buff)
 		}
 
-		encodedAuth := base64.StdEncoding.EncodeToString([]byte(remoteWriteConfig.BasicAuth.Username + ":" + password))
+		encodedAuth := base64.StdEncoding.EncodeToString([]byte(rwCfg.BasicAuth.Username + ":" + password))
 		headers["authorization"] = "Basic " + encodedAuth
 	}
 
-	compression := remoteWriteConfig.Compression
+	compression := rwCfg.Compression
 	if compression == compressionNone {
 		compression = ""
 	}
 
 	otlpExporter := map[string]interface{}{
-		"endpoint":             remoteWriteConfig.Endpoint,
-		"compression":          compression,
-		"headers":              headers,
-		"insecure":             remoteWriteConfig.Insecure,
-		"insecure_skip_verify": remoteWriteConfig.InsecureSkipVerify,
-		"sending_queue":        remoteWriteConfig.SendingQueue,
-		"retry_on_failure":     remoteWriteConfig.RetryOnFailure,
+		"endpoint":         rwCfg.Endpoint,
+		"compression":      compression,
+		"headers":          headers,
+		"insecure":         rwCfg.Insecure,
+		"sending_queue":    rwCfg.SendingQueue,
+		"retry_on_failure": rwCfg.RetryOnFailure,
+	}
+
+	if !rwCfg.Insecure {
+		// If there is a TLSConfig use it
+		if rwCfg.TLSConfig != nil {
+			otlpExporter["ca_file"] = rwCfg.TLSConfig.CAFile
+			otlpExporter["cert_file"] = rwCfg.TLSConfig.CertFile
+			otlpExporter["key_file"] = rwCfg.TLSConfig.KeyFile
+			otlpExporter["insecure_skip_verify"] = rwCfg.TLSConfig.InsecureSkipVerify
+		} else {
+			// If not, set whatever value is specified in the old config.
+			otlpExporter["insecure_skip_verify"] = rwCfg.InsecureSkipVerify
+		}
 	}
 
 	// Apply some sane defaults to the exporter. The
@@ -306,13 +320,15 @@ func exporter(remoteWriteConfig RemoteWriteConfig) (map[string]interface{}, erro
 func (c *InstanceConfig) exporters() (map[string]interface{}, error) {
 	if len(c.RemoteWrite) == 0 {
 		otlpExporter, err := exporter(RemoteWriteConfig{
-			Endpoint:           c.PushConfig.Endpoint,
-			Compression:        c.PushConfig.Compression,
-			Insecure:           c.PushConfig.Insecure,
-			InsecureSkipVerify: c.PushConfig.InsecureSkipVerify,
-			BasicAuth:          c.PushConfig.BasicAuth,
-			SendingQueue:       c.PushConfig.SendingQueue,
-			RetryOnFailure:     c.PushConfig.RetryOnFailure,
+			Endpoint:    c.PushConfig.Endpoint,
+			Compression: c.PushConfig.Compression,
+			Insecure:    c.PushConfig.Insecure,
+			TLSConfig: &prom_config.TLSConfig{
+				InsecureSkipVerify: c.PushConfig.InsecureSkipVerify,
+			},
+			BasicAuth:      c.PushConfig.BasicAuth,
+			SendingQueue:   c.PushConfig.SendingQueue,
+			RetryOnFailure: c.PushConfig.RetryOnFailure,
 		})
 		return map[string]interface{}{
 			"otlp": otlpExporter,
@@ -350,11 +366,11 @@ func resolver(config map[string]interface{}) (map[string]interface{}, error) {
 func (c *InstanceConfig) loadBalancingExporter() (map[string]interface{}, error) {
 	exporter, err := exporter(RemoteWriteConfig{
 		// Endpoint is omitted in OTel load balancing exporter
-		Endpoint:           "noop",
-		Compression:        c.TailSampling.LoadBalancing.Exporter.Compression,
-		Insecure:           c.TailSampling.LoadBalancing.Exporter.Insecure,
-		InsecureSkipVerify: c.TailSampling.LoadBalancing.Exporter.InsecureSkipVerify,
-		BasicAuth:          c.TailSampling.LoadBalancing.Exporter.BasicAuth,
+		Endpoint:    "noop",
+		Compression: c.TailSampling.LoadBalancing.Exporter.Compression,
+		Insecure:    c.TailSampling.LoadBalancing.Exporter.Insecure,
+		TLSConfig:   &prom_config.TLSConfig{InsecureSkipVerify: c.TailSampling.LoadBalancing.Exporter.InsecureSkipVerify},
+		BasicAuth:   c.TailSampling.LoadBalancing.Exporter.BasicAuth,
 	})
 	if err != nil {
 		return nil, err

pkg/tempo/config_test.go

@@ -700,6 +700,44 @@ service:
       receivers: ["jaeger"]
       `,
 		},
+		{
+			name: "tls config",
+			cfg: `
+receivers:
+  jaeger:
+    protocols:
+      grpc:
+remote_write:
+  - insecure: false
+    tls_config:
+      ca_file: server.crt
+      cert_file: client.crt
+      key_file: client.key
+    endpoint: example.com:12345
+`,
+			expectedConfig: `
+receivers:
+  jaeger:
+    protocols:
+      grpc:
+exporters:
+  otlp/0:
+    endpoint: example.com:12345
+    insecure: false
+    ca_file: server.crt
+    cert_file: client.crt
+    key_file: client.key
+    compression: gzip
+    retry_on_failure:
+      max_elapsed_time: 60s
+service:
+  pipelines:
+    traces:
+      exporters: ["otlp/0"]
+      processors: []
+      receivers: ["jaeger"]
+`,
+		},
 	}
 
 	for _, tc := range tt {

pkg/tempo/instance.go

@@ -138,6 +138,14 @@ func (i *Instance) buildAndStartPipeline(ctx context.Context, cfg InstanceConfig
 	if cfg.PushConfig.Endpoint != "" {
 		i.logger.Warn("Configuring exporter with deprecated push_config. Use remote_write and batch instead")
 	}
+	for _, rw := range cfg.RemoteWrite {
+		if rw.InsecureSkipVerify {
+			i.logger.Warn("Configuring TLS with insecure_skip_verify. Use tls_config.insecure_skip_verify instead")
+		}
+		if rw.TLSConfig != nil && rw.TLSConfig.ServerName != "" {
+			i.logger.Warn("Configuring unsupported tls_config.server_name")
+		}
+	}
 
 	if cfg.SpanMetrics != nil && len(cfg.SpanMetrics.PromInstance) != 0 {
 		ctx = context.WithValue(ctx, contextkeys.Prometheus, promManager)