Redact headers when exporting internal telemetry through OTLP

.chloggen/redact-internal-telemetry-headers.yaml

@@ -0,0 +1,25 @@
+# Use this changelog template to create an entry for release notes.
+
+# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix'
+change_type: bug_fix
+
+# The name of the component, or a single word describing the area of concern, (e.g. receiver/otlp)
+component: pkg/service
+
+# A brief description of the change.  Surround your text with quotes ("") if it needs to start with a backtick (`).
+note: Headers on the internal telemetry OTLP exporter are now redacted when the configuration is marshaled
+
+# One or more tracking issues or pull requests related to the change
+issues: [14756]
+
+# (Optional) One or more lines of additional information to render under the primary note.
+# These lines will be padded with 2 spaces and then inserted directly into the document.
+# Use pipe (|) for multiline entries.
+subtext:
+
+# Optional: The change log or logs in which this entry should be included.
+# e.g. '[user]' or '[user, api]'
+# Include 'user' if the change is relevant to end users.
+# Include 'api' if there is a change to a library API.
+# Default: '[user]'
+change_logs: []

service/telemetry/otelconftelemetry/internal/migration/redact.go

@@ -0,0 +1,34 @@
+// Copyright The OpenTelemetry Authors
+// SPDX-License-Identifier: Apache-2.0
+
+package migration // import "go.opentelemetry.io/collector/service/telemetry/otelconftelemetry/internal/migration"
+
+import "strings"
+
+const maskedString = "[REDACTED]" // Same as configopaque.String
+
+func redactHeaderPath(config any, path []string) {
+	if len(path) == 0 {
+		return
+	}
+	next, rest := path[0], path[1:]
+	if next == "*" {
+		if configArray, ok := config.([]any); ok {
+			for _, elem := range configArray {
+				redactHeaderPath(elem, rest)
+			}
+		}
+	} else if configMap, ok := config.(map[string]any); ok {
+		for nextKey := range strings.SplitSeq(next, "|") {
+			if len(path) == 1 {
+				configMap[nextKey] = maskedString
+			} else if elem, ok := configMap[nextKey]; ok {
+				redactHeaderPath(elem, rest)
+			}
+		}
+	}
+}
+
+func redactHeaders(config any, path string) {
+	redactHeaderPath(config, strings.Split(path, "."))
+}

service/telemetry/otelconftelemetry/internal/migration/testdata/v0.3.0_logs.yaml

@@ -19,3 +19,5 @@ processors:
           headers:
           - name: "key1"
             value: "value1"
+error_output_paths: []
+output_paths: []

service/telemetry/otelconftelemetry/internal/migration/testdata/v0.3.0_logs_marshaled.yaml

@@ -0,0 +1,25 @@
+level: "info"
+processors:
+  - batch:
+      exporter:
+        otlp:
+          protocol: http/protobuf
+          endpoint: https://127.0.0.1:4317
+          headers:
+          - name: "key1"
+            value: "[REDACTED]"
+  - simple:
+      exporter:
+        console: {}
+  - simple:
+      exporter:
+        otlp:
+          protocol: http/protobuf
+          endpoint: http://127.0.0.1:4317
+          headers:
+          - name: "key1"
+            value: "[REDACTED]"
+error_output_paths: []
+output_paths: []
+encoding: ""
+sampling: null

service/telemetry/otelconftelemetry/internal/migration/testdata/v0.3.0_metrics.yaml

@@ -1,4 +1,4 @@
-level: detailed
+level: Detailed
 readers:
   - periodic:
       exporter:

service/telemetry/otelconftelemetry/internal/migration/testdata/v0.3.0_metrics_marshaled.yaml

@@ -0,0 +1,15 @@
+level: Detailed
+readers:
+  - periodic:
+      exporter:
+        otlp:
+          protocol: http/protobuf
+          endpoint: https://127.0.0.1:4317
+          headers:
+          - name: "key1"
+            value: "[REDACTED]"
+  - pull:
+      exporter:
+        prometheus:
+          host: 127.0.0.1
+          port: 8902

service/telemetry/otelconftelemetry/internal/migration/testdata/v0.3.0_traces.yaml

@@ -1,4 +1,4 @@
-level: "none"
+level: "None"
 processors:
   - batch:
       exporter:
@@ -19,6 +19,7 @@ processors:
           headers:
           - name: "key1"
             value: "value1"
+propagators: []
 sampler:
   parent_based:
     root:

service/telemetry/otelconftelemetry/internal/migration/testdata/v0.3.0_traces_marshaled.yaml

@@ -0,0 +1,27 @@
+level: "None"
+processors:
+  - batch:
+      exporter:
+        otlp:
+          protocol: http/protobuf
+          endpoint: https://127.0.0.1:4317
+          headers:
+          - name: "key1"
+            value: "[REDACTED]"
+  - simple:
+      exporter:
+        console: {}
+  - simple:
+      exporter:
+        otlp:
+          protocol: http/protobuf
+          endpoint: http://127.0.0.1:4317
+          headers:
+          - name: "key1"
+            value: "[REDACTED]"
+propagators: []
+sampler:
+  parent_based:
+    root:
+      trace_id_ratio_based:
+        ratio: 0.01

service/telemetry/otelconftelemetry/internal/migration/v0.3.0.go

@@ -4,6 +4,7 @@
 package migration // import "go.opentelemetry.io/collector/service/telemetry/otelconftelemetry/internal/migration"
 
 import (
+	"fmt"
 	"time"
 
 	config "go.opentelemetry.io/contrib/otelconf/v0.3.0"
@@ -58,6 +59,19 @@ func (c *TracesConfigV030) Unmarshal(conf *confmap.Conf) error {
 	return nil
 }
 
+var _ confmap.Marshaler = TracesConfigV030{}
+
+func (c TracesConfigV030) Marshal(conf *confmap.Conf) error {
+	if err := conf.Marshal(c); err != nil {
+		return fmt.Errorf("otelconftelemetry: failed to marshal traces configuration: %w", err)
+	}
+
+	// Redact header values the way configopaque would
+	sm := conf.ToStringMap()
+	redactHeaders(sm, "processors.*.simple|batch.exporter.otlp.headers.*.value")
+	return conf.Marshal(sm)
+}
+
 type MetricsConfigV030 struct {
 	// Level is the level of telemetry metrics, the possible values are:
 	//  - "none" indicates that no telemetry data should be collected;
@@ -95,6 +109,19 @@ func (c *MetricsConfigV030) Unmarshal(conf *confmap.Conf) error {
 	return nil
 }
 
+var _ confmap.Marshaler = MetricsConfigV030{}
+
+func (c MetricsConfigV030) Marshal(conf *confmap.Conf) error {
+	if err := conf.Marshal(c); err != nil {
+		return fmt.Errorf("otelconftelemetry: failed to marshal metrics configuration: %w", err)
+	}
+
+	// Redact header values the way configopaque would
+	sm := conf.ToStringMap()
+	redactHeaders(sm, "readers.*.periodic.exporter.otlp.headers.*.value")
+	return conf.Marshal(sm)
+}
+
 type LogsConfigV030 struct {
 	// Level is the minimum enabled logging level.
 	// (default = "INFO")
@@ -220,3 +247,16 @@ func (c *LogsConfigV030) Unmarshal(conf *confmap.Conf) error {
 	}
 	return nil
 }
+
+var _ confmap.Marshaler = LogsConfigV030{}
+
+func (c LogsConfigV030) Marshal(conf *confmap.Conf) error {
+	if err := conf.Marshal(c); err != nil {
+		return fmt.Errorf("otelconftelemetry: failed to marshal logs configuration: %w", err)
+	}
+
+	// Redact header values the way configopaque would
+	sm := conf.ToStringMap()
+	redactHeaders(sm, "processors.*.simple|batch.exporter.otlp.headers.*.value")
+	return conf.Marshal(sm)
+}

service/telemetry/otelconftelemetry/internal/migration/v0.3.0_test.go

@@ -7,8 +7,10 @@ import (
 	"path/filepath"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
+	"go.opentelemetry.io/collector/confmap"
 	"go.opentelemetry.io/collector/confmap/confmaptest"
 )
 
@@ -25,6 +27,20 @@ func TestUnmarshalLogsConfigV030(t *testing.T) {
 	require.Equal(t, "http://127.0.0.1:4317", *cfg.Processors[2].Simple.Exporter.OTLP.Endpoint)
 }
 
+func TestMarshalLogsConfigV030(t *testing.T) {
+	cm, err := confmaptest.LoadConf(filepath.Join("testdata", "v0.3.0_logs.yaml"))
+	require.NoError(t, err)
+	cm2, err := confmaptest.LoadConf(filepath.Join("testdata", "v0.3.0_logs_marshaled.yaml"))
+	require.NoError(t, err)
+
+	cfg := LogsConfigV030{}
+	require.NoError(t, cm.Unmarshal(&cfg))
+	cm3 := confmap.New()
+	require.NoError(t, cm3.Marshal(&cfg))
+
+	assert.Equal(t, cm2.ToStringMap(), cm3.ToStringMap())
+}
+
 func TestUnmarshalTracesConfigV030(t *testing.T) {
 	cm, err := confmaptest.LoadConf(filepath.Join("testdata", "v0.3.0_traces.yaml"))
 	require.NoError(t, err)
@@ -38,6 +54,20 @@ func TestUnmarshalTracesConfigV030(t *testing.T) {
 	require.Equal(t, "http://127.0.0.1:4317", *cfg.Processors[2].Simple.Exporter.OTLP.Endpoint)
 }
 
+func TestMarshalTracesConfigV030(t *testing.T) {
+	cm, err := confmaptest.LoadConf(filepath.Join("testdata", "v0.3.0_traces.yaml"))
+	require.NoError(t, err)
+	cm2, err := confmaptest.LoadConf(filepath.Join("testdata", "v0.3.0_traces_marshaled.yaml"))
+	require.NoError(t, err)
+
+	cfg := TracesConfigV030{}
+	require.NoError(t, cm.Unmarshal(&cfg))
+	cm3 := confmap.New()
+	require.NoError(t, cm3.Marshal(&cfg))
+
+	assert.Equal(t, cm2.ToStringMap(), cm3.ToStringMap())
+}
+
 func TestUnmarshalMetricsConfigV030(t *testing.T) {
 	cm, err := confmaptest.LoadConf(filepath.Join("testdata", "v0.3.0_metrics.yaml"))
 	require.NoError(t, err)
@@ -49,3 +79,17 @@ func TestUnmarshalMetricsConfigV030(t *testing.T) {
 	// check the endpoint is prefixed w/ https
 	require.Equal(t, "https://127.0.0.1:4317", *cfg.Readers[0].Periodic.Exporter.OTLP.Endpoint)
 }
+
+func TestMarshalMetricsConfigV030(t *testing.T) {
+	cm, err := confmaptest.LoadConf(filepath.Join("testdata", "v0.3.0_metrics.yaml"))
+	require.NoError(t, err)
+	cm2, err := confmaptest.LoadConf(filepath.Join("testdata", "v0.3.0_metrics_marshaled.yaml"))
+	require.NoError(t, err)
+
+	cfg := MetricsConfigV030{}
+	require.NoError(t, cm.Unmarshal(&cfg))
+	cm3 := confmap.New()
+	require.NoError(t, cm3.Marshal(&cfg))
+
+	assert.Equal(t, cm2.ToStringMap(), cm3.ToStringMap())
+}