diff --git a/api/v1alpha1/envoyproxy_accesslogging_types.go b/api/v1alpha1/envoyproxy_accesslogging_types.go index faf78e53e5..5447b2502e 100644 --- a/api/v1alpha1/envoyproxy_accesslogging_types.go +++ b/api/v1alpha1/envoyproxy_accesslogging_types.go @@ -5,6 +5,8 @@ package v1alpha1 +import gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + type ProxyAccessLog struct { // Disable disables access logging for managed proxies if set to true. // @@ -208,6 +210,12 @@ type OpenTelemetryEnvoyProxyAccessLog struct { // It's recommended to follow [semantic conventions](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/). // +optional Resources map[string]string `json:"resources,omitempty"` + // Headers is a list of additional headers to send with OTLP export requests. + // These headers are added as gRPC initial metadata for the OTLP gRPC service. + // +optional + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=32 + Headers []gwapiv1.HTTPHeader `json:"headers,omitempty"` // TODO: support more OpenTelemetry accesslog options(e.g. TLS, auth etc.) in the future. } diff --git a/api/v1alpha1/envoyproxy_metric_types.go b/api/v1alpha1/envoyproxy_metric_types.go index 4c4f1d0ea4..d760cdd09b 100644 --- a/api/v1alpha1/envoyproxy_metric_types.go +++ b/api/v1alpha1/envoyproxy_metric_types.go @@ -5,6 +5,8 @@ package v1alpha1 +import gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" + type MetricSinkType string const ( @@ -109,6 +111,12 @@ type ProxyOpenTelemetrySink struct { // // +optional ReportHistogramsAsDeltas *bool `json:"reportHistogramsAsDeltas,omitempty"` + // Headers is a list of additional headers to send with OTLP export requests. + // These headers are added as gRPC initial metadata for the OTLP gRPC service. + // +optional + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=32 + Headers []gwapiv1.HTTPHeader `json:"headers,omitempty"` } type ProxyPrometheusProvider struct { diff --git a/api/v1alpha1/envoyproxy_tracing_types.go b/api/v1alpha1/envoyproxy_tracing_types.go index ca3d089b6c..7cc9d51610 100644 --- a/api/v1alpha1/envoyproxy_tracing_types.go +++ b/api/v1alpha1/envoyproxy_tracing_types.go @@ -61,6 +61,7 @@ const ( // +kubebuilder:validation:XValidation:message="BackendRefs must be used, backendRef is not supported.",rule="!has(self.backendRef)" // +kubebuilder:validation:XValidation:message="BackendRefs only support Service and Backend kind.",rule="has(self.backendRefs) ? self.backendRefs.all(f, f.kind == 'Service' || f.kind == 'Backend') : true" // +kubebuilder:validation:XValidation:message="BackendRefs only support Core and gateway.envoyproxy.io group.",rule="has(self.backendRefs) ? (self.backendRefs.all(f, f.group == \"\" || f.group == 'gateway.envoyproxy.io')) : true" +// +kubebuilder:validation:XValidation:message="openTelemetry can only be used with type OpenTelemetry",rule="has(self.openTelemetry) ? self.type == 'OpenTelemetry' : true" type TracingProvider struct { BackendCluster `json:",inline"` // Type defines the tracing provider type. @@ -92,6 +93,9 @@ type TracingProvider struct { // Zipkin defines the Zipkin tracing provider configuration // +optional Zipkin *ZipkinTracingProvider `json:"zipkin,omitempty"` + // OpenTelemetry defines the OpenTelemetry tracing provider configuration + // +optional + OpenTelemetry *OpenTelemetryTracingProvider `json:"openTelemetry,omitempty"` } type CustomTagType string @@ -158,3 +162,13 @@ type ZipkinTracingProvider struct { // +optional DisableSharedSpanContext *bool `json:"disableSharedSpanContext,omitempty"` } + +// OpenTelemetryTracingProvider defines the OpenTelemetry tracing provider configuration. +type OpenTelemetryTracingProvider struct { + // Headers is a list of additional headers to send with OTLP export requests. + // These headers are added as gRPC initial metadata for the OTLP gRPC service. + // +optional + // +kubebuilder:validation:MinItems=1 + // +kubebuilder:validation:MaxItems=32 + Headers []gwapiv1.HTTPHeader `json:"headers,omitempty"` +} diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 5dc1a2f891..a4e97ff1e7 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -5260,6 +5260,11 @@ func (in *OpenTelemetryEnvoyProxyAccessLog) DeepCopyInto(out *OpenTelemetryEnvoy (*out)[key] = val } } + if in.Headers != nil { + in, out := &in.Headers, &out.Headers + *out = make([]v1.HTTPHeader, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenTelemetryEnvoyProxyAccessLog. @@ -5272,6 +5277,26 @@ func (in *OpenTelemetryEnvoyProxyAccessLog) DeepCopy() *OpenTelemetryEnvoyProxyA return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *OpenTelemetryTracingProvider) DeepCopyInto(out *OpenTelemetryTracingProvider) { + *out = *in + if in.Headers != nil { + in, out := &in.Headers, &out.Headers + *out = make([]v1.HTTPHeader, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OpenTelemetryTracingProvider. +func (in *OpenTelemetryTracingProvider) DeepCopy() *OpenTelemetryTracingProvider { + if in == nil { + return nil + } + out := new(OpenTelemetryTracingProvider) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *Operation) DeepCopyInto(out *Operation) { *out = *in @@ -5897,6 +5922,11 @@ func (in *ProxyOpenTelemetrySink) DeepCopyInto(out *ProxyOpenTelemetrySink) { *out = new(bool) **out = **in } + if in.Headers != nil { + in, out := &in.Headers, &out.Headers + *out = make([]v1.HTTPHeader, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyOpenTelemetrySink. @@ -7280,6 +7310,11 @@ func (in *TracingProvider) DeepCopyInto(out *TracingProvider) { *out = new(ZipkinTracingProvider) (*in).DeepCopyInto(*out) } + if in.OpenTelemetry != nil { + in, out := &in.OpenTelemetry, &out.OpenTelemetry + *out = new(OpenTelemetryTracingProvider) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TracingProvider. diff --git a/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml index 434d325c2b..99fbf0f4f4 100644 --- a/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -13503,6 +13503,42 @@ spec: && !(has(self.loadBalancer) && has(self.loadBalancer.type) && self.loadBalancer.type in [''Random'', ''RoundRobin'']))' + headers: + description: |- + Headers is a list of additional headers to send with OTLP export requests. + These headers are added as gRPC initial metadata for the OTLP gRPC service. + items: + description: HTTPHeader represents an HTTP + Header name and value as defined by RFC + 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP + Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 32 + minItems: 1 + type: array host: description: |- Host define the extension service hostname. @@ -14831,6 +14867,41 @@ spec: && has(self.connection.preconnect.predictivePercent)) && !(has(self.loadBalancer) && has(self.loadBalancer.type) && self.loadBalancer.type in [''Random'', ''RoundRobin'']))' + headers: + description: |- + Headers is a list of additional headers to send with OTLP export requests. + These headers are added as gRPC initial metadata for the OTLP gRPC service. + items: + description: HTTPHeader represents an HTTP Header + name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header + to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 32 + minItems: 1 + type: array host: description: |- Host define the service hostname. @@ -16091,6 +16162,46 @@ spec: Host define the provider service hostname. Deprecated: Use BackendRefs instead. type: string + openTelemetry: + description: OpenTelemetry defines the OpenTelemetry tracing + provider configuration + properties: + headers: + description: |- + Headers is a list of additional headers to send with OTLP export requests. + These headers are added as gRPC initial metadata for the OTLP gRPC service. + items: + description: HTTPHeader represents an HTTP Header + name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header + to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 32 + minItems: 1 + type: array + type: object port: default: 4317 description: |- @@ -16151,6 +16262,9 @@ spec: rule: 'has(self.backendRefs) ? (self.backendRefs.all(f, f.group == "" || f.group == ''gateway.envoyproxy.io'')) : true' + - message: openTelemetry can only be used with type OpenTelemetry + rule: 'has(self.openTelemetry) ? self.type == ''OpenTelemetry'' + : true' samplingFraction: description: |- SamplingFraction represents the fraction of requests that should be diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index b0629515be..a4f7568198 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -13502,6 +13502,42 @@ spec: && !(has(self.loadBalancer) && has(self.loadBalancer.type) && self.loadBalancer.type in [''Random'', ''RoundRobin'']))' + headers: + description: |- + Headers is a list of additional headers to send with OTLP export requests. + These headers are added as gRPC initial metadata for the OTLP gRPC service. + items: + description: HTTPHeader represents an HTTP + Header name and value as defined by RFC + 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP + Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 32 + minItems: 1 + type: array host: description: |- Host define the extension service hostname. @@ -14830,6 +14866,41 @@ spec: && has(self.connection.preconnect.predictivePercent)) && !(has(self.loadBalancer) && has(self.loadBalancer.type) && self.loadBalancer.type in [''Random'', ''RoundRobin'']))' + headers: + description: |- + Headers is a list of additional headers to send with OTLP export requests. + These headers are added as gRPC initial metadata for the OTLP gRPC service. + items: + description: HTTPHeader represents an HTTP Header + name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header + to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 32 + minItems: 1 + type: array host: description: |- Host define the service hostname. @@ -16090,6 +16161,46 @@ spec: Host define the provider service hostname. Deprecated: Use BackendRefs instead. type: string + openTelemetry: + description: OpenTelemetry defines the OpenTelemetry tracing + provider configuration + properties: + headers: + description: |- + Headers is a list of additional headers to send with OTLP export requests. + These headers are added as gRPC initial metadata for the OTLP gRPC service. + items: + description: HTTPHeader represents an HTTP Header + name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header + to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 32 + minItems: 1 + type: array + type: object port: default: 4317 description: |- @@ -16150,6 +16261,9 @@ spec: rule: 'has(self.backendRefs) ? (self.backendRefs.all(f, f.group == "" || f.group == ''gateway.envoyproxy.io'')) : true' + - message: openTelemetry can only be used with type OpenTelemetry + rule: 'has(self.openTelemetry) ? self.type == ''OpenTelemetry'' + : true' samplingFraction: description: |- SamplingFraction represents the fraction of requests that should be diff --git a/examples/otel-headers/README.md b/examples/otel-headers/README.md new file mode 100644 index 0000000000..3a44616672 --- /dev/null +++ b/examples/otel-headers/README.md @@ -0,0 +1,51 @@ +# OpenTelemetry Example + +This example demonstrates OpenTelemetry metrics, tracing, and access logging +with custom headers. It uses Envoy Gateway's standalone mode (no Kubernetes +required). + +Note: This uses a fake token to demonstrate how headers are sent to the OTLP +collector. Production systems should use proper secret management. + +## Architecture + +``` +┌──────────┐ ┌───────────────────┐ ┌─────────────┐ +│ curl │───────▶│ Envoy │───────▶│ httpbingo.org │ +└──────────┘ :10080 └───────────────────┘ └─────────────┘ + │ + │ OTLP/gRPC :4317 + │ metrics + traces + access logs + │ Authorization: Bearer fake + ▼ + ┌────────────────────────┐ + │ AUTH_TOKEN=fake │ + │ otel-tui │ + └────────────────────────┘ +``` + +## Running the Example + +1. Start otel-tui with auth token validation: + ```bash + AUTH_TOKEN=fake otel-tui + ``` + +2. From this directory, start Envoy Gateway: + ```bash + envoy-gateway server --config-path envoy-gateway.yaml + ``` + +3. Send a request: + ```bash + curl localhost:10080/get + ``` + +4. In otel-tui, you should see metrics, traces, and access logs. Traces and + access logs share the same trace ID. If the Authorization header was missing + or wrong, otel-tui would reject the OTLP export. + +## Files + +- [envoy-gateway.yaml](envoy-gateway.yaml) - EnvoyGateway configuration +- [resources/gateway.yaml](resources/gateway.yaml) - Gateway API resources diff --git a/examples/otel-headers/envoy-gateway.yaml b/examples/otel-headers/envoy-gateway.yaml new file mode 100644 index 0000000000..0756f1585b --- /dev/null +++ b/examples/otel-headers/envoy-gateway.yaml @@ -0,0 +1,19 @@ +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyGateway +gateway: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +provider: + type: Custom + custom: + resource: + type: File + file: + paths: ["./resources"] + infrastructure: + type: Host + host: {} +logging: + level: + default: info +extensionApis: + enableBackend: true diff --git a/examples/otel-headers/resources/gateway.yaml b/examples/otel-headers/resources/gateway.yaml new file mode 100644 index 0000000000..d540fc8288 --- /dev/null +++ b/examples/otel-headers/resources/gateway.yaml @@ -0,0 +1,109 @@ +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: otel-example +spec: + telemetry: + metrics: + # TODO: Metrics will have the wrong service name until EG exposes + # resource_detectors present in Envoy. + sinks: + - type: OpenTelemetry + openTelemetry: + host: localhost + port: 4317 + # Delta temporality required for backends like otel-tui and Elastic + reportCountersAsDeltas: true + reportHistogramsAsDeltas: true + # Authorization header sent as gRPC initial metadata + headers: + - name: Authorization + value: Bearer fake + tracing: + samplingRate: 100 + provider: + type: OpenTelemetry + host: localhost + port: 4317 + # Authorization header sent as gRPC initial metadata + openTelemetry: + headers: + - name: Authorization + value: Bearer fake + accessLog: + settings: + - format: + # Text format becomes the OTLP log body + text: "%REQ(:METHOD)% %REQ(:PATH)% %PROTOCOL% %RESPONSE_CODE%" + # JSON format becomes OTLP log attributes + json: + response_code_details: "%RESPONSE_CODE_DETAILS%" + upstream_host: "%UPSTREAM_HOST%" + upstream_cluster: "%UPSTREAM_CLUSTER%" + upstream_transport_failure_reason: "%UPSTREAM_TRANSPORT_FAILURE_REASON%" + response_flags: "%RESPONSE_FLAGS%" + connection_termination_details: "%CONNECTION_TERMINATION_DETAILS%" + sinks: + - type: OpenTelemetry + openTelemetry: + host: localhost + port: 4317 + resources: + service.name: envoy-gateway-example + # Authorization header sent as gRPC initial metadata + headers: + - name: Authorization + value: Bearer fake +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: eg +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: eg +spec: + gatewayClassName: eg + infrastructure: + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: otel-example + listeners: + - name: http + protocol: HTTP + port: 10080 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: backend +spec: + parentRefs: + - name: eg + rules: + - backendRefs: + - group: gateway.envoyproxy.io + kind: Backend + name: httpbin + matches: + - path: + type: PathPrefix + value: / +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: Backend +metadata: + name: httpbin +spec: + endpoints: + - fqdn: + hostname: httpbingo.org + port: 443 + tls: + sni: httpbingo.org + wellKnownCACertificates: System diff --git a/internal/gatewayapi/listener.go b/internal/gatewayapi/listener.go index 20720832f8..6f9514f7c5 100644 --- a/internal/gatewayapi/listener.go +++ b/internal/gatewayapi/listener.go @@ -691,6 +691,7 @@ func (t *Translator) processAccessLog(envoyproxy *egv1a1.EnvoyProxy, resources * al := &ir.OpenTelemetryAccessLog{ CELMatches: validExprs, Resources: sink.OpenTelemetry.Resources, + Headers: sink.OpenTelemetry.Headers, Destination: ir.RouteDestination{ Name: destName, Settings: ds, @@ -789,6 +790,7 @@ func (t *Translator) processTracing(gw *gwapiv1.Gateway, envoyproxy *egv1a1.Envo }, Provider: tracing.Provider, Traffic: traffic, + Headers: getOpenTelemetryTracingHeaders(&tracing.Provider), }, nil } @@ -811,6 +813,13 @@ func proxySamplingRate(tracing *egv1a1.ProxyTracing) float64 { return rate } +func getOpenTelemetryTracingHeaders(provider *egv1a1.TracingProvider) []gwapiv1.HTTPHeader { + if provider != nil && provider.OpenTelemetry != nil { + return provider.OpenTelemetry.Headers + } + return nil +} + func (t *Translator) processMetrics(envoyproxy *egv1a1.EnvoyProxy, resources *resource.Resources) (*ir.Metrics, error) { if envoyproxy == nil || envoyproxy.Spec.Telemetry == nil || diff --git a/internal/ir/xds.go b/internal/ir/xds.go index be33746eda..e74f2fab58 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -2442,14 +2442,15 @@ type ALSAccessLogHTTP struct { // OpenTelemetryAccessLog holds the configuration for OpenTelemetry access logging. // +k8s:deepcopy-gen=true type OpenTelemetryAccessLog struct { - CELMatches []string `json:"celMatches,omitempty" yaml:"celMatches,omitempty"` - Authority string `json:"authority,omitempty" yaml:"authority,omitempty"` - Text *string `json:"text,omitempty" yaml:"text,omitempty"` - Attributes map[string]string `json:"attributes,omitempty" yaml:"attributes,omitempty"` - Resources map[string]string `json:"resources,omitempty" yaml:"resources,omitempty"` - Destination RouteDestination `json:"destination,omitempty" yaml:"destination,omitempty"` - Traffic *TrafficFeatures `json:"traffic,omitempty" yaml:"traffic,omitempty"` - LogType *ProxyAccessLogType `json:"logType,omitempty" yaml:"logType,omitempty"` + CELMatches []string `json:"celMatches,omitempty" yaml:"celMatches,omitempty"` + Authority string `json:"authority,omitempty" yaml:"authority,omitempty"` + Text *string `json:"text,omitempty" yaml:"text,omitempty"` + Attributes map[string]string `json:"attributes,omitempty" yaml:"attributes,omitempty"` + Resources map[string]string `json:"resources,omitempty" yaml:"resources,omitempty"` + Headers []gwapiv1.HTTPHeader `json:"headers,omitempty" yaml:"headers,omitempty"` + Destination RouteDestination `json:"destination,omitempty" yaml:"destination,omitempty"` + Traffic *TrafficFeatures `json:"traffic,omitempty" yaml:"traffic,omitempty"` + LogType *ProxyAccessLogType `json:"logType,omitempty" yaml:"logType,omitempty"` } // EnvoyPatchPolicy defines the intermediate representation of the EnvoyPatchPolicy resource. @@ -2585,6 +2586,7 @@ type Tracing struct { Destination RouteDestination `json:"destination,omitempty"` Traffic *TrafficFeatures `json:"traffic,omitempty"` Provider egv1a1.TracingProvider `json:"provider"` + Headers []gwapiv1.HTTPHeader `json:"headers,omitempty" yaml:"headers,omitempty"` } // Metrics defines the configuration for metrics generated by Envoy diff --git a/internal/ir/zz_generated.deepcopy.go b/internal/ir/zz_generated.deepcopy.go index caf0a2dae3..52e21f1daa 100644 --- a/internal/ir/zz_generated.deepcopy.go +++ b/internal/ir/zz_generated.deepcopy.go @@ -2743,6 +2743,11 @@ func (in *OpenTelemetryAccessLog) DeepCopyInto(out *OpenTelemetryAccessLog) { (*out)[key] = val } } + if in.Headers != nil { + in, out := &in.Headers, &out.Headers + *out = make([]apisv1.HTTPHeader, len(*in)) + copy(*out, *in) + } in.Destination.DeepCopyInto(&out.Destination) if in.Traffic != nil { in, out := &in.Traffic, &out.Traffic @@ -4252,6 +4257,11 @@ func (in *Tracing) DeepCopyInto(out *Tracing) { (*in).DeepCopyInto(*out) } in.Provider.DeepCopyInto(&out.Provider) + if in.Headers != nil { + in, out := &in.Headers, &out.Headers + *out = make([]apisv1.HTTPHeader, len(*in)) + copy(*out, *in) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Tracing. diff --git a/internal/xds/bootstrap/bootstrap.go b/internal/xds/bootstrap/bootstrap.go index 33e7fd828d..f200d53527 100644 --- a/internal/xds/bootstrap/bootstrap.go +++ b/internal/xds/bootstrap/bootstrap.go @@ -16,6 +16,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/utils/ptr" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1" netutils "github.com/envoyproxy/gateway/internal/utils/net" @@ -124,6 +125,13 @@ type metricSink struct { ReportCountersAsDeltas bool // ReportHistogramsAsDeltas configures histograms to use delta temporality. ReportHistogramsAsDeltas bool + // Headers is a list of headers to send with OTLP export requests. + Headers []header +} + +type header struct { + Key string + Value string } type adminServerParameters struct { @@ -227,6 +235,7 @@ func GetRenderedBootstrapConfig(opts *RenderBootstrapConfigOptions) (string, err Port: port, ReportCountersAsDeltas: ptr.Deref(sink.OpenTelemetry.ReportCountersAsDeltas, false), ReportHistogramsAsDeltas: ptr.Deref(sink.OpenTelemetry.ReportHistogramsAsDeltas, false), + Headers: convertHeaders(sink.OpenTelemetry.Headers), }) } @@ -338,3 +347,15 @@ func GetRenderedBootstrapConfig(opts *RenderBootstrapConfigOptions) (string, err return cfg.rendered, nil } + +// convertHeaders converts gateway-api HTTPHeaders to bootstrap header format. +func convertHeaders(headers []gwapiv1.HTTPHeader) []header { + if len(headers) == 0 { + return nil + } + result := make([]header, len(headers)) + for i, h := range headers { + result[i] = header{Key: string(h.Name), Value: h.Value} + } + return result +} diff --git a/internal/xds/bootstrap/bootstrap.yaml.tpl b/internal/xds/bootstrap/bootstrap.yaml.tpl index 08e14df220..f873068ca0 100644 --- a/internal/xds/bootstrap/bootstrap.yaml.tpl +++ b/internal/xds/bootstrap/bootstrap.yaml.tpl @@ -75,6 +75,13 @@ stats_sinks: grpc_service: envoy_grpc: cluster_name: otel_metric_sink_{{ $idx }} + {{- if $sink.Headers }} + initial_metadata: + {{- range $sink.Headers }} + - key: "{{ .Key }}" + value: "{{ .Value }}" + {{- end }} + {{- end }} {{- if $sink.ReportCountersAsDeltas }} report_counters_as_deltas: true {{- end }} diff --git a/internal/xds/bootstrap/bootstrap_test.go b/internal/xds/bootstrap/bootstrap_test.go index c4c2d7d61b..1228a902cc 100644 --- a/internal/xds/bootstrap/bootstrap_test.go +++ b/internal/xds/bootstrap/bootstrap_test.go @@ -162,6 +162,36 @@ func TestGetRenderedBootstrapConfig(t *testing.T) { SdsConfig: sds, }, }, + { + name: "otel-metrics-headers", + opts: &RenderBootstrapConfigOptions{ + ProxyMetrics: &egv1a1.ProxyMetrics{ + Prometheus: &egv1a1.ProxyPrometheusProvider{ + Disable: true, + }, + Sinks: []egv1a1.ProxyMetricSink{ + { + Type: egv1a1.MetricSinkTypeOpenTelemetry, + OpenTelemetry: &egv1a1.ProxyOpenTelemetrySink{ + Host: ptr.To("otel-collector.monitoring.svc"), + Port: 4317, + Headers: []gwapiv1.HTTPHeader{ + { + Name: "Authorization", + Value: "Bearer my-api-key", + }, + { + Name: "X-Tenant-ID", + Value: "tenant-123", + }, + }, + }, + }, + }, + }, + SdsConfig: sds, + }, + }, { name: "custom-stats-matcher", opts: &RenderBootstrapConfigOptions{ diff --git a/internal/xds/bootstrap/testdata/render/otel-metrics-headers.yaml b/internal/xds/bootstrap/testdata/render/otel-metrics-headers.yaml new file mode 100644 index 0000000000..45f1751f5d --- /dev/null +++ b/internal/xds/bootstrap/testdata/render/otel-metrics-headers.yaml @@ -0,0 +1,144 @@ +admin: + access_log: + - name: envoy.access_loggers.file + typed_config: + "@type": type.googleapis.com/envoy.extensions.access_loggers.file.v3.FileAccessLog + path: /dev/null + address: + socket_address: + address: 127.0.0.1 + port_value: 19000 +cluster_manager: + local_cluster_name: local_cluster +node: + locality: + zone: $(ENVOY_SERVICE_ZONE) +stats_config: + use_all_default_tags: true + stats_tags: + - regex: \.zone(\.(([^\.]+)\.)) + tag_name: from_zone + - regex: \.zone\.[^\.]+\.(([^\.]+)\.) + tag_name: to_zone + - regex: "^cluster(\\..+\\.(.+))\\.total_match_count$" + tag_name: socket_match_name + - regex: "circuit_breakers\\.((.+?)\\.).+" + tag_name: priority +layered_runtime: + layers: + - name: global_config + static_layer: + envoy.restart_features.use_eds_cache_for_ads: true + re2.max_program_size.error_level: 4294967295 + re2.max_program_size.warn_level: 1000 +dynamic_resources: + ads_config: + api_type: DELTA_GRPC + transport_api_version: V3 + grpc_services: + - envoy_grpc: + cluster_name: xds_cluster + set_node_on_first_message_only: true + lds_config: + ads: {} + resource_api_version: V3 + cds_config: + ads: {} + resource_api_version: V3 +stats_sinks: +- name: "envoy.stat_sinks.open_telemetry" + typed_config: + "@type": type.googleapis.com/envoy.extensions.stat_sinks.open_telemetry.v3.SinkConfig + grpc_service: + envoy_grpc: + cluster_name: otel_metric_sink_0 + initial_metadata: + - key: "Authorization" + value: "Bearer my-api-key" + - key: "X-Tenant-ID" + value: "tenant-123" +static_resources: + clusters: + - name: otel_metric_sink_0 + connect_timeout: 0.250s + type: STRICT_DNS + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: {} + lb_policy: ROUND_ROBIN + load_assignment: + cluster_name: otel_metric_sink_0 + endpoints: + - lb_endpoints: + - endpoint: + address: + socket_address: + address: otel-collector.monitoring.svc + port_value: 4317 + - connect_timeout: 10s + eds_cluster_config: + eds_config: + ads: {} + resource_api_version: 'V3' + service_name: local_cluster + load_balancing_policy: + policies: + - typed_extension_config: + name: 'envoy.load_balancing_policies.least_request' + typed_config: + '@type': 'type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest' + locality_lb_config: + zone_aware_lb_config: + min_cluster_size: '1' + name: local_cluster + type: EDS + - connect_timeout: 10s + load_assignment: + cluster_name: xds_cluster + endpoints: + - load_balancing_weight: 1 + lb_endpoints: + - load_balancing_weight: 1 + endpoint: + address: + socket_address: + address: envoy-gateway + port_value: 18000 + typed_extension_protocol_options: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + "@type": "type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions" + explicit_http_config: + http2_protocol_options: + connection_keepalive: + interval: 30s + timeout: 5s + name: xds_cluster + type: STRICT_DNS + transport_socket: + name: envoy.transport_sockets.tls + typed_config: + "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext + common_tls_context: + tls_params: + tls_maximum_protocol_version: TLSv1_3 + tls_certificate_sds_secret_configs: + - name: xds_certificate + sds_config: + path_config_source: + path: /sds/xds-certificate.json + resource_api_version: V3 + validation_context_sds_secret_config: + name: xds_trusted_ca + sds_config: + path_config_source: + path: /sds/xds-trusted-ca.json + resource_api_version: V3 +overload_manager: + refresh_interval: 0.25s + resource_monitors: + - name: "envoy.resource_monitors.global_downstream_max_connections" + typed_config: + "@type": type.googleapis.com/envoy.extensions.resource_monitors.downstream_connections.v3.DownstreamConnectionsConfig + max_active_downstream_connections: 50000 diff --git a/internal/xds/translator/accesslog.go b/internal/xds/translator/accesslog.go index 5cd7ef6116..9f68539c28 100644 --- a/internal/xds/translator/accesslog.go +++ b/internal/xds/translator/accesslog.go @@ -298,6 +298,7 @@ func buildXdsAccessLog(al *ir.AccessLog, accessLogType ir.ProxyAccessLogType) ([ Authority: otel.Authority, }, }, + InitialMetadata: buildGrpcInitialMetadata(otel.Headers), }, TransportApiVersion: cfgcore.ApiVersion_V3, }, diff --git a/internal/xds/translator/otel.go b/internal/xds/translator/otel.go new file mode 100644 index 0000000000..4fe2960967 --- /dev/null +++ b/internal/xds/translator/otel.go @@ -0,0 +1,26 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package translator + +import ( + corev3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +// buildGrpcInitialMetadata converts HTTP headers to gRPC initial metadata. +func buildGrpcInitialMetadata(headers []gwapiv1.HTTPHeader) []*corev3.HeaderValue { + if len(headers) == 0 { + return nil + } + result := make([]*corev3.HeaderValue, len(headers)) + for i, h := range headers { + result[i] = &corev3.HeaderValue{ + Key: string(h.Name), + Value: h.Value, + } + } + return result +} diff --git a/internal/xds/translator/otel_test.go b/internal/xds/translator/otel_test.go new file mode 100644 index 0000000000..62e3d8df0c --- /dev/null +++ b/internal/xds/translator/otel_test.go @@ -0,0 +1,60 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package translator + +import ( + "testing" + + cfgcore "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + "github.com/stretchr/testify/require" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" +) + +func TestBuildGrpcInitialMetadata(t *testing.T) { + tests := []struct { + name string + headers []gwapiv1.HTTPHeader + expected []*cfgcore.HeaderValue + }{ + { + name: "nil headers", + headers: nil, + expected: nil, + }, + { + name: "empty headers", + headers: []gwapiv1.HTTPHeader{}, + expected: nil, + }, + { + name: "single header", + headers: []gwapiv1.HTTPHeader{ + {Name: "X-Custom-Header", Value: "custom-value"}, + }, + expected: []*cfgcore.HeaderValue{ + {Key: "X-Custom-Header", Value: "custom-value"}, + }, + }, + { + name: "multiple headers", + headers: []gwapiv1.HTTPHeader{ + {Name: "X-Foo", Value: "bar"}, + {Name: "X-Custom-Header", Value: "custom-value"}, + }, + expected: []*cfgcore.HeaderValue{ + {Key: "X-Foo", Value: "bar"}, + {Key: "X-Custom-Header", Value: "custom-value"}, + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + actual := buildGrpcInitialMetadata(tc.headers) + require.Equal(t, tc.expected, actual) + }) + } +} diff --git a/internal/xds/translator/testdata/in/xds-ir/accesslog-otel-headers.yaml b/internal/xds/translator/testdata/in/xds-ir/accesslog-otel-headers.yaml new file mode 100644 index 0000000000..4c120b10c8 --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/accesslog-otel-headers.yaml @@ -0,0 +1,43 @@ +name: "accesslog-otel-headers" +accesslog: + openTelemetry: + - text: | + [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% + attributes: + "response_code": "%RESPONSE_CODE%" + resources: + "cluster_name": "cluster1" + authority: "otel-collector.default.svc.cluster.local" + headers: + - name: Authorization + value: Bearer my-api-key + - name: X-Tenant-ID + value: tenant-123 + destination: + name: "accesslog-0" + settings: + - endpoints: + - host: "otel-collector.default.svc.cluster.local" + port: 4317 + protocol: "GRPC" + addressType: FQDN + name: "accesslog-0/backend/0" +http: +- name: "first-listener" + address: "::" + port: 10080 + hostnames: + - "*" + path: + mergeSlashes: true + escapedSlashesAction: UnescapeAndRedirect + routes: + - name: "direct-route" + hostname: "*" + destination: + name: "direct-route-dest" + settings: + - endpoints: + - host: "1.2.3.4" + port: 50000 + name: "direct-route-dest/backend/0" diff --git a/internal/xds/translator/testdata/in/xds-ir/tracing-otel-headers.yaml b/internal/xds/translator/testdata/in/xds-ir/tracing-otel-headers.yaml new file mode 100644 index 0000000000..3cfc2592e4 --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/tracing-otel-headers.yaml @@ -0,0 +1,42 @@ +name: "tracing-otel-headers" +tracing: + serviceName: "fake-name.fake-ns" + samplingRate: 90 + authority: "otel-collector.default.svc.cluster.local" + headers: + - name: Authorization + value: Bearer my-api-key + - name: X-Tenant-ID + value: tenant-123 + destination: + name: "tracing-0" + settings: + - endpoints: + - host: "otel-collector.default.svc.cluster.local" + port: 4317 + protocol: "GRPC" + addressType: FQDN + name: "tracing-0/backend/0" + provider: + host: otel-collector.monitoring.svc.cluster.local + port: 4317 + type: OpenTelemetry +http: +- name: "first-listener" + address: "::" + port: 10080 + hostnames: + - "*" + path: + mergeSlashes: true + escapedSlashesAction: UnescapeAndRedirect + routes: + - name: "direct-route" + hostname: "*" + destination: + name: "direct-route-dest" + settings: + - endpoints: + - host: "1.2.3.4" + port: 50000 + name: "direct-route-dest/backend/0" diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog-otel-headers.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog-otel-headers.clusters.yaml new file mode 100644 index 0000000000..a46650172b --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog-otel-headers.clusters.yaml @@ -0,0 +1,65 @@ +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: direct-route-dest + ignoreHealthOnHostRemoval: true + lbPolicy: LEAST_REQUEST + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + name: direct-route-dest + perConnectionBufferLimitBytes: 32768 + type: EDS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + dnsRefreshRate: 30s + ignoreHealthOnHostRemoval: true + lbPolicy: LEAST_REQUEST + loadAssignment: + clusterName: accesslog-0 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: otel-collector.default.svc.cluster.local + portValue: 4317 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: accesslog-0/backend/0 + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + name: accesslog-0 + perConnectionBufferLimitBytes: 32768 + respectDnsTtl: true + type: STRICT_DNS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicitHttpConfig: + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog-otel-headers.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog-otel-headers.endpoints.yaml new file mode 100644 index 0000000000..20c80b3aaa --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog-otel-headers.endpoints.yaml @@ -0,0 +1,12 @@ +- clusterName: direct-route-dest + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: direct-route-dest/backend/0 diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog-otel-headers.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog-otel-headers.listeners.yaml new file mode 100644 index 0000000000..30e3251a89 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog-otel-headers.listeners.yaml @@ -0,0 +1,109 @@ +- accessLog: + - filter: + responseFlagFilter: + flags: + - NR + name: envoy.access_loggers.open_telemetry + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.open_telemetry.v3.OpenTelemetryAccessLogConfig + attributes: + values: + - key: k8s.namespace.name + value: + stringValue: '%ENVIRONMENT(ENVOY_POD_NAMESPACE)%' + - key: k8s.pod.name + value: + stringValue: '%ENVIRONMENT(ENVOY_POD_NAME)%' + - key: response_code + value: + stringValue: '%RESPONSE_CODE%' + body: + stringValue: | + [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% + commonConfig: + grpcService: + envoyGrpc: + authority: otel-collector.default.svc.cluster.local + clusterName: accesslog-0 + initialMetadata: + - key: Authorization + value: Bearer my-api-key + - key: X-Tenant-ID + value: tenant-123 + logName: otel_envoy_accesslog + transportApiVersion: V3 + resourceAttributes: + values: + - key: cluster_name + value: + stringValue: cluster1 + address: + socketAddress: + address: '::' + portValue: 10080 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + accessLog: + - name: envoy.access_loggers.open_telemetry + typedConfig: + '@type': type.googleapis.com/envoy.extensions.access_loggers.open_telemetry.v3.OpenTelemetryAccessLogConfig + attributes: + values: + - key: k8s.namespace.name + value: + stringValue: '%ENVIRONMENT(ENVOY_POD_NAMESPACE)%' + - key: k8s.pod.name + value: + stringValue: '%ENVIRONMENT(ENVOY_POD_NAME)%' + - key: response_code + value: + stringValue: '%RESPONSE_CODE%' + body: + stringValue: | + [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% + commonConfig: + grpcService: + envoyGrpc: + authority: otel-collector.default.svc.cluster.local + clusterName: accesslog-0 + initialMetadata: + - key: Authorization + value: Bearer my-api-key + - key: X-Tenant-ID + value: tenant-123 + logName: otel_envoy_accesslog + transportApiVersion: V3 + resourceAttributes: + values: + - key: cluster_name + value: + stringValue: cluster1 + commonHttpProtocolOptions: + headersWithUnderscoresAction: REJECT_REQUEST + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + maxConcurrentStreams: 100 + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + suppressEnvoyHeaders: true + mergeSlashes: true + normalizePath: true + pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT + rds: + configSource: + ads: {} + resourceApiVersion: V3 + routeConfigName: first-listener + serverHeaderTransformation: PASS_THROUGH + statPrefix: http-10080 + useRemoteAddress: true + name: first-listener + maxConnectionsToAcceptPerSocketEvent: 1 + name: first-listener + perConnectionBufferLimitBytes: 32768 diff --git a/internal/xds/translator/testdata/out/xds-ir/accesslog-otel-headers.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/accesslog-otel-headers.routes.yaml new file mode 100644 index 0000000000..ea343799ac --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/accesslog-otel-headers.routes.yaml @@ -0,0 +1,14 @@ +- ignorePortInHostMatching: true + name: first-listener + virtualHosts: + - domains: + - '*' + name: first-listener/* + routes: + - match: + prefix: / + name: direct-route + route: + cluster: direct-route-dest + upgradeConfigs: + - upgradeType: websocket diff --git a/internal/xds/translator/testdata/out/xds-ir/tracing-otel-headers.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/tracing-otel-headers.clusters.yaml new file mode 100644 index 0000000000..0a03375ae4 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/tracing-otel-headers.clusters.yaml @@ -0,0 +1,65 @@ +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: direct-route-dest + ignoreHealthOnHostRemoval: true + lbPolicy: LEAST_REQUEST + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + name: direct-route-dest + perConnectionBufferLimitBytes: 32768 + type: EDS +- circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + dnsRefreshRate: 30s + ignoreHealthOnHostRemoval: true + lbPolicy: LEAST_REQUEST + loadAssignment: + clusterName: tracing-0 + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: otel-collector.default.svc.cluster.local + portValue: 4317 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: tracing-0/backend/0 + loadBalancingPolicy: + policies: + - typedExtensionConfig: + name: envoy.load_balancing_policies.least_request + typedConfig: + '@type': type.googleapis.com/envoy.extensions.load_balancing_policies.least_request.v3.LeastRequest + localityLbConfig: + localityWeightedLbConfig: {} + name: tracing-0 + perConnectionBufferLimitBytes: 32768 + respectDnsTtl: true + type: STRICT_DNS + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicitHttpConfig: + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 diff --git a/internal/xds/translator/testdata/out/xds-ir/tracing-otel-headers.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/tracing-otel-headers.endpoints.yaml new file mode 100644 index 0000000000..20c80b3aaa --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/tracing-otel-headers.endpoints.yaml @@ -0,0 +1,12 @@ +- clusterName: direct-route-dest + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.2.3.4 + portValue: 50000 + loadBalancingWeight: 1 + loadBalancingWeight: 1 + locality: + region: direct-route-dest/backend/0 diff --git a/internal/xds/translator/testdata/out/xds-ir/tracing-otel-headers.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/tracing-otel-headers.listeners.yaml new file mode 100644 index 0000000000..7e6489bf88 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/tracing-otel-headers.listeners.yaml @@ -0,0 +1,57 @@ +- address: + socketAddress: + address: '::' + portValue: 10080 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + commonHttpProtocolOptions: + headersWithUnderscoresAction: REJECT_REQUEST + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + maxConcurrentStreams: 100 + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + suppressEnvoyHeaders: true + mergeSlashes: true + normalizePath: true + pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT + rds: + configSource: + ads: {} + resourceApiVersion: V3 + routeConfigName: first-listener + serverHeaderTransformation: PASS_THROUGH + statPrefix: http-10080 + tracing: + clientSampling: + value: 100 + overallSampling: + value: 100 + provider: + name: envoy.tracers.opentelemetry + typedConfig: + '@type': type.googleapis.com/envoy.config.trace.v3.OpenTelemetryConfig + grpcService: + envoyGrpc: + authority: otel-collector.default.svc.cluster.local + clusterName: tracing-0 + initialMetadata: + - key: Authorization + value: Bearer my-api-key + - key: X-Tenant-ID + value: tenant-123 + serviceName: fake-name.fake-ns + randomSampling: + value: 90 + spawnUpstreamSpan: true + useRemoteAddress: true + name: first-listener + maxConnectionsToAcceptPerSocketEvent: 1 + name: first-listener + perConnectionBufferLimitBytes: 32768 diff --git a/internal/xds/translator/testdata/out/xds-ir/tracing-otel-headers.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/tracing-otel-headers.routes.yaml new file mode 100644 index 0000000000..ea343799ac --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/tracing-otel-headers.routes.yaml @@ -0,0 +1,14 @@ +- ignorePortInHostMatching: true + name: first-listener + virtualHosts: + - domains: + - '*' + name: first-listener/* + routes: + - match: + prefix: / + name: direct-route + route: + cluster: direct-route-dest + upgradeConfigs: + - upgradeType: websocket diff --git a/internal/xds/translator/tracing.go b/internal/xds/translator/tracing.go index 56c5a8f101..998f566004 100644 --- a/internal/xds/translator/tracing.go +++ b/internal/xds/translator/tracing.go @@ -63,6 +63,7 @@ func buildHCMTracing(tracing *ir.Tracing) (*hcm.HttpConnectionManager_Tracing, e Authority: tracing.Authority, }, }, + InitialMetadata: buildGrpcInitialMetadata(tracing.Headers), }, ServiceName: tracing.ServiceName, } diff --git a/release-notes/current.yaml b/release-notes/current.yaml index 148a855833..f1b1fb8b7b 100644 --- a/release-notes/current.yaml +++ b/release-notes/current.yaml @@ -21,6 +21,7 @@ new features: | Added support for specifying both text (body) and attributes in access log format by making the type field optional. Set warning status condition for deprecated fields in xPolicy CRDs. Added support for URLRewrite filter on individual backendRefs. + Added support for custom headers on OTLP exports (metrics, tracing, access logs). bug fixes: | Fixed configured OIDC authorization endpoint being overridden by discovered endpoints from issuer's well-known URL. diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 992a1ae325..1b62124bd3 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -3648,6 +3648,21 @@ _Appears in:_ | `host` | _string_ | false | | Host define the extension service hostname.
Deprecated: Use BackendRefs instead. | | `port` | _integer_ | false | 4317 | Port defines the port the extension service is exposed on.
Deprecated: Use BackendRefs instead. | | `resources` | _object (keys:string, values:string)_ | false | | Resources is a set of labels that describe the source of a log entry, including envoy node info.
It's recommended to follow [semantic conventions](https://opentelemetry.io/docs/reference/specification/resource/semantic_conventions/). | +| `headers` | _[HTTPHeader](#httpheader) array_ | false | | Headers is a list of additional headers to send with OTLP export requests.
These headers are added as gRPC initial metadata for the OTLP gRPC service. | + + +#### OpenTelemetryTracingProvider + + + +OpenTelemetryTracingProvider defines the OpenTelemetry tracing provider configuration. + +_Appears in:_ +- [TracingProvider](#tracingprovider) + +| Field | Type | Required | Default | Description | +| --- | --- | --- | --- | --- | +| `headers` | _[HTTPHeader](#httpheader) array_ | false | | Headers is a list of additional headers to send with OTLP export requests.
These headers are added as gRPC initial metadata for the OTLP gRPC service. | #### Operation @@ -4138,6 +4153,7 @@ _Appears in:_ | `port` | _integer_ | false | 4317 | Port defines the port the service is exposed on.
Deprecated: Use BackendRefs instead. | | `reportCountersAsDeltas` | _boolean_ | false | | ReportCountersAsDeltas configures the OpenTelemetry sink to report
counters as delta temporality instead of cumulative. | | `reportHistogramsAsDeltas` | _boolean_ | false | | ReportHistogramsAsDeltas configures the OpenTelemetry sink to report
histograms as delta temporality instead of cumulative.
Required for backends like Elastic that drop cumulative histograms. | +| `headers` | _[HTTPHeader](#httpheader) array_ | false | | Headers is a list of additional headers to send with OTLP export requests.
These headers are added as gRPC initial metadata for the OTLP gRPC service. | #### ProxyPrometheusProvider @@ -5328,6 +5344,7 @@ _Appears in:_ | `port` | _integer_ | false | 4317 | Port defines the port the provider service is exposed on.
Deprecated: Use BackendRefs instead. | | `serviceName` | _string_ | false | | ServiceName defines the service name to use in tracing configuration.
If not set, Envoy Gateway will use a default service name set as
"name.namespace" (e.g., "my-gateway.default").
Note: This field is only supported for OpenTelemetry and Datadog tracing providers.
For Zipkin, the service name in traces is always derived from the Envoy --service-cluster flag
(typically "namespace/name" format). Setting this field has no effect for Zipkin. | | `zipkin` | _[ZipkinTracingProvider](#zipkintracingprovider)_ | false | | Zipkin defines the Zipkin tracing provider configuration | +| `openTelemetry` | _[OpenTelemetryTracingProvider](#opentelemetrytracingprovider)_ | false | | OpenTelemetry defines the OpenTelemetry tracing provider configuration | #### TracingProviderType diff --git a/test/helm/gateway-crds-helm/all.out.yaml b/test/helm/gateway-crds-helm/all.out.yaml index 2e8927e3ca..e77f496b5e 100644 --- a/test/helm/gateway-crds-helm/all.out.yaml +++ b/test/helm/gateway-crds-helm/all.out.yaml @@ -42095,6 +42095,42 @@ spec: && !(has(self.loadBalancer) && has(self.loadBalancer.type) && self.loadBalancer.type in [''Random'', ''RoundRobin'']))' + headers: + description: |- + Headers is a list of additional headers to send with OTLP export requests. + These headers are added as gRPC initial metadata for the OTLP gRPC service. + items: + description: HTTPHeader represents an HTTP + Header name and value as defined by RFC + 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP + Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 32 + minItems: 1 + type: array host: description: |- Host define the extension service hostname. @@ -43423,6 +43459,41 @@ spec: && has(self.connection.preconnect.predictivePercent)) && !(has(self.loadBalancer) && has(self.loadBalancer.type) && self.loadBalancer.type in [''Random'', ''RoundRobin'']))' + headers: + description: |- + Headers is a list of additional headers to send with OTLP export requests. + These headers are added as gRPC initial metadata for the OTLP gRPC service. + items: + description: HTTPHeader represents an HTTP Header + name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header + to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 32 + minItems: 1 + type: array host: description: |- Host define the service hostname. @@ -44683,6 +44754,46 @@ spec: Host define the provider service hostname. Deprecated: Use BackendRefs instead. type: string + openTelemetry: + description: OpenTelemetry defines the OpenTelemetry tracing + provider configuration + properties: + headers: + description: |- + Headers is a list of additional headers to send with OTLP export requests. + These headers are added as gRPC initial metadata for the OTLP gRPC service. + items: + description: HTTPHeader represents an HTTP Header + name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header + to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 32 + minItems: 1 + type: array + type: object port: default: 4317 description: |- @@ -44743,6 +44854,9 @@ spec: rule: 'has(self.backendRefs) ? (self.backendRefs.all(f, f.group == "" || f.group == ''gateway.envoyproxy.io'')) : true' + - message: openTelemetry can only be used with type OpenTelemetry + rule: 'has(self.openTelemetry) ? self.type == ''OpenTelemetry'' + : true' samplingFraction: description: |- SamplingFraction represents the fraction of requests that should be diff --git a/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml b/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml index af0d50f0ac..0c912f5cd9 100644 --- a/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml +++ b/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml @@ -21275,6 +21275,42 @@ spec: && !(has(self.loadBalancer) && has(self.loadBalancer.type) && self.loadBalancer.type in [''Random'', ''RoundRobin'']))' + headers: + description: |- + Headers is a list of additional headers to send with OTLP export requests. + These headers are added as gRPC initial metadata for the OTLP gRPC service. + items: + description: HTTPHeader represents an HTTP + Header name and value as defined by RFC + 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP + Header to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 32 + minItems: 1 + type: array host: description: |- Host define the extension service hostname. @@ -22603,6 +22639,41 @@ spec: && has(self.connection.preconnect.predictivePercent)) && !(has(self.loadBalancer) && has(self.loadBalancer.type) && self.loadBalancer.type in [''Random'', ''RoundRobin'']))' + headers: + description: |- + Headers is a list of additional headers to send with OTLP export requests. + These headers are added as gRPC initial metadata for the OTLP gRPC service. + items: + description: HTTPHeader represents an HTTP Header + name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header + to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 32 + minItems: 1 + type: array host: description: |- Host define the service hostname. @@ -23863,6 +23934,46 @@ spec: Host define the provider service hostname. Deprecated: Use BackendRefs instead. type: string + openTelemetry: + description: OpenTelemetry defines the OpenTelemetry tracing + provider configuration + properties: + headers: + description: |- + Headers is a list of additional headers to send with OTLP export requests. + These headers are added as gRPC initial metadata for the OTLP gRPC service. + items: + description: HTTPHeader represents an HTTP Header + name and value as defined by RFC 7230. + properties: + name: + description: |- + Name is the name of the HTTP Header to be matched. Name matching MUST be + case-insensitive. (See https://tools.ietf.org/html/rfc7230#section-3.2). + + If multiple entries specify equivalent header names, the first entry with + an equivalent name MUST be considered for a match. Subsequent entries + with an equivalent header name MUST be ignored. Due to the + case-insensitivity of header names, "foo" and "Foo" are considered + equivalent. + maxLength: 256 + minLength: 1 + pattern: ^[A-Za-z0-9!#$%&'*+\-.^_\x60|~]+$ + type: string + value: + description: Value is the value of HTTP Header + to be matched. + maxLength: 4096 + minLength: 1 + type: string + required: + - name + - value + type: object + maxItems: 32 + minItems: 1 + type: array + type: object port: default: 4317 description: |- @@ -23923,6 +24034,9 @@ spec: rule: 'has(self.backendRefs) ? (self.backendRefs.all(f, f.group == "" || f.group == ''gateway.envoyproxy.io'')) : true' + - message: openTelemetry can only be used with type OpenTelemetry + rule: 'has(self.openTelemetry) ? self.type == ''OpenTelemetry'' + : true' samplingFraction: description: |- SamplingFraction represents the fraction of requests that should be diff --git a/test/standalone/standalone_test.go b/test/standalone/standalone_test.go new file mode 100644 index 0000000000..a78d6bb501 --- /dev/null +++ b/test/standalone/standalone_test.go @@ -0,0 +1,135 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build standalone + +package standalone + +import ( + _ "embed" + "fmt" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" + + testutils "github.com/envoyproxy/gateway/test/utils" + "github.com/envoyproxy/gateway/test/utils/testotel" +) + +//go:embed testdata/otel-grpc-headers/envoy-gateway.yaml +var otelGRPCHeadersEnvoyGateway string + +//go:embed testdata/otel-grpc-headers/resources/gateway.yaml +var otelGRPCHeadersGateway string + +// gatewayBin is the path to the compiled envoy-gateway binary. +var gatewayBin string + +// TestMain builds the binary once for all tests. +func TestMain(m *testing.M) { + var err error + if gatewayBin, err = testutils.BuildGoBinaryOnDemand("ENVOY_GATEWAY_BIN", "envoy-gateway", "./cmd/envoy-gateway"); err != nil { + fmt.Fprintf(os.Stderr, "failed to build envoy-gateway: %v\n", err) + os.Exit(1) + } + os.Exit(m.Run()) +} + +func TestOTELGRPCHeaders(t *testing.T) { + ports := testutils.RequireRandomPorts(t, 2) + listenerPort := ports[0] + adminPort := ports[1] + + // Start a local backend server + backendListener, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + backendPort := backendListener.Addr().(*net.TCPAddr).Port + backendServer := &http.Server{ + Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(`{"status": "ok"}`)) + }), + } + go func() { _ = backendServer.Serve(backendListener) }() + defer backendServer.Close() + + collector, err := testotel.StartGRPCCollector() + require.NoError(t, err) + defer collector.Close() + + testDir := t.TempDir() + resourcesDir := filepath.Join(testDir, "resources") + require.NoError(t, os.Mkdir(resourcesDir, 0o755)) + + // Replace default ports with actual ports + replacements := map[string]string{ + "port: 10080": "port: " + strconv.Itoa(listenerPort), + "port: 19000": "port: " + strconv.Itoa(adminPort), + "port: 4317": "port: " + strconv.Itoa(collector.Port()), + "port: 8080": "port: " + strconv.Itoa(backendPort), + } + + envoyGatewayYAML := replaceTokens(otelGRPCHeadersEnvoyGateway, replacements) + gatewayYAML := replaceTokens(otelGRPCHeadersGateway, replacements) + + require.NoError(t, os.WriteFile(filepath.Join(testDir, "envoy-gateway.yaml"), []byte(envoyGatewayYAML), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(resourcesDir, "gateway.yaml"), []byte(gatewayYAML), 0o600)) + + cmd := exec.Command(gatewayBin, "server", "--config-path", filepath.Join(testDir, "envoy-gateway.yaml")) + cmd.Dir = testDir + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + require.NoError(t, cmd.Start()) + defer func() { + _ = cmd.Process.Kill() + _ = cmd.Wait() + }() + + // Wait for Envoy to be ready + require.Eventually(t, func() bool { + resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/", listenerPort)) + if err != nil { + return false + } + resp.Body.Close() + return true + }, 30*time.Second, 500*time.Millisecond, "envoy not ready on port %d", listenerPort) + + resp, err := http.Get(fmt.Sprintf("http://127.0.0.1:%d/get", listenerPort)) + require.NoError(t, err) + resp.Body.Close() + + // Verify collector received an access log with the expected format and Authorization header + log := collector.TakeLog() + require.NotNil(t, log) + require.Contains(t, log.Body.GetStringValue(), `HTTP/1.1" 200`) + require.Equal(t, "Bearer test-api-key", testotel.GetAttributeString(log.Attributes, "grpc.metadata.authorization")) + + // Verify collector received a trace span with Authorization header + span := collector.TakeSpan() + require.NotNil(t, span) + require.Equal(t, "Bearer test-api-key", testotel.GetAttributeString(span.Attributes, "grpc.metadata.authorization")) + + // Verify collector received a metric with Authorization header + resourceMetrics := collector.TakeMetric() + require.NotNil(t, resourceMetrics) + require.Equal(t, "Bearer test-api-key", testotel.GetAttributeString(resourceMetrics.Resource.Attributes, "grpc.metadata.authorization")) +} + +func replaceTokens(content string, replacements map[string]string) string { + result := content + for token, value := range replacements { + result = strings.ReplaceAll(result, token, value) + } + return result +} diff --git a/test/standalone/testdata/otel-grpc-headers/envoy-gateway.yaml b/test/standalone/testdata/otel-grpc-headers/envoy-gateway.yaml new file mode 100644 index 0000000000..627bf803d0 --- /dev/null +++ b/test/standalone/testdata/otel-grpc-headers/envoy-gateway.yaml @@ -0,0 +1,23 @@ +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyGateway +gateway: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +provider: + type: Custom + custom: + resource: + type: File + file: + paths: ["./resources"] + infrastructure: + type: Host + host: {} +logging: + level: + default: info +extensionApis: + enableBackend: true +admin: + address: + host: 127.0.0.1 + port: 19000 diff --git a/test/standalone/testdata/otel-grpc-headers/resources/gateway.yaml b/test/standalone/testdata/otel-grpc-headers/resources/gateway.yaml new file mode 100644 index 0000000000..df7ac1e808 --- /dev/null +++ b/test/standalone/testdata/otel-grpc-headers/resources/gateway.yaml @@ -0,0 +1,89 @@ +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: otel-headers +spec: + telemetry: + metrics: + sinks: + - type: OpenTelemetry + openTelemetry: + host: 127.0.0.1 + port: 4317 + headers: + - name: Authorization + value: Bearer test-api-key + tracing: + samplingRate: 100 + provider: + type: OpenTelemetry + host: 127.0.0.1 + port: 4317 + openTelemetry: + headers: + - name: Authorization + value: Bearer test-api-key + accessLog: + settings: + - format: + type: Text + text: | + [%START_TIME%] "%REQ(:METHOD)% %REQ(X-ENVOY-ORIGINAL-PATH?:PATH)% %PROTOCOL%" %RESPONSE_CODE% + sinks: + - type: OpenTelemetry + openTelemetry: + host: 127.0.0.1 + port: 4317 + headers: + - name: Authorization + value: Bearer test-api-key +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: eg +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: eg +spec: + gatewayClassName: eg + infrastructure: + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: otel-headers + listeners: + - name: http + protocol: HTTP + port: 10080 +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: backend +spec: + parentRefs: + - name: eg + rules: + - backendRefs: + - group: gateway.envoyproxy.io + kind: Backend + name: httpbin + matches: + - path: + type: PathPrefix + value: / +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: Backend +metadata: + name: httpbin +spec: + endpoints: + - ip: + address: 127.0.0.1 + port: 8080 diff --git a/test/utils/build.go b/test/utils/build.go new file mode 100644 index 0000000000..3e2a26f69a --- /dev/null +++ b/test/utils/build.go @@ -0,0 +1,76 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package utils + +import ( + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" +) + +// BuildGoBinaryOnDemand builds the binary unless the env variable is set. +// If the environment variable is set, it will use that path instead. +func BuildGoBinaryOnDemand(env, binaryName, packagePath string) (string, error) { + if envPath := os.Getenv(env); envPath != "" { + if !filepath.IsAbs(envPath) { + envPath = filepath.Join(FindProjectRoot(), envPath) + } + if _, err := os.Stat(envPath); err != nil { + return "", fmt.Errorf("%s path does not exist: %s", env, envPath) + } + fmt.Fprintf(os.Stderr, "Using %s : %s\n", env, envPath) + return envPath, nil + } + + return BuildGoBinary(binaryName, packagePath) +} + +// BuildGoBinary builds a Go binary with the given name and package path. +func BuildGoBinary(binaryName, packagePath string) (string, error) { + projectRoot := FindProjectRoot() + outputDir := filepath.Join(projectRoot, "out") + if err := os.MkdirAll(outputDir, 0o755); err != nil { + return "", fmt.Errorf("failed to create output directory: %w", err) + } + + platformBinaryName := fmt.Sprintf("%s-%s-%s", binaryName, runtime.GOOS, runtime.GOARCH) + binaryPath := filepath.Join(outputDir, platformBinaryName) + + cmd := exec.Command("go", "build", "-o", binaryPath, packagePath) + cmd.Dir = projectRoot + cmd.Env = append(os.Environ(), "CGO_ENABLED=0") + var stderr strings.Builder + cmd.Stdout = io.Discard + cmd.Stderr = &stderr + + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("failed to build %s: %w\nstderr: %s", binaryName, err, stderr.String()) + } + + if err := os.Chmod(binaryPath, 0o755); err != nil { + return "", fmt.Errorf("failed to make binary executable: %w", err) + } + return binaryPath, nil +} + +// FindProjectRoot finds the root of the project by looking for go.mod. +func FindProjectRoot() string { + dir, _ := os.Getwd() + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + panic("could not find project root (go.mod)") + } + dir = parent + } +} diff --git a/test/utils/ports.go b/test/utils/ports.go new file mode 100644 index 0000000000..7137ed8bd4 --- /dev/null +++ b/test/utils/ports.go @@ -0,0 +1,57 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package utils + +import ( + "context" + "fmt" + "net" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +// RequireRandomPorts returns random available ports. +func RequireRandomPorts(t testing.TB, count int) []int { + t.Helper() + + ports := make([]int, count) + + listeners := make([]net.Listener, 0, count) + for i := range count { + lc := net.ListenConfig{} + lis, err := lc.Listen(context.Background(), "tcp", "127.0.0.1:0") + require.NoError(t, err, "failed to listen on random port %d", i) + listeners = append(listeners, lis) + addr := lis.Addr().(*net.TCPAddr) + ports[i] = addr.Port + } + for _, lis := range listeners { + require.NoError(t, lis.Close()) + } + return ports +} + +// AwaitPortClosed waits until the port is no longer listening. +func AwaitPortClosed(port int, timeout time.Duration) error { + deadline := time.Now().Add(timeout) + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + for { + conn, err := net.Dial("tcp", fmt.Sprintf("127.0.0.1:%d", port)) + if err != nil { + return nil // Port closed + } + conn.Close() + + if time.Now().After(deadline) { + return fmt.Errorf("port %d still listening after %v", port, timeout) + } + <-ticker.C + } +} diff --git a/test/utils/testotel/collector.go b/test/utils/testotel/collector.go new file mode 100644 index 0000000000..317fea18d6 --- /dev/null +++ b/test/utils/testotel/collector.go @@ -0,0 +1,220 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +// Package testotel provides test utilities for OpenTelemetry tests. +package testotel + +import ( + "context" + "fmt" + "net" + "time" + + collectlogsv1 "go.opentelemetry.io/proto/otlp/collector/logs/v1" + collectmetricsv1 "go.opentelemetry.io/proto/otlp/collector/metrics/v1" + collecttracev1 "go.opentelemetry.io/proto/otlp/collector/trace/v1" + commonv1 "go.opentelemetry.io/proto/otlp/common/v1" + logsv1 "go.opentelemetry.io/proto/otlp/logs/v1" + metricsv1 "go.opentelemetry.io/proto/otlp/metrics/v1" + resourcev1 "go.opentelemetry.io/proto/otlp/resource/v1" + tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" +) + +const otlpTimeout = 5 * time.Second + +// GRPCCollector is a test OTLP gRPC collector that captures logs, traces, and metrics. +// gRPC metadata is injected as attributes on each signal for easy assertion. +type GRPCCollector struct { + server *grpc.Server + listener net.Listener + port int + + logCh chan *logsv1.ResourceLogs + traceCh chan *tracev1.ResourceSpans + metricsCh chan *metricsv1.ResourceMetrics +} + +type logsServer struct { + collectlogsv1.UnimplementedLogsServiceServer + collector *GRPCCollector +} + +type traceServer struct { + collecttracev1.UnimplementedTraceServiceServer + collector *GRPCCollector +} + +type metricsServer struct { + collectmetricsv1.UnimplementedMetricsServiceServer + collector *GRPCCollector +} + +// StartGRPCCollector starts a test OTLP gRPC collector on an available port. +func StartGRPCCollector() (*GRPCCollector, error) { + listener, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + return nil, fmt.Errorf("failed to listen: %w", err) + } + + c := &GRPCCollector{ + listener: listener, + port: listener.Addr().(*net.TCPAddr).Port, + logCh: make(chan *logsv1.ResourceLogs, 100), + traceCh: make(chan *tracev1.ResourceSpans, 100), + metricsCh: make(chan *metricsv1.ResourceMetrics, 100), + } + + c.server = grpc.NewServer() + + collectlogsv1.RegisterLogsServiceServer(c.server, &logsServer{collector: c}) + collecttracev1.RegisterTraceServiceServer(c.server, &traceServer{collector: c}) + collectmetricsv1.RegisterMetricsServiceServer(c.server, &metricsServer{collector: c}) + + go func() { + _ = c.server.Serve(listener) + }() + + return c, nil +} + +// Export implements the LogsService Export RPC. +// gRPC metadata is injected as attributes on each log record. +func (s *logsServer) Export(ctx context.Context, req *collectlogsv1.ExportLogsServiceRequest) (*collectlogsv1.ExportLogsServiceResponse, error) { + md, _ := metadata.FromIncomingContext(ctx) + attrs := metadataToAttributes(md) + for _, resourceLogs := range req.ResourceLogs { + for _, scopeLogs := range resourceLogs.ScopeLogs { + for _, log := range scopeLogs.LogRecords { + log.Attributes = append(log.Attributes, attrs...) + } + } + select { + case s.collector.logCh <- resourceLogs: + case <-time.After(otlpTimeout): + fmt.Println("Warning: Dropping logs due to timeout") + } + } + return &collectlogsv1.ExportLogsServiceResponse{}, nil +} + +// Export implements the TraceService Export RPC. +// gRPC metadata is injected as attributes on each span. +func (s *traceServer) Export(ctx context.Context, req *collecttracev1.ExportTraceServiceRequest) (*collecttracev1.ExportTraceServiceResponse, error) { + md, _ := metadata.FromIncomingContext(ctx) + attrs := metadataToAttributes(md) + for _, resourceSpans := range req.ResourceSpans { + for _, scopeSpans := range resourceSpans.ScopeSpans { + for _, span := range scopeSpans.Spans { + span.Attributes = append(span.Attributes, attrs...) + } + } + select { + case s.collector.traceCh <- resourceSpans: + case <-time.After(otlpTimeout): + fmt.Println("Warning: Dropping spans due to timeout") + } + } + return &collecttracev1.ExportTraceServiceResponse{}, nil +} + +// Export implements the MetricsService Export RPC. +// gRPC metadata is injected as resource attributes. +func (s *metricsServer) Export(ctx context.Context, req *collectmetricsv1.ExportMetricsServiceRequest) (*collectmetricsv1.ExportMetricsServiceResponse, error) { + md, _ := metadata.FromIncomingContext(ctx) + attrs := metadataToAttributes(md) + for _, resourceMetrics := range req.ResourceMetrics { + if resourceMetrics.Resource == nil { + resourceMetrics.Resource = &resourcev1.Resource{} + } + resourceMetrics.Resource.Attributes = append(resourceMetrics.Resource.Attributes, attrs...) + select { + case s.collector.metricsCh <- resourceMetrics: + case <-time.After(otlpTimeout): + fmt.Println("Warning: Dropping metrics due to timeout") + } + } + return &collectmetricsv1.ExportMetricsServiceResponse{}, nil +} + +// metadataToAttributes converts gRPC metadata to OTel attributes with "grpc.metadata." prefix. +func metadataToAttributes(md metadata.MD) []*commonv1.KeyValue { + var attrs []*commonv1.KeyValue + for key, values := range md { + if len(values) > 0 { + attrs = append(attrs, &commonv1.KeyValue{ + Key: "grpc.metadata." + key, + Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_StringValue{StringValue: values[0]}}, + }) + } + } + return attrs +} + +// GetAttributeString returns the string value for the given key, or empty string if not found. +func GetAttributeString(attrs []*commonv1.KeyValue, key string) string { + for _, attr := range attrs { + if attr.Key == key { + return attr.Value.GetStringValue() + } + } + return "" +} + +// Port returns the port the collector is listening on. +func (c *GRPCCollector) Port() int { + return c.port +} + +// Address returns the full address (host:port) the collector is listening on. +func (c *GRPCCollector) Address() string { + return fmt.Sprintf("127.0.0.1:%d", c.port) +} + +// TakeLog returns a single log record or nil if none were recorded within the timeout. +func (c *GRPCCollector) TakeLog() *logsv1.LogRecord { + select { + case resourceLogs := <-c.logCh: + if len(resourceLogs.ScopeLogs) == 0 || len(resourceLogs.ScopeLogs[0].LogRecords) == 0 { + return nil + } + return resourceLogs.ScopeLogs[0].LogRecords[0] + case <-time.After(otlpTimeout): + return nil + } +} + +// TakeSpan returns a single span or nil if none were recorded within the timeout. +func (c *GRPCCollector) TakeSpan() *tracev1.Span { + select { + case resourceSpans := <-c.traceCh: + if len(resourceSpans.ScopeSpans) == 0 || len(resourceSpans.ScopeSpans[0].Spans) == 0 { + return nil + } + return resourceSpans.ScopeSpans[0].Spans[0] + case <-time.After(otlpTimeout): + return nil + } +} + +// TakeMetric returns resource metrics or nil if none were recorded within the timeout. +func (c *GRPCCollector) TakeMetric() *metricsv1.ResourceMetrics { + select { + case resourceMetrics := <-c.metricsCh: + return resourceMetrics + case <-time.After(otlpTimeout): + return nil + } +} + +// Close shuts down the collector and cleans up resources. +func (c *GRPCCollector) Close() { + c.server.GracefulStop() + c.listener.Close() + close(c.logCh) + close(c.traceCh) + close(c.metricsCh) +} diff --git a/test/utils/testotel/collector_test.go b/test/utils/testotel/collector_test.go new file mode 100644 index 0000000000..ae723cbba6 --- /dev/null +++ b/test/utils/testotel/collector_test.go @@ -0,0 +1,136 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +package testotel + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + collectlogsv1 "go.opentelemetry.io/proto/otlp/collector/logs/v1" + collectmetricsv1 "go.opentelemetry.io/proto/otlp/collector/metrics/v1" + collecttracev1 "go.opentelemetry.io/proto/otlp/collector/trace/v1" + logsv1 "go.opentelemetry.io/proto/otlp/logs/v1" + metricsv1 "go.opentelemetry.io/proto/otlp/metrics/v1" + tracev1 "go.opentelemetry.io/proto/otlp/trace/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/metadata" +) + +func TestGRPCCollector_InjectsMetadataAsLogAttributes(t *testing.T) { + collector, err := StartGRPCCollector() + require.NoError(t, err) + defer collector.Close() + + conn, err := grpc.NewClient( + collector.Address(), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + require.NoError(t, err) + defer conn.Close() + + logsClient := collectlogsv1.NewLogsServiceClient(conn) + ctx := metadata.NewOutgoingContext(context.Background(), metadata.Pairs( + "authorization", "Bearer log-api-key", + "x-tenant-id", "tenant-123", + )) + + _, err = logsClient.Export(ctx, &collectlogsv1.ExportLogsServiceRequest{ + ResourceLogs: []*logsv1.ResourceLogs{ + { + ScopeLogs: []*logsv1.ScopeLogs{ + { + LogRecords: []*logsv1.LogRecord{ + {}, + }, + }, + }, + }, + }, + }) + require.NoError(t, err) + + log := collector.TakeLog() + require.NotNil(t, log) + require.Equal(t, "Bearer log-api-key", GetAttributeString(log.Attributes, "grpc.metadata.authorization")) + require.Equal(t, "tenant-123", GetAttributeString(log.Attributes, "grpc.metadata.x-tenant-id")) +} + +func TestGRPCCollector_InjectsMetadataAsSpanAttributes(t *testing.T) { + collector, err := StartGRPCCollector() + require.NoError(t, err) + defer collector.Close() + + conn, err := grpc.NewClient( + collector.Address(), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + require.NoError(t, err) + defer conn.Close() + + traceClient := collecttracev1.NewTraceServiceClient(conn) + ctx := metadata.NewOutgoingContext(context.Background(), metadata.Pairs( + "authorization", "Bearer trace-api-key", + )) + + _, err = traceClient.Export(ctx, &collecttracev1.ExportTraceServiceRequest{ + ResourceSpans: []*tracev1.ResourceSpans{ + { + ScopeSpans: []*tracev1.ScopeSpans{ + { + Spans: []*tracev1.Span{ + {Name: "test-span"}, + }, + }, + }, + }, + }, + }) + require.NoError(t, err) + + span := collector.TakeSpan() + require.NotNil(t, span) + require.Equal(t, "Bearer trace-api-key", GetAttributeString(span.Attributes, "grpc.metadata.authorization")) +} + +func TestGRPCCollector_InjectsMetadataAsResourceAttributes(t *testing.T) { + collector, err := StartGRPCCollector() + require.NoError(t, err) + defer collector.Close() + + conn, err := grpc.NewClient( + collector.Address(), + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) + require.NoError(t, err) + defer conn.Close() + + metricsClient := collectmetricsv1.NewMetricsServiceClient(conn) + ctx := metadata.NewOutgoingContext(context.Background(), metadata.Pairs( + "authorization", "Bearer metrics-api-key", + )) + + _, err = metricsClient.Export(ctx, &collectmetricsv1.ExportMetricsServiceRequest{ + ResourceMetrics: []*metricsv1.ResourceMetrics{ + { + ScopeMetrics: []*metricsv1.ScopeMetrics{ + { + Metrics: []*metricsv1.Metric{ + {Name: "test-metric"}, + }, + }, + }, + }, + }, + }) + require.NoError(t, err) + + resourceMetrics := collector.TakeMetric() + require.NotNil(t, resourceMetrics) + require.NotNil(t, resourceMetrics.Resource) + require.Equal(t, "Bearer metrics-api-key", GetAttributeString(resourceMetrics.Resource.Attributes, "grpc.metadata.authorization")) +}