diff --git a/apis/projectcontour/v1alpha1/contourconfig.go b/apis/projectcontour/v1alpha1/contourconfig.go index f810a0cfbd0..0375642b75a 100644 --- a/apis/projectcontour/v1alpha1/contourconfig.go +++ b/apis/projectcontour/v1alpha1/contourconfig.go @@ -81,6 +81,9 @@ type ContourConfigurationSpec struct { // Contour's default is { address: "0.0.0.0", port: 8000 }. // +optional Metrics *MetricsConfig `json:"metrics,omitempty"` + + // Tracing defines properties for exporting trace data to OpenTelemetry. + Tracing *TracingConfig `json:"tracing,omitempty"` } // XDSServerType is the type of xDS server implementation. @@ -659,6 +662,56 @@ type RateLimitServiceConfig struct { EnableResourceExhaustedCode *bool `json:"enableResourceExhaustedCode,omitempty"` } +// TracingConfig defines properties for exporting trace data to OpenTelemetry. +type TracingConfig struct { + // IncludePodDetail defines a flag. + // If it is true, contour will add the pod name and namespace to the span of the trace. + // the default is true. + // Note: The Envoy pods MUST have the HOSTNAME and CONTOUR_NAMESPACE environment variables set for this to work properly. + // +optional + IncludePodDetail *bool `json:"includePodDetail,omitempty"` + + // ServiceName defines the name for the service. + // contour's default is contour. + ServiceName *string `json:"serviceName,omitempty"` + + // OverallSampling defines the sampling rate of trace data. + // contour's default is 100. + // +optional + OverallSampling *string `json:"overallSampling,omitempty"` + + // MaxPathTagLength defines maximum length of the request path + // to extract and include in the HttpUrl tag. + // contour's default is 256. + // +optional + MaxPathTagLength *uint32 `json:"maxPathTagLength,omitempty"` + + // CustomTags defines a list of custom tags with unique tag name. + // +optional + CustomTags []*CustomTag `json:"customTags,omitempty"` + + // ExtensionService identifies the extension service defining the otel-collector. + ExtensionService *NamespacedName `json:"extensionService"` +} + +// CustomTag defines custom tags with unique tag name +// to create tags for the active span. +type CustomTag struct { + // TagName is the unique name of the custom tag. + TagName string `json:"tagName"` + + // Literal is a static custom tag value. + // Precisely one of Literal, RequestHeaderName must be set. + // +optional + Literal string `json:"literal,omitempty"` + + // RequestHeaderName indicates which request header + // the label value is obtained from. + // Precisely one of Literal, RequestHeaderName must be set. + // +optional + RequestHeaderName string `json:"requestHeaderName,omitempty"` +} + // PolicyConfig holds default policy used if not explicitly set by the user type PolicyConfig struct { // RequestHeadersPolicy defines the request headers set/removed on all routes diff --git a/apis/projectcontour/v1alpha1/contourconfig_helpers.go b/apis/projectcontour/v1alpha1/contourconfig_helpers.go index 738e73a69d9..57b01ce6105 100644 --- a/apis/projectcontour/v1alpha1/contourconfig_helpers.go +++ b/apis/projectcontour/v1alpha1/contourconfig_helpers.go @@ -15,6 +15,7 @@ package v1alpha1 import ( "fmt" + "strconv" "k8s.io/apimachinery/pkg/util/sets" ) @@ -38,6 +39,9 @@ func (c *ContourConfigurationSpec) Validate() error { if c.Gateway != nil { validateFuncs = append(validateFuncs, c.Gateway.Validate) } + if c.Tracing != nil { + validateFuncs = append(validateFuncs, c.Tracing.Validate) + } for _, validate := range validateFuncs { if err := validate(); err != nil { @@ -48,6 +52,47 @@ func (c *ContourConfigurationSpec) Validate() error { return nil } +func (t *TracingConfig) Validate() error { + if t.ExtensionService == nil { + return fmt.Errorf("tracing.extensionService must be defined") + } + + if t.OverallSampling != nil { + _, err := strconv.ParseFloat(*t.OverallSampling, 64) + if err != nil { + return fmt.Errorf("invalid tracing sampling: %v", err) + } + } + + var customTagNames []string + + for _, customTag := range t.CustomTags { + var fieldCount int + if customTag.TagName == "" { + return fmt.Errorf("tracing.customTag.tagName must be defined") + } + + for _, customTagName := range customTagNames { + if customTagName == customTag.TagName { + return fmt.Errorf("tagName %s is duplicate", customTagName) + } + } + + if customTag.Literal != "" { + fieldCount++ + } + + if customTag.RequestHeaderName != "" { + fieldCount++ + } + if fieldCount != 1 { + return fmt.Errorf("must set exactly one of Literal or RequestHeaderName") + } + customTagNames = append(customTagNames, customTag.TagName) + } + return nil +} + func (x XDSServerType) Validate() error { switch x { case ContourServerType, EnvoyServerType: diff --git a/apis/projectcontour/v1alpha1/contourconfig_helpers_test.go b/apis/projectcontour/v1alpha1/contourconfig_helpers_test.go index 65b8fe7b45c..d9e5f5b3ccc 100644 --- a/apis/projectcontour/v1alpha1/contourconfig_helpers_test.go +++ b/apis/projectcontour/v1alpha1/contourconfig_helpers_test.go @@ -17,6 +17,7 @@ import ( "testing" "github.com/projectcontour/contour/apis/projectcontour/v1alpha1" + "github.com/projectcontour/contour/internal/ref" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -158,6 +159,60 @@ func TestContourConfigurationSpecValidate(t *testing.T) { c.Gateway.GatewayRef = &v1alpha1.NamespacedName{Namespace: "ns", Name: "name"} require.Error(t, c.Validate()) }) + + t.Run("tracing validation", func(t *testing.T) { + c := v1alpha1.ContourConfigurationSpec{ + Tracing: &v1alpha1.TracingConfig{}, + } + + require.Error(t, c.Validate()) + + c.Tracing.ExtensionService = &v1alpha1.NamespacedName{ + Name: "otel-collector", + Namespace: "projectcontour", + } + require.NoError(t, c.Validate()) + + c.Tracing.OverallSampling = ref.To("number") + require.Error(t, c.Validate()) + + c.Tracing.OverallSampling = ref.To("10") + require.NoError(t, c.Validate()) + + customTags := []*v1alpha1.CustomTag{ + { + TagName: "first tag", + Literal: "literal", + }, + } + c.Tracing.CustomTags = customTags + require.NoError(t, c.Validate()) + + customTags = append(customTags, &v1alpha1.CustomTag{ + TagName: "second tag", + RequestHeaderName: "x-custom-header", + }) + c.Tracing.CustomTags = customTags + require.NoError(t, c.Validate()) + + customTags = append(customTags, &v1alpha1.CustomTag{ + TagName: "first tag", + RequestHeaderName: "x-custom-header", + }) + c.Tracing.CustomTags = customTags + require.Error(t, c.Validate()) + + customTags = []*v1alpha1.CustomTag{ + { + TagName: "first tag", + Literal: "literal", + RequestHeaderName: "x-custom-header", + }, + } + c.Tracing.CustomTags = customTags + require.Error(t, c.Validate()) + + }) } func TestSanitizeCipherSuites(t *testing.T) { diff --git a/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go b/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go index 5cc64d74f5f..63ad9b433a0 100644 --- a/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go +++ b/apis/projectcontour/v1alpha1/zz_generated.deepcopy.go @@ -185,6 +185,11 @@ func (in *ContourConfigurationSpec) DeepCopyInto(out *ContourConfigurationSpec) *out = new(MetricsConfig) (*in).DeepCopyInto(*out) } + if in.Tracing != nil { + in, out := &in.Tracing, &out.Tracing + *out = new(TracingConfig) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ContourConfigurationSpec. @@ -363,6 +368,21 @@ func (in *ContourSettings) DeepCopy() *ContourSettings { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CustomTag) DeepCopyInto(out *CustomTag) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CustomTag. +func (in *CustomTag) DeepCopy() *CustomTag { + if in == nil { + return nil + } + out := new(CustomTag) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DaemonSetSettings) DeepCopyInto(out *DaemonSetSettings) { *out = *in @@ -1145,6 +1165,57 @@ func (in *TimeoutParameters) DeepCopy() *TimeoutParameters { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TracingConfig) DeepCopyInto(out *TracingConfig) { + *out = *in + if in.IncludePodDetail != nil { + in, out := &in.IncludePodDetail, &out.IncludePodDetail + *out = new(bool) + **out = **in + } + if in.ServiceName != nil { + in, out := &in.ServiceName, &out.ServiceName + *out = new(string) + **out = **in + } + if in.OverallSampling != nil { + in, out := &in.OverallSampling, &out.OverallSampling + *out = new(string) + **out = **in + } + if in.MaxPathTagLength != nil { + in, out := &in.MaxPathTagLength, &out.MaxPathTagLength + *out = new(uint32) + **out = **in + } + if in.CustomTags != nil { + in, out := &in.CustomTags, &out.CustomTags + *out = make([]*CustomTag, len(*in)) + for i := range *in { + if (*in)[i] != nil { + in, out := &(*in)[i], &(*out)[i] + *out = new(CustomTag) + **out = **in + } + } + } + if in.ExtensionService != nil { + in, out := &in.ExtensionService, &out.ExtensionService + *out = new(NamespacedName) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TracingConfig. +func (in *TracingConfig) DeepCopy() *TracingConfig { + if in == nil { + return nil + } + out := new(TracingConfig) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *XDSServerConfig) DeepCopyInto(out *XDSServerConfig) { *out = *in diff --git a/changelogs/unreleased/5043-yangyy93-major.md b/changelogs/unreleased/5043-yangyy93-major.md new file mode 100644 index 00000000000..96ea4258de6 --- /dev/null +++ b/changelogs/unreleased/5043-yangyy93-major.md @@ -0,0 +1,14 @@ +## Add Tracing Support + +Contour now supports exporting tracing data to [OpenTelemetry][1] + +The Contour configuration file and ContourConfiguration CRD will be extended with a new optional `tracing` section. This configuration block, if present, will enable tracing and will define the trace properties needed to generate and export trace data. + +### Contour supports the following configurations +- Custom service name, the default is `contour`. +- Custom sampling rate, the default is `100`. +- Custom the maximum length of the request path, the default is `256`. +- Customize span tags from literal and request headers. +- Customize whether to include the pod's hostname and namespace. + +[1]: https://opentelemetry.io/ diff --git a/cmd/contour/serve.go b/cmd/contour/serve.go index e2aaaeef8ee..9b9275610b9 100644 --- a/cmd/contour/serve.go +++ b/cmd/contour/serve.go @@ -354,6 +354,10 @@ func (s *Server) doServe() error { ConnectionBalancer: contourConfiguration.Envoy.Listener.ConnectionBalancer, } + if listenerConfig.TracingConfig, err = s.setupTracingService(contourConfiguration.Tracing); err != nil { + return err + } + if listenerConfig.RateLimitConfig, err = s.setupRateLimitService(contourConfiguration); err != nil { return err } @@ -594,6 +598,75 @@ func (s *Server) doServe() error { return s.mgr.Start(signals.SetupSignalHandler()) } +func (s *Server) setupTracingService(tracingConfig *contour_api_v1alpha1.TracingConfig) (*xdscache_v3.TracingConfig, error) { + if tracingConfig == nil { + return nil, nil + } + + // ensure the specified ExtensionService exists + extensionSvc := &contour_api_v1alpha1.ExtensionService{} + key := client.ObjectKey{ + Namespace: tracingConfig.ExtensionService.Namespace, + Name: tracingConfig.ExtensionService.Name, + } + // Using GetAPIReader() here because the manager's caches won't be started yet, + // so reads from the manager's client (which uses the caches for reads) will fail. + if err := s.mgr.GetAPIReader().Get(context.Background(), key, extensionSvc); err != nil { + return nil, fmt.Errorf("error getting tracing extension service %s: %v", key, err) + } + // get the response timeout from the ExtensionService + var responseTimeout timeout.Setting + var err error + + if tp := extensionSvc.Spec.TimeoutPolicy; tp != nil { + responseTimeout, err = timeout.Parse(tp.Response) + if err != nil { + return nil, fmt.Errorf("error parsing tracing extension service %s response timeout: %v", key, err) + } + } + + var sni string + if extensionSvc.Spec.UpstreamValidation != nil { + sni = extensionSvc.Spec.UpstreamValidation.SubjectName + } + + var customTags []*xdscache_v3.CustomTag + + if ref.Val(tracingConfig.IncludePodDetail, true) { + customTags = append(customTags, &xdscache_v3.CustomTag{ + TagName: "podName", + EnvironmentName: "HOSTNAME", + }, &xdscache_v3.CustomTag{ + TagName: "podNamespace", + EnvironmentName: "CONTOUR_NAMESPACE", + }) + } + + for _, customTag := range tracingConfig.CustomTags { + customTags = append(customTags, &xdscache_v3.CustomTag{ + TagName: customTag.TagName, + Literal: customTag.Literal, + RequestHeaderName: customTag.RequestHeaderName, + }) + } + + overallSampling, err := strconv.ParseFloat(ref.Val(tracingConfig.OverallSampling, "100"), 64) + if err != nil || overallSampling == 0 { + overallSampling = 100.0 + } + + return &xdscache_v3.TracingConfig{ + ServiceName: ref.Val(tracingConfig.ServiceName, "contour"), + ExtensionService: key, + SNI: sni, + Timeout: responseTimeout, + OverallSampling: overallSampling, + MaxPathTagLength: ref.Val(tracingConfig.MaxPathTagLength, 256), + CustomTags: customTags, + }, nil + +} + func (s *Server) setupRateLimitService(contourConfiguration contour_api_v1alpha1.ContourConfigurationSpec) (*xdscache_v3.RateLimitConfig, error) { if contourConfiguration.RateLimitService == nil { return nil, nil diff --git a/cmd/contour/servecontext.go b/cmd/contour/servecontext.go index 00b07e7900c..d4a72a74fbb 100644 --- a/cmd/contour/servecontext.go +++ b/cmd/contour/servecontext.go @@ -30,7 +30,6 @@ import ( "github.com/projectcontour/contour/internal/ref" xdscache_v3 "github.com/projectcontour/contour/internal/xdscache/v3" "github.com/projectcontour/contour/pkg/config" - "github.com/sirupsen/logrus" "google.golang.org/grpc" "google.golang.org/grpc/credentials" @@ -366,6 +365,30 @@ func (ctx *serveContext) convertToContourConfigurationSpec() contour_api_v1alpha dnsLookupFamily = contour_api_v1alpha1.AllClusterDNSFamily } + var tracingConfig *contour_api_v1alpha1.TracingConfig + if ctx.Config.Tracing != nil { + namespacedName := k8s.NamespacedNameFrom(ctx.Config.Tracing.ExtensionService) + var customTags []*contour_api_v1alpha1.CustomTag + for _, customTag := range ctx.Config.Tracing.CustomTags { + customTags = append(customTags, &contour_api_v1alpha1.CustomTag{ + TagName: customTag.TagName, + Literal: customTag.Literal, + RequestHeaderName: customTag.RequestHeaderName, + }) + } + tracingConfig = &contour_api_v1alpha1.TracingConfig{ + IncludePodDetail: ctx.Config.Tracing.IncludePodDetail, + ServiceName: ctx.Config.Tracing.ServiceName, + OverallSampling: ctx.Config.Tracing.OverallSampling, + MaxPathTagLength: ctx.Config.Tracing.MaxPathTagLength, + CustomTags: customTags, + ExtensionService: &contour_api_v1alpha1.NamespacedName{ + Name: namespacedName.Name, + Namespace: namespacedName.Namespace, + }, + } + } + var rateLimitService *contour_api_v1alpha1.RateLimitServiceConfig if ctx.Config.RateLimitService.ExtensionService != "" { @@ -538,6 +561,7 @@ func (ctx *serveContext) convertToContourConfigurationSpec() contour_api_v1alpha RateLimitService: rateLimitService, Policy: policy, Metrics: &contourMetrics, + Tracing: tracingConfig, } xdsServerType := contour_api_v1alpha1.ContourServerType diff --git a/cmd/contour/servecontext_test.go b/cmd/contour/servecontext_test.go index cc537a78715..34516f442b9 100644 --- a/cmd/contour/servecontext_test.go +++ b/cmd/contour/servecontext_test.go @@ -24,16 +24,15 @@ import ( "testing" "time" - "github.com/projectcontour/contour/pkg/config" - "github.com/tsaarni/certyaml" - contour_api_v1 "github.com/projectcontour/contour/apis/projectcontour/v1" contour_api_v1alpha1 "github.com/projectcontour/contour/apis/projectcontour/v1alpha1" "github.com/projectcontour/contour/internal/contourconfig" envoy_v3 "github.com/projectcontour/contour/internal/envoy/v3" "github.com/projectcontour/contour/internal/fixture" "github.com/projectcontour/contour/internal/ref" + "github.com/projectcontour/contour/pkg/config" "github.com/stretchr/testify/assert" + "github.com/tsaarni/certyaml" "google.golang.org/grpc" ) @@ -741,6 +740,68 @@ func TestConvertServeContext(t *testing.T) { return cfg }, }, + "tracing config normal": { + getServeContext: func(ctx *serveContext) *serveContext { + ctx.Config.Tracing = &config.Tracing{ + IncludePodDetail: ref.To(false), + ServiceName: ref.To("contour"), + OverallSampling: ref.To("100"), + MaxPathTagLength: ref.To(uint32(256)), + CustomTags: []config.CustomTag{ + { + TagName: "literal", + Literal: "this is literal", + }, + { + TagName: "header", + RequestHeaderName: ":method", + }, + }, + ExtensionService: "otel/otel-collector", + } + return ctx + }, + getContourConfiguration: func(cfg contour_api_v1alpha1.ContourConfigurationSpec) contour_api_v1alpha1.ContourConfigurationSpec { + cfg.Tracing = &contour_api_v1alpha1.TracingConfig{ + IncludePodDetail: ref.To(false), + ServiceName: ref.To("contour"), + OverallSampling: ref.To("100"), + MaxPathTagLength: ref.To(uint32(256)), + CustomTags: []*contour_api_v1alpha1.CustomTag{ + { + TagName: "literal", + Literal: "this is literal", + }, + { + TagName: "header", + RequestHeaderName: ":method", + }, + }, + ExtensionService: &contour_api_v1alpha1.NamespacedName{ + Name: "otel-collector", + Namespace: "otel", + }, + } + return cfg + }, + }, + "tracing config only extensionService": { + getServeContext: func(ctx *serveContext) *serveContext { + ctx.Config.Tracing = &config.Tracing{ + ExtensionService: "otel/otel-collector", + } + return ctx + }, + getContourConfiguration: func(cfg contour_api_v1alpha1.ContourConfigurationSpec) contour_api_v1alpha1.ContourConfigurationSpec { + cfg.Tracing = &contour_api_v1alpha1.TracingConfig{ + ExtensionService: &contour_api_v1alpha1.NamespacedName{ + Name: "otel-collector", + Namespace: "otel", + }, + } + return cfg + }, + }, } for name, tc := range cases { diff --git a/examples/contour/01-crds.yaml b/examples/contour/01-crds.yaml index 6395cd2d569..b3d13356256 100644 --- a/examples/contour/01-crds.yaml +++ b/examples/contour/01-crds.yaml @@ -646,6 +646,69 @@ spec: required: - extensionService type: object + tracing: + description: Tracing defines properties for exporting trace data to + OpenTelemetry. + properties: + customTags: + description: CustomTags defines a list of custom tags with unique + tag name. + items: + description: CustomTag defines custom tags with unique tag name + to create tags for the active span. + properties: + literal: + description: Literal is a static custom tag value. Precisely + one of Literal, RequestHeaderName must be set. + type: string + requestHeaderName: + description: RequestHeaderName indicates which request header + the label value is obtained from. Precisely one of Literal, + RequestHeaderName must be set. + type: string + tagName: + description: TagName is the unique name of the custom tag. + type: string + required: + - tagName + type: object + type: array + extensionService: + description: ExtensionService identifies the extension service + defining the otel-collector. + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + includePodDetail: + description: 'IncludePodDetail defines a flag. If it is true, + contour will add the pod name and namespace to the span of the + trace. the default is true. Note: The Envoy pods MUST have the + HOSTNAME and CONTOUR_NAMESPACE environment variables set for + this to work properly.' + type: boolean + maxPathTagLength: + description: MaxPathTagLength defines maximum length of the request + path to extract and include in the HttpUrl tag. contour's default + is 256. + format: int32 + type: integer + overallSampling: + description: OverallSampling defines the sampling rate of trace + data. contour's default is 100. + type: string + serviceName: + description: ServiceName defines the name for the service. contour's + default is contour. + type: string + required: + - extensionService + type: object xdsServer: description: XDSServer contains parameters for the xDS server. properties: @@ -3755,6 +3818,70 @@ spec: required: - extensionService type: object + tracing: + description: Tracing defines properties for exporting trace data + to OpenTelemetry. + properties: + customTags: + description: CustomTags defines a list of custom tags with + unique tag name. + items: + description: CustomTag defines custom tags with unique tag + name to create tags for the active span. + properties: + literal: + description: Literal is a static custom tag value. Precisely + one of Literal, RequestHeaderName must be set. + type: string + requestHeaderName: + description: RequestHeaderName indicates which request + header the label value is obtained from. Precisely + one of Literal, RequestHeaderName must be set. + type: string + tagName: + description: TagName is the unique name of the custom + tag. + type: string + required: + - tagName + type: object + type: array + extensionService: + description: ExtensionService identifies the extension service + defining the otel-collector. + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + includePodDetail: + description: 'IncludePodDetail defines a flag. If it is true, + contour will add the pod name and namespace to the span + of the trace. the default is true. Note: The Envoy pods + MUST have the HOSTNAME and CONTOUR_NAMESPACE environment + variables set for this to work properly.' + type: boolean + maxPathTagLength: + description: MaxPathTagLength defines maximum length of the + request path to extract and include in the HttpUrl tag. + contour's default is 256. + format: int32 + type: integer + overallSampling: + description: OverallSampling defines the sampling rate of + trace data. contour's default is 100. + type: string + serviceName: + description: ServiceName defines the name for the service. + contour's default is contour. + type: string + required: + - extensionService + type: object xdsServer: description: XDSServer contains parameters for the xDS server. properties: diff --git a/examples/render/contour-deployment.yaml b/examples/render/contour-deployment.yaml index 063e42afef1..14eca24b300 100644 --- a/examples/render/contour-deployment.yaml +++ b/examples/render/contour-deployment.yaml @@ -859,6 +859,69 @@ spec: required: - extensionService type: object + tracing: + description: Tracing defines properties for exporting trace data to + OpenTelemetry. + properties: + customTags: + description: CustomTags defines a list of custom tags with unique + tag name. + items: + description: CustomTag defines custom tags with unique tag name + to create tags for the active span. + properties: + literal: + description: Literal is a static custom tag value. Precisely + one of Literal, RequestHeaderName must be set. + type: string + requestHeaderName: + description: RequestHeaderName indicates which request header + the label value is obtained from. Precisely one of Literal, + RequestHeaderName must be set. + type: string + tagName: + description: TagName is the unique name of the custom tag. + type: string + required: + - tagName + type: object + type: array + extensionService: + description: ExtensionService identifies the extension service + defining the otel-collector. + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + includePodDetail: + description: 'IncludePodDetail defines a flag. If it is true, + contour will add the pod name and namespace to the span of the + trace. the default is true. Note: The Envoy pods MUST have the + HOSTNAME and CONTOUR_NAMESPACE environment variables set for + this to work properly.' + type: boolean + maxPathTagLength: + description: MaxPathTagLength defines maximum length of the request + path to extract and include in the HttpUrl tag. contour's default + is 256. + format: int32 + type: integer + overallSampling: + description: OverallSampling defines the sampling rate of trace + data. contour's default is 100. + type: string + serviceName: + description: ServiceName defines the name for the service. contour's + default is contour. + type: string + required: + - extensionService + type: object xdsServer: description: XDSServer contains parameters for the xDS server. properties: @@ -3968,6 +4031,70 @@ spec: required: - extensionService type: object + tracing: + description: Tracing defines properties for exporting trace data + to OpenTelemetry. + properties: + customTags: + description: CustomTags defines a list of custom tags with + unique tag name. + items: + description: CustomTag defines custom tags with unique tag + name to create tags for the active span. + properties: + literal: + description: Literal is a static custom tag value. Precisely + one of Literal, RequestHeaderName must be set. + type: string + requestHeaderName: + description: RequestHeaderName indicates which request + header the label value is obtained from. Precisely + one of Literal, RequestHeaderName must be set. + type: string + tagName: + description: TagName is the unique name of the custom + tag. + type: string + required: + - tagName + type: object + type: array + extensionService: + description: ExtensionService identifies the extension service + defining the otel-collector. + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + includePodDetail: + description: 'IncludePodDetail defines a flag. If it is true, + contour will add the pod name and namespace to the span + of the trace. the default is true. Note: The Envoy pods + MUST have the HOSTNAME and CONTOUR_NAMESPACE environment + variables set for this to work properly.' + type: boolean + maxPathTagLength: + description: MaxPathTagLength defines maximum length of the + request path to extract and include in the HttpUrl tag. + contour's default is 256. + format: int32 + type: integer + overallSampling: + description: OverallSampling defines the sampling rate of + trace data. contour's default is 100. + type: string + serviceName: + description: ServiceName defines the name for the service. + contour's default is contour. + type: string + required: + - extensionService + type: object xdsServer: description: XDSServer contains parameters for the xDS server. properties: diff --git a/examples/render/contour-gateway-provisioner.yaml b/examples/render/contour-gateway-provisioner.yaml index 6307c404a5e..ce335bd33e0 100644 --- a/examples/render/contour-gateway-provisioner.yaml +++ b/examples/render/contour-gateway-provisioner.yaml @@ -660,6 +660,69 @@ spec: required: - extensionService type: object + tracing: + description: Tracing defines properties for exporting trace data to + OpenTelemetry. + properties: + customTags: + description: CustomTags defines a list of custom tags with unique + tag name. + items: + description: CustomTag defines custom tags with unique tag name + to create tags for the active span. + properties: + literal: + description: Literal is a static custom tag value. Precisely + one of Literal, RequestHeaderName must be set. + type: string + requestHeaderName: + description: RequestHeaderName indicates which request header + the label value is obtained from. Precisely one of Literal, + RequestHeaderName must be set. + type: string + tagName: + description: TagName is the unique name of the custom tag. + type: string + required: + - tagName + type: object + type: array + extensionService: + description: ExtensionService identifies the extension service + defining the otel-collector. + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + includePodDetail: + description: 'IncludePodDetail defines a flag. If it is true, + contour will add the pod name and namespace to the span of the + trace. the default is true. Note: The Envoy pods MUST have the + HOSTNAME and CONTOUR_NAMESPACE environment variables set for + this to work properly.' + type: boolean + maxPathTagLength: + description: MaxPathTagLength defines maximum length of the request + path to extract and include in the HttpUrl tag. contour's default + is 256. + format: int32 + type: integer + overallSampling: + description: OverallSampling defines the sampling rate of trace + data. contour's default is 100. + type: string + serviceName: + description: ServiceName defines the name for the service. contour's + default is contour. + type: string + required: + - extensionService + type: object xdsServer: description: XDSServer contains parameters for the xDS server. properties: @@ -3769,6 +3832,70 @@ spec: required: - extensionService type: object + tracing: + description: Tracing defines properties for exporting trace data + to OpenTelemetry. + properties: + customTags: + description: CustomTags defines a list of custom tags with + unique tag name. + items: + description: CustomTag defines custom tags with unique tag + name to create tags for the active span. + properties: + literal: + description: Literal is a static custom tag value. Precisely + one of Literal, RequestHeaderName must be set. + type: string + requestHeaderName: + description: RequestHeaderName indicates which request + header the label value is obtained from. Precisely + one of Literal, RequestHeaderName must be set. + type: string + tagName: + description: TagName is the unique name of the custom + tag. + type: string + required: + - tagName + type: object + type: array + extensionService: + description: ExtensionService identifies the extension service + defining the otel-collector. + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + includePodDetail: + description: 'IncludePodDetail defines a flag. If it is true, + contour will add the pod name and namespace to the span + of the trace. the default is true. Note: The Envoy pods + MUST have the HOSTNAME and CONTOUR_NAMESPACE environment + variables set for this to work properly.' + type: boolean + maxPathTagLength: + description: MaxPathTagLength defines maximum length of the + request path to extract and include in the HttpUrl tag. + contour's default is 256. + format: int32 + type: integer + overallSampling: + description: OverallSampling defines the sampling rate of + trace data. contour's default is 100. + type: string + serviceName: + description: ServiceName defines the name for the service. + contour's default is contour. + type: string + required: + - extensionService + type: object xdsServer: description: XDSServer contains parameters for the xDS server. properties: diff --git a/examples/render/contour-gateway.yaml b/examples/render/contour-gateway.yaml index 37211a40e61..7645a528a72 100644 --- a/examples/render/contour-gateway.yaml +++ b/examples/render/contour-gateway.yaml @@ -865,6 +865,69 @@ spec: required: - extensionService type: object + tracing: + description: Tracing defines properties for exporting trace data to + OpenTelemetry. + properties: + customTags: + description: CustomTags defines a list of custom tags with unique + tag name. + items: + description: CustomTag defines custom tags with unique tag name + to create tags for the active span. + properties: + literal: + description: Literal is a static custom tag value. Precisely + one of Literal, RequestHeaderName must be set. + type: string + requestHeaderName: + description: RequestHeaderName indicates which request header + the label value is obtained from. Precisely one of Literal, + RequestHeaderName must be set. + type: string + tagName: + description: TagName is the unique name of the custom tag. + type: string + required: + - tagName + type: object + type: array + extensionService: + description: ExtensionService identifies the extension service + defining the otel-collector. + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + includePodDetail: + description: 'IncludePodDetail defines a flag. If it is true, + contour will add the pod name and namespace to the span of the + trace. the default is true. Note: The Envoy pods MUST have the + HOSTNAME and CONTOUR_NAMESPACE environment variables set for + this to work properly.' + type: boolean + maxPathTagLength: + description: MaxPathTagLength defines maximum length of the request + path to extract and include in the HttpUrl tag. contour's default + is 256. + format: int32 + type: integer + overallSampling: + description: OverallSampling defines the sampling rate of trace + data. contour's default is 100. + type: string + serviceName: + description: ServiceName defines the name for the service. contour's + default is contour. + type: string + required: + - extensionService + type: object xdsServer: description: XDSServer contains parameters for the xDS server. properties: @@ -3974,6 +4037,70 @@ spec: required: - extensionService type: object + tracing: + description: Tracing defines properties for exporting trace data + to OpenTelemetry. + properties: + customTags: + description: CustomTags defines a list of custom tags with + unique tag name. + items: + description: CustomTag defines custom tags with unique tag + name to create tags for the active span. + properties: + literal: + description: Literal is a static custom tag value. Precisely + one of Literal, RequestHeaderName must be set. + type: string + requestHeaderName: + description: RequestHeaderName indicates which request + header the label value is obtained from. Precisely + one of Literal, RequestHeaderName must be set. + type: string + tagName: + description: TagName is the unique name of the custom + tag. + type: string + required: + - tagName + type: object + type: array + extensionService: + description: ExtensionService identifies the extension service + defining the otel-collector. + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + includePodDetail: + description: 'IncludePodDetail defines a flag. If it is true, + contour will add the pod name and namespace to the span + of the trace. the default is true. Note: The Envoy pods + MUST have the HOSTNAME and CONTOUR_NAMESPACE environment + variables set for this to work properly.' + type: boolean + maxPathTagLength: + description: MaxPathTagLength defines maximum length of the + request path to extract and include in the HttpUrl tag. + contour's default is 256. + format: int32 + type: integer + overallSampling: + description: OverallSampling defines the sampling rate of + trace data. contour's default is 100. + type: string + serviceName: + description: ServiceName defines the name for the service. + contour's default is contour. + type: string + required: + - extensionService + type: object xdsServer: description: XDSServer contains parameters for the xDS server. properties: diff --git a/examples/render/contour.yaml b/examples/render/contour.yaml index 3089ef047a7..d5ed976b944 100644 --- a/examples/render/contour.yaml +++ b/examples/render/contour.yaml @@ -859,6 +859,69 @@ spec: required: - extensionService type: object + tracing: + description: Tracing defines properties for exporting trace data to + OpenTelemetry. + properties: + customTags: + description: CustomTags defines a list of custom tags with unique + tag name. + items: + description: CustomTag defines custom tags with unique tag name + to create tags for the active span. + properties: + literal: + description: Literal is a static custom tag value. Precisely + one of Literal, RequestHeaderName must be set. + type: string + requestHeaderName: + description: RequestHeaderName indicates which request header + the label value is obtained from. Precisely one of Literal, + RequestHeaderName must be set. + type: string + tagName: + description: TagName is the unique name of the custom tag. + type: string + required: + - tagName + type: object + type: array + extensionService: + description: ExtensionService identifies the extension service + defining the otel-collector. + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + includePodDetail: + description: 'IncludePodDetail defines a flag. If it is true, + contour will add the pod name and namespace to the span of the + trace. the default is true. Note: The Envoy pods MUST have the + HOSTNAME and CONTOUR_NAMESPACE environment variables set for + this to work properly.' + type: boolean + maxPathTagLength: + description: MaxPathTagLength defines maximum length of the request + path to extract and include in the HttpUrl tag. contour's default + is 256. + format: int32 + type: integer + overallSampling: + description: OverallSampling defines the sampling rate of trace + data. contour's default is 100. + type: string + serviceName: + description: ServiceName defines the name for the service. contour's + default is contour. + type: string + required: + - extensionService + type: object xdsServer: description: XDSServer contains parameters for the xDS server. properties: @@ -3968,6 +4031,70 @@ spec: required: - extensionService type: object + tracing: + description: Tracing defines properties for exporting trace data + to OpenTelemetry. + properties: + customTags: + description: CustomTags defines a list of custom tags with + unique tag name. + items: + description: CustomTag defines custom tags with unique tag + name to create tags for the active span. + properties: + literal: + description: Literal is a static custom tag value. Precisely + one of Literal, RequestHeaderName must be set. + type: string + requestHeaderName: + description: RequestHeaderName indicates which request + header the label value is obtained from. Precisely + one of Literal, RequestHeaderName must be set. + type: string + tagName: + description: TagName is the unique name of the custom + tag. + type: string + required: + - tagName + type: object + type: array + extensionService: + description: ExtensionService identifies the extension service + defining the otel-collector. + properties: + name: + type: string + namespace: + type: string + required: + - name + - namespace + type: object + includePodDetail: + description: 'IncludePodDetail defines a flag. If it is true, + contour will add the pod name and namespace to the span + of the trace. the default is true. Note: The Envoy pods + MUST have the HOSTNAME and CONTOUR_NAMESPACE environment + variables set for this to work properly.' + type: boolean + maxPathTagLength: + description: MaxPathTagLength defines maximum length of the + request path to extract and include in the HttpUrl tag. + contour's default is 256. + format: int32 + type: integer + overallSampling: + description: OverallSampling defines the sampling rate of + trace data. contour's default is 100. + type: string + serviceName: + description: ServiceName defines the name for the service. + contour's default is contour. + type: string + required: + - extensionService + type: object xdsServer: description: XDSServer contains parameters for the xDS server. properties: diff --git a/examples/tracing/01-opentelemetry-collector.yaml b/examples/tracing/01-opentelemetry-collector.yaml new file mode 100644 index 00000000000..4b56f2cee2b --- /dev/null +++ b/examples/tracing/01-opentelemetry-collector.yaml @@ -0,0 +1,82 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: otel-collector + name: otel-collector + namespace: projectcontour +spec: + selector: + matchLabels: + app: otel-collector + template: + metadata: + labels: + app: otel-collector + spec: + containers: + - args: + - --config=/etc/otel-collector-config.yaml + image: otel/opentelemetry-collector:latest + imagePullPolicy: IfNotPresent + name: otel-collector + ports: + - containerPort: 4317 + name: grpc + protocol: TCP + volumeMounts: + - name: config + mountPath: /etc/otel-collector-config.yaml + subPath: otel-collector-config.yaml + volumes: + - name: config + configMap: + name: otel-collector-config + restartPolicy: Always + +--- + +apiVersion: v1 +kind: Service +metadata: + name: otel-collector + namespace: projectcontour + labels: + app: otel-collector +spec: + ports: + - port: 4317 + targetPort: grpc + protocol: TCP + name: grpc + selector: + app: otel-collector + type: ClusterIP + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: otel-collector-config + namespace: projectcontour + labels: + app: otel-collector +data: + otel-collector-config.yaml: |+ + receivers: + otlp: + protocols: + grpc: + http: + + exporters: + logging: + loglevel: debug + + service: + pipelines: + traces: + receivers: [otlp] + exporters: [logging] + diff --git a/examples/tracing/02-opentelemetry-extsvc.yaml b/examples/tracing/02-opentelemetry-extsvc.yaml new file mode 100644 index 00000000000..4c3d4a6eddf --- /dev/null +++ b/examples/tracing/02-opentelemetry-extsvc.yaml @@ -0,0 +1,10 @@ +apiVersion: projectcontour.io/v1alpha1 +kind: ExtensionService +metadata: + namespace: projectcontour + name: otel-collector +spec: + protocol: h2c + services: + - name: otel-collector + port: 4317 diff --git a/examples/tracing/03-contour-config.yaml b/examples/tracing/03-contour-config.yaml new file mode 100644 index 00000000000..ce0f1831c4c --- /dev/null +++ b/examples/tracing/03-contour-config.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: contour + namespace: projectcontour +data: + contour.yaml: | + tracing: + # Whether to send the namespace and instance where envoy is located to open, the default is true. + includePodDetail: true + # The extensionService and namespace and name defined above in the format of namespace/name. + extensionService: projectcontour/otel-collector + # The service name that envoy sends to openTelemetry-collector, the default is contour. + serviceName: some-service-name + # A custom set of tags. + customTags: + # envoy will send the tagName to the collector. + - tagName: custom-tag + # fixed tag value. + literal: foo + - tagName: header-tag + # The tag value obtained from the request header, + # if the request header does not exist, this tag will not be sent. + requestHeaderName: X-Custom-Header diff --git a/internal/envoy/v3/listener.go b/internal/envoy/v3/listener.go index f4a2caef4ae..5629f17ec86 100644 --- a/internal/envoy/v3/listener.go +++ b/internal/envoy/v3/listener.go @@ -165,6 +165,7 @@ type httpConnectionManagerBuilder struct { serverHeaderTransformation http.HttpConnectionManager_ServerHeaderTransformation forwardClientCertificate *dag.ClientCertificateDetails numTrustedHops uint32 + tracingConfig *http.HttpConnectionManager_Tracing } // RouteConfigName sets the name of the RDS element that contains @@ -401,6 +402,14 @@ func (b *httpConnectionManagerBuilder) AddFilter(f *http.HttpFilter) *httpConnec return b } +func (b *httpConnectionManagerBuilder) Tracing(tracing *http.HttpConnectionManager_Tracing) *httpConnectionManagerBuilder { + if tracing == nil { + return b + } + b.tracingConfig = tracing + return b +} + // Validate runs builtin validation rules against the current builder state. func (b *httpConnectionManagerBuilder) Validate() error { @@ -442,6 +451,7 @@ func (b *httpConnectionManagerBuilder) Get() *envoy_listener_v3.Filter { ConfigSource: ConfigSource("contour"), }, }, + Tracing: b.tracingConfig, HttpFilters: b.filters, CommonHttpProtocolOptions: &envoy_core_v3.HttpProtocolOptions{ IdleTimeout: envoy.Timeout(b.connectionIdleTimeout), diff --git a/internal/envoy/v3/tracing.go b/internal/envoy/v3/tracing.go new file mode 100644 index 00000000000..ffd44084968 --- /dev/null +++ b/internal/envoy/v3/tracing.go @@ -0,0 +1,112 @@ +// Copyright Project Contour Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v3 + +import ( + envoy_config_trace_v3 "github.com/envoyproxy/go-control-plane/envoy/config/trace/v3" + http "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + envoy_trace_v3 "github.com/envoyproxy/go-control-plane/envoy/type/tracing/v3" + envoy_type "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "github.com/projectcontour/contour/internal/dag" + "github.com/projectcontour/contour/internal/protobuf" + "github.com/projectcontour/contour/internal/timeout" + "google.golang.org/protobuf/types/known/wrapperspb" + "k8s.io/apimachinery/pkg/types" +) + +// TracingConfig returns a tracing config, +// or nil if config is nil. +func TracingConfig(tracing *EnvoyTracingConfig) *http.HttpConnectionManager_Tracing { + if tracing == nil { + return nil + } + + var customTags []*envoy_trace_v3.CustomTag + for _, tag := range tracing.CustomTags { + if traceCustomTag := customTag(tag); traceCustomTag != nil { + customTags = append(customTags, traceCustomTag) + } + } + + return &http.HttpConnectionManager_Tracing{ + OverallSampling: &envoy_type.Percent{ + Value: tracing.OverallSampling, + }, + MaxPathTagLength: wrapperspb.UInt32(tracing.MaxPathTagLength), + CustomTags: customTags, + Provider: &envoy_config_trace_v3.Tracing_Http{ + Name: "envoy.tracers.opentelemetry", + ConfigType: &envoy_config_trace_v3.Tracing_Http_TypedConfig{ + TypedConfig: protobuf.MustMarshalAny(&envoy_config_trace_v3.OpenTelemetryConfig{ + GrpcService: GrpcService(dag.ExtensionClusterName(tracing.ExtensionService), tracing.SNI, tracing.Timeout), + ServiceName: tracing.ServiceName, + }), + }, + }, + } +} + +func customTag(tag *CustomTag) *envoy_trace_v3.CustomTag { + if tag == nil { + return nil + } + if tag.Literal != "" { + return &envoy_trace_v3.CustomTag{ + Tag: tag.TagName, + Type: &envoy_trace_v3.CustomTag_Literal_{ + Literal: &envoy_trace_v3.CustomTag_Literal{ + Value: tag.Literal, + }, + }, + } + } + if tag.EnvironmentName != "" { + return &envoy_trace_v3.CustomTag{ + Tag: tag.TagName, + Type: &envoy_trace_v3.CustomTag_Environment_{ + Environment: &envoy_trace_v3.CustomTag_Environment{ + Name: tag.EnvironmentName, + }, + }, + } + } + if tag.RequestHeaderName != "" { + return &envoy_trace_v3.CustomTag{ + Tag: tag.TagName, + Type: &envoy_trace_v3.CustomTag_RequestHeader{ + RequestHeader: &envoy_trace_v3.CustomTag_Header{ + Name: tag.RequestHeaderName, + }, + }, + } + } + return nil +} + +type EnvoyTracingConfig struct { + ExtensionService types.NamespacedName + ServiceName string + SNI string + Timeout timeout.Setting + OverallSampling float64 + MaxPathTagLength uint32 + CustomTags []*CustomTag +} + +type CustomTag struct { + TagName string + Literal string + EnvironmentName string + RequestHeaderName string +} diff --git a/internal/envoy/v3/tracing_test.go b/internal/envoy/v3/tracing_test.go new file mode 100644 index 00000000000..72cffe7269e --- /dev/null +++ b/internal/envoy/v3/tracing_test.go @@ -0,0 +1,192 @@ +// Copyright Project Contour Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v3 + +import ( + "testing" + "time" + + envoy_core_v3 "github.com/envoyproxy/go-control-plane/envoy/config/core/v3" + envoy_config_trace_v3 "github.com/envoyproxy/go-control-plane/envoy/config/trace/v3" + http "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3" + envoy_trace_v3 "github.com/envoyproxy/go-control-plane/envoy/type/tracing/v3" + envoy_type_v3 "github.com/envoyproxy/go-control-plane/envoy/type/v3" + "github.com/projectcontour/contour/internal/k8s" + "github.com/projectcontour/contour/internal/protobuf" + "github.com/projectcontour/contour/internal/timeout" + "github.com/stretchr/testify/assert" + "google.golang.org/protobuf/types/known/durationpb" + "google.golang.org/protobuf/types/known/wrapperspb" +) + +func TestTracingConfig(t *testing.T) { + tests := map[string]struct { + tracing *EnvoyTracingConfig + want *http.HttpConnectionManager_Tracing + }{ + "nil config": { + tracing: nil, + want: nil, + }, + "normal config": { + tracing: &EnvoyTracingConfig{ + ExtensionService: k8s.NamespacedNameFrom("projectcontour/otel-collector"), + ServiceName: "contour", + SNI: "some-server.com", + Timeout: timeout.DurationSetting(5 * time.Second), + OverallSampling: 100, + MaxPathTagLength: 256, + CustomTags: []*CustomTag{ + { + TagName: "literal", + Literal: "this is literal", + }, + { + TagName: "podName", + EnvironmentName: "HOSTNAME", + }, + { + TagName: "requestHeaderName", + RequestHeaderName: ":path", + }, + }, + }, + want: &http.HttpConnectionManager_Tracing{ + OverallSampling: &envoy_type_v3.Percent{ + Value: 100.0, + }, + MaxPathTagLength: wrapperspb.UInt32(256), + CustomTags: []*envoy_trace_v3.CustomTag{ + { + Tag: "literal", + Type: &envoy_trace_v3.CustomTag_Literal_{ + Literal: &envoy_trace_v3.CustomTag_Literal{ + Value: "this is literal", + }, + }, + }, + { + Tag: "podName", + Type: &envoy_trace_v3.CustomTag_Environment_{ + Environment: &envoy_trace_v3.CustomTag_Environment{ + Name: "HOSTNAME", + }, + }, + }, + { + Tag: "requestHeaderName", + Type: &envoy_trace_v3.CustomTag_RequestHeader{ + RequestHeader: &envoy_trace_v3.CustomTag_Header{ + Name: ":path", + }, + }, + }, + }, + Provider: &envoy_config_trace_v3.Tracing_Http{ + Name: "envoy.tracers.opentelemetry", + ConfigType: &envoy_config_trace_v3.Tracing_Http_TypedConfig{ + TypedConfig: protobuf.MustMarshalAny(&envoy_config_trace_v3.OpenTelemetryConfig{ + GrpcService: &envoy_core_v3.GrpcService{ + TargetSpecifier: &envoy_core_v3.GrpcService_EnvoyGrpc_{ + EnvoyGrpc: &envoy_core_v3.GrpcService_EnvoyGrpc{ + ClusterName: "extension/projectcontour/otel-collector", + Authority: "some-server.com", + }, + }, + Timeout: durationpb.New(5 * time.Second), + }, + ServiceName: "contour", + }), + }, + }, + }, + }, + "no custom tag": { + tracing: &EnvoyTracingConfig{ + ExtensionService: k8s.NamespacedNameFrom("projectcontour/otel-collector"), + ServiceName: "contour", + SNI: "some-server.com", + Timeout: timeout.DurationSetting(5 * time.Second), + OverallSampling: 100, + MaxPathTagLength: 256, + CustomTags: nil, + }, + want: &http.HttpConnectionManager_Tracing{ + OverallSampling: &envoy_type_v3.Percent{ + Value: 100.0, + }, + MaxPathTagLength: wrapperspb.UInt32(256), + CustomTags: nil, + Provider: &envoy_config_trace_v3.Tracing_Http{ + Name: "envoy.tracers.opentelemetry", + ConfigType: &envoy_config_trace_v3.Tracing_Http_TypedConfig{ + TypedConfig: protobuf.MustMarshalAny(&envoy_config_trace_v3.OpenTelemetryConfig{ + GrpcService: &envoy_core_v3.GrpcService{ + TargetSpecifier: &envoy_core_v3.GrpcService_EnvoyGrpc_{ + EnvoyGrpc: &envoy_core_v3.GrpcService_EnvoyGrpc{ + ClusterName: "extension/projectcontour/otel-collector", + Authority: "some-server.com", + }, + }, + Timeout: durationpb.New(5 * time.Second), + }, + ServiceName: "contour", + }), + }, + }, + }, + }, + "no SNI set": { + tracing: &EnvoyTracingConfig{ + ExtensionService: k8s.NamespacedNameFrom("projectcontour/otel-collector"), + ServiceName: "contour", + SNI: "", + Timeout: timeout.DurationSetting(5 * time.Second), + OverallSampling: 100, + MaxPathTagLength: 256, + CustomTags: nil, + }, + want: &http.HttpConnectionManager_Tracing{ + OverallSampling: &envoy_type_v3.Percent{ + Value: 100.0, + }, + MaxPathTagLength: wrapperspb.UInt32(256), + CustomTags: nil, + Provider: &envoy_config_trace_v3.Tracing_Http{ + Name: "envoy.tracers.opentelemetry", + ConfigType: &envoy_config_trace_v3.Tracing_Http_TypedConfig{ + TypedConfig: protobuf.MustMarshalAny(&envoy_config_trace_v3.OpenTelemetryConfig{ + GrpcService: &envoy_core_v3.GrpcService{ + TargetSpecifier: &envoy_core_v3.GrpcService_EnvoyGrpc_{ + EnvoyGrpc: &envoy_core_v3.GrpcService_EnvoyGrpc{ + ClusterName: "extension/projectcontour/otel-collector", + Authority: "extension.projectcontour.otel-collector", + }, + }, + Timeout: durationpb.New(5 * time.Second), + }, + ServiceName: "contour", + }), + }, + }, + }, + }, + } + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + got := TracingConfig(tc.tracing) + assert.Equal(t, tc.want, got) + }) + } +} diff --git a/internal/featuretests/v3/tracing_test.go b/internal/featuretests/v3/tracing_test.go new file mode 100644 index 00000000000..ca339dca092 --- /dev/null +++ b/internal/featuretests/v3/tracing_test.go @@ -0,0 +1,131 @@ +// Copyright Project Contour Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package v3 + +import ( + "testing" + + envoy_discovery_v3 "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v3" + contour_api_v1 "github.com/projectcontour/contour/apis/projectcontour/v1" + "github.com/projectcontour/contour/apis/projectcontour/v1alpha1" + envoy_v3 "github.com/projectcontour/contour/internal/envoy/v3" + "github.com/projectcontour/contour/internal/featuretests" + "github.com/projectcontour/contour/internal/fixture" + "github.com/projectcontour/contour/internal/k8s" + "github.com/projectcontour/contour/internal/ref" + "github.com/projectcontour/contour/internal/timeout" + xdscache_v3 "github.com/projectcontour/contour/internal/xdscache/v3" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestTracing(t *testing.T) { + tracingConfig := &xdscache_v3.TracingConfig{ + ExtensionService: k8s.NamespacedNameFrom("projectcontour/otel-collector"), + ServiceName: "contour", + Timeout: timeout.DefaultSetting(), + OverallSampling: 100, + MaxPathTagLength: 256, + } + withTrace := func(conf *xdscache_v3.ListenerConfig) { + conf.TracingConfig = tracingConfig + } + rh, c, done := setup(t, withTrace) + defer done() + + rh.OnAdd(fixture.NewService("projectcontour/otel-collector"). + WithPorts(corev1.ServicePort{Port: 4317})) + + rh.OnAdd(featuretests.Endpoints("projectcontour", "otel-collector", corev1.EndpointSubset{ + Addresses: featuretests.Addresses("10.244.41.241"), + Ports: featuretests.Ports(featuretests.Port("", 4317)), + })) + + rh.OnAdd(&v1alpha1.ExtensionService{ + ObjectMeta: fixture.ObjectMeta("projectcontour/otel-collector"), + Spec: v1alpha1.ExtensionServiceSpec{ + Services: []v1alpha1.ExtensionServiceTarget{ + {Name: "otel-collector", Port: 4317}, + }, + Protocol: ref.To("h2c"), + TimeoutPolicy: &contour_api_v1.TimeoutPolicy{ + Response: defaultResponseTimeout.String(), + }, + }, + }) + + rh.OnAdd(fixture.NewService("projectcontour/app-server"). + WithPorts(corev1.ServicePort{Port: 80})) + + rh.OnAdd(featuretests.Endpoints("projectcontour", "app-server", corev1.EndpointSubset{ + Addresses: featuretests.Addresses("10.244.184.102"), + Ports: featuretests.Ports(featuretests.Port("", 80)), + })) + + p := &contour_api_v1.HTTPProxy{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "projectcontour", + Name: "app-server", + }, + Spec: contour_api_v1.HTTPProxySpec{ + VirtualHost: &contour_api_v1.VirtualHost{ + Fqdn: "foo.com", + }, + Routes: []contour_api_v1.Route{ + { + Services: []contour_api_v1.Service{ + { + Name: "app-server", + Port: 80, + }, + }, + }, + }, + }, + } + rh.OnAdd(p) + + httpListener := defaultHTTPListener() + httpListener.FilterChains = envoy_v3.FilterChains(envoy_v3.HTTPConnectionManagerBuilder(). + RouteConfigName(xdscache_v3.ENVOY_HTTP_LISTENER). + MetricsPrefix(xdscache_v3.ENVOY_HTTP_LISTENER). + AccessLoggers(envoy_v3.FileAccessLogEnvoy(xdscache_v3.DEFAULT_HTTP_ACCESS_LOG, "", nil, v1alpha1.LogLevelInfo)). + DefaultFilters(). + Tracing(envoy_v3.TracingConfig(&envoy_v3.EnvoyTracingConfig{ + ExtensionService: tracingConfig.ExtensionService, + ServiceName: tracingConfig.ServiceName, + Timeout: tracingConfig.Timeout, + OverallSampling: tracingConfig.OverallSampling, + MaxPathTagLength: tracingConfig.MaxPathTagLength, + })). + Get(), + ) + + c.Request(listenerType, xdscache_v3.ENVOY_HTTP_LISTENER).Equals(&envoy_discovery_v3.DiscoveryResponse{ + TypeUrl: listenerType, + Resources: resources(t, httpListener), + }) + + c.Request(clusterType).Equals(&envoy_discovery_v3.DiscoveryResponse{ + TypeUrl: clusterType, + Resources: resources(t, + DefaultCluster( + h2cCluster(cluster("extension/projectcontour/otel-collector", "extension/projectcontour/otel-collector", "extension_projectcontour_otel-collector")), + ), + DefaultCluster( + cluster("projectcontour/app-server/80/da39a3ee5e", "projectcontour/app-server", "projectcontour_app-server_80"), + ), + ), + }) +} diff --git a/internal/xdscache/v3/listener.go b/internal/xdscache/v3/listener.go index 6a24fcddc45..cc986830120 100644 --- a/internal/xdscache/v3/listener.go +++ b/internal/xdscache/v3/listener.go @@ -128,6 +128,42 @@ type ListenerConfig struct { // GlobalExternalAuthConfig optionally configures the global external authorization Service to be // used. GlobalExternalAuthConfig *GlobalExternalAuthConfig + + // TracingConfig optionally configures the tracing collector Service to be + // used. + TracingConfig *TracingConfig +} + +type TracingConfig struct { + ExtensionService types.NamespacedName + + ServiceName string + + SNI string + + Timeout timeout.Setting + + OverallSampling float64 + + MaxPathTagLength uint32 + + CustomTags []*CustomTag +} + +type CustomTag struct { + // TagName is the unique name of the custom tag. + TagName string + + // Literal is a static custom tag value. + Literal string + + // EnvironmentName indicates that the label value is obtained + // from the environment variable. + EnvironmentName string + + // RequestHeaderName indicates which request header + // the label value is obtained from. + RequestHeaderName string } type RateLimitConfig struct { @@ -331,6 +367,7 @@ func (c *ListenerCache) OnChange(root *dag.DAG) { MergeSlashes(cfg.MergeSlashes). ServerHeaderTransformation(cfg.ServerHeaderTransformation). NumTrustedHops(cfg.XffNumTrustedHops). + Tracing(envoy_v3.TracingConfig(envoyTracingConfig(cfg.TracingConfig))). AddFilter(envoy_v3.GlobalRateLimitFilter(envoyGlobalRateLimitConfig(cfg.RateLimitConfig))). AddFilter(httpGlobalExternalAuthConfig(cfg.GlobalExternalAuthConfig)). Get() @@ -398,6 +435,7 @@ func (c *ListenerCache) OnChange(root *dag.DAG) { MergeSlashes(cfg.MergeSlashes). ServerHeaderTransformation(cfg.ServerHeaderTransformation). NumTrustedHops(cfg.XffNumTrustedHops). + Tracing(envoy_v3.TracingConfig(envoyTracingConfig(cfg.TracingConfig))). AddFilter(envoy_v3.GlobalRateLimitFilter(envoyGlobalRateLimitConfig(cfg.RateLimitConfig))). ForwardClientCertificate(forwardClientCertificate). Get() @@ -461,6 +499,7 @@ func (c *ListenerCache) OnChange(root *dag.DAG) { MergeSlashes(cfg.MergeSlashes). ServerHeaderTransformation(cfg.ServerHeaderTransformation). NumTrustedHops(cfg.XffNumTrustedHops). + Tracing(envoy_v3.TracingConfig(envoyTracingConfig(cfg.TracingConfig))). AddFilter(envoy_v3.GlobalRateLimitFilter(envoyGlobalRateLimitConfig(cfg.RateLimitConfig))). ForwardClientCertificate(forwardClientCertificate). Get() @@ -531,6 +570,38 @@ func envoyGlobalRateLimitConfig(config *RateLimitConfig) *envoy_v3.GlobalRateLim } } +func envoyTracingConfig(config *TracingConfig) *envoy_v3.EnvoyTracingConfig { + if config == nil { + return nil + } + + return &envoy_v3.EnvoyTracingConfig{ + ExtensionService: config.ExtensionService, + ServiceName: config.ServiceName, + SNI: config.SNI, + Timeout: config.Timeout, + OverallSampling: config.OverallSampling, + MaxPathTagLength: config.MaxPathTagLength, + CustomTags: envoyTracingConfigCustomTag(config.CustomTags), + } +} + +func envoyTracingConfigCustomTag(tags []*CustomTag) []*envoy_v3.CustomTag { + if tags == nil { + return nil + } + var customTags = make([]*envoy_v3.CustomTag, len(tags)) + for i, tag := range tags { + customTags[i] = &envoy_v3.CustomTag{ + TagName: tag.TagName, + Literal: tag.Literal, + EnvironmentName: tag.EnvironmentName, + RequestHeaderName: tag.RequestHeaderName, + } + } + return customTags +} + func proxyProtocol(useProxy bool) []*envoy_listener_v3.ListenerFilter { if useProxy { return envoy_v3.ListenerFilters( diff --git a/pkg/config/parameters.go b/pkg/config/parameters.go index d3235b002e9..78ed5d4aaf8 100644 --- a/pkg/config/parameters.go +++ b/pkg/config/parameters.go @@ -560,6 +560,54 @@ type Parameters struct { // MetricsParameters holds configurable parameters for Contour and Envoy metrics. Metrics MetricsParameters `yaml:"metrics,omitempty"` + + // Tracing holds the relevant configuration for exporting trace data to OpenTelemetry. + Tracing *Tracing `yaml:"tracing,omitempty"` +} + +// Tracing defines properties for exporting trace data to OpenTelemetry. +type Tracing struct { + // IncludePodDetail defines a flag. + // If it is true, contour will add the pod name and namespace to the span of the trace. + // the default is true. + // Note: The Envoy pods MUST have the HOSTNAME and CONTOUR_NAMESPACE environment variables set for this to work properly. + IncludePodDetail *bool `yaml:"includePodDetail,omitempty"` + + // ServiceName defines the name for the service + // contour's default is contour. + ServiceName *string `yaml:"serviceName,omitempty"` + + // OverallSampling defines the sampling rate of trace data. + // the default value is 100. + OverallSampling *string `yaml:"overallSampling,omitempty"` + + // MaxPathTagLength defines maximum length of the request path + // to extract and include in the HttpUrl tag. + // the default value is 256. + MaxPathTagLength *uint32 `yaml:"maxPathTagLength,omitempty"` + + // CustomTags defines a list of custom tags with unique tag name. + CustomTags []CustomTag `yaml:"customTags,omitempty"` + + // ExtensionService identifies the extension service defining the otel-collector, + // formatted as /. + ExtensionService string `yaml:"extensionService"` +} + +// CustomTag defines custom tags with unique tag name +// to create tags for the active span. +type CustomTag struct { + // TagName is the unique name of the custom tag. + TagName string `yaml:"tagName"` + + // Literal is a static custom tag value. + // Precisely one of Literal, RequestHeaderName must be set. + Literal string `yaml:"literal,omitempty"` + + // RequestHeaderName indicates which request header + // the label value is obtained from. + // Precisely one of Literal, RequestHeaderName must be set. + RequestHeaderName string `yaml:"requestHeaderName,omitempty"` } // GlobalExternalAuthorizationConfig defines properties of global external authorization. @@ -689,6 +737,44 @@ func (p *MetricsParameters) Validate() error { return nil } +func (t *Tracing) Validate() error { + if t == nil { + return nil + } + + if t.ExtensionService == "" { + return errors.New("tracing.extensionService must be defined") + } + + var customTagNames []string + + for _, customTag := range t.CustomTags { + var fieldCount int + if customTag.TagName == "" { + return errors.New("tracing.customTag.tagName must be defined") + } + + for _, customTagName := range customTagNames { + if customTagName == customTag.TagName { + return fmt.Errorf("tagName %s is duplicate", customTagName) + } + } + + if customTag.Literal != "" { + fieldCount++ + } + + if customTag.RequestHeaderName != "" { + fieldCount++ + } + if fieldCount != 1 { + return errors.New("must set exactly one of Literal or RequestHeaderName") + } + customTagNames = append(customTagNames, customTag.TagName) + } + return nil +} + func (p *MetricsServerParameters) Validate() error { // Check that both certificate and key are provided if either one is provided. if (p.ServerCert != "") != (p.ServerKey != "") { @@ -770,6 +856,10 @@ func (p *Parameters) Validate() error { return err } + if err := p.Tracing.Validate(); err != nil { + return err + } + return p.Listener.Validate() } diff --git a/pkg/config/parameters_test.go b/pkg/config/parameters_test.go index d136eea3467..ba262651963 100644 --- a/pkg/config/parameters_test.go +++ b/pkg/config/parameters_test.go @@ -18,6 +18,7 @@ import ( "strings" "testing" + "github.com/projectcontour/contour/internal/ref" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gopkg.in/yaml.v3" @@ -492,3 +493,80 @@ func TestListenerValidation(t *testing.T) { } require.Error(t, l.Validate()) } + +func TestTracingConfigValidation(t *testing.T) { + var trace *Tracing + require.NoError(t, trace.Validate()) + + trace = &Tracing{ + IncludePodDetail: ref.To(false), + ServiceName: ref.To("contour"), + OverallSampling: ref.To("100"), + MaxPathTagLength: ref.To(uint32(256)), + CustomTags: nil, + ExtensionService: "projectcontour/otel-collector", + } + require.NoError(t, trace.Validate()) + + trace = &Tracing{ + IncludePodDetail: ref.To(false), + ServiceName: ref.To("contour"), + OverallSampling: ref.To("100"), + MaxPathTagLength: ref.To(uint32(256)), + CustomTags: nil, + } + require.Error(t, trace.Validate()) + + trace = &Tracing{ + IncludePodDetail: ref.To(false), + OverallSampling: ref.To("100"), + MaxPathTagLength: ref.To(uint32(256)), + CustomTags: nil, + ExtensionService: "projectcontour/otel-collector", + } + require.NoError(t, trace.Validate()) + + trace = &Tracing{ + OverallSampling: ref.To("100"), + MaxPathTagLength: ref.To(uint32(256)), + CustomTags: []CustomTag{ + { + TagName: "first", + Literal: "literal", + RequestHeaderName: ":path", + }, + }, + ExtensionService: "projectcontour/otel-collector", + } + require.Error(t, trace.Validate()) + + trace = &Tracing{ + OverallSampling: ref.To("100"), + MaxPathTagLength: ref.To(uint32(256)), + CustomTags: []CustomTag{ + { + Literal: "literal", + }, + }, + ExtensionService: "projectcontour/otel-collector", + } + require.Error(t, trace.Validate()) + + trace = &Tracing{ + IncludePodDetail: ref.To(true), + OverallSampling: ref.To("100"), + MaxPathTagLength: ref.To(uint32(256)), + CustomTags: []CustomTag{ + { + TagName: "first", + Literal: "literal", + }, + { + TagName: "first", + RequestHeaderName: ":path", + }, + }, + ExtensionService: "projectcontour/otel-collector", + } + require.Error(t, trace.Validate()) +} diff --git a/site/content/docs/main/config/api-reference.html b/site/content/docs/main/config/api-reference.html index 9905210fff6..14e8169f9af 100644 --- a/site/content/docs/main/config/api-reference.html +++ b/site/content/docs/main/config/api-reference.html @@ -4790,6 +4790,20 @@

ContourConfiguration

Contour’s default is { address: “0.0.0.0”, port: 8000 }.

+ + +tracing +
+ + +TracingConfig + + + + +

Tracing defines properties for exporting trace data to OpenTelemetry.

+ + @@ -5484,6 +5498,20 @@

ContourConfiguratio

Contour’s default is { address: “0.0.0.0”, port: 8000 }.

+ + +tracing +
+ + +TracingConfig + + + + +

Tracing defines properties for exporting trace data to OpenTelemetry.

+ +

ContourConfigurationStatus @@ -5757,6 +5785,63 @@

ContourSettings +

CustomTag +

+

+

CustomTag defines custom tags with unique tag name +to create tags for the active span.

+

+ + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+tagName +
+ +string + +		 +

TagName is the unique name of the custom tag.

+
+literal +
+ +string + +		 +(Optional) +

Literal is a static custom tag value. +Precisely one of Literal, RequestHeaderName must be set.

+
+requestHeaderName +
+ +string + +		 +(Optional) +

RequestHeaderName indicates which request header +the label value is obtained from. +Precisely one of Literal, RequestHeaderName must be set.

+

DaemonSetSettings

@@ -7311,7 +7396,8 @@

NamespacedName EnvoyConfig, GatewayConfig, HTTPProxyConfig, -RateLimitServiceConfig) +RateLimitServiceConfig, +TracingConfig)

NamespacedName defines the namespace/name of the Kubernetes resource referred from the config file. @@ -7981,6 +8067,112 @@

TimeoutParameters +

TracingConfig +

+

+(Appears on: +ContourConfigurationSpec) +

+

+

TracingConfig defines properties for exporting trace data to OpenTelemetry.

+

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+includePodDetail +
+ +bool + +		 +(Optional) +

IncludePodDetail defines a flag. +If it is true, contour will add the pod name and namespace to the span of the trace. +the default is true. +Note: The Envoy pods MUST have the HOSTNAME and CONTOUR_NAMESPACE environment variables set for this to work properly.

+
+serviceName +
+ +string + +		 +

ServiceName defines the name for the service. +contour’s default is contour.

+
+overallSampling +
+ +string + +		 +(Optional) +

OverallSampling defines the sampling rate of trace data. +contour’s default is 100.

+
+maxPathTagLength +
+ +uint32 + +		 +(Optional) +

MaxPathTagLength defines maximum length of the request path +to extract and include in the HttpUrl tag. +contour’s default is 256.

+
+customTags +
+ + +[]*github.com/projectcontour/contour/apis/projectcontour/v1alpha1.CustomTag + + +		 +(Optional) +

CustomTags defines a list of custom tags with unique tag name.

+
+extensionService +
+ + +NamespacedName + + +		 +

ExtensionService identifies the extension service defining the otel-collector.

+

WorkloadType (string alias)

diff --git a/site/content/docs/main/config/tracing.md b/site/content/docs/main/config/tracing.md new file mode 100644 index 00000000000..5500c26d547 --- /dev/null +++ b/site/content/docs/main/config/tracing.md @@ -0,0 +1,117 @@ +# Tracing Support + +- [Overview](#overview) +- [Tracing-config](#tracing-config) + +## Overview + +Envoy has rich support for [distributed tracing][1]，and supports exporting data to third-party providers (Zipkin, Jaeger, Datadog, etc.) + +[OpenTelemetry][2] is a CNCF project which is working to become a standard in the space. It was formed as a merger of the OpenTracing and OpenCensus projects. + +Contour supports configuring envoy to export data to OpenTelemetry, and allows users to customize some configurations. + +- Custom service name, the default is `contour`. +- Custom sampling rate, the default is `100`. +- Custom the maximum length of the request path, the default is `256`. +- Customize span tags from literal or request headers. +- Customize whether to include the pod's hostname and namespace. + +## Tracing-config + +In order to use this feature, you must first select and deploy an opentelemetry-collector to receive the tracing data exported by envoy. + +First we should deploy an opentelemetry-collector to receive the tracing data exported by envoy +```bash +# install operator +kubectl apply -f https://github.com/open-telemetry/opentelemetry-operator/releases/latest/download/opentelemetry-operator.yaml +``` + +Install an otel collector instance, with verbose logging exporter enabled: +```shell +kubectl apply -f - <