diff --git a/api/v1alpha1/envoyproxy_metric_types.go b/api/v1alpha1/envoyproxy_metric_types.go index 69ce7bdfa8..320b7436ca 100644 --- a/api/v1alpha1/envoyproxy_metric_types.go +++ b/api/v1alpha1/envoyproxy_metric_types.go @@ -41,6 +41,22 @@ type ProxyMetrics struct { // // +optional EnableRequestResponseSizesStats *bool `json:"enableRequestResponseSizesStats,omitempty"` + + // ClusterStatName defines the value of cluster alt_stat_name, determining how cluster stats are named. + // For more details, see envoy docs: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto.html + // The supported operators for this pattern are: + // %ROUTE_NAME%: name of Gateway API xRoute resource + // %ROUTE_NAMESPACE%: namespace of Gateway API xRoute resource + // %ROUTE_KIND%: kind of Gateway API xRoute resource + // %ROUTE_RULE_NAME%: name of the Gateway API xRoute section + // %ROUTE_RULE_NUMBER%: name of the Gateway API xRoute section + // %BACKEND_REFS%: names of all backends referenced in /|/|... format + // Only xDS Clusters created for HTTPRoute and GRPCRoute are currently supported. + // Default: %ROUTE_KIND%/%ROUTE_NAMESPACE%/%ROUTE_NAME%/rule/%ROUTE_RULE_NUMBER% + // Example: httproute/my-ns/my-route/rule/0 + // + // +optional + ClusterStatName *string `json:"clusterStatName,omitempty"` } // ProxyMetricSink defines the sink of metrics. diff --git a/api/v1alpha1/envoyproxy_types.go b/api/v1alpha1/envoyproxy_types.go index 97ee9dea3b..c81fe1c5da 100644 --- a/api/v1alpha1/envoyproxy_types.go +++ b/api/v1alpha1/envoyproxy_types.go @@ -266,6 +266,24 @@ const ( // EnvoyFilterBuffer defines the Envoy HTTP buffer filter EnvoyFilterBuffer EnvoyFilter = "envoy.filters.http.buffer" + + // StatFormatterRouteName defines the Route Name formatter for stats + StatFormatterRouteName string = "%ROUTE_NAME%" + + // StatFormatterRouteNamespace defines the Route Name formatter for stats + StatFormatterRouteNamespace string = "%ROUTE_NAMESPACE%" + + // StatFormatterRouteKind defines the Route Name formatter for stats + StatFormatterRouteKind string = "%ROUTE_KIND%" + + // StatFormatterRouteRuleName defines the Route Name formatter for stats + StatFormatterRouteRuleName string = "%ROUTE_RULE_NAME%" + + // StatFormatterRouteRuleNumber defines the Route Name formatter for stats + StatFormatterRouteRuleNumber string = "%ROUTE_RULE_NUMBER%" + + // StatFormatterBackendRefs defines the Route Name formatter for stats + StatFormatterBackendRefs string = "%BACKEND_REFS%" ) type ProxyTelemetry struct { diff --git a/api/v1alpha1/validation/envoyproxy_validate.go b/api/v1alpha1/validation/envoyproxy_validate.go index 482d38b217..12226c3044 100644 --- a/api/v1alpha1/validation/envoyproxy_validate.go +++ b/api/v1alpha1/validation/envoyproxy_validate.go @@ -10,6 +10,7 @@ import ( "fmt" "net" "net/netip" + "regexp" "github.com/dominikbraun/graph" utilerrors "k8s.io/apimachinery/pkg/util/errors" @@ -204,6 +205,12 @@ func validateProxyTelemetry(spec *egv1a1.EnvoyProxySpec) []error { } } } + + if spec.Telemetry.Metrics.ClusterStatName != nil { + if clusterStatErrs := validateClusterStatName(*spec.Telemetry.Metrics.ClusterStatName); clusterStatErrs != nil { + errs = append(errs, clusterStatErrs...) + } + } } return errs @@ -283,3 +290,26 @@ func validateFilterOrder(filterOrder []egv1a1.FilterPosition) error { return nil } + +func validateClusterStatName(clusterStatName string) []error { + supportedOperators := map[string]bool{ + egv1a1.StatFormatterRouteName: true, + egv1a1.StatFormatterRouteNamespace: true, + egv1a1.StatFormatterRouteKind: true, + egv1a1.StatFormatterRouteRuleName: true, + egv1a1.StatFormatterRouteRuleNumber: true, + egv1a1.StatFormatterBackendRefs: true, + } + + var errs []error + re := regexp.MustCompile("%[^%]*%") + matches := re.FindAllString(clusterStatName, -1) + for _, operator := range matches { + if _, ok := supportedOperators[operator]; !ok { + err := fmt.Errorf("unable to configure Cluster Stat Name with unsupported operator: %s", operator) + errs = append(errs, err) + } + } + + return errs +} diff --git a/api/v1alpha1/validation/envoyproxy_validate_test.go b/api/v1alpha1/validation/envoyproxy_validate_test.go index f6ff1c6b23..0834bbe1a5 100644 --- a/api/v1alpha1/validation/envoyproxy_validate_test.go +++ b/api/v1alpha1/validation/envoyproxy_validate_test.go @@ -6,6 +6,7 @@ package validation import ( + "fmt" "reflect" "testing" @@ -759,6 +760,42 @@ func TestValidateEnvoyProxy(t *testing.T) { }, expected: false, }, + { + name: "valid operators in ClusterStatName", + proxy: &egv1a1.EnvoyProxy{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test", + Name: "test", + }, + Spec: egv1a1.EnvoyProxySpec{ + Telemetry: &egv1a1.ProxyTelemetry{ + Metrics: &egv1a1.ProxyMetrics{ + ClusterStatName: ptr.To(fmt.Sprintf("%s/%s/%s/%s/%s/%s/%s", egv1a1.StatFormatterRouteName, + egv1a1.StatFormatterRouteName, egv1a1.StatFormatterRouteNamespace, egv1a1.StatFormatterRouteKind, + egv1a1.StatFormatterRouteRuleName, egv1a1.StatFormatterRouteRuleNumber, egv1a1.StatFormatterBackendRefs)), + }, + }, + }, + }, + expected: true, + }, + { + name: "invalid operators in ClusterStatName", + proxy: &egv1a1.EnvoyProxy{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "test", + Name: "test", + }, + Spec: egv1a1.EnvoyProxySpec{ + Telemetry: &egv1a1.ProxyTelemetry{ + Metrics: &egv1a1.ProxyMetrics{ + ClusterStatName: ptr.To("%ROUTE_NAME%.%FOO%.%BAR%/my/%BACKEND_REFS%/%FOOBAR%"), + }, + }, + }, + }, + expected: false, + }, } for i := range testCases { diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index fd5f76573e..0feeda2285 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -5096,6 +5096,11 @@ func (in *ProxyMetrics) DeepCopyInto(out *ProxyMetrics) { *out = new(bool) **out = **in } + if in.ClusterStatName != nil { + in, out := &in.ClusterStatName, &out.ClusterStatName + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProxyMetrics. diff --git a/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml index 4122ba3254..329cf2733d 100644 --- a/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -12599,6 +12599,21 @@ spec: description: Metrics defines metrics configuration for managed proxies. properties: + clusterStatName: + description: |- + ClusterStatName defines the value of cluster alt_stat_name, determining how cluster stats are named. + For more details, see envoy docs: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto.html + The supported operators for this pattern are: + %ROUTE_NAME%: name of Gateway API xRoute resource + %ROUTE_NAMESPACE%: namespace of Gateway API xRoute resource + %ROUTE_KIND%: kind of Gateway API xRoute resource + %ROUTE_RULE_NAME%: name of the Gateway API xRoute section + %ROUTE_RULE_NUMBER%: name of the Gateway API xRoute section + %BACKEND_REFS%: names of all backends referenced in /|/|... format + Only xDS Clusters created for HTTPRoute and GRPCRoute are currently supported. + Default: %ROUTE_KIND%/%ROUTE_NAMESPACE%/%ROUTE_NAME%/rule/%ROUTE_RULE_NUMBER% + Example: httproute/my-ns/my-route/rule/0 + type: string enablePerEndpointStats: description: |- EnablePerEndpointStats enables per endpoint envoy stats metrics. diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml index 0c2dae98f6..0a066f367f 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyproxies.yaml @@ -12598,6 +12598,21 @@ spec: description: Metrics defines metrics configuration for managed proxies. properties: + clusterStatName: + description: |- + ClusterStatName defines the value of cluster alt_stat_name, determining how cluster stats are named. + For more details, see envoy docs: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto.html + The supported operators for this pattern are: + %ROUTE_NAME%: name of Gateway API xRoute resource + %ROUTE_NAMESPACE%: namespace of Gateway API xRoute resource + %ROUTE_KIND%: kind of Gateway API xRoute resource + %ROUTE_RULE_NAME%: name of the Gateway API xRoute section + %ROUTE_RULE_NUMBER%: name of the Gateway API xRoute section + %BACKEND_REFS%: names of all backends referenced in /|/|... format + Only xDS Clusters created for HTTPRoute and GRPCRoute are currently supported. + Default: %ROUTE_KIND%/%ROUTE_NAMESPACE%/%ROUTE_NAME%/rule/%ROUTE_RULE_NUMBER% + Example: httproute/my-ns/my-route/rule/0 + type: string enablePerEndpointStats: description: |- EnablePerEndpointStats enables per endpoint envoy stats metrics. diff --git a/internal/gatewayapi/route.go b/internal/gatewayapi/route.go index 9c95e9d879..05edb5a00e 100644 --- a/internal/gatewayapi/route.go +++ b/internal/gatewayapi/route.go @@ -187,6 +187,7 @@ func (t *Translator) processHTTPRouteRules(httpRoute *HTTPRouteContext, parentRe irRoutes []*ir.HTTPRoute errs = &status.MultiStatusError{} ) + pattern := getStatPattern(httpRoute, parentRef) // process each HTTPRouteRule, generate a unique Xds IR HTTPRoute per match of the rule for ruleIdx, rule := range httpRoute.Spec.Rules { @@ -221,7 +222,7 @@ func (t *Translator) processHTTPRouteRules(httpRoute *HTTPRouteContext, parentRe allDs := []*ir.DestinationSetting{} failedProcessDestination := false hasDynamicResolver := false - + backendRefNames := make([]string, len(rule.BackendRefs)) // process each backendRef, and calculate the destination settings for this rule for i, backendRef := range rule.BackendRefs { settingName := irDestinationSettingName(destName, i) @@ -245,6 +246,8 @@ func (t *Translator) processHTTPRouteRules(httpRoute *HTTPRouteContext, parentRe if ds.IsDynamicResolver { hasDynamicResolver = true } + backendNamespace := NamespaceDerefOr(backendRef.Namespace, httpRoute.GetNamespace()) + backendRefNames[i] = fmt.Sprintf("%s/%s", backendNamespace, backendRef.Name) } // process each ir route @@ -280,6 +283,10 @@ func (t *Translator) processHTTPRouteRules(httpRoute *HTTPRouteContext, parentRe destination.Settings = allDs irRoute.Destination = destination } + + if pattern != "" { + destination.StatName = ptr.To(buildStatName(pattern, httpRoute, rule.Name, ruleIdx, backendRefNames)) + } } if hasDynamicResolver && len(rule.BackendRefs) > 1 { @@ -627,6 +634,7 @@ func (t *Translator) processGRPCRouteRules(grpcRoute *GRPCRouteContext, parentRe irRoutes []*ir.HTTPRoute errs = &status.MultiStatusError{} ) + pattern := getStatPattern(grpcRoute, parentRef) // compute matches, filters, backends for ruleIdx, rule := range grpcRoute.Spec.Rules { @@ -654,6 +662,7 @@ func (t *Translator) processGRPCRouteRules(grpcRoute *GRPCRouteContext, parentRe allDs := []*ir.DestinationSetting{} failedProcessDestination := false + backendRefNames := make([]string, len(rule.BackendRefs)) for i, backendRef := range rule.BackendRefs { settingName := irDestinationSettingName(destName, i) ds, err := t.processDestination(settingName, backendRef, parentRef, grpcRoute, resources) @@ -670,6 +679,8 @@ func (t *Translator) processGRPCRouteRules(grpcRoute *GRPCRouteContext, parentRe continue } allDs = append(allDs, ds) + backendNamespace := NamespaceDerefOr(backendRef.Namespace, grpcRoute.GetNamespace()) + backendRefNames[i] = fmt.Sprintf("%s/%s", backendNamespace, backendRef.Name) } // process each ir route @@ -700,6 +711,10 @@ func (t *Translator) processGRPCRouteRules(grpcRoute *GRPCRouteContext, parentRe destination.Settings = allDs irRoute.Destination = destination } + + if pattern != "" { + destination.StatName = ptr.To(buildStatName(pattern, grpcRoute, rule.Name, ruleIdx, backendRefNames)) + } } // TODO handle: @@ -1973,3 +1988,31 @@ func backendAppProtocolToIRAppProtocol(ap egv1a1.AppProtocolType, defaultProtoco return defaultProtocol } } + +func getStatPattern(routeContext RouteContext, parentRef *RouteParentContext) string { + var pattern string + var envoyProxy *egv1a1.EnvoyProxy + gatewayCtx := GetRouteParentContext(routeContext, *parentRef.ParentReference).GetGateway() + if gatewayCtx != nil { + envoyProxy = gatewayCtx.envoyProxy + } + if envoyProxy != nil && envoyProxy.Spec.Telemetry != nil && envoyProxy.Spec.Telemetry.Metrics != nil && + envoyProxy.Spec.Telemetry.Metrics.ClusterStatName != nil { + pattern = *envoyProxy.Spec.Telemetry.Metrics.ClusterStatName + } + return pattern +} + +func buildStatName(pattern string, route RouteContext, ruleName *gwapiv1.SectionName, idx int, refs []string) string { + statName := strings.ReplaceAll(pattern, egv1a1.StatFormatterRouteName, route.GetName()) + statName = strings.ReplaceAll(statName, egv1a1.StatFormatterRouteNamespace, route.GetNamespace()) + statName = strings.ReplaceAll(statName, egv1a1.StatFormatterRouteKind, route.GetObjectKind().GroupVersionKind().Kind) + if ruleName == nil { + statName = strings.ReplaceAll(statName, egv1a1.StatFormatterRouteRuleName, "-") + } else { + statName = strings.ReplaceAll(statName, egv1a1.StatFormatterRouteRuleName, string(*ruleName)) + } + statName = strings.ReplaceAll(statName, egv1a1.StatFormatterRouteRuleNumber, fmt.Sprintf("%d", idx)) + statName = strings.ReplaceAll(statName, egv1a1.StatFormatterBackendRefs, strings.Join(refs, "|")) + return statName +} diff --git a/internal/gatewayapi/testdata/envoyproxy-with-statname.in.yaml b/internal/gatewayapi/testdata/envoyproxy-with-statname.in.yaml new file mode 100644 index 0000000000..54dfc32ab7 --- /dev/null +++ b/internal/gatewayapi/testdata/envoyproxy-with-statname.in.yaml @@ -0,0 +1,78 @@ +envoyProxyForGatewayClass: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + namespace: envoy-gateway-system + name: test + spec: + telemetry: + metrics: + clusterStatName: "%ROUTE_KIND%/%ROUTE_NAMESPACE%/%ROUTE_NAME%/%ROUTE_RULE_NAME%/%ROUTE_RULE_NUMBER%/%BACKEND_REFS%" + provider: + type: Kubernetes +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + namespace: envoy-gateway + name: gateway-1 + spec: + gatewayClassName: envoy-gateway-class + listeners: + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + namespaces: + from: All +grpcRoutes: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: GRPCRoute + metadata: + namespace: default + name: grpcroute-1 + spec: + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + sectionName: http + rules: + - backendRefs: + - name: service-1 + port: 8080 + - name: service-2 + port: 8080 + - name: service-3 + port: 8080 + - name: service-4 + port: 8080 +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + namespace: default + name: httproute-1 + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - namespace: envoy-gateway + name: gateway-1 + sectionName: http + rules: + - matches: + - path: + value: "/foo" + name: "foo" + backendRefs: + - name: service-3 + port: 8080 + - name: service-4 + port: 8080 + - matches: + - path: + value: "/" + name: "fallback" + backendRefs: + - name: service-1 + port: 8080 diff --git a/internal/gatewayapi/testdata/envoyproxy-with-statname.out.yaml b/internal/gatewayapi/testdata/envoyproxy-with-statname.out.yaml new file mode 100644 index 0000000000..07d92ece8b --- /dev/null +++ b/internal/gatewayapi/testdata/envoyproxy-with-statname.out.yaml @@ -0,0 +1,326 @@ +gateways: +- apiVersion: gateway.networking.k8s.io/v1 + kind: Gateway + metadata: + creationTimestamp: null + name: gateway-1 + namespace: envoy-gateway + spec: + gatewayClassName: envoy-gateway-class + listeners: + - allowedRoutes: + namespaces: + from: All + name: http + port: 80 + protocol: HTTP + status: + listeners: + - attachedRoutes: 2 + conditions: + - lastTransitionTime: null + message: Sending translated listener configuration to the data plane + reason: Programmed + status: "True" + type: Programmed + - lastTransitionTime: null + message: Listener has been successfully translated + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Listener references have been resolved + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + name: http + supportedKinds: + - group: gateway.networking.k8s.io + kind: HTTPRoute + - group: gateway.networking.k8s.io + kind: GRPCRoute +grpcRoutes: +- apiVersion: gateway.networking.k8s.io/v1alpha2 + kind: GRPCRoute + metadata: + creationTimestamp: null + name: grpcroute-1 + namespace: default + spec: + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - name: service-1 + port: 8080 + - name: service-2 + port: 8080 + - name: service-3 + port: 8080 + - name: service-4 + port: 8080 + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway + sectionName: http +httpRoutes: +- apiVersion: gateway.networking.k8s.io/v1 + kind: HTTPRoute + metadata: + creationTimestamp: null + name: httproute-1 + namespace: default + spec: + hostnames: + - gateway.envoyproxy.io + parentRefs: + - name: gateway-1 + namespace: envoy-gateway + sectionName: http + rules: + - backendRefs: + - name: service-3 + port: 8080 + - name: service-4 + port: 8080 + matches: + - path: + value: /foo + name: foo + - backendRefs: + - name: service-1 + port: 8080 + matches: + - path: + value: / + name: fallback + status: + parents: + - conditions: + - lastTransitionTime: null + message: Route is accepted + reason: Accepted + status: "True" + type: Accepted + - lastTransitionTime: null + message: Resolved all the Object references for the Route + reason: ResolvedRefs + status: "True" + type: ResolvedRefs + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parentRef: + name: gateway-1 + namespace: envoy-gateway + sectionName: http +infraIR: + envoy-gateway/gateway-1: + proxy: + config: + apiVersion: gateway.envoyproxy.io/v1alpha1 + kind: EnvoyProxy + metadata: + creationTimestamp: null + name: test + namespace: envoy-gateway-system + spec: + logging: {} + provider: + type: Kubernetes + telemetry: + metrics: + clusterStatName: '%ROUTE_KIND%/%ROUTE_NAMESPACE%/%ROUTE_NAME%/%ROUTE_RULE_NAME%/%ROUTE_RULE_NUMBER%/%BACKEND_REFS%' + status: {} + listeners: + - address: null + name: envoy-gateway/gateway-1/http + ports: + - containerPort: 10080 + name: http-80 + protocol: HTTP + servicePort: 80 + metadata: + labels: + gateway.envoyproxy.io/owning-gateway-name: gateway-1 + gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway + name: envoy-gateway/gateway-1 + namespace: envoy-gateway-system +xdsIR: + envoy-gateway/gateway-1: + accessLog: + json: + - path: /dev/stdout + http: + - address: 0.0.0.0 + hostnames: + - '*' + isHTTP2: true + metadata: + kind: Gateway + name: gateway-1 + namespace: envoy-gateway + sectionName: http + name: envoy-gateway/gateway-1/http + path: + escapedSlashesAction: UnescapeAndRedirect + mergeSlashes: true + port: 10080 + routes: + - destination: + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + sectionName: foo + name: httproute/default/httproute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + metadata: + name: service-3 + namespace: default + sectionName: "8080" + name: httproute/default/httproute-1/rule/0/backend/0 + protocol: HTTP + weight: 1 + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + metadata: + name: service-4 + namespace: default + sectionName: "8080" + name: httproute/default/httproute-1/rule/0/backend/1 + protocol: HTTP + weight: 1 + statName: HTTPRoute/default/httproute-1/foo/0/default/service-3|default/service-4 + hostname: gateway.envoyproxy.io + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + sectionName: foo + name: httproute/default/httproute-1/rule/0/match/0/gateway_envoyproxy_io + pathMatch: + distinct: false + name: "" + prefix: /foo + - destination: + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + sectionName: fallback + name: httproute/default/httproute-1/rule/1 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + metadata: + name: service-1 + namespace: default + sectionName: "8080" + name: httproute/default/httproute-1/rule/1/backend/0 + protocol: HTTP + weight: 1 + statName: HTTPRoute/default/httproute-1/fallback/1/default/service-1 + hostname: gateway.envoyproxy.io + isHTTP2: false + metadata: + kind: HTTPRoute + name: httproute-1 + namespace: default + sectionName: fallback + name: httproute/default/httproute-1/rule/1/match/0/gateway_envoyproxy_io + pathMatch: + distinct: false + name: "" + prefix: / + - destination: + metadata: + kind: GRPCRoute + name: grpcroute-1 + namespace: default + name: grpcroute/default/grpcroute-1/rule/0 + settings: + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + metadata: + name: service-1 + namespace: default + sectionName: "8080" + name: grpcroute/default/grpcroute-1/rule/0/backend/0 + protocol: GRPC + weight: 1 + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + metadata: + name: service-2 + namespace: default + sectionName: "8080" + name: grpcroute/default/grpcroute-1/rule/0/backend/1 + protocol: GRPC + weight: 1 + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + metadata: + name: service-3 + namespace: default + sectionName: "8080" + name: grpcroute/default/grpcroute-1/rule/0/backend/2 + protocol: GRPC + weight: 1 + - addressType: IP + endpoints: + - host: 7.7.7.7 + port: 8080 + metadata: + name: service-4 + namespace: default + sectionName: "8080" + name: grpcroute/default/grpcroute-1/rule/0/backend/3 + protocol: GRPC + weight: 1 + statName: GRPCRoute/default/grpcroute-1/-/0/default/service-1|default/service-2|default/service-3|default/service-4 + hostname: '*' + isHTTP2: true + metadata: + kind: GRPCRoute + name: grpcroute-1 + namespace: default + name: grpcroute/default/grpcroute-1/rule/0/match/-1/* + metrics: + enablePerEndpointStats: false + enableRequestResponseSizesStats: false + enableVirtualHostStats: false + readyListener: + address: 0.0.0.0 + ipFamily: IPv4 + path: /ready + port: 19003 diff --git a/internal/ir/xds.go b/internal/ir/xds.go index c2c3c39cc5..02f21f673b 100644 --- a/internal/ir/xds.go +++ b/internal/ir/xds.go @@ -1497,6 +1497,7 @@ type RouteDestination struct { // to check if this route destination already exists and can be // reused Name string `json:"name" yaml:"name"` + StatName *string `json:"statName,omitempty" yaml:"statName,omitempty"` Settings []*DestinationSetting `json:"settings,omitempty" yaml:"settings,omitempty"` // Metadata is used to enrich envoy route metadata with user and provider-specific information // RouteDestination metadata is primarily derived from the xRoute resources. In some cases, diff --git a/internal/ir/zz_generated.deepcopy.go b/internal/ir/zz_generated.deepcopy.go index de924f1d52..344ed66ee9 100644 --- a/internal/ir/zz_generated.deepcopy.go +++ b/internal/ir/zz_generated.deepcopy.go @@ -3065,6 +3065,11 @@ func (in *RoundRobin) DeepCopy() *RoundRobin { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *RouteDestination) DeepCopyInto(out *RouteDestination) { *out = *in + if in.StatName != nil { + in, out := &in.StatName, &out.StatName + *out = new(string) + **out = **in + } if in.Settings != nil { in, out := &in.Settings, &out.Settings *out = make([]*DestinationSetting, len(*in)) diff --git a/internal/xds/translator/cluster.go b/internal/xds/translator/cluster.go index b17e48379c..8d811c2d6a 100644 --- a/internal/xds/translator/cluster.go +++ b/internal/xds/translator/cluster.go @@ -69,6 +69,7 @@ type xdsClusterArgs struct { useClientProtocol bool ipFamily *egv1a1.IPFamily metadata *ir.ResourceMetadata + statName *string } type EndpointType int @@ -148,6 +149,10 @@ func buildXdsCluster(args *xdsClusterArgs) (*buildClusterResult, error) { Metadata: buildXdsMetadata(args.metadata), } + if args.statName != nil { + cluster.AltStatName = *args.statName + } + // 50% is the Envoy default value for panic threshold. No need to explicitly set it in this case. if args.healthCheck != nil && args.healthCheck.PanicThreshold != nil && *args.healthCheck.PanicThreshold != 50 { cluster.CommonLbConfig.HealthyPanicThreshold = &xdstype.Percent{ @@ -1042,6 +1047,7 @@ type ExtraArgs struct { http1Settings *ir.HTTP1Settings http2Settings *ir.HTTP2Settings ipFamily *egv1a1.IPFamily + statName *string } type clusterArgs interface { @@ -1116,6 +1122,7 @@ func (httpRoute *HTTPRouteTranslator) asClusterArgs(name string, useClientProtocol: ptr.Deref(httpRoute.UseClientProtocol, false), ipFamily: extra.ipFamily, metadata: metadata, + statName: extra.statName, } // Populate traffic features. diff --git a/internal/xds/translator/testdata/in/xds-ir/http-route-stat-name.yaml b/internal/xds/translator/testdata/in/xds-ir/http-route-stat-name.yaml new file mode 100644 index 0000000000..d09a10b3c5 --- /dev/null +++ b/internal/xds/translator/testdata/in/xds-ir/http-route-stat-name.yaml @@ -0,0 +1,36 @@ +http: +- name: "first-listener" + address: "::" + port: 10080 + hostnames: + - "*" + path: + mergeSlashes: true + escapedSlashesAction: UnescapeAndRedirect + routes: + - name: "first-route" + hostname: "*" + destination: + statName: "custom-stat-name" + name: "first-route-dest" + settings: + - endpoints: + - host: "1.1.1.1" + port: 50001 + weight: 20 + name: "first-route-dest/backend/0" + - endpoints: + - host: "2.2.2.2" + port: 50002 + weight: 40 + name: "first-route-dest/backend/1" + - endpoints: + - host: "3.3.3.3" + port: 50003 + weight: 20 + name: "first-route-dest/backend/2" + - endpoints: + - host: "4.4.4.4" + port: 50004 + weight: 20 + name: "first-route-dest/backend/3" diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-stat-name.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-stat-name.clusters.yaml new file mode 100644 index 0000000000..6d9c237e17 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-stat-name.clusters.yaml @@ -0,0 +1,18 @@ +- altStatName: custom-stat-name + circuitBreakers: + thresholds: + - maxRetries: 1024 + commonLbConfig: + localityWeightedLbConfig: {} + connectTimeout: 10s + dnsLookupFamily: V4_PREFERRED + edsClusterConfig: + edsConfig: + ads: {} + resourceApiVersion: V3 + serviceName: first-route-dest + ignoreHealthOnHostRemoval: true + lbPolicy: LEAST_REQUEST + name: first-route-dest + perConnectionBufferLimitBytes: 32768 + type: EDS diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-stat-name.endpoints.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-stat-name.endpoints.yaml new file mode 100644 index 0000000000..f5633bd158 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-stat-name.endpoints.yaml @@ -0,0 +1,42 @@ +- clusterName: first-route-dest + endpoints: + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 1.1.1.1 + portValue: 50001 + loadBalancingWeight: 1 + loadBalancingWeight: 20 + locality: + region: first-route-dest/backend/0 + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 2.2.2.2 + portValue: 50002 + loadBalancingWeight: 1 + loadBalancingWeight: 40 + locality: + region: first-route-dest/backend/1 + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 3.3.3.3 + portValue: 50003 + loadBalancingWeight: 1 + loadBalancingWeight: 20 + locality: + region: first-route-dest/backend/2 + - lbEndpoints: + - endpoint: + address: + socketAddress: + address: 4.4.4.4 + portValue: 50004 + loadBalancingWeight: 1 + loadBalancingWeight: 20 + locality: + region: first-route-dest/backend/3 diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-stat-name.listeners.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-stat-name.listeners.yaml new file mode 100644 index 0000000000..80ae84fd10 --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-stat-name.listeners.yaml @@ -0,0 +1,34 @@ +- address: + socketAddress: + address: '::' + portValue: 10080 + defaultFilterChain: + filters: + - name: envoy.filters.network.http_connection_manager + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager + commonHttpProtocolOptions: + headersWithUnderscoresAction: REJECT_REQUEST + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + maxConcurrentStreams: 100 + httpFilters: + - name: envoy.filters.http.router + typedConfig: + '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router + suppressEnvoyHeaders: true + mergeSlashes: true + normalizePath: true + pathWithEscapedSlashesAction: UNESCAPE_AND_REDIRECT + rds: + configSource: + ads: {} + resourceApiVersion: V3 + routeConfigName: first-listener + serverHeaderTransformation: PASS_THROUGH + statPrefix: http-10080 + useRemoteAddress: true + name: first-listener + name: first-listener + perConnectionBufferLimitBytes: 32768 diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-stat-name.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-stat-name.routes.yaml new file mode 100644 index 0000000000..0b5b4bee7b --- /dev/null +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-stat-name.routes.yaml @@ -0,0 +1,14 @@ +- ignorePortInHostMatching: true + name: first-listener + virtualHosts: + - domains: + - '*' + name: first-listener/* + routes: + - match: + prefix: / + name: first-route + route: + cluster: first-route-dest + upgradeConfigs: + - upgradeType: websocket diff --git a/internal/xds/translator/translator.go b/internal/xds/translator/translator.go index b89731fc13..80221babd3 100644 --- a/internal/xds/translator/translator.go +++ b/internal/xds/translator/translator.go @@ -523,6 +523,7 @@ func (t *Translator) addRouteToRouteConfig( metrics: metrics, http1Settings: httpListener.HTTP1, ipFamily: determineIPFamily(httpRoute.Destination.Settings), + statName: httpRoute.Destination.StatName, } if httpRoute.Traffic != nil && httpRoute.Traffic.HTTP2 != nil { diff --git a/release-notes/current.yaml b/release-notes/current.yaml index 62338fa9ed..ba3dcf22ca 100644 --- a/release-notes/current.yaml +++ b/release-notes/current.yaml @@ -18,6 +18,7 @@ new features: | Added support for configuring hostname in active HTTP healthchecks. Added support for GatewayInfrastructure in gateway namespace mode. Added support for using extension server policies to in PostTranslateModify hook. + Added support for configuring cluster stat name for HTTPRoute and GRPCRoute in EnvoyProxy CRD. bug fixes: | Handle integer zone annotation values diff --git a/site/content/en/latest/api/extension_types.md b/site/content/en/latest/api/extension_types.md index 53b146792f..275c77879c 100644 --- a/site/content/en/latest/api/extension_types.md +++ b/site/content/en/latest/api/extension_types.md @@ -3660,6 +3660,7 @@ _Appears in:_ | `enableVirtualHostStats` | _boolean_ | false | | EnableVirtualHostStats enables envoy stat metrics for virtual hosts. | | `enablePerEndpointStats` | _boolean_ | false | | EnablePerEndpointStats enables per endpoint envoy stats metrics.
Please use with caution. | | `enableRequestResponseSizesStats` | _boolean_ | false | | EnableRequestResponseSizesStats enables publishing of histograms tracking header and body sizes of requests and responses. | +| `clusterStatName` | _string_ | false | | ClusterStatName defines the value of cluster alt_stat_name, determining how cluster stats are named.
For more details, see envoy docs: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto.html
The supported operators for this pattern are:
%ROUTE_NAME%: name of Gateway API xRoute resource
%ROUTE_NAMESPACE%: namespace of Gateway API xRoute resource
%ROUTE_KIND%: kind of Gateway API xRoute resource
%ROUTE_RULE_NAME%: name of the Gateway API xRoute section
%ROUTE_RULE_NUMBER%: name of the Gateway API xRoute section
%BACKEND_REFS%: names of all backends referenced in /\|/\|... format
Only xDS Clusters created for HTTPRoute and GRPCRoute are currently supported.
Default: %ROUTE_KIND%/%ROUTE_NAMESPACE%/%ROUTE_NAME%/rule/%ROUTE_RULE_NUMBER%
Example: httproute/my-ns/my-route/rule/0 | #### ProxyOpenTelemetrySink diff --git a/test/e2e/testdata/envoyproxy-stat-name.yaml b/test/e2e/testdata/envoyproxy-stat-name.yaml new file mode 100644 index 0000000000..de15930772 --- /dev/null +++ b/test/e2e/testdata/envoyproxy-stat-name.yaml @@ -0,0 +1,57 @@ +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: stat-name-gtw + namespace: gateway-conformance-infra +spec: + gatewayClassName: "{GATEWAY_CLASS_NAME}" + listeners: + - name: http + port: 80 + protocol: HTTP + allowedRoutes: + namespaces: + from: Same + infrastructure: + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: stat-name-ep +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: stat-name-ep + namespace: gateway-conformance-infra +spec: + ipFamily: IPv4 + telemetry: + metrics: + clusterStatName: "%ROUTE_NAMESPACE%/%ROUTE_NAME%/%BACKEND_REFS%" + shutdown: + drainTimeout: 5s + minDrainDuration: 1s +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: stat-name-route + namespace: gateway-conformance-infra +spec: + parentRefs: + - name: stat-name-gtw + rules: + - matches: + - path: + type: PathPrefix + value: /foo + backendRefs: + - name: infra-backend-v1 + port: 8080 + - matches: + - path: + type: PathPrefix + value: /bar + backendRefs: + - name: infra-backend-v1 + port: 8080 diff --git a/test/e2e/tests/stat_name.go b/test/e2e/tests/stat_name.go new file mode 100644 index 0000000000..99441a5169 --- /dev/null +++ b/test/e2e/tests/stat_name.go @@ -0,0 +1,86 @@ +// Copyright Envoy Gateway Authors +// SPDX-License-Identifier: Apache-2.0 +// The full text of the Apache license is available in the LICENSE file at +// the root of the repo. + +//go:build e2e + +package tests + +import ( + "context" + "testing" + "time" + + "github.com/prometheus/common/model" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/wait" + httputils "sigs.k8s.io/gateway-api/conformance/utils/http" + "sigs.k8s.io/gateway-api/conformance/utils/kubernetes" + "sigs.k8s.io/gateway-api/conformance/utils/suite" + "sigs.k8s.io/gateway-api/conformance/utils/tlog" + + "github.com/envoyproxy/gateway/test/utils/prometheus" +) + +func init() { + ConformanceTests = append(ConformanceTests, StatNameTest) +} + +var StatNameTest = suite.ConformanceTest{ + ShortName: "StatName", + Description: "Make sure metric is working", + Manifests: []string{"testdata/envoyproxy-stat-name.yaml"}, + Test: func(t *testing.T, suite *suite.ConformanceTestSuite) { + ns := "gateway-conformance-infra" + routeNN := types.NamespacedName{Name: "stat-name-route", Namespace: ns} + gwNN := types.NamespacedName{Name: "stat-name-gtw", Namespace: ns} + gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN) + + t.Run("prometheus", func(t *testing.T) { + expectedResponse := httputils.ExpectedResponse{ + Request: httputils.Request{ + Path: "/foo", + }, + Response: httputils.Response{ + StatusCode: 200, + }, + Namespace: ns, + } + // make sure listener is ready + httputils.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expectedResponse) + + expectedResponse = httputils.ExpectedResponse{ + Request: httputils.Request{ + Path: "/bar", + }, + Response: httputils.Response{ + StatusCode: 200, + }, + Namespace: ns, + } + // make sure listener is ready + httputils.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expectedResponse) + + // make sure that a metrics for alt_stat_name exists in test gateway and they collapse stats from multiple clusters + // expect to find 2 cluster members, since there are two routes with the same alt_stat_name and each cluster has a single member + if err := wait.PollUntilContextTimeout(context.TODO(), 3*time.Second, time.Minute, true, func(ctx context.Context) (done bool, err error) { + v, err := prometheus.QueryPrometheus(suite.Client, `envoy_cluster_membership_healthy{envoy_cluster_name="gateway-conformance-infra/stat-name-route/gateway-conformance-infra/infra-backend-v1"}`) + if err != nil { + tlog.Logf(t, "failed to query prometheus: %v", err) + return false, err + } + if v != nil && v.Type() == model.ValVector { + vectorVal := v.(model.Vector) + if len(vectorVal) == 1 && vectorVal[0].Value == 2 { + tlog.Logf(t, "got expected value: %v", v) + return true, nil + } + } + return false, nil + }); err != nil { + t.Errorf("failed to get expected response for the last (fourth) request: %v", err) + } + }) + }, +} diff --git a/test/helm/gateway-crds-helm/all.out.yaml b/test/helm/gateway-crds-helm/all.out.yaml index 3da0d85af4..353b7c87d8 100644 --- a/test/helm/gateway-crds-helm/all.out.yaml +++ b/test/helm/gateway-crds-helm/all.out.yaml @@ -36126,6 +36126,21 @@ spec: description: Metrics defines metrics configuration for managed proxies. properties: + clusterStatName: + description: |- + ClusterStatName defines the value of cluster alt_stat_name, determining how cluster stats are named. + For more details, see envoy docs: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto.html + The supported operators for this pattern are: + %ROUTE_NAME%: name of Gateway API xRoute resource + %ROUTE_NAMESPACE%: namespace of Gateway API xRoute resource + %ROUTE_KIND%: kind of Gateway API xRoute resource + %ROUTE_RULE_NAME%: name of the Gateway API xRoute section + %ROUTE_RULE_NUMBER%: name of the Gateway API xRoute section + %BACKEND_REFS%: names of all backends referenced in /|/|... format + Only xDS Clusters created for HTTPRoute and GRPCRoute are currently supported. + Default: %ROUTE_KIND%/%ROUTE_NAMESPACE%/%ROUTE_NAME%/rule/%ROUTE_RULE_NUMBER% + Example: httproute/my-ns/my-route/rule/0 + type: string enablePerEndpointStats: description: |- EnablePerEndpointStats enables per endpoint envoy stats metrics. diff --git a/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml b/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml index c045a76b99..cd32c60ecd 100644 --- a/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml +++ b/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml @@ -18814,6 +18814,21 @@ spec: description: Metrics defines metrics configuration for managed proxies. properties: + clusterStatName: + description: |- + ClusterStatName defines the value of cluster alt_stat_name, determining how cluster stats are named. + For more details, see envoy docs: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/cluster/v3/cluster.proto.html + The supported operators for this pattern are: + %ROUTE_NAME%: name of Gateway API xRoute resource + %ROUTE_NAMESPACE%: namespace of Gateway API xRoute resource + %ROUTE_KIND%: kind of Gateway API xRoute resource + %ROUTE_RULE_NAME%: name of the Gateway API xRoute section + %ROUTE_RULE_NUMBER%: name of the Gateway API xRoute section + %BACKEND_REFS%: names of all backends referenced in /|/|... format + Only xDS Clusters created for HTTPRoute and GRPCRoute are currently supported. + Default: %ROUTE_KIND%/%ROUTE_NAMESPACE%/%ROUTE_NAME%/rule/%ROUTE_RULE_NUMBER% + Example: httproute/my-ns/my-route/rule/0 + type: string enablePerEndpointStats: description: |- EnablePerEndpointStats enables per endpoint envoy stats metrics.