From 1f7bdf95961a0563e5928d9acba6d5f05f37c92f Mon Sep 17 00:00:00 2001 From: "Huabing (Robin) Zhao" Date: Tue, 6 May 2025 08:25:33 +0000 Subject: [PATCH 1/3] support app protocol for dynamic resolver backend Signed-off-by: Huabing (Robin) Zhao --- api/v1alpha1/backend_types.go | 2 +- .../gateway.envoyproxy.io_backends.yaml | 5 ++-- .../gateway.envoyproxy.io_backends.yaml | 5 ++-- internal/gatewayapi/backend.go | 4 +-- internal/gatewayapi/route.go | 9 ++++--- .../httproute-dynamic-resolver.in.yaml | 2 ++ .../httproute-dynamic-resolver.out.yaml | 4 +++ internal/xds/translator/cluster.go | 9 +++++++ .../xds-ir/http-route-dynamic-resolver.yaml | 1 + .../http-route-dynamic-resolver.clusters.yaml | 10 +++++++ test/cel-validation/backend_test.go | 14 +++++++--- ...tproute-with-dynamic-resolver-backend.yaml | 27 +++++++++++++++++++ ...httproute_with_dynamic_resolver_backend.go | 22 +++++++++++++++ test/helm/gateway-crds-helm/all.out.yaml | 5 ++-- .../envoy-gateway-crds.out.yaml | 5 ++-- 15 files changed, 101 insertions(+), 23 deletions(-) diff --git a/api/v1alpha1/backend_types.go b/api/v1alpha1/backend_types.go index 544ff79944..32bd64ab10 100644 --- a/api/v1alpha1/backend_types.go +++ b/api/v1alpha1/backend_types.go @@ -114,7 +114,7 @@ type UnixSocket struct { } // BackendSpec describes the desired state of BackendSpec. -// +kubebuilder:validation:XValidation:rule="self.type != 'DynamicResolver' || !has(self.endpoints) && !has(self.appProtocols)",message="DynamicResolver type cannot have endpoints and appProtocols specified" +// +kubebuilder:validation:XValidation:rule="self.type != 'DynamicResolver' || !has(self.endpoints)",message="DynamicResolver type cannot have endpoints specified" // +kubebuilder:validation:XValidation:rule="has(self.tls) ? self.type == 'DynamicResolver' : true",message="TLS settings can only be specified for DynamicResolver backends" type BackendSpec struct { // Type defines the type of the backend. Defaults to "Endpoints" diff --git a/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_backends.yaml b/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_backends.yaml index 6cf3e37a38..6910ad3f9e 100644 --- a/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_backends.yaml +++ b/charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_backends.yaml @@ -231,9 +231,8 @@ spec: type: string type: object x-kubernetes-validations: - - message: DynamicResolver type cannot have endpoints and appProtocols - specified - rule: self.type != 'DynamicResolver' || !has(self.endpoints) && !has(self.appProtocols) + - message: DynamicResolver type cannot have endpoints specified + rule: self.type != 'DynamicResolver' || !has(self.endpoints) - message: TLS settings can only be specified for DynamicResolver backends rule: 'has(self.tls) ? self.type == ''DynamicResolver'' : true' status: diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_backends.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_backends.yaml index cf4a6eb50b..9c90883b42 100644 --- a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_backends.yaml +++ b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_backends.yaml @@ -230,9 +230,8 @@ spec: type: string type: object x-kubernetes-validations: - - message: DynamicResolver type cannot have endpoints and appProtocols - specified - rule: self.type != 'DynamicResolver' || !has(self.endpoints) && !has(self.appProtocols) + - message: DynamicResolver type cannot have endpoints specified + rule: self.type != 'DynamicResolver' || !has(self.endpoints) - message: TLS settings can only be specified for DynamicResolver backends rule: 'has(self.tls) ? self.type == ''DynamicResolver'' : true' status: diff --git a/internal/gatewayapi/backend.go b/internal/gatewayapi/backend.go index 715504d495..e5168ca42a 100644 --- a/internal/gatewayapi/backend.go +++ b/internal/gatewayapi/backend.go @@ -42,9 +42,9 @@ func (t *Translator) ProcessBackends(backends []*egv1a1.Backend) []*egv1a1.Backe func validateBackend(backend *egv1a1.Backend) status.Error { if backend.Spec.Type != nil && *backend.Spec.Type == egv1a1.BackendTypeDynamicResolver { - if len(backend.Spec.Endpoints) > 0 || len(backend.Spec.AppProtocols) > 0 { + if len(backend.Spec.Endpoints) > 0 { return status.NewRouteStatusError( - fmt.Errorf("DynamicResolver type cannot have endpoints or appProtocols specified"), + fmt.Errorf("DynamicResolver type cannot have endpoints specified"), status.RouteReasonInvalidBackendRef, ) } diff --git a/internal/gatewayapi/route.go b/internal/gatewayapi/route.go index b98e81a465..335f3c73ac 100644 --- a/internal/gatewayapi/route.go +++ b/internal/gatewayapi/route.go @@ -1886,13 +1886,17 @@ func (t *Translator) processBackendDestinationSetting( ) addrTypeMap := make(map[ir.DestinationAddressType]int) - backend := resources.GetBackend(backendNamespace, string(backendRef.Name)) + for _, ap := range backend.Spec.AppProtocols { + protocol = backendAppProtocolToIRAppProtocol(ap, protocol) + } + ds := &ir.DestinationSetting{Name: name} // There is only one backend if it is a dynamic resolver if backend.Spec.Type != nil && *backend.Spec.Type == egv1a1.BackendTypeDynamicResolver { ds.IsDynamicResolver = true + ds.Protocol = protocol return ds } @@ -1935,9 +1939,6 @@ func (t *Translator) processBackendDestinationSetting( dstAddrType = ptr.To(ir.MIXED) } - for _, ap := range backend.Spec.AppProtocols { - protocol = backendAppProtocolToIRAppProtocol(ap, protocol) - } ds.Endpoints = dstEndpoints ds.AddressType = dstAddrType ds.Protocol = protocol diff --git a/internal/gatewayapi/testdata/httproute-dynamic-resolver.in.yaml b/internal/gatewayapi/testdata/httproute-dynamic-resolver.in.yaml index 03f9e16bf9..2e124969fd 100644 --- a/internal/gatewayapi/testdata/httproute-dynamic-resolver.in.yaml +++ b/internal/gatewayapi/testdata/httproute-dynamic-resolver.in.yaml @@ -56,6 +56,8 @@ backends: namespace: default spec: type: DynamicResolver + AppProtocols: + - "gateway.envoyproxy.io/h2c" tls: caCertificateRefs: - name: ca-cmap diff --git a/internal/gatewayapi/testdata/httproute-dynamic-resolver.out.yaml b/internal/gatewayapi/testdata/httproute-dynamic-resolver.out.yaml index 8164485596..7b5bef409c 100644 --- a/internal/gatewayapi/testdata/httproute-dynamic-resolver.out.yaml +++ b/internal/gatewayapi/testdata/httproute-dynamic-resolver.out.yaml @@ -6,6 +6,8 @@ backends: name: backend-with-custom-ca namespace: default spec: + appProtocols: + - gateway.envoyproxy.io/h2c tls: caCertificateRefs: - group: "" @@ -193,6 +195,7 @@ xdsIR: settings: - isDynamicResolver: true name: httproute/default/httproute-1/rule/0/backend/0 + protocol: HTTP2 tls: alpnProtocols: null caCertificate: @@ -211,6 +214,7 @@ xdsIR: settings: - isDynamicResolver: true name: httproute/default/httproute-2/rule/0/backend/0 + protocol: HTTP tls: alpnProtocols: null caCertificate: diff --git a/internal/xds/translator/cluster.go b/internal/xds/translator/cluster.go index 4b9e01ea51..305990562d 100644 --- a/internal/xds/translator/cluster.go +++ b/internal/xds/translator/cluster.go @@ -305,6 +305,7 @@ func buildXdsCluster(args *xdsClusterArgs) (*buildClusterResult, error) { TypedConfig: dfpAny, }} cluster.LbPolicy = clusterv3.Cluster_CLUSTER_PROVIDED + case EndpointTypeStatic: cluster.ClusterDiscoveryType = &clusterv3.Cluster_Type{Type: clusterv3.Cluster_EDS} cluster.EdsClusterConfig = &clusterv3.Cluster_EdsClusterConfig{ @@ -841,6 +842,14 @@ func buildTypedExtensionProtocolOptions(args *xdsClusterArgs) (map[string]*anypb } } + // Envoy proxy requires AutoSNI and AutoSanValidation to be set to true for dynamic_forward_proxy clusters + if args.endpointType == EndpointTypeDynamicResolver { + protocolOptions.UpstreamHttpProtocolOptions = &corev3.UpstreamHttpProtocolOptions{ + AutoSni: true, + AutoSanValidation: true, + } + } + var ( filters []*hcmv3.HttpFilter secrets []*tlsv3.Secret diff --git a/internal/xds/translator/testdata/in/xds-ir/http-route-dynamic-resolver.yaml b/internal/xds/translator/testdata/in/xds-ir/http-route-dynamic-resolver.yaml index 11c3aa8fb9..b996416c43 100644 --- a/internal/xds/translator/testdata/in/xds-ir/http-route-dynamic-resolver.yaml +++ b/internal/xds/translator/testdata/in/xds-ir/http-route-dynamic-resolver.yaml @@ -19,6 +19,7 @@ http: settings: - isDynamicResolver: true name: httproute/default/httproute-1/rule/0/backend/0 + protocol: HTTP2 tls: alpnProtocols: null caCertificate: diff --git a/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver.clusters.yaml b/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver.clusters.yaml index 818cf07425..10a5acd0d8 100644 --- a/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver.clusters.yaml +++ b/internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver.clusters.yaml @@ -28,3 +28,13 @@ sdsConfig: ads: {} resourceApiVersion: V3 + typedExtensionProtocolOptions: + envoy.extensions.upstreams.http.v3.HttpProtocolOptions: + '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions + explicitHttpConfig: + http2ProtocolOptions: + initialConnectionWindowSize: 1048576 + initialStreamWindowSize: 65536 + upstreamHttpProtocolOptions: + autoSanValidation: true + autoSni: true diff --git a/test/cel-validation/backend_test.go b/test/cel-validation/backend_test.go index a29c36169d..25035cd763 100644 --- a/test/cel-validation/backend_test.go +++ b/test/cel-validation/backend_test.go @@ -268,12 +268,18 @@ func TestBackend(t *testing.T) { desc: "dynamic resolver invalid", mutate: func(backend *egv1a1.Backend) { backend.Spec = egv1a1.BackendSpec{ - Type: ptr.To(egv1a1.BackendTypeDynamicResolver), - Endpoints: []egv1a1.BackendEndpoint{}, - AppProtocols: []egv1a1.AppProtocolType{egv1a1.AppProtocolTypeH2C}, + Type: ptr.To(egv1a1.BackendTypeDynamicResolver), + Endpoints: []egv1a1.BackendEndpoint{ + { + FQDN: &egv1a1.FQDNEndpoint{ + Hostname: "example.com", + Port: 443, + }, + }, + }, } }, - wantErrors: []string{"DynamicResolver type cannot have endpoints and appProtocols specified"}, + wantErrors: []string{"DynamicResolver type cannot have endpoints specified"}, }, { desc: "tls settings on non-dynamic resolver", diff --git a/test/e2e/testdata/httproute-with-dynamic-resolver-backend.yaml b/test/e2e/testdata/httproute-with-dynamic-resolver-backend.yaml index abb97d608c..b91952a990 100644 --- a/test/e2e/testdata/httproute-with-dynamic-resolver-backend.yaml +++ b/test/e2e/testdata/httproute-with-dynamic-resolver-backend.yaml @@ -12,6 +12,23 @@ spec: kind: Backend name: backend-dynamic-resolver --- +apiVersion: gateway.networking.k8s.io/v1 +kind: HTTPRoute +metadata: + name: httproute-with-dynamic-resolver-backend-with-app-protocol + namespace: gateway-conformance-infra +spec: + parentRefs: + - name: same-namespace + rules: + - backendRefs: + - group: gateway.envoyproxy.io + kind: Backend + name: backend-dynamic-resolver-with-app-protocol + match: + type: PathPrefix + value: /status +--- apiVersion: gateway.envoyproxy.io/v1alpha1 kind: Backend metadata: @@ -20,6 +37,16 @@ metadata: spec: type: DynamicResolver --- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: Backend +metadata: + name: backend-dynamic-resolver-with-app-protocol + namespace: gateway-conformance-infra +spec: + type: DynamicResolver + appProtocols: + - gateway.envoyproxy.io/h2c +--- apiVersion: v1 kind: Service metadata: diff --git a/test/e2e/tests/httproute_with_dynamic_resolver_backend.go b/test/e2e/tests/httproute_with_dynamic_resolver_backend.go index 9168431c46..ec4fca430d 100644 --- a/test/e2e/tests/httproute_with_dynamic_resolver_backend.go +++ b/test/e2e/tests/httproute_with_dynamic_resolver_backend.go @@ -63,6 +63,28 @@ var DynamicResolverBackendTest = suite.ConformanceTest{ http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expectedResponse) }) + t.Run("route to external service with app protocol", func(t *testing.T) { + routeNN := types.NamespacedName{Name: "httproute-with-dynamic-resolver-backend-with-app-protocol", Namespace: ns} + gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN) + BackendMustBeAccepted(t, suite.Client, types.NamespacedName{Name: "backend-dynamic-resolver-with-app-protocol", Namespace: ns}) + + expectedResponse := http.ExpectedResponse{ + Request: http.Request{ + Host: "httpbin.org", + Path: "/status/200", + }, + ExpectedRequest: &http.ExpectedRequest{ + Request: http.Request{ + Host: "", + }, + }, + Response: http.Response{ + StatusCode: 200, + }, + } + + http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expectedResponse) + }) t.Run("route to service with TLS", func(t *testing.T) { routeNN := types.NamespacedName{Name: "httproute-with-dynamic-resolver-backend-tls", Namespace: ns} gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN) diff --git a/test/helm/gateway-crds-helm/all.out.yaml b/test/helm/gateway-crds-helm/all.out.yaml index 8d3a3a2403..ec3cacabc5 100644 --- a/test/helm/gateway-crds-helm/all.out.yaml +++ b/test/helm/gateway-crds-helm/all.out.yaml @@ -17543,9 +17543,8 @@ spec: type: string type: object x-kubernetes-validations: - - message: DynamicResolver type cannot have endpoints and appProtocols - specified - rule: self.type != 'DynamicResolver' || !has(self.endpoints) && !has(self.appProtocols) + - message: DynamicResolver type cannot have endpoints specified + rule: self.type != 'DynamicResolver' || !has(self.endpoints) - message: TLS settings can only be specified for DynamicResolver backends rule: 'has(self.tls) ? self.type == ''DynamicResolver'' : true' status: diff --git a/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml b/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml index a0e7257100..bcaf49047a 100644 --- a/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml +++ b/test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml @@ -231,9 +231,8 @@ spec: type: string type: object x-kubernetes-validations: - - message: DynamicResolver type cannot have endpoints and appProtocols - specified - rule: self.type != 'DynamicResolver' || !has(self.endpoints) && !has(self.appProtocols) + - message: DynamicResolver type cannot have endpoints specified + rule: self.type != 'DynamicResolver' || !has(self.endpoints) - message: TLS settings can only be specified for DynamicResolver backends rule: 'has(self.tls) ? self.type == ''DynamicResolver'' : true' status: From cad01717464984e3a6e02259b17d61e49a16995b Mon Sep 17 00:00:00 2001 From: "Huabing (Robin) Zhao" Date: Tue, 20 May 2025 02:01:45 +0000 Subject: [PATCH 2/3] address comment Signed-off-by: Huabing (Robin) Zhao --- test/cel-validation/backend_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/cel-validation/backend_test.go b/test/cel-validation/backend_test.go index 25035cd763..801fcb8742 100644 --- a/test/cel-validation/backend_test.go +++ b/test/cel-validation/backend_test.go @@ -260,7 +260,10 @@ func TestBackend(t *testing.T) { { desc: "dynamic resolver ok", mutate: func(backend *egv1a1.Backend) { - backend.Spec = egv1a1.BackendSpec{Type: ptr.To(egv1a1.BackendTypeDynamicResolver)} + backend.Spec = egv1a1.BackendSpec{ + Type: ptr.To(egv1a1.BackendTypeDynamicResolver), + AppProtocols: []egv1a1.AppProtocolType{egv1a1.AppProtocolTypeH2C}, + } }, wantErrors: []string{}, }, From 96789b581fa2358a702c18c61f113825a12bbf11 Mon Sep 17 00:00:00 2001 From: "Huabing (Robin) Zhao" Date: Tue, 20 May 2025 02:26:21 +0000 Subject: [PATCH 3/3] fix lint Signed-off-by: Huabing (Robin) Zhao --- test/cel-validation/backend_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/cel-validation/backend_test.go b/test/cel-validation/backend_test.go index 801fcb8742..1d744a0d51 100644 --- a/test/cel-validation/backend_test.go +++ b/test/cel-validation/backend_test.go @@ -261,7 +261,7 @@ func TestBackend(t *testing.T) { desc: "dynamic resolver ok", mutate: func(backend *egv1a1.Backend) { backend.Spec = egv1a1.BackendSpec{ - Type: ptr.To(egv1a1.BackendTypeDynamicResolver), + Type: ptr.To(egv1a1.BackendTypeDynamicResolver), AppProtocols: []egv1a1.AppProtocolType{egv1a1.AppProtocolTypeH2C}, } },