feat: support app protocol for dynamic resolver backend

api/v1alpha1/backend_types.go

@@ -114,7 +114,7 @@ type UnixSocket struct {
 }
 
 // BackendSpec describes the desired state of BackendSpec.
-// +kubebuilder:validation:XValidation:rule="self.type != 'DynamicResolver' || !has(self.endpoints) && !has(self.appProtocols)",message="DynamicResolver type cannot have endpoints and appProtocols specified"
+// +kubebuilder:validation:XValidation:rule="self.type != 'DynamicResolver' || !has(self.endpoints)",message="DynamicResolver type cannot have endpoints specified"
 // +kubebuilder:validation:XValidation:rule="has(self.tls) ? self.type == 'DynamicResolver' : true",message="TLS settings can only be specified for DynamicResolver backends"
 type BackendSpec struct {
 	// Type defines the type of the backend. Defaults to "Endpoints"

charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_backends.yaml

@@ -231,9 +231,8 @@ spec:
                 type: string
             type: object
             x-kubernetes-validations:
-            - message: DynamicResolver type cannot have endpoints and appProtocols
-                specified
-              rule: self.type != 'DynamicResolver' || !has(self.endpoints) && !has(self.appProtocols)
+            - message: DynamicResolver type cannot have endpoints specified
+              rule: self.type != 'DynamicResolver' || !has(self.endpoints)
             - message: TLS settings can only be specified for DynamicResolver backends
               rule: 'has(self.tls) ? self.type == ''DynamicResolver'' : true'
           status:

charts/gateway-helm/crds/generated/gateway.envoyproxy.io_backends.yaml

@@ -230,9 +230,8 @@ spec:
                 type: string
             type: object
             x-kubernetes-validations:
-            - message: DynamicResolver type cannot have endpoints and appProtocols
-                specified
-              rule: self.type != 'DynamicResolver' || !has(self.endpoints) && !has(self.appProtocols)
+            - message: DynamicResolver type cannot have endpoints specified
+              rule: self.type != 'DynamicResolver' || !has(self.endpoints)
             - message: TLS settings can only be specified for DynamicResolver backends
               rule: 'has(self.tls) ? self.type == ''DynamicResolver'' : true'
           status:

internal/gatewayapi/backend.go

@@ -42,9 +42,9 @@
 func validateBackend(backend *egv1a1.Backend) status.Error {
 	if backend.Spec.Type != nil &&
 		*backend.Spec.Type == egv1a1.BackendTypeDynamicResolver {
-		if len(backend.Spec.Endpoints) > 0 || len(backend.Spec.AppProtocols) > 0 {
+		if len(backend.Spec.Endpoints) > 0 {
 			return status.NewRouteStatusError(
-				fmt.Errorf("DynamicResolver type cannot have endpoints or appProtocols specified"),
+				fmt.Errorf("DynamicResolver type cannot have endpoints specified"),
 				status.RouteReasonInvalidBackendRef,
 			)
 		}

internal/gatewayapi/route.go

@@ -1886,13 +1886,17 @@ func (t *Translator) processBackendDestinationSetting(
 	)
 
 	addrTypeMap := make(map[ir.DestinationAddressType]int)
-
 	backend := resources.GetBackend(backendNamespace, string(backendRef.Name))
+	for _, ap := range backend.Spec.AppProtocols {
+		protocol = backendAppProtocolToIRAppProtocol(ap, protocol)
+	}
+
 	ds := &ir.DestinationSetting{Name: name}
 
 	// There is only one backend if it is a dynamic resolver
 	if backend.Spec.Type != nil && *backend.Spec.Type == egv1a1.BackendTypeDynamicResolver {
 		ds.IsDynamicResolver = true
+		ds.Protocol = protocol
 		return ds
 	}
 
@@ -1935,9 +1939,6 @@ func (t *Translator) processBackendDestinationSetting(
 		dstAddrType = ptr.To(ir.MIXED)
 	}
 
-	for _, ap := range backend.Spec.AppProtocols {
-		protocol = backendAppProtocolToIRAppProtocol(ap, protocol)
-	}
 	ds.Endpoints = dstEndpoints
 	ds.AddressType = dstAddrType
 	ds.Protocol = protocol

internal/gatewayapi/testdata/httproute-dynamic-resolver.in.yaml

@@ -56,6 +56,8 @@ backends:
     namespace: default
   spec:
     type: DynamicResolver
+    AppProtocols:
+    - "gateway.envoyproxy.io/h2c"
     tls:
       caCertificateRefs:
       - name: ca-cmap

internal/gatewayapi/testdata/httproute-dynamic-resolver.out.yaml

@@ -6,6 +6,8 @@ backends:
     name: backend-with-custom-ca
     namespace: default
   spec:
+    appProtocols:
+    - gateway.envoyproxy.io/h2c
     tls:
       caCertificateRefs:
       - group: ""
@@ -193,6 +195,7 @@ xdsIR:
           settings:
           - isDynamicResolver: true
             name: httproute/default/httproute-1/rule/0/backend/0
+            protocol: HTTP2
             tls:
               alpnProtocols: null
               caCertificate:
@@ -211,6 +214,7 @@ xdsIR:
           settings:
           - isDynamicResolver: true
             name: httproute/default/httproute-2/rule/0/backend/0
+            protocol: HTTP
             tls:
               alpnProtocols: null
               caCertificate:

internal/xds/translator/cluster.go

@@ -305,6 +305,7 @@ func buildXdsCluster(args *xdsClusterArgs) (*buildClusterResult, error) {
 			TypedConfig: dfpAny,
 		}}
 		cluster.LbPolicy = clusterv3.Cluster_CLUSTER_PROVIDED
+
 	case EndpointTypeStatic:
 		cluster.ClusterDiscoveryType = &clusterv3.Cluster_Type{Type: clusterv3.Cluster_EDS}
 		cluster.EdsClusterConfig = &clusterv3.Cluster_EdsClusterConfig{
@@ -841,6 +842,14 @@ func buildTypedExtensionProtocolOptions(args *xdsClusterArgs) (map[string]*anypb
 		}
 	}
 
+	// Envoy proxy requires AutoSNI and AutoSanValidation to be set to true for dynamic_forward_proxy clusters
+	if args.endpointType == EndpointTypeDynamicResolver {
+		protocolOptions.UpstreamHttpProtocolOptions = &corev3.UpstreamHttpProtocolOptions{
+			AutoSni:           true,
+			AutoSanValidation: true,
+		}
+	}
+
 	var (
 		filters []*hcmv3.HttpFilter
 		secrets []*tlsv3.Secret

internal/xds/translator/testdata/in/xds-ir/http-route-dynamic-resolver.yaml

@@ -19,6 +19,7 @@ http:
       settings:
       - isDynamicResolver: true
         name: httproute/default/httproute-1/rule/0/backend/0
+        protocol: HTTP2
         tls:
           alpnProtocols: null
           caCertificate:

internal/xds/translator/testdata/out/xds-ir/http-route-dynamic-resolver.clusters.yaml

@@ -28,3 +28,13 @@
             sdsConfig:
               ads: {}
               resourceApiVersion: V3
+  typedExtensionProtocolOptions:
+    envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
+      '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
+      explicitHttpConfig:
+        http2ProtocolOptions:
+          initialConnectionWindowSize: 1048576
+          initialStreamWindowSize: 65536
+      upstreamHttpProtocolOptions:
+        autoSanValidation: true
+        autoSni: true

test/cel-validation/backend_test.go

@@ -260,20 +260,29 @@ func TestBackend(t *testing.T) {
 		{
 			desc: "dynamic resolver ok",
 			mutate: func(backend *egv1a1.Backend) {
-				backend.Spec = egv1a1.BackendSpec{Type: ptr.To(egv1a1.BackendTypeDynamicResolver)}
+				backend.Spec = egv1a1.BackendSpec{
+					Type:         ptr.To(egv1a1.BackendTypeDynamicResolver),
+					AppProtocols: []egv1a1.AppProtocolType{egv1a1.AppProtocolTypeH2C},
+				}
 			},
 			wantErrors: []string{},
 		},
 		{
 			desc: "dynamic resolver invalid",
 			mutate: func(backend *egv1a1.Backend) {
 				backend.Spec = egv1a1.BackendSpec{
-					Type:         ptr.To(egv1a1.BackendTypeDynamicResolver),
-					Endpoints:    []egv1a1.BackendEndpoint{},
-					AppProtocols: []egv1a1.AppProtocolType{egv1a1.AppProtocolTypeH2C},
+					Type: ptr.To(egv1a1.BackendTypeDynamicResolver),
+					Endpoints: []egv1a1.BackendEndpoint{
+						{
+							FQDN: &egv1a1.FQDNEndpoint{
+								Hostname: "example.com",
+								Port:     443,
+							},
+						},
+					},
 				}
 			},
-			wantErrors: []string{"DynamicResolver type cannot have endpoints and appProtocols specified"},
+			wantErrors: []string{"DynamicResolver type cannot have endpoints specified"},
 		},
 		{
 			desc: "tls settings on non-dynamic resolver",

test/e2e/testdata/httproute-with-dynamic-resolver-backend.yaml

@@ -12,6 +12,23 @@ spec:
       kind: Backend
       name: backend-dynamic-resolver
 ---
+apiVersion: gateway.networking.k8s.io/v1
+kind: HTTPRoute
+metadata:
+  name: httproute-with-dynamic-resolver-backend-with-app-protocol
+  namespace: gateway-conformance-infra
+spec:
+  parentRefs:
+  - name: same-namespace
+  rules:
+  - backendRefs:
+    - group: gateway.envoyproxy.io
+      kind: Backend
+      name: backend-dynamic-resolver-with-app-protocol
+      match:
+        type: PathPrefix
+        value: /status
+---
 apiVersion: gateway.envoyproxy.io/v1alpha1
 kind: Backend
 metadata:
@@ -20,6 +37,16 @@ metadata:
 spec:
   type: DynamicResolver
 ---
+apiVersion: gateway.envoyproxy.io/v1alpha1
+kind: Backend
+metadata:
+  name: backend-dynamic-resolver-with-app-protocol
+  namespace: gateway-conformance-infra
+spec:
+  type: DynamicResolver
+  appProtocols:
+  - gateway.envoyproxy.io/h2c
+---
 apiVersion: v1
 kind: Service
 metadata:

test/e2e/tests/httproute_with_dynamic_resolver_backend.go

@@ -63,6 +63,28 @@ var DynamicResolverBackendTest = suite.ConformanceTest{
 
 			http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expectedResponse)
 		})
+		t.Run("route to external service with app protocol", func(t *testing.T) {
+			routeNN := types.NamespacedName{Name: "httproute-with-dynamic-resolver-backend-with-app-protocol", Namespace: ns}
+			gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN)
+			BackendMustBeAccepted(t, suite.Client, types.NamespacedName{Name: "backend-dynamic-resolver-with-app-protocol", Namespace: ns})
+
+			expectedResponse := http.ExpectedResponse{
+				Request: http.Request{
+					Host: "httpbin.org",
+					Path: "/status/200",
+				},
+				ExpectedRequest: &http.ExpectedRequest{
+					Request: http.Request{
+						Host: "",
+					},
+				},
+				Response: http.Response{
+					StatusCode: 200,
+				},
+			}
+
+			http.MakeRequestAndExpectEventuallyConsistentResponse(t, suite.RoundTripper, suite.TimeoutConfig, gwAddr, expectedResponse)
+		})
 		t.Run("route to service with TLS", func(t *testing.T) {
 			routeNN := types.NamespacedName{Name: "httproute-with-dynamic-resolver-backend-tls", Namespace: ns}
 			gwAddr := kubernetes.GatewayAndHTTPRoutesMustBeAccepted(t, suite.Client, suite.TimeoutConfig, suite.ControllerName, kubernetes.NewGatewayRef(gwNN), routeNN)

test/helm/gateway-crds-helm/all.out.yaml

@@ -17543,9 +17543,8 @@ spec:
                 type: string
             type: object
             x-kubernetes-validations:
-            - message: DynamicResolver type cannot have endpoints and appProtocols
-                specified
-              rule: self.type != 'DynamicResolver' || !has(self.endpoints) && !has(self.appProtocols)
+            - message: DynamicResolver type cannot have endpoints specified
+              rule: self.type != 'DynamicResolver' || !has(self.endpoints)
             - message: TLS settings can only be specified for DynamicResolver backends
               rule: 'has(self.tls) ? self.type == ''DynamicResolver'' : true'
           status:

test/helm/gateway-crds-helm/envoy-gateway-crds.out.yaml

@@ -231,9 +231,8 @@ spec:
                 type: string
             type: object
             x-kubernetes-validations:
-            - message: DynamicResolver type cannot have endpoints and appProtocols
-                specified
-              rule: self.type != 'DynamicResolver' || !has(self.endpoints) && !has(self.appProtocols)
+            - message: DynamicResolver type cannot have endpoints specified
+              rule: self.type != 'DynamicResolver' || !has(self.endpoints)
             - message: TLS settings can only be specified for DynamicResolver backends
               rule: 'has(self.tls) ? self.type == ''DynamicResolver'' : true'
           status: