fix(translator): ext-proc full duplex streamed trailers and validation

api/v1alpha1/ext_proc_types.go

@@ -17,7 +17,8 @@ const (
 	StreamedExtProcBodyProcessingMode ExtProcBodyProcessingMode = "Streamed"
 	// BufferedExtProcBodyProcessingMode will buffer the message body in memory and send the entire body at once. If the body exceeds the configured buffer limit, then the downstream system will receive an error.
 	BufferedExtProcBodyProcessingMode ExtProcBodyProcessingMode = "Buffered"
-	// FullDuplexStreamedExtBodyProcessingMode will send the body in pieces, to be read in a stream. Full details here: https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/filters/http/ext_proc/v3/processing_mode.proto.html#enum-extensions-filters-http-ext-proc-v3-processingmode-bodysendmode
+	// FullDuplexStreamedExtBodyProcessingMode will send the body in pieces, to be read in a stream. When enabled, trailers are also sent, and failOpen must be false.
+	// Full details here: https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/filters/http/ext_proc/v3/processing_mode.proto.html#enum-extensions-filters-http-ext-proc-v3-processingmode-bodysendmode
 	FullDuplexStreamedExtBodyProcessingMode ExtProcBodyProcessingMode = "FullDuplexStreamed"
 	// BufferedPartialExtBodyHeaderProcessingMode will buffer the message body in memory and send the entire body in one chunk. If the body exceeds the configured buffer limit, then the body contents up to the buffer limit will be sent.
 	BufferedPartialExtBodyHeaderProcessingMode ExtProcBodyProcessingMode = "BufferedPartial"
@@ -67,6 +68,7 @@ type ExtProcProcessingMode struct {
 // +kubebuilder:validation:XValidation:message="BackendRefs must be used, backendRef is not supported.",rule="!has(self.backendRef)"
 // +kubebuilder:validation:XValidation:message="BackendRefs only supports Service and Backend kind.",rule="has(self.backendRefs) ? self.backendRefs.all(f, f.kind == 'Service' || f.kind == 'Backend') : true"
 // +kubebuilder:validation:XValidation:message="BackendRefs only supports Core and gateway.envoyproxy.io group.",rule="has(self.backendRefs) ? (self.backendRefs.all(f, f.group == \"\" || f.group == 'gateway.envoyproxy.io')) : true"
+// +kubebuilder:validation:XValidation:message="If FullDuplexStreamed body processing mode is used, FailOpen must be false.",rule="!(has(self.failOpen) && self.failOpen == true && ((has(self.processingMode.request.body) && self.processingMode.request.body == 'FullDuplexStreamed') || (has(self.processingMode.response.body) && self.processingMode.response.body == 'FullDuplexStreamed')))"
 type ExtProc struct {
 	BackendCluster `json:",inline"`
 

charts/gateway-crds-helm/templates/generated/gateway.envoyproxy.io_envoyextensionpolicies.yaml

@@ -1057,6 +1057,12 @@ spec:
                       group.
                     rule: 'has(self.backendRefs) ? (self.backendRefs.all(f, f.group
                       == "" || f.group == ''gateway.envoyproxy.io'')) : true'
+                  - message: If FullDuplexStreamed body processing mode is used, FailOpen
+                      must be false.
+                    rule: '!(has(self.failOpen) && self.failOpen == true && ((has(self.processingMode.request.body)
+                      && self.processingMode.request.body == ''FullDuplexStreamed'')
+                      || (has(self.processingMode.response.body) && self.processingMode.response.body
+                      == ''FullDuplexStreamed'')))'
                 maxItems: 16
                 type: array
               lua:

charts/gateway-helm/crds/generated/gateway.envoyproxy.io_envoyextensionpolicies.yaml

@@ -1056,6 +1056,12 @@ spec:
                       group.
                     rule: 'has(self.backendRefs) ? (self.backendRefs.all(f, f.group
                       == "" || f.group == ''gateway.envoyproxy.io'')) : true'
+                  - message: If FullDuplexStreamed body processing mode is used, FailOpen
+                      must be false.
+                    rule: '!(has(self.failOpen) && self.failOpen == true && ((has(self.processingMode.request.body)
+                      && self.processingMode.request.body == ''FullDuplexStreamed'')
+                      || (has(self.processingMode.response.body) && self.processingMode.response.body
+                      == ''FullDuplexStreamed'')))'
                 maxItems: 16
                 type: array
               lua:

internal/xds/translator/extproc.go

@@ -99,16 +99,10 @@ func extProcConfig(extProc ir.ExtProc) *extprocv3.ExternalProcessor {
 				Seconds: defaultExtServiceRequestTimeout,
 			},
 		},
-		ProcessingMode: &extprocv3.ProcessingMode{
-			RequestHeaderMode:   extprocv3.ProcessingMode_SKIP,
-			ResponseHeaderMode:  extprocv3.ProcessingMode_SKIP,
-			RequestBodyMode:     extprocv3.ProcessingMode_NONE,
-			ResponseBodyMode:    extprocv3.ProcessingMode_NONE,
-			RequestTrailerMode:  extprocv3.ProcessingMode_SKIP,
-			ResponseTrailerMode: extprocv3.ProcessingMode_SKIP,
-		},
 	}
 
+	config.ProcessingMode = buildProcessingMode(extProc)
+
 	if extProc.FailOpen != nil {
 		config.FailureModeAllow = *extProc.FailOpen
 	}
@@ -117,22 +111,6 @@ func extProcConfig(extProc ir.ExtProc) *extprocv3.ExternalProcessor {
 		config.MessageTimeout = durationpb.New(extProc.MessageTimeout.Duration)
 	}
 
-	if extProc.RequestBodyProcessingMode != nil {
-		config.ProcessingMode.RequestBodyMode = buildExtProcBodyProcessingMode(extProc.RequestBodyProcessingMode)
-	}
-
-	if extProc.RequestHeaderProcessing {
-		config.ProcessingMode.RequestHeaderMode = extprocv3.ProcessingMode_SEND
-	}
-
-	if extProc.ResponseBodyProcessingMode != nil {
-		config.ProcessingMode.ResponseBodyMode = buildExtProcBodyProcessingMode(extProc.ResponseBodyProcessingMode)
-	}
-
-	if extProc.ResponseHeaderProcessing {
-		config.ProcessingMode.ResponseHeaderMode = extprocv3.ProcessingMode_SEND
-	}
-
 	if extProc.RequestAttributes != nil {
 		var attrs []string
 		attrs = append(attrs, extProc.RequestAttributes...)
@@ -232,7 +210,43 @@ func (*extProc) patchRoute(route *routev3.Route, irRoute *ir.HTTPRoute, _ *ir.HT
 	return nil
 }
 
-func buildExtProcBodyProcessingMode(mode *ir.ExtProcBodyProcessingMode) extprocv3.ProcessingMode_BodySendMode {
+func buildProcessingMode(extProc ir.ExtProc) *extprocv3.ProcessingMode {
+	processingMode := &extprocv3.ProcessingMode{
+		RequestHeaderMode:   extprocv3.ProcessingMode_SKIP,
+		ResponseHeaderMode:  extprocv3.ProcessingMode_SKIP,
+		RequestBodyMode:     extprocv3.ProcessingMode_NONE,
+		ResponseBodyMode:    extprocv3.ProcessingMode_NONE,
+		RequestTrailerMode:  extprocv3.ProcessingMode_SKIP,
+		ResponseTrailerMode: extprocv3.ProcessingMode_SKIP,
+	}
+
+	if extProc.RequestBodyProcessingMode != nil {
+		processingMode.RequestBodyMode = translateExtProcBodyProcessingMode(extProc.RequestBodyProcessingMode)
+		//
+		if processingMode.RequestBodyMode == extprocv3.ProcessingMode_FULL_DUPLEX_STREAMED {
+			processingMode.RequestTrailerMode = extprocv3.ProcessingMode_SEND
+		}
+	}
+
+	if extProc.RequestHeaderProcessing {
+		processingMode.RequestHeaderMode = extprocv3.ProcessingMode_SEND
+	}
+
+	if extProc.ResponseBodyProcessingMode != nil {
+		processingMode.ResponseBodyMode = translateExtProcBodyProcessingMode(extProc.ResponseBodyProcessingMode)
+		if processingMode.ResponseBodyMode == extprocv3.ProcessingMode_FULL_DUPLEX_STREAMED {
+			processingMode.ResponseTrailerMode = extprocv3.ProcessingMode_SEND
+		}
+	}
+
+	if extProc.ResponseHeaderProcessing {
+		processingMode.ResponseHeaderMode = extprocv3.ProcessingMode_SEND
+	}
+
+	return processingMode
+}
+
+func translateExtProcBodyProcessingMode(mode *ir.ExtProcBodyProcessingMode) extprocv3.ProcessingMode_BodySendMode {
 	lookup := map[ir.ExtProcBodyProcessingMode]extprocv3.ProcessingMode_BodySendMode{
 		ir.ExtProcBodyBuffered:           extprocv3.ProcessingMode_BUFFERED,
 		ir.ExtProcBodyBufferedPartial:    extprocv3.ProcessingMode_BUFFERED_PARTIAL,

internal/xds/translator/testdata/in/xds-ir/ext-proc.yaml

@@ -102,6 +102,28 @@ http:
                   - protocol: GRPC
                     weight: 1
                     name: envoyextensionpolicy/envoy-gateway/policy-for-gateway-1/0/grpc-backend/backend/0
+      - destination:
+          name: httproute/default/httproute-3/rule/0
+          settings:
+            - addressType: IP
+              endpoints:
+                - host: 7.7.7.7
+                  port: 8080
+              protocol: HTTP
+              weight: 1
+              name: httproute/default/httproute-3/rule/0/backend/0
+        envoyExtensions:
+          extProcs:
+            - name: envoyextensionpolicy/envoy-gateway/policy-for-route-3/extproc/0
+              authority: grpc-backend-3.envoy-gateway:3000
+              requestBodyProcessingMode: FullDuplexStreamed
+              responseBodyProcessingMode: FullDuplexStreamed
+              destination:
+                name: envoyextensionpolicy/envoy-gateway/policy-for-route-3/0/grpc-backend-3
+                settings:
+                  - protocol: GRPC
+                    weight: 1
+                    name: envoyextensionpolicy/envoy-gateway/policy-for-route-3/0/grpc-backend-3/backend/0
         hostname: gateway.envoyproxy.io
         isHTTP2: false
         name: httproute/default/httproute-2/rule/0/match/0/gateway_envoyproxy_io

internal/xds/translator/testdata/out/xds-ir/ext-proc.clusters.yaml

@@ -32,6 +32,23 @@
   name: httproute/default/httproute-2/rule/0
   perConnectionBufferLimitBytes: 32768
   type: EDS
+- circuitBreakers:
+    thresholds:
+    - maxRetries: 1024
+  commonLbConfig:
+    localityWeightedLbConfig: {}
+  connectTimeout: 10s
+  dnsLookupFamily: V4_PREFERRED
+  edsClusterConfig:
+    edsConfig:
+      ads: {}
+      resourceApiVersion: V3
+    serviceName: httproute/default/httproute-3/rule/0
+  ignoreHealthOnHostRemoval: true
+  lbPolicy: LEAST_REQUEST
+  name: httproute/default/httproute-3/rule/0
+  perConnectionBufferLimitBytes: 32768
+  type: EDS
 - circuitBreakers:
     thresholds:
     - maxRetries: 1024
@@ -128,3 +145,27 @@
         http2ProtocolOptions:
           initialConnectionWindowSize: 1048576
           initialStreamWindowSize: 65536
+- circuitBreakers:
+    thresholds:
+    - maxRetries: 1024
+  commonLbConfig:
+    localityWeightedLbConfig: {}
+  connectTimeout: 10s
+  dnsLookupFamily: V4_PREFERRED
+  edsClusterConfig:
+    edsConfig:
+      ads: {}
+      resourceApiVersion: V3
+    serviceName: envoyextensionpolicy/envoy-gateway/policy-for-route-3/0/grpc-backend-3
+  ignoreHealthOnHostRemoval: true
+  lbPolicy: LEAST_REQUEST
+  name: envoyextensionpolicy/envoy-gateway/policy-for-route-3/0/grpc-backend-3
+  perConnectionBufferLimitBytes: 32768
+  type: EDS
+  typedExtensionProtocolOptions:
+    envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
+      '@type': type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
+      explicitHttpConfig:
+        http2ProtocolOptions:
+          initialConnectionWindowSize: 1048576
+          initialStreamWindowSize: 65536

internal/xds/translator/testdata/out/xds-ir/ext-proc.endpoints.yaml

@@ -22,6 +22,18 @@
     loadBalancingWeight: 1
     locality:
       region: httproute/default/httproute-2/rule/0/backend/0
+- clusterName: httproute/default/httproute-3/rule/0
+  endpoints:
+  - lbEndpoints:
+    - endpoint:
+        address:
+          socketAddress:
+            address: 7.7.7.7
+            portValue: 8080
+      loadBalancingWeight: 1
+    loadBalancingWeight: 1
+    locality:
+      region: httproute/default/httproute-3/rule/0/backend/0
 - clusterName: envoyextensionpolicy/default/policy-for-route-2/0/grpc-backend-4
   endpoints:
   - loadBalancingWeight: 1
@@ -42,3 +54,8 @@
   - loadBalancingWeight: 1
     locality:
       region: envoyextensionpolicy/envoy-gateway/policy-for-gateway-1/0/grpc-backend/backend/0
+- clusterName: envoyextensionpolicy/envoy-gateway/policy-for-route-3/0/grpc-backend-3
+  endpoints:
+  - loadBalancingWeight: 1
+    locality:
+      region: envoyextensionpolicy/envoy-gateway/policy-for-route-3/0/grpc-backend-3/backend/0

internal/xds/translator/testdata/out/xds-ir/ext-proc.listeners.yaml

@@ -103,6 +103,22 @@
             - connection.requested_server_name
             responseAttributes:
             - request.path
+        - disabled: true
+          name: envoy.filters.http.ext_proc/envoyextensionpolicy/envoy-gateway/policy-for-route-3/extproc/0
+          typedConfig:
+            '@type': type.googleapis.com/envoy.extensions.filters.http.ext_proc.v3.ExternalProcessor
+            grpcService:
+              envoyGrpc:
+                authority: grpc-backend-3.envoy-gateway:3000
+                clusterName: envoyextensionpolicy/envoy-gateway/policy-for-route-3/0/grpc-backend-3
+              timeout: 10s
+            processingMode:
+              requestBodyMode: FULL_DUPLEX_STREAMED
+              requestHeaderMode: SKIP
+              requestTrailerMode: SEND
+              responseBodyMode: FULL_DUPLEX_STREAMED
+              responseHeaderMode: SKIP
+              responseTrailerMode: SEND
         - name: envoy.filters.http.router
           typedConfig:
             '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router

internal/xds/translator/testdata/out/xds-ir/ext-proc.routes.yaml

@@ -22,6 +22,20 @@
     - match:
         pathSeparatedPrefix: /bar
       name: httproute/default/httproute-2/rule/0/match/0/gateway_envoyproxy_io
+      route:
+        cluster: httproute/default/httproute-3/rule/0
+        upgradeConfigs:
+        - upgradeType: websocket
+      typedPerFilterConfig:
+        envoy.filters.http.ext_proc/envoyextensionpolicy/envoy-gateway/policy-for-route-3/extproc/0:
+          '@type': type.googleapis.com/envoy.config.route.v3.FilterConfig
+          config: {}
+  - domains:
+    - ""
+    name: envoy-gateway/gateway-1/http/
+    routes:
+    - match:
+        prefix: /
       route:
         cluster: httproute/default/httproute-2/rule/0
         upgradeConfigs:

release-notes/current.yaml

@@ -36,6 +36,9 @@ bug fixes: |
   Fixed issue that switch on wrong SubjectAltNameType enum value in BackendTLSPolicy.
   Fixed issue that BackendTLSPolicy should not reference ConfigMap or Secret across namespace.
   Fixed bug in certificate SANs overlap detection in listeners.
+  Fixed issue where EnvoyExtensionPolicy ExtProc body processing mode is set to FullDuplexStreamed, but trailers were not sent.
+  Fixed validation issue where EnvoyExtensionPolicy ExtProc failOpen is true, and body processing mode FullDuplexStreamed is not rejected.
+
 
 # Enhancements that improve performance.
 performance improvements: |

site/content/en/latest/api/extension_types.md

@@ -1715,7 +1715,7 @@ _Appears in:_
 | ----- | ----------- |
 | `Streamed` | StreamedExtProcBodyProcessingMode will stream the body to the server in pieces as they arrive at the proxy.<br /> | 
 | `Buffered` | BufferedExtProcBodyProcessingMode will buffer the message body in memory and send the entire body at once. If the body exceeds the configured buffer limit, then the downstream system will receive an error.<br /> | 
-| `FullDuplexStreamed` | FullDuplexStreamedExtBodyProcessingMode will send the body in pieces, to be read in a stream. Full details here: https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/filters/http/ext_proc/v3/processing_mode.proto.html#enum-extensions-filters-http-ext-proc-v3-processingmode-bodysendmode<br /> | 
+| `FullDuplexStreamed` | FullDuplexStreamedExtBodyProcessingMode will send the body in pieces, to be read in a stream. When enabled, trailers are also sent, and failOpen must be false.<br />Full details here: https://www.envoyproxy.io/docs/envoy/latest/api-v3/extensions/filters/http/ext_proc/v3/processing_mode.proto.html#enum-extensions-filters-http-ext-proc-v3-processingmode-bodysendmode<br /> | 
 | `BufferedPartial` | BufferedPartialExtBodyHeaderProcessingMode will buffer the message body in memory and send the entire body in one chunk. If the body exceeds the configured buffer limit, then the body contents up to the buffer limit will be sent.<br /> |