Add configurability for HTTP requests per IO cycle

apis/projectcontour/v1alpha1/contourconfig.go

@@ -391,6 +391,16 @@ type EnvoyListenerConfig struct {
 	// Single set of options are applied to all listeners.
 	// +optional
 	SocketOptions *SocketOptions `json:"socketOptions,omitempty"`
+
+	// Defines the limit on number of HTTP requests that Envoy will process from a single
+	// connection in a single I/O cycle. Requests over this limit are processed in subsequent
+	// I/O cycles. Can be used as a mitigation for CVE-2023-44487 when abusive traffic is
+	// detected. Configures the http.max_requests_per_io_cycle Envoy runtime setting. The default
+	// value when this is not set is no limit.
+	//
+	// +kubebuilder:validation:Minimum=1
+	// +optional
+	MaxRequestsPerIOCycle *uint32 `json:"maxRequestsPerIOCycle"`
 }
 
 // SocketOptions defines configurable socket options for Envoy listeners.

changelogs/unreleased/5827-sunjayBhatia-minor.md

@@ -0,0 +1,16 @@
+## Max HTTP requests per IO cycle is configurable as an additional mitigation for HTTP/2 CVE-2023-44487
+
+Envoy v1.27.1 mitigates CVE-2023-44487 with some default runtime settings, however the `http.max_requests_per_io_cycle` does not have a default value.
+This change allows configuring this runtime setting via Contour configuration to allow administrators of Contour to prevent abusive connections from starving resources from other valid connections.
+The default is left as the existing behavior (no limit) so as not to impact existing valid traffic.
+
+The Contour ConfigMap can be modified similar to the following (and Contour restarted) to set this value:
+
+```
+listener:
+  max-requests-per-io-cycle: 10
+```
+
+(Note this can be used in addition to the existing Listener configuration field `listener.max-requests-per-connection` which is used primarily for HTTP/1.1 connections and is an approximate limit for HTTP/2)
+
+See the [Envoy release notes](https://www.envoyproxy.io/docs/envoy/v1.27.1/version_history/v1.27/v1.27.1) for more details.

cmd/contour/serve.go

@@ -465,7 +465,9 @@ func (s *Server) doServe() error {
 		&xdscache_v3.RouteCache{},
 		&xdscache_v3.ClusterCache{},
 		endpointHandler,
-		&xdscache_v3.RuntimeCache{},
+		xdscache_v3.NewRuntimeCache(xdscache_v3.ConfigurableRuntimeSettings{
+			MaxRequestsPerIOCycle: contourConfiguration.Envoy.Listener.MaxRequestsPerIOCycle,
+		}),
 	}
 
 	// snapshotHandler is used to produce new snapshots when the internal state changes for any xDS resource.

cmd/contour/servecontext.go

@@ -528,6 +528,7 @@ func (ctx *serveContext) convertToContourConfigurationSpec() contour_api_v1alpha
 				ConnectionBalancer:            ctx.Config.Listener.ConnectionBalancer,
 				PerConnectionBufferLimitBytes: ctx.Config.Listener.PerConnectionBufferLimitBytes,
 				MaxRequestsPerConnection:      ctx.Config.Listener.MaxRequestsPerConnection,
+				MaxRequestsPerIOCycle:         ctx.Config.Listener.MaxRequestsPerIOCycle,
 				TLS: &contour_api_v1alpha1.EnvoyTLS{
 					MinimumProtocolVersion: ctx.Config.TLS.MinimumProtocolVersion,
 					MaximumProtocolVersion: ctx.Config.TLS.MaximumProtocolVersion,

cmd/contour/servecontext_test.go

@@ -874,6 +874,16 @@ func TestConvertServeContext(t *testing.T) {
 				return cfg
 			},
 		},
+		"envoy listener settings": {
+			getServeContext: func(ctx *serveContext) *serveContext {
+				ctx.Config.Listener.MaxRequestsPerIOCycle = ref.To(uint32(10))
+				return ctx
+			},
+			getContourConfiguration: func(cfg contour_api_v1alpha1.ContourConfigurationSpec) contour_api_v1alpha1.ContourConfigurationSpec {
+				cfg.Envoy.Listener.MaxRequestsPerIOCycle = ref.To(uint32(10))
+				return cfg
+			},
+		},
 	}
 
 	for name, tc := range cases {

examples/contour/01-crds.yaml

@@ -203,6 +203,17 @@ spec:
                         format: int32
                         minimum: 1
                         type: integer
+                      maxRequestsPerIOCycle:
+                        description: Defines the limit on number of HTTP requests
+                          that Envoy will process from a single connection in a single
+                          I/O cycle. Requests over this limit are processed in subsequent
+                          I/O cycles. Can be used as a mitigation for CVE-2023-44487
+                          when abusive traffic is detected. Configures the http.max_requests_per_io_cycle
+                          Envoy runtime setting. The default value when this is not
+                          set is no limit.
+                        format: int32
+                        minimum: 1
+                        type: integer
                       per-connection-buffer-limit-bytes:
                         description: Defines the soft limit on size of the listener’s
                           new connection read and write buffers in bytes. If unspecified,
@@ -3643,6 +3654,18 @@ spec:
                             format: int32
                             minimum: 1
                             type: integer
+                          maxRequestsPerIOCycle:
+                            description: Defines the limit on number of HTTP requests
+                              that Envoy will process from a single connection in
+                              a single I/O cycle. Requests over this limit are processed
+                              in subsequent I/O cycles. Can be used as a mitigation
+                              for CVE-2023-44487 when abusive traffic is detected.
+                              Configures the http.max_requests_per_io_cycle Envoy
+                              runtime setting. The default value when this is not
+                              set is no limit.
+                            format: int32
+                            minimum: 1
+                            type: integer
                           per-connection-buffer-limit-bytes:
                             description: Defines the soft limit on size of the listener’s
                               new connection read and write buffers in bytes. If unspecified,

examples/render/contour-deployment.yaml

@@ -422,6 +422,17 @@ spec:
                         format: int32
                         minimum: 1
                         type: integer
+                      maxRequestsPerIOCycle:
+                        description: Defines the limit on number of HTTP requests
+                          that Envoy will process from a single connection in a single
+                          I/O cycle. Requests over this limit are processed in subsequent
+                          I/O cycles. Can be used as a mitigation for CVE-2023-44487
+                          when abusive traffic is detected. Configures the http.max_requests_per_io_cycle
+                          Envoy runtime setting. The default value when this is not
+                          set is no limit.
+                        format: int32
+                        minimum: 1
+                        type: integer
                       per-connection-buffer-limit-bytes:
                         description: Defines the soft limit on size of the listener’s
                           new connection read and write buffers in bytes. If unspecified,
@@ -3862,6 +3873,18 @@ spec:
                             format: int32
                             minimum: 1
                             type: integer
+                          maxRequestsPerIOCycle:
+                            description: Defines the limit on number of HTTP requests
+                              that Envoy will process from a single connection in
+                              a single I/O cycle. Requests over this limit are processed
+                              in subsequent I/O cycles. Can be used as a mitigation
+                              for CVE-2023-44487 when abusive traffic is detected.
+                              Configures the http.max_requests_per_io_cycle Envoy
+                              runtime setting. The default value when this is not
+                              set is no limit.
+                            format: int32
+                            minimum: 1
+                            type: integer
                           per-connection-buffer-limit-bytes:
                             description: Defines the soft limit on size of the listener’s
                               new connection read and write buffers in bytes. If unspecified,

examples/render/contour-gateway-provisioner.yaml

@@ -214,6 +214,17 @@ spec:
                         format: int32
                         minimum: 1
                         type: integer
+                      maxRequestsPerIOCycle:
+                        description: Defines the limit on number of HTTP requests
+                          that Envoy will process from a single connection in a single
+                          I/O cycle. Requests over this limit are processed in subsequent
+                          I/O cycles. Can be used as a mitigation for CVE-2023-44487
+                          when abusive traffic is detected. Configures the http.max_requests_per_io_cycle
+                          Envoy runtime setting. The default value when this is not
+                          set is no limit.
+                        format: int32
+                        minimum: 1
+                        type: integer
                       per-connection-buffer-limit-bytes:
                         description: Defines the soft limit on size of the listener’s
                           new connection read and write buffers in bytes. If unspecified,
@@ -3654,6 +3665,18 @@ spec:
                             format: int32
                             minimum: 1
                             type: integer
+                          maxRequestsPerIOCycle:
+                            description: Defines the limit on number of HTTP requests
+                              that Envoy will process from a single connection in
+                              a single I/O cycle. Requests over this limit are processed
+                              in subsequent I/O cycles. Can be used as a mitigation
+                              for CVE-2023-44487 when abusive traffic is detected.
+                              Configures the http.max_requests_per_io_cycle Envoy
+                              runtime setting. The default value when this is not
+                              set is no limit.
+                            format: int32
+                            minimum: 1
+                            type: integer
                           per-connection-buffer-limit-bytes:
                             description: Defines the soft limit on size of the listener’s
                               new connection read and write buffers in bytes. If unspecified,

examples/render/contour-gateway.yaml

@@ -425,6 +425,17 @@ spec:
                         format: int32
                         minimum: 1
                         type: integer
+                      maxRequestsPerIOCycle:
+                        description: Defines the limit on number of HTTP requests
+                          that Envoy will process from a single connection in a single
+                          I/O cycle. Requests over this limit are processed in subsequent
+                          I/O cycles. Can be used as a mitigation for CVE-2023-44487
+                          when abusive traffic is detected. Configures the http.max_requests_per_io_cycle
+                          Envoy runtime setting. The default value when this is not
+                          set is no limit.
+                        format: int32
+                        minimum: 1
+                        type: integer
                       per-connection-buffer-limit-bytes:
                         description: Defines the soft limit on size of the listener’s
                           new connection read and write buffers in bytes. If unspecified,
@@ -3865,6 +3876,18 @@ spec:
                             format: int32
                             minimum: 1
                             type: integer
+                          maxRequestsPerIOCycle:
+                            description: Defines the limit on number of HTTP requests
+                              that Envoy will process from a single connection in
+                              a single I/O cycle. Requests over this limit are processed
+                              in subsequent I/O cycles. Can be used as a mitigation
+                              for CVE-2023-44487 when abusive traffic is detected.
+                              Configures the http.max_requests_per_io_cycle Envoy
+                              runtime setting. The default value when this is not
+                              set is no limit.
+                            format: int32
+                            minimum: 1
+                            type: integer
                           per-connection-buffer-limit-bytes:
                             description: Defines the soft limit on size of the listener’s
                               new connection read and write buffers in bytes. If unspecified,

examples/render/contour.yaml

@@ -422,6 +422,17 @@ spec:
                         format: int32
                         minimum: 1
                         type: integer
+                      maxRequestsPerIOCycle:
+                        description: Defines the limit on number of HTTP requests
+                          that Envoy will process from a single connection in a single
+                          I/O cycle. Requests over this limit are processed in subsequent
+                          I/O cycles. Can be used as a mitigation for CVE-2023-44487
+                          when abusive traffic is detected. Configures the http.max_requests_per_io_cycle
+                          Envoy runtime setting. The default value when this is not
+                          set is no limit.
+                        format: int32
+                        minimum: 1
+                        type: integer
                       per-connection-buffer-limit-bytes:
                         description: Defines the soft limit on size of the listener’s
                           new connection read and write buffers in bytes. If unspecified,
@@ -3862,6 +3873,18 @@ spec:
                             format: int32
                             minimum: 1
                             type: integer
+                          maxRequestsPerIOCycle:
+                            description: Defines the limit on number of HTTP requests
+                              that Envoy will process from a single connection in
+                              a single I/O cycle. Requests over this limit are processed
+                              in subsequent I/O cycles. Can be used as a mitigation
+                              for CVE-2023-44487 when abusive traffic is detected.
+                              Configures the http.max_requests_per_io_cycle Envoy
+                              runtime setting. The default value when this is not
+                              set is no limit.
+                            format: int32
+                            minimum: 1
+                            type: integer
                           per-connection-buffer-limit-bytes:
                             description: Defines the soft limit on size of the listener’s
                               new connection read and write buffers in bytes. If unspecified,

internal/envoy/v3/runtime.go

@@ -24,20 +24,24 @@ const (
 	maxRegexProgramSizeWarn  = 1000
 )
 
-func RuntimeLayers() []*envoy_service_runtime_v3.Runtime {
+func RuntimeLayers(configurableRuntimeFields map[string]*structpb.Value) []*envoy_service_runtime_v3.Runtime {
+	baseLayer := baseRuntimeLayer()
+	for k, v := range configurableRuntimeFields {
+		baseLayer.Fields[k] = v
+	}
 	return []*envoy_service_runtime_v3.Runtime{
 		{
 			Name:  DynamicRuntimeLayerName,
-			Layer: baseRuntimeLayer(),
+			Layer: baseLayer,
 		},
 	}
 }
 
 func baseRuntimeLayer() *structpb.Struct {
 	return &structpb.Struct{
 		Fields: map[string]*structpb.Value{
-			"re2.max_program_size.error_level": {Kind: &structpb.Value_NumberValue{NumberValue: maxRegexProgramSizeError}},
-			"re2.max_program_size.warn_level":  {Kind: &structpb.Value_NumberValue{NumberValue: maxRegexProgramSizeWarn}},
+			"re2.max_program_size.error_level": structpb.NewNumberValue(maxRegexProgramSizeError),
+			"re2.max_program_size.warn_level":  structpb.NewNumberValue(maxRegexProgramSizeWarn),
 		},
 	}
 }

internal/envoy/v3/runtime_test.go

@@ -22,15 +22,38 @@ import (
 )
 
 func TestRuntimeLayers(t *testing.T) {
-	require.Equal(t, []*envoy_service_runtime_v3.Runtime{
-		{
-			Name: "dynamic",
-			Layer: &structpb.Struct{
-				Fields: map[string]*structpb.Value{
-					"re2.max_program_size.error_level": {Kind: &structpb.Value_NumberValue{NumberValue: 1 << 20}},
-					"re2.max_program_size.warn_level":  {Kind: &structpb.Value_NumberValue{NumberValue: 1000}},
-				},
+	testCases := map[string]struct {
+		configurableFields map[string]*structpb.Value
+	}{
+		"nil configurable fields": {},
+		"empty configurable fields": {
+			configurableFields: map[string]*structpb.Value{},
+		},
+		"some configurable fields": {
+			configurableFields: map[string]*structpb.Value{
+				"some.value1": structpb.NewBoolValue(true),
+				"some.value2": structpb.NewNumberValue(1000),
 			},
 		},
-	}, RuntimeLayers())
+	}
+	for name, tc := range testCases {
+		t.Run(name, func(t *testing.T) {
+			expectedFields := map[string]*structpb.Value{
+				"re2.max_program_size.error_level": structpb.NewNumberValue(1 << 20),
+				"re2.max_program_size.warn_level":  structpb.NewNumberValue(1000),
+			}
+			for k, v := range tc.configurableFields {
+				expectedFields[k] = v
+			}
+			layers := RuntimeLayers(tc.configurableFields)
+			require.Equal(t, []*envoy_service_runtime_v3.Runtime{
+				{
+					Name: "dynamic",
+					Layer: &structpb.Struct{
+						Fields: expectedFields,
+					},
+				},
+			}, layers)
+		})
+	}
 }

internal/xdscache/v3/runtime.go

@@ -20,30 +20,46 @@ import (
 	envoy_v3 "github.com/projectcontour/contour/internal/envoy/v3"
 	"github.com/projectcontour/contour/internal/protobuf"
 	"google.golang.org/protobuf/proto"
+	"google.golang.org/protobuf/types/known/structpb"
 )
 
+type ConfigurableRuntimeSettings struct {
+	MaxRequestsPerIOCycle *uint32
+}
+
 // RuntimeCache manages the contents of the gRPC RTDS cache.
-type RuntimeCache struct {
+type runtimeCache struct {
 	contour.Cond
+	runtimeKV map[string]*structpb.Value
+}
+
+// NewRuntimeCache builds a RuntimeCache with the provided runtime
+// settings that will be set in the runtime layer configured by Contour.
+func NewRuntimeCache(runtimeSettings ConfigurableRuntimeSettings) *runtimeCache {
+	runtimeKV := make(map[string]*structpb.Value)
+	if runtimeSettings.MaxRequestsPerIOCycle != nil && *runtimeSettings.MaxRequestsPerIOCycle > 0 {
+		runtimeKV["http.max_requests_per_io_cycle"] = structpb.NewNumberValue(float64(*runtimeSettings.MaxRequestsPerIOCycle))
+	}
+	return &runtimeCache{runtimeKV: runtimeKV}
 }
 
 // Contents returns all Runtime layers.
-func (c *RuntimeCache) Contents() []proto.Message {
-	return protobuf.AsMessages(envoy_v3.RuntimeLayers())
+func (c *runtimeCache) Contents() []proto.Message {
+	return protobuf.AsMessages(envoy_v3.RuntimeLayers(c.runtimeKV))
 }
 
 // Query returns only the "dynamic" layer if requested, otherwise empty.
-func (c *RuntimeCache) Query(names []string) []proto.Message {
+func (c *runtimeCache) Query(names []string) []proto.Message {
 	for _, name := range names {
 		if name == envoy_v3.DynamicRuntimeLayerName {
-			return protobuf.AsMessages(envoy_v3.RuntimeLayers())
+			return protobuf.AsMessages(envoy_v3.RuntimeLayers(c.runtimeKV))
 		}
 	}
 	return []proto.Message{}
 }
 
-func (*RuntimeCache) TypeURL() string { return resource.RuntimeType }
+func (*runtimeCache) TypeURL() string { return resource.RuntimeType }
 
-func (c *RuntimeCache) OnChange(root *dag.DAG) {
+func (c *runtimeCache) OnChange(root *dag.DAG) {
 	// DAG changes do not affect runtime layers at the moment.
 }