diff --git a/operator/v1/0000_50_ingress-operator_00-ingresscontroller.crd.yaml b/operator/v1/0000_50_ingress-operator_00-ingresscontroller.crd.yaml index 189fb4e7bbd..5e39f7544f3 100644 --- a/operator/v1/0000_50_ingress-operator_00-ingresscontroller.crd.yaml +++ b/operator/v1/0000_50_ingress-operator_00-ingresscontroller.crd.yaml @@ -1135,6 +1135,17 @@ spec: type: string type: object tuningOptions: + anyOf: + - properties: + maxConnections: + enum: + - -1 + - 0 + - properties: + maxConnections: + format: int32 + maximum: 2000000 + minimum: 2000 description: "tuningOptions defines parameters for adjusting the performance of ingress controller pods. All fields are optional and will use their respective defaults if not set. See specific tuningOptions @@ -1203,6 +1214,35 @@ spec: are subject to change over time." format: duration type: string + maxConnections: + description: "maxConnections defines the maximum number of simultaneous + connections that can be established per HAProxy process. Increasing + this value allows each ingress controller pod to handle more + connections but at the cost of additional system resources being + consumed. \n Permitted values are: empty, 0, -1, and the range + 2000-2000000. \n If this field is empty or 0, the IngressController + will use the default value of 20000, but the default is subject + to change in future releases. \n If the value is -1 then HAProxy + will dynamically compute a maximum value based on the available + ulimits in the running container. Selecting -1 (i.e., auto) + will result in a large value being computed (~520000 on OpenShift + >=4.10 clusters) and therefore each HAProxy process will incur + significant memory usage compared to the current default of + 20000. \n Setting a value that is greater than the current operating + system limit will prevent the HAProxy process from starting. + \n If you choose a discrete value (e.g., 750000) and the router + pod is migrated to a new node, there's no guarantee that that + new node has identical ulimits configured. In such a scenario + the pod would fail to start. If you have nodes with different + ulimits configured (e.g., different tuned profiles) and you + choose a discrete value then the guidance is to use -1 and let + the value be computed dynamically at runtime. \n You can monitor + memory usage for router containers with the following metric: + 'container_memory_working_set_bytes{container=\"router\",namespace=\"openshift-ingress\"}'. + \n You can monitor memory usage of individual HAProxy processes + in router containers with the following metric: 'container_memory_working_set_bytes{container=\"router\",namespace=\"openshift-ingress\"}/container_processes{container=\"router\",namespace=\"openshift-ingress\"}'." + format: int32 + type: integer serverFinTimeout: description: "serverFinTimeout defines how long a connection will be held open while waiting for the server/backend response to diff --git a/operator/v1/0000_50_ingress-operator_00-ingresscontroller.crd.yaml-patch b/operator/v1/0000_50_ingress-operator_00-ingresscontroller.crd.yaml-patch index 6076c3a31b8..f9336304f81 100644 --- a/operator/v1/0000_50_ingress-operator_00-ingresscontroller.crd.yaml-patch +++ b/operator/v1/0000_50_ingress-operator_00-ingresscontroller.crd.yaml-patch @@ -7,3 +7,26 @@ - properties: address: format: ipv6 +- op: add + path: /spec/versions/name=v1/schema/openAPIV3Schema/properties/spec/properties/tuningOptions/anyOf + # We explicitly choose anyOf to allow: + # + # - the enum + # - the range + # - and null + # + # If we specify oneOf that only allows for one of 'the enum' or 'the + # range'. Anything outside of that is invalid. However, we want to + # allow 'null' because maxConnections is an optional field. Using + # anyOf allows for 'the enum', 'the range', or 'null'. By allowing + # 'null' we provide a consistent user experience given that there + # are other optional integer fields in tuningOptions. + value: + - properties: + maxConnections: + enum: [ -1, 0 ] + - properties: + maxConnections: + format: int32 + minimum: 2000 + maximum: 2000000 diff --git a/operator/v1/types_ingress.go b/operator/v1/types_ingress.go index 4a7ddf082ec..7e023929b25 100644 --- a/operator/v1/types_ingress.go +++ b/operator/v1/types_ingress.go @@ -1392,6 +1392,51 @@ type IngressControllerTuningOptions struct { // +kubebuilder:validation:Format=duration // +optional HealthCheckInterval *metav1.Duration `json:"healthCheckInterval,omitempty"` + + // maxConnections defines the maximum number of simultaneous + // connections that can be established per HAProxy process. + // Increasing this value allows each ingress controller pod to + // handle more connections but at the cost of additional + // system resources being consumed. + // + // Permitted values are: empty, 0, -1, and the range + // 2000-2000000. + // + // If this field is empty or 0, the IngressController will use + // the default value of 20000, but the default is subject to + // change in future releases. + // + // If the value is -1 then HAProxy will dynamically compute a + // maximum value based on the available ulimits in the running + // container. Selecting -1 (i.e., auto) will result in a large + // value being computed (~520000 on OpenShift >=4.10 clusters) + // and therefore each HAProxy process will incur significant + // memory usage compared to the current default of 20000. + // + // Setting a value that is greater than the current operating + // system limit will prevent the HAProxy process from + // starting. + // + // If you choose a discrete value (e.g., 750000) and the + // router pod is migrated to a new node, there's no guarantee + // that that new node has identical ulimits configured. In + // such a scenario the pod would fail to start. If you have + // nodes with different ulimits configured (e.g., different + // tuned profiles) and you choose a discrete value then the + // guidance is to use -1 and let the value be computed + // dynamically at runtime. + // + // You can monitor memory usage for router containers with the + // following metric: + // 'container_memory_working_set_bytes{container="router",namespace="openshift-ingress"}'. + // + // You can monitor memory usage of individual HAProxy + // processes in router containers with the following metric: + // 'container_memory_working_set_bytes{container="router",namespace="openshift-ingress"}/container_processes{container="router",namespace="openshift-ingress"}'. + // + // +kubebuilder:validation:Optional + // +optional + MaxConnections int32 `json:"maxConnections,omitempty"` } // HTTPEmptyRequestsPolicy indicates how HTTP connections for which no request diff --git a/operator/v1/zz_generated.swagger_doc_generated.go b/operator/v1/zz_generated.swagger_doc_generated.go index c2ecf7a9d40..ace4296d66d 100644 --- a/operator/v1/zz_generated.swagger_doc_generated.go +++ b/operator/v1/zz_generated.swagger_doc_generated.go @@ -771,6 +771,7 @@ var map_IngressControllerTuningOptions = map[string]string{ "tunnelTimeout": "tunnelTimeout defines how long a tunnel connection (including websockets) will be held open while the tunnel is idle.\n\nIf unset, the default timeout is 1h", "tlsInspectDelay": "tlsInspectDelay defines how long the router can hold data to find a matching route.\n\nSetting this too short can cause the router to fall back to the default certificate for edge-terminated or reencrypt routes even when a better matching certificate could be used.\n\nIf unset, the default inspect delay is 5s", "healthCheckInterval": "healthCheckInterval defines how long the router waits between two consecutive health checks on its configured backends. This value is applied globally as a default for all routes, but may be overridden per-route by the route annotation \"router.openshift.io/haproxy.health.check.interval\".\n\nSetting this to less than 5s can cause excess traffic due to too frequent TCP health checks and accompanying SYN packet storms. Alternatively, setting this too high can result in increased latency, due to backend servers that are no longer available, but haven't yet been detected as such.\n\nAn empty or zero healthCheckInterval means no opinion and IngressController chooses a default, which is subject to change over time. Currently the default healthCheckInterval value is 5s.\n\nCurrently the minimum allowed value is 1s and the maximum allowed value is 2147483647ms (24.85 days). Both are subject to change over time.", + "maxConnections": "maxConnections defines the maximum number of simultaneous connections that can be established per HAProxy process. Increasing this value allows each ingress controller pod to handle more connections but at the cost of additional system resources being consumed.\n\nPermitted values are: empty, 0, -1, and the range 2000-2000000.\n\nIf this field is empty or 0, the IngressController will use the default value of 20000, but the default is subject to change in future releases.\n\nIf the value is -1 then HAProxy will dynamically compute a maximum value based on the available ulimits in the running container. Selecting -1 (i.e., auto) will result in a large value being computed (~520000 on OpenShift >=4.10 clusters) and therefore each HAProxy process will incur significant memory usage compared to the current default of 20000.\n\nSetting a value that is greater than the current operating system limit will prevent the HAProxy process from starting.\n\nIf you choose a discrete value (e.g., 750000) and the router pod is migrated to a new node, there's no guarantee that that new node has identical ulimits configured. In such a scenario the pod would fail to start. If you have nodes with different ulimits configured (e.g., different tuned profiles) and you choose a discrete value then the guidance is to use -1 and let the value be computed dynamically at runtime.\n\nYou can monitor memory usage for router containers with the following metric: 'container_memory_working_set_bytes{container=\"router\",namespace=\"openshift-ingress\"}'.\n\nYou can monitor memory usage of individual HAProxy processes in router containers with the following metric: 'container_memory_working_set_bytes{container=\"router\",namespace=\"openshift-ingress\"}/container_processes{container=\"router\",namespace=\"openshift-ingress\"}'.", } func (IngressControllerTuningOptions) SwaggerDoc() map[string]string {