Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ require (
github.com/go-logr/zapr v1.2.0
github.com/google/go-cmp v0.5.6
github.com/kevinburke/go-bindata v3.11.0+incompatible
github.com/openshift/api v0.0.0-20220413080212-716dc5660b74
github.com/openshift/api v0.0.0-20220429082003-1469fac35c75
github.com/openshift/library-go v0.0.0-20220401140922-0716d0ba0657
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.11.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -608,8 +608,8 @@ github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQ
github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
github.com/openshift/api v0.0.0-20211209135129-c58d9f695577/go.mod h1:DoslCwtqUpr3d/gsbq4ZlkaMEdYqKxuypsDjorcHhME=
github.com/openshift/api v0.0.0-20220315184754-d7c10d0b647e/go.mod h1:F/eU6jgr6Q2VhMu1mSpMmygxAELd7+BUxs3NHZ25jV4=
github.com/openshift/api v0.0.0-20220413080212-716dc5660b74 h1:MowyFJybvMnUrONS5qWdYlntWt3H3kCfYOC1zzCCSEk=
github.com/openshift/api v0.0.0-20220413080212-716dc5660b74/go.mod h1:F/eU6jgr6Q2VhMu1mSpMmygxAELd7+BUxs3NHZ25jV4=
github.com/openshift/api v0.0.0-20220429082003-1469fac35c75 h1:pPQx0SKvxsVb02NaBNKS3yPT/rwoCIsVO/Et4F7cI14=
github.com/openshift/api v0.0.0-20220429082003-1469fac35c75/go.mod h1:F/eU6jgr6Q2VhMu1mSpMmygxAELd7+BUxs3NHZ25jV4=
github.com/openshift/build-machinery-go v0.0.0-20210712174854-1bb7fd1518d3/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE=
github.com/openshift/build-machinery-go v0.0.0-20211213093930-7e33a7eb4ce3/go.mod h1:b1BuldmJlbA/xYtdZvKi+7j5YGB44qJUJDZ9zwiNCfE=
github.com/openshift/client-go v0.0.0-20211209144617-7385dd6338e3/go.mod h1:cwhyki5lqBmrT0m8Im+9I7PGFaraOzcYPtEz93RcsGY=
Expand Down
40 changes: 40 additions & 0 deletions manifests/00-custom-resource-definition.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -1135,6 +1135,17 @@ spec:
type: string
type: object
tuningOptions:
anyOf:
- properties:
maxConnections:
enum:
- -1
- 0
- properties:
maxConnections:
format: int32
maximum: 2000000
minimum: 2000
description: "tuningOptions defines parameters for adjusting the performance
of ingress controller pods. All fields are optional and will use
their respective defaults if not set. See specific tuningOptions
Expand Down Expand Up @@ -1203,6 +1214,35 @@ spec:
are subject to change over time."
format: duration
type: string
maxConnections:
description: "maxConnections defines the maximum number of simultaneous
connections that can be established per HAProxy process. Increasing
this value allows each ingress controller pod to handle more
connections but at the cost of additional system resources being
consumed. \n Permitted values are: empty, 0, -1, and the range
2000-2000000. \n If this field is empty or 0, the IngressController
will use the default value of 20000, but the default is subject
to change in future releases. \n If the value is -1 then HAProxy
will dynamically compute a maximum value based on the available
ulimits in the running container. Selecting -1 (i.e., auto)
will result in a large value being computed (~520000 on OpenShift
>=4.10 clusters) and therefore each HAProxy process will incur
significant memory usage compared to the current default of
20000. \n Setting a value that is greater than the current operating
system limit will prevent the HAProxy process from starting.
\n If you choose a discrete value (e.g., 750000) and the router
pod is migrated to a new node, there's no guarantee that that
new node has identical ulimits configured. In such a scenario
the pod would fail to start. If you have nodes with different
ulimits configured (e.g., different tuned profiles) and you
choose a discrete value then the guidance is to use -1 and let
the value be computed dynamically at runtime. \n You can monitor
memory usage for router containers with the following metric:
'container_memory_working_set_bytes{container=\"router\",namespace=\"openshift-ingress\"}'.
\n You can monitor memory usage of individual HAProxy processes
in router containers with the following metric: 'container_memory_working_set_bytes{container=\"router\",namespace=\"openshift-ingress\"}/container_processes{container=\"router\",namespace=\"openshift-ingress\"}'."
format: int32
type: integer
serverFinTimeout:
description: "serverFinTimeout defines how long a connection will
be held open while waiting for the server/backend response to
Expand Down
8 changes: 4 additions & 4 deletions pkg/manifests/bindata.go

Large diffs are not rendered by default.

3 changes: 1 addition & 2 deletions pkg/operator/controller/ingress/deployment.go
Original file line number Diff line number Diff line change
Expand Up @@ -483,7 +483,6 @@ func desiredRouterDeployment(ci *operatorv1.IngressController, ingressController
var unsupportedConfigOverrides struct {
LoadBalancingAlgorithm string `json:"loadBalancingAlgorithm"`
DynamicConfigManager string `json:"dynamicConfigManager"`
MaxConnections int32 `json:"maxConnections"`
ReloadInterval int32 `json:"reloadInterval"`
}
if len(ci.Spec.UnsupportedConfigOverrides.Raw) > 0 {
Expand Down Expand Up @@ -514,7 +513,7 @@ func desiredRouterDeployment(ci *operatorv1.IngressController, ingressController
Value: "source",
})

switch v := unsupportedConfigOverrides.MaxConnections; {
switch v := ci.Spec.TuningOptions.MaxConnections; {
case v == -1:
env = append(env, corev1.EnvVar{
Name: RouterMaxConnectionsEnvName,
Expand Down
6 changes: 4 additions & 2 deletions pkg/operator/controller/ingress/deployment_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -431,6 +431,7 @@ func TestDesiredRouterDeploymentSpecAndNetwork(t *testing.T) {
HeaderBufferBytes: 16384,
HeaderBufferMaxRewriteBytes: 4096,
ThreadCount: RouterHAProxyThreadsDefaultValue * 2,
MaxConnections: -1,
}
ic.Spec.HTTPEmptyRequestsPolicy = "Ignore"

Expand All @@ -454,7 +455,7 @@ func TestDesiredRouterDeploymentSpecAndNetwork(t *testing.T) {
var expectedReplicas int32 = 8
ic.Spec.Replicas = &expectedReplicas
ic.Spec.UnsupportedConfigOverrides = runtime.RawExtension{
Raw: []byte(`{"loadBalancingAlgorithm":"random","dynamicConfigManager":"false","maxConnections":-1,"reloadInterval":15}`),
Raw: []byte(`{"loadBalancingAlgorithm":"random","dynamicConfigManager":"false","reloadInterval":15}`),
}
ic.Spec.HttpErrorCodePages = configv1.ConfigMapNameReference{
Name: "my-custom-error-code-pages",
Expand Down Expand Up @@ -538,8 +539,9 @@ func TestDesiredRouterDeploymentSpecAndNetwork(t *testing.T) {
// Any value for loadBalancingAlgorithm other than "random" should be
// ignored.
ic.Spec.UnsupportedConfigOverrides = runtime.RawExtension{
Raw: []byte(`{"loadBalancingAlgorithm":"source","dynamicConfigManager":"true","maxConnections":40000}`),
Raw: []byte(`{"loadBalancingAlgorithm":"source","dynamicConfigManager":"true"}`),
}
ic.Spec.TuningOptions.MaxConnections = 40000
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TestDesiredRouterDeploymentSpecAndNetwork feels like it should be at least two separate tests, but that can be a further cleanup for another day.

ic.Status.EndpointPublishingStrategy.LoadBalancer = &operatorv1.LoadBalancerStrategy{
Scope: operatorv1.ExternalLoadBalancer,
ProviderParameters: &operatorv1.ProviderLoadBalancerParameters{
Expand Down
152 changes: 152 additions & 0 deletions test/e2e/maxconn_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
//go:build e2e
// +build e2e

package e2e

import (
"context"
"strings"
"testing"
"time"

operatorv1 "github.com/openshift/api/operator/v1"
"github.com/openshift/cluster-ingress-operator/pkg/operator/controller"
"github.com/openshift/cluster-ingress-operator/pkg/operator/controller/ingress"
"sigs.k8s.io/controller-runtime/pkg/client"

appsv1 "k8s.io/api/apps/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/wait"
)

func TestTunableMaxConnectionsValidValues(t *testing.T) {
updateMaxConnections := func(t *testing.T, client client.Client, timeout time.Duration, maxConnections int32, name types.NamespacedName) error {
return wait.PollImmediate(time.Second, timeout, func() (bool, error) {
ic := operatorv1.IngressController{}
if err := client.Get(context.TODO(), name, &ic); err != nil {
t.Logf("Get %q failed: %v, retrying...", name, err)
return false, nil
}
ic.Spec.TuningOptions.MaxConnections = maxConnections
if err := client.Update(context.TODO(), &ic); err != nil {
t.Logf("Update %q failed: %v, retrying...", name, err)
return false, nil
}
return true, nil
})
}

icName := types.NamespacedName{Namespace: operatorNamespace, Name: "maxconnections-valid-values"}
domain := icName.Name + "." + dnsConfig.Spec.BaseDomain
ic := newPrivateController(icName, domain)
if err := kclient.Create(context.TODO(), ic); err != nil {
t.Fatalf("failed to create ingresscontroller %s: %v", icName, err)
}
defer assertIngressControllerDeleted(t, kclient, ic)
t.Logf("waiting for ingresscontroller %v", icName)
if err := waitForIngressControllerCondition(t, kclient, 5*time.Minute, icName, availableConditionsForPrivateIngressController...); err != nil {
t.Fatalf("failed to observe expected conditions: %v", err)
}

ic, err := getIngressController(t, kclient, icName, 1*time.Minute)
if err != nil {
t.Fatalf("failed to get ingress controller: %v", err)
}

deployment := &appsv1.Deployment{}
if err := kclient.Get(context.TODO(), controller.RouterDeploymentName(ic), deployment); err != nil {
t.Fatalf("failed to get ingresscontroller deployment: %v", err)
}

if err := waitForDeploymentEnvVar(t, kclient, deployment, time.Minute, ingress.RouterMaxConnectionsEnvName, ""); err != nil {
t.Fatalf("expected router deployment to have %s unset: %v", ingress.RouterMaxConnectionsEnvName, err)
}

for _, testCase := range []struct {
description string
maxConnections int32
expectedEnvVar string
}{
{"set maxconn 12345", 12345, "12345"},
{"set maxconn auto", -1, "auto"},
{"set maxconn to default", 0, ""},
} {
t.Log(testCase.description)
if err := updateMaxConnections(t, kclient, time.Minute, testCase.maxConnections, icName); err != nil {
t.Fatalf("failed to update ingresscontroller with maxConnections=%v: %v", testCase.maxConnections, err)
}
if err := waitForIngressControllerCondition(t, kclient, 5*time.Minute, icName, availableConditionsForPrivateIngressController...); err != nil {
t.Fatalf("failed to observe expected conditions: %v", err)
}
if err := waitForDeploymentEnvVar(t, kclient, deployment, time.Minute, ingress.RouterMaxConnectionsEnvName, testCase.expectedEnvVar); err != nil {
t.Fatalf("router deployment not updated with %s=%v: %v", ingress.RouterMaxConnectionsEnvName, testCase.expectedEnvVar, err)
}
}
}

// TestTunableMaxConnectionsInvalidValues tests that invalid values
// cannot be set for tuningOptions.maxConnections.
//
// Valid values are:
// 0
// -1
// 2000-2000000
//
// so we test outside of those value and expect validation failures
// when we attempt to set them.
func TestTunableMaxConnectionsInvalidValues(t *testing.T) {
updateMaxConnections := func(t *testing.T, client client.Client, maxConnections int32, name types.NamespacedName) error {
ic := operatorv1.IngressController{}
if err := client.Get(context.TODO(), name, &ic); err != nil {
t.Logf("Get %q failed: %v, retrying...", name, err)
return err
}
ic.Spec.TuningOptions.MaxConnections = maxConnections
return client.Update(context.TODO(), &ic)
}

icName := types.NamespacedName{Namespace: operatorNamespace, Name: "maxconnections-invalid-values"}
domain := icName.Name + "." + dnsConfig.Spec.BaseDomain
ic := newPrivateController(icName, domain)
if err := kclient.Create(context.TODO(), ic); err != nil {
t.Fatalf("failed to create ingresscontroller %s: %v", icName, err)
}
defer assertIngressControllerDeleted(t, kclient, ic)
t.Logf("waiting for ingresscontroller %v", icName)
if err := waitForIngressControllerCondition(t, kclient, 5*time.Minute, icName, availableConditionsForPrivateIngressController...); err != nil {
t.Fatalf("failed to observe expected conditions: %v", err)
}

ic, err := getIngressController(t, kclient, icName, 1*time.Minute)
if err != nil {
t.Fatalf("failed to get ingress controller: %v", err)
}

deployment := &appsv1.Deployment{}
if err := kclient.Get(context.TODO(), controller.RouterDeploymentName(ic), deployment); err != nil {
t.Fatalf("failed to get ingresscontroller deployment: %v", err)
}

if err := waitForDeploymentEnvVar(t, kclient, deployment, time.Minute, ingress.RouterMaxConnectionsEnvName, ""); err != nil {
t.Fatalf("expected router deployment to have %s unset: %v", ingress.RouterMaxConnectionsEnvName, err)
}

for _, testCase := range []struct {
description string
maxConnections int32
}{
{"set maxconn -2", -2},
{"set maxconn 1999", 1999},
{"set maxconn 2000001", 2000001},
} {
t.Log(testCase.description)
const expectedErr string = `"spec.tuningOptions" must validate at least one schema (anyOf), spec.tuningOptions.maxConnections`
err := updateMaxConnections(t, kclient, testCase.maxConnections, icName)
if err == nil {
t.Fatal("expected an error")
}
if !strings.Contains(err.Error(), expectedErr) {
t.Fatalf("expected error message %q, got %v", expectedErr, err)
}
}
}
52 changes: 0 additions & 52 deletions test/e2e/operator_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2494,58 +2494,6 @@ func TestLocalWithFallbackOverrideForNodePortService(t *testing.T) {
}
}

// TestMaxConnectionsUnsupportedConfigOverride verifies that the operator
// configures router pod replicas with the appropriate value for
// ROUTER_MAX_CONNECTIONS if a value is specified using an unsupported config
// override on the ingresscontroller.
func TestMaxConnectionsUnsupportedConfigOverride(t *testing.T) {
icName := types.NamespacedName{Namespace: operatorNamespace, Name: "max-connections"}
domain := icName.Name + "." + dnsConfig.Spec.BaseDomain
ic := newPrivateController(icName, domain)
if err := kclient.Create(context.TODO(), ic); err != nil {
t.Fatalf("failed to create ingresscontroller: %v", err)
}
defer assertIngressControllerDeleted(t, kclient, ic)

if err := waitForIngressControllerCondition(t, kclient, 5*time.Minute, icName, availableConditionsForPrivateIngressController...); err != nil {
t.Fatalf("failed to observe expected conditions: %v", err)
}

deployment := &appsv1.Deployment{}
if err := kclient.Get(context.TODO(), controller.RouterDeploymentName(ic), deployment); err != nil {
t.Fatalf("failed to get ingresscontroller deployment: %v", err)
}
if err := waitForDeploymentEnvVar(t, kclient, deployment, 30*time.Second, "ROUTER_MAX_CONNECTIONS", ""); err != nil {
t.Fatalf("expected initial deployment not to set ROUTER_MAX_CONNECTIONS: %v", err)
}

if err := kclient.Get(context.TODO(), icName, ic); err != nil {
t.Fatalf("failed to get ingresscontroller: %v", err)
}
ic.Spec.UnsupportedConfigOverrides = runtime.RawExtension{
Raw: []byte(`{"maxConnections":-1}`),
}
if err := kclient.Update(context.TODO(), ic); err != nil {
t.Fatalf("failed to update ingresscontroller: %v", err)
}
if err := waitForDeploymentEnvVar(t, kclient, deployment, 1*time.Minute, "ROUTER_MAX_CONNECTIONS", "auto"); err != nil {
t.Fatalf("expected updated deployment to set ROUTER_MAX_CONNECTIONS=auto: %v", err)
}

if err := kclient.Get(context.TODO(), icName, ic); err != nil {
t.Fatalf("failed to get ingresscontroller: %v", err)
}
ic.Spec.UnsupportedConfigOverrides = runtime.RawExtension{
Raw: []byte(`{"maxConnections":40000}`),
}
if err := kclient.Update(context.TODO(), ic); err != nil {
t.Fatalf("failed to update ingresscontroller: %v", err)
}
if err := waitForDeploymentEnvVar(t, kclient, deployment, 1*time.Minute, "ROUTER_MAX_CONNECTIONS", "40000"); err != nil {
t.Fatalf("expected twice-updated deployment to set ROUTER_MAX_CONNECTIONS=40000: %v", err)
}
}

// TestReloadIntervalUnsupportedConfigOverride verifies that the operator
// configures router pod replicas with the specified value for RELOAD_INTERVAL
// if one is specified using an unsupported config override on the
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading