OCPBUGS-98462: add jitter to AWS endpoint service requeue delay

hypershift-operator/controllers/platform/aws/controller.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"math/rand"
 	"strings"
 	"time"
 
@@ -52,9 +53,14 @@ import (
 const (
 	finalizer                              = "hypershift.openshift.io/hypershift-operator-finalizer"
 	endpointServiceDeletionRequeueDuration = 5 * time.Second
-	lbNotActiveRequeueDuration             = 20 * time.Second
+	lbNotActiveRequeueBase                 = 15 * time.Second
+	lbNotActiveJitterMax                   = 10
 )
 
+func lbNotActiveRequeueDelay() time.Duration {
+	return lbNotActiveRequeueBase + time.Duration(rand.Intn(lbNotActiveJitterMax)+1)*time.Second
+}
+
 // AWSEndpointServiceReconciler watches HC/NodePools/awsEndpointService and reconcile the awsEndpointService
 // CRs existing for the KubeAPIServerPrivateService and the PrivateRouterService.
 // It creates the endpoint service in AWS and keeps the SubnetIDs up to date so NodePools are able to attach to the service endpoint.
@@ -354,8 +360,9 @@ func (r *AWSEndpointServiceReconciler) Reconcile(ctx context.Context, req ctrl.R
 		}
 		// Most likely cause of error here is the NLB is not yet active.  This can take ~2m so
 		// a longer requeue time is warranted.  This ratelimits AWS calls and updates to the CR.
-		log.Info("reconciliation failed, retrying in 20s", "err", err)
-		return ctrl.Result{RequeueAfter: lbNotActiveRequeueDuration}, nil
+		requeueAfter := lbNotActiveRequeueDelay()
+		log.Info("reconciliation failed, retrying with jitter", "err", err, "requeueAfter", requeueAfter)
+		return ctrl.Result{RequeueAfter: requeueAfter}, nil
 	}
 
 	meta.SetStatusCondition(&awsEndpointService.Status.Conditions, metav1.Condition{

hypershift-operator/controllers/platform/aws/controller_test.go

@@ -4,14 +4,17 @@ import (
 	"context"
 	"fmt"
 	"testing"
+	"time"
 
 	. "github.com/onsi/gomega"
 
 	hyperv1 "github.com/openshift/hypershift/api/hypershift/v1beta1"
 	hyperapi "github.com/openshift/hypershift/support/api"
 	"github.com/openshift/hypershift/support/awsapi"
 	"github.com/openshift/hypershift/support/capabilities"
+	"github.com/openshift/hypershift/support/k8sutil"
 	karpenterutil "github.com/openshift/hypershift/support/karpenter"
+	"github.com/openshift/hypershift/support/upsert"
 
 	configv1 "github.com/openshift/api/config/v1"
 
@@ -1261,3 +1264,88 @@ type fakeManager struct {
 func (m *fakeManager) GetLogger() logr.Logger {
 	return logr.Discard()
 }
+
+func TestLbNotActiveRequeueDelay(t *testing.T) {
+	g := NewWithT(t)
+	min := lbNotActiveRequeueBase + 1*time.Second
+	max := lbNotActiveRequeueBase + time.Duration(lbNotActiveJitterMax)*time.Second
+
+	for i := 0; i < 50; i++ {
+		d := lbNotActiveRequeueDelay()
+		g.Expect(d).To(And(
+			BeNumerically(">=", min),
+			BeNumerically("<=", max),
+		), "iteration %d returned %s", i, d)
+	}
+}
+
+func TestReconcileRequeuesWithJitter(t *testing.T) {
+	g := NewWithT(t)
+	min := lbNotActiveRequeueBase + 1*time.Second
+	max := lbNotActiveRequeueBase + time.Duration(lbNotActiveJitterMax)*time.Second
+
+	mockCtrl := gomock.NewController(t)
+	elbClient := awsapi.NewMockELBV2API(mockCtrl)
+	ec2Client := awsapi.NewMockEC2API(mockCtrl)
+
+	elbClient.EXPECT().DescribeLoadBalancers(gomock.Any(), gomock.Any()).Return(
+		&elasticloadbalancingv2.DescribeLoadBalancersOutput{LoadBalancers: []elbv2types.LoadBalancer{}}, nil,
+	).AnyTimes()
+
+	hcpNs := "test-ns-test-hc"
+	hcp := &hyperv1.HostedControlPlane{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:        "test-hc",
+			Namespace:   hcpNs,
+			Annotations: map[string]string{k8sutil.HostedClusterAnnotation: "test-ns/test-hc"},
+		},
+		Spec: hyperv1.HostedControlPlaneSpec{
+			Platform: hyperv1.PlatformSpec{
+				AWS: &hyperv1.AWSPlatformSpec{
+					RolesRef: hyperv1.AWSRolesRef{ControlPlaneOperatorARN: "arn:aws:iam::role/fake"},
+				},
+			},
+		},
+	}
+	hc := &hyperv1.HostedCluster{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "test-hc", Namespace: "test-ns",
+		},
+		Spec: hyperv1.HostedClusterSpec{
+			Platform: hyperv1.PlatformSpec{
+				AWS: &hyperv1.AWSPlatformSpec{
+					RolesRef: hyperv1.AWSRolesRef{ControlPlaneOperatorARN: "arn:aws:iam::role/fake"},
+				},
+			},
+		},
+	}
+	awsES := &hyperv1.AWSEndpointService{
+		ObjectMeta: metav1.ObjectMeta{Name: "test-ep", Namespace: hcpNs},
+		Spec:       hyperv1.AWSEndpointServiceSpec{NetworkLoadBalancerName: "test-nlb"},
+	}
+
+	fakeClient := fake.NewClientBuilder().
+		WithScheme(hyperapi.Scheme).
+		WithObjects(hcp, hc, awsES).
+		WithStatusSubresource(awsES).
+		Build()
+
+	r := &AWSEndpointServiceReconciler{
+		Client:                        fakeClient,
+		CreateOrUpdateProvider:        upsert.New(false),
+		ec2Client:                     ec2Client,
+		elbv2Client:                   elbClient,
+		ManagementClusterCapabilities: &capabilities.ManagementClusterCapabilities{},
+	}
+
+	ctx := log.IntoContext(context.Background(), testr.New(t))
+	result, err := r.Reconcile(ctx, ctrl.Request{
+		NamespacedName: client.ObjectKeyFromObject(awsES),
+	})
+
+	g.Expect(err).NotTo(HaveOccurred())
+	g.Expect(result.RequeueAfter).To(And(
+		BeNumerically(">=", min),
+		BeNumerically("<=", max),
+	))
+}