OCPBUGS-60620: e2e: Deflake tests by using ReplicaSet for test workload

test/e2e/idle_connection_test.go

@@ -14,6 +14,7 @@ import (
 	"testing"
 	"time"
 
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -109,9 +110,9 @@ type idleConnectionTestConfig struct {
 }
 
 type idleConnectionTestAction struct {
-	description      string                                                                                                  // A human-readable description of the step
-	fetchResponse    func(httpClient *http.Client, elbAddress string, cfg *idleConnectionTestConfig) (string, string, error) // Function to fetch a response
-	expectedResponse string                                                                                                  // The expected response value for verification
+	description            string                                                                                                  // A human-readable description of the step
+	fetchResponse          func(httpClient *http.Client, elbAddress string, cfg *idleConnectionTestConfig) (string, string, error) // Function to fetch a response
+	expectedResponsePrefix string                                                                                                  // The expected response value for verification
 }
 
 func idleConnectionCreateBackendService(ctx context.Context, t *testing.T, namespace, name, image string) error {
@@ -124,13 +125,13 @@ func idleConnectionCreateBackendService(ctx context.Context, t *testing.T, names
 		return fmt.Errorf("failed to create service %s/%s: %w", namespace, name, err)
 	}
 
-	pod, err := idleConnectionCreatePod(ctx, namespace, name, image, labels)
+	rs, err := idleConnectionCreateReplicaSet(ctx, namespace, name, image, labels)
 	if err != nil {
-		return fmt.Errorf("failed to create pod %s: %w", name, err)
+		return fmt.Errorf("failed to create replicaset %s: %w", name, err)
 	}
 
-	if err := waitForPodReady(t, kclient, pod, 2*time.Minute); err != nil {
-		return fmt.Errorf("pod %s is not ready: %w", name, err)
+	if err := waitForReplicaSetReady(t, kclient, rs, 2*time.Minute); err != nil {
+		return fmt.Errorf("replicaset %s is not ready: %w", name, err)
 	}
 
 	return nil
@@ -154,15 +155,15 @@ func idleConnectionCreateService(ctx context.Context, namespace, name string, la
 		},
 	}
 
-	if err := kclient.Create(ctx, service); err != nil {
+	if err := createWithRetryOnError(ctx, service, 2*time.Minute); err != nil {
 		return nil, fmt.Errorf("failed to create service %s/%s: %w", service.Namespace, service.Name, err)
 	}
 
 	return service, nil
 }
 
-func idleConnectionCreatePod(ctx context.Context, namespace, name, image string, labels map[string]string) (*corev1.Pod, error) {
-	pod := &corev1.Pod{
+func idleConnectionBuildPod(namespace, name, image string, labels map[string]string) *corev1.Pod {
+	return &corev1.Pod{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,
 			Namespace: namespace,
@@ -236,12 +237,33 @@ func idleConnectionCreatePod(ctx context.Context, namespace, name, image string,
 			},
 		},
 	}
+}
 
-	if err := kclient.Create(ctx, pod); err != nil {
-		return nil, fmt.Errorf("failed to create pod %s/%s: %w", pod.Namespace, pod.Name, err)
+func idleConnectionCreateReplicaSet(ctx context.Context, namespace, name, image string, labels map[string]string) (*appsv1.ReplicaSet, error) {
+	pod := idleConnectionBuildPod(namespace, name, image, labels)
+	one := int32(1)
+	rs := &appsv1.ReplicaSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Spec: appsv1.ReplicaSetSpec{
+			Replicas: &one,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: pod.Labels,
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: pod.Labels,
+				},
+				Spec: pod.Spec,
+			},
+		},
 	}
-
-	return pod, nil
+	if err := createWithRetryOnError(ctx, rs, 2*time.Minute); err != nil {
+		return nil, fmt.Errorf("failed to create replicaset %s/%s: %w", rs.Namespace, rs.Name, err)
+	}
+	return rs, nil
 }
 
 func idleConnectionSwitchRouteService(t *testing.T, routeName types.NamespacedName, routerName, serviceName string) error {
@@ -499,8 +521,8 @@ func idleConnectionTerminationPolicyRunTest(t *testing.T, policy operatorv1.Ingr
 			t.Fatalf("step %d: failed: %v", step+1, err)
 		}
 
-		if response != action.expectedResponse {
-			t.Fatalf("step %d: unexpected response: got %q, want %q", step+1, response, action.expectedResponse)
+		if !strings.HasPrefix(response, action.expectedResponsePrefix) {
+			t.Fatalf("step %d: unexpected response prefix: got response %q, want prefix %q", step+1, response, action.expectedResponsePrefix)
 		}
 
 		localAddresses = append(localAddresses, localAddr)
@@ -550,7 +572,7 @@ func Test_IdleConnectionTerminationPolicyImmediate(t *testing.T) {
 			fetchResponse: func(httpClient *http.Client, elbAddr string, cfg *idleConnectionTestConfig) (string, string, error) {
 				return idleConnectionFetchResponse(t, httpClient, elbAddr, cfg.routeHost)
 			},
-			expectedResponse: "web-service-1",
+			expectedResponsePrefix: "web-service-1",
 		},
 		{
 			description: "Switch route to web-service-2 and verify Immediate policy ensures new responses are served by web-service-2",
@@ -560,14 +582,14 @@ func Test_IdleConnectionTerminationPolicyImmediate(t *testing.T) {
 				}
 				return idleConnectionFetchResponse(t, httpClient, elbAddr, cfg.routeHost)
 			},
-			expectedResponse: "web-service-2",
+			expectedResponsePrefix: "web-service-2",
 		},
 		{
 			description: "Ensure subsequent responses are served by web-service-2",
 			fetchResponse: func(httpClient *http.Client, elbAddr string, cfg *idleConnectionTestConfig) (string, string, error) {
 				return idleConnectionFetchResponse(t, httpClient, elbAddr, cfg.routeHost)
 			},
-			expectedResponse: "web-service-2",
+			expectedResponsePrefix: "web-service-2",
 		},
 	})
 }
@@ -610,7 +632,7 @@ func Test_IdleConnectionTerminationPolicyDeferred(t *testing.T) {
 			fetchResponse: func(httpClient *http.Client, elbAddr string, cfg *idleConnectionTestConfig) (string, string, error) {
 				return idleConnectionFetchResponse(t, httpClient, elbAddr, cfg.routeHost)
 			},
-			expectedResponse: "web-service-1",
+			expectedResponsePrefix: "web-service-1",
 		},
 		{
 			description: "Switch route to web-service-2 and validate Deferred policy allows one final response to be served by web-service-1",
@@ -620,14 +642,14 @@ func Test_IdleConnectionTerminationPolicyDeferred(t *testing.T) {
 				}
 				return idleConnectionFetchResponse(t, httpClient, elbAddr, cfg.routeHost)
 			},
-			expectedResponse: "web-service-1",
+			expectedResponsePrefix: "web-service-1",
 		},
 		{
 			description: "Ensure subsequent responses are now served by web-service-2",
 			fetchResponse: func(httpClient *http.Client, elbAddr string, cfg *idleConnectionTestConfig) (string, string, error) {
 				return idleConnectionFetchResponse(t, httpClient, elbAddr, cfg.routeHost)
 			},
-			expectedResponse: "web-service-2",
+			expectedResponsePrefix: "web-service-2",
 		},
 	})
 }

test/e2e/operator_test.go

@@ -4275,6 +4275,27 @@ func waitForPodReady(t *testing.T, cl client.Client, pod *corev1.Pod, timeout ti
 	return nil
 }
 
+func waitForReplicaSetReady(t *testing.T, cl client.Client, replicaset *appsv1.ReplicaSet, timeout time.Duration) error {
+	t.Helper()
+	name := types.NamespacedName{Namespace: replicaset.Namespace, Name: replicaset.Name}
+	rs := &appsv1.ReplicaSet{}
+	err := wait.PollUntilContextTimeout(context.Background(), 2*time.Second, timeout, true, func(ctx context.Context) (bool, error) {
+		if err := cl.Get(ctx, name, rs); err != nil {
+			t.Logf("error getting replicaset %s: %v", name, err)
+			return false, nil
+		}
+		if rs.Status.AvailableReplicas == *rs.Spec.Replicas {
+			return true, nil
+		}
+		t.Logf("replicaset %s not ready", name)
+		return false, nil
+	})
+	if err != nil {
+		return fmt.Errorf("failed to wait for replicaset %s to become ready", name)
+	}
+	return nil
+}
+
 func clusterOperatorConditionMap(conditions ...configv1.ClusterOperatorStatusCondition) map[string]string {
 	conds := map[string]string{}
 	for _, cond := range conditions {

test/e2e/util_gatewayapi_test.go

@@ -189,14 +189,14 @@ func createHttpRoute(namespace, routeName, parentNamespace, hostname, backendRef
 
 	// Create the backend (service and pod) needed for the route to have resolvedRefs=true.
 	// The http route, service, and pod are cleaned up when the namespace is automatically deleted.
-	// buildEchoPod builds a pod that listens on port 8080.
-	echoPod := buildEchoPod(backendRefname, namespace)
-	if err := kclient.Create(context.TODO(), echoPod); err != nil {
-		return nil, fmt.Errorf("failed to create pod %s/%s: %v", namespace, echoPod.Name, err)
+	// buildEchoReplicaSet builds a replicaset which creates a pod that listens on port 8080.
+	echoRs := buildEchoReplicaSet(backendRefname, namespace)
+	if err := createWithRetryOnError(context.TODO(), echoRs, 2*time.Minute); err != nil {
+		return nil, fmt.Errorf("failed to create replicaset %s/%s: %v", namespace, echoRs.Name, err)
 	}
 	// buildEchoService builds a service that targets port 8080.
-	echoService := buildEchoService(echoPod.Name, namespace, echoPod.ObjectMeta.Labels)
-	if err := kclient.Create(context.TODO(), echoService); err != nil {
+	echoService := buildEchoService(echoRs.Name, namespace, echoRs.Spec.Template.ObjectMeta.Labels)
+	if err := createWithRetryOnError(context.TODO(), echoService, 2*time.Minute); err != nil {
 		return nil, fmt.Errorf("failed to create service %s/%s: %v", echoService.Namespace, echoService.Name, err)
 	}
 

test/e2e/util_test.go

@@ -297,6 +297,30 @@ func buildDelayConnectHTTPPod(name, namespace, initImage, image string) *corev1.
 	}
 }
 
+// buildEchoReplicaSet returns a replicaset definition for an socat-based echo server.
+func buildEchoReplicaSet(name, namespace string) *appsv1.ReplicaSet {
+	pod := buildEchoPod(name, namespace)
+	one := int32(1)
+	return &appsv1.ReplicaSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Spec: appsv1.ReplicaSetSpec{
+			Replicas: &one,
+			Selector: &metav1.LabelSelector{
+				MatchLabels: pod.Labels,
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: pod.Labels,
+				},
+				Spec: pod.Spec,
+			},
+		},
+	}
+}
+
 // buildRoute returns a route definition targeting the specified service.
 func buildRoute(name, namespace, serviceName string) *routev1.Route {
 	return &routev1.Route{
@@ -649,6 +673,17 @@ func updateInfrastructureConfigStatusWithRetryOnConflict(t *testing.T, timeout t
 	})
 }
 
+// createWithRetryOnError creates the given object. If there is an error on create
+// apart from "AlreadyExists" then the create is retried until the timeout is reached.
+func createWithRetryOnError(ctx context.Context, obj client.Object, timeout time.Duration) error {
+	return wait.PollUntilContextTimeout(ctx, 2*time.Second, timeout, true, func(ctx context.Context) (bool, error) {
+		if err := kclient.Create(ctx, obj); err != nil && !errors.IsAlreadyExists(err) {
+			return false, nil
+		}
+		return true, nil
+	})
+}
+
 // verifyExternalIngressController verifies connectivity between the router
 // and a test workload by making a http call using the hostname passed to it.
 // This hostname must be the domain associated with the ingresscontroller under test.