E2E: Fixes to stabilize the multicluster tests further

tests/e2e/multicluster/common.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/istio-ecosystem/sail-operator/pkg/kube"
+	. "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/certs"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/common"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl"
@@ -32,6 +33,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
+var controlPlaneNamespace = common.ControlPlaneNamespace
+
 // ClusterDeployment represents a cluster along with its sample app version.
 type ClusterDeployment struct {
 	Kubectl    kubectl.Kubectl
@@ -66,15 +69,18 @@ func deploySampleAppToClusters(ns, profile string, clusters []ClusterDeployment)
 }
 
 // verifyResponsesAreReceivedFromBothClusters checks that when the sleep pod in the sample namespace
-// sends a request to the helloworld service, it receives responses from expectedVersions,
-// which can be either "v1" or "v2" on on different clusters.
+// sends requests to the helloworld service, it receives responses from expectedVersions,
+// which can be either "v1" or "v2" on different clusters.
 func verifyResponsesAreReceivedFromExpectedVersions(k kubectl.Kubectl, expectedVersions ...string) {
+	Step(fmt.Sprintf("Checking app connectivity from %s", k.ClusterName))
 	if len(expectedVersions) == 0 {
 		expectedVersions = []string{"v1", "v2"}
 	}
 	for _, v := range expectedVersions {
+		// Each curl will fetch the URL multiple times, minimizing time wasted waiting for different responses.
+		// This is because running the exec is much more expensive than having curl fetch the URL several times.
 		Eventually(k.WithNamespace("sample").Exec, 10*time.Minute, 2*time.Second).
-			WithArguments("deploy/sleep", "sleep", "curl -sS helloworld.sample:5000/hello").
+			WithArguments("deploy/sleep", "sleep", "curl -sS -Z helloworld.sample:5000/hello{,,,,,,,,,}").
 			Should(ContainSubstring(fmt.Sprintf("Hello version: %s", v)),
 				fmt.Sprintf("sleep pod in %s did not receive any response from %s", k.ClusterName, v))
 	}
@@ -154,3 +160,21 @@ func awaitSecretCreation(cluster string, cl client.Client) {
 		return err
 	}).ShouldNot(HaveOccurred(), fmt.Sprintf("Secret is not created on %s cluster", cluster))
 }
+
+// expectLoadBalancerAddress to be assigned.
+// If the address is not assigned, cross-cluster traffic will not be sent since Istio doesn't know where to send it for the remote cluster(s).
+func expectLoadBalancerAddress(ctx context.Context, k kubectl.Kubectl, cl client.Client, name string) {
+	address := common.GetSVCLoadBalancerAddress(ctx, cl, controlPlaneNamespace, name)
+	Expect(address).ToNot(BeEmpty(), fmt.Sprintf("Gateway service %q does not have an external address on %s", name, k.ClusterName))
+	Success(fmt.Sprintf("Gateway service %q has an external address %s on %s", name, address, k.ClusterName))
+}
+
+// eventuallyLoadBalancerIsReachable to make sure that cross cluster communication is working on the infrastructure level
+func eventuallyLoadBalancerIsReachable(ctx context.Context, kSrc kubectl.Kubectl, kDest kubectl.Kubectl, clDest client.Client, name string) {
+	address := common.GetSVCLoadBalancerAddress(ctx, clDest, controlPlaneNamespace, name)
+	Eventually(ctx, kSrc.WithNamespace("sample").Exec).WithArguments(
+		"deploy/sleep", "sleep", fmt.Sprintf("curl -sS -o /dev/null -w '%%{http_code}' %s:15021/healthz/ready", address)).
+		Should(Equal("200"), fmt.Sprintf("Gateway %q(%s) on %s is not reachable from %s, this might indicate a problem with the underlying infrastructure",
+			name, address, kDest.ClusterName, kSrc.ClusterName))
+	Success(fmt.Sprintf("Gateway %q(%s) on %s is reachable from %s", name, address, kDest.ClusterName, kSrc.ClusterName))
+}

tests/e2e/multicluster/multicluster_externalcontrolplane_test.go

@@ -92,6 +92,10 @@ values:
 					It("updates Gateway status to Available", func(ctx SpecContext) {
 						common.AwaitDeployment(ctx, "istio-ingressgateway", k1, clPrimary)
 					})
+
+					It("has an external IP assigned", func(ctx SpecContext) {
+						expectLoadBalancerAddress(ctx, k1, clPrimary, "istio-ingressgateway")
+					})
 				})
 
 				When("Istio external is installed in Cluster #2", func() {
@@ -354,6 +358,10 @@ spec:
 						Success("Sample pods has expected annotation")
 					})
 
+					It("can reach the ingress gateway from remote cluster", func(ctx SpecContext) {
+						eventuallyLoadBalancerIsReachable(ctx, k2, k1, clPrimary, "istio-ingressgateway")
+					})
+
 					It("can access the sample app from the local service", func(ctx SpecContext) {
 						verifyResponsesAreReceivedFromExpectedVersions(k2, "v1")
 						Success("Sample app is accessible from hello service in Cluster #2")

tests/e2e/multicluster/multicluster_multiprimary_test.go

@@ -126,6 +126,11 @@ func generateMultiPrimaryTestCases(profile string) {
 						common.AwaitDeployment(ctx, "istio-eastwestgateway", k2, clRemote)
 						Success("Gateway is created and available in both clusters")
 					})
+
+					It("has external IPs assigned", func(ctx SpecContext) {
+						expectLoadBalancerAddress(ctx, k1, clPrimary, "istio-eastwestgateway")
+						expectLoadBalancerAddress(ctx, k2, clRemote, "istio-eastwestgateway")
+					})
 				})
 
 				When("are installed remote secrets on each cluster", func() {
@@ -177,6 +182,11 @@ func generateMultiPrimaryTestCases(profile string) {
 						Success("Sample app is created in both clusters and Running")
 					})
 
+					It("can reach target east-west gateway from each cluster", func(ctx SpecContext) {
+						eventuallyLoadBalancerIsReachable(ctx, k1, k2, clRemote, "istio-eastwestgateway")
+						eventuallyLoadBalancerIsReachable(ctx, k2, k1, clPrimary, "istio-eastwestgateway")
+					})
+
 					It("can access the sample app from both clusters", func(ctx SpecContext) {
 						verifyResponsesAreReceivedFromExpectedVersions(k1)
 						verifyResponsesAreReceivedFromExpectedVersions(k2)

tests/e2e/multicluster/multicluster_primaryremote_test.go

@@ -109,6 +109,10 @@ pilot:
 					It("updates Gateway status to Available", func(ctx SpecContext) {
 						common.AwaitDeployment(ctx, "istio-eastwestgateway", k1, clPrimary)
 					})
+
+					It("has an external IP assigned", func(ctx SpecContext) {
+						expectLoadBalancerAddress(ctx, k1, clPrimary, "istio-eastwestgateway")
+					})
 				})
 
 				When("Istio and IstioCNI are created in Remote cluster", func() {
@@ -179,6 +183,10 @@ values:
 					It("updates Gateway status to Available", func(ctx SpecContext) {
 						common.AwaitDeployment(ctx, "istio-eastwestgateway", k2, clRemote)
 					})
+
+					It("has an external IP assigned", func(ctx SpecContext) {
+						expectLoadBalancerAddress(ctx, k2, clRemote, "istio-eastwestgateway")
+					})
 				})
 
 				When("sample apps are deployed in both clusters", func() {
@@ -196,6 +204,11 @@ values:
 						Success("Sample app is created in both clusters and Running")
 					})
 
+					It("can reach target east-west gateway from each cluster", func(ctx SpecContext) {
+						eventuallyLoadBalancerIsReachable(ctx, k1, k2, clRemote, "istio-eastwestgateway")
+						eventuallyLoadBalancerIsReachable(ctx, k2, k1, clPrimary, "istio-eastwestgateway")
+					})
+
 					It("can access the sample app from both clusters", func(ctx SpecContext) {
 						verifyResponsesAreReceivedFromExpectedVersions(k1)
 						verifyResponsesAreReceivedFromExpectedVersions(k2)

tests/e2e/multicluster/multicluster_suite_test.go

@@ -37,7 +37,6 @@ var (
 	clRemote                      client.Client
 	err                           error
 	debugInfoLogged               bool
-	controlPlaneNamespace         = common.ControlPlaneNamespace
 	externalControlPlaneNamespace = env.Get("EXTERNAL_CONTROL_PLANE_NS", "external-istiod")
 	istioName                     = env.Get("ISTIO_NAME", "default")
 	istioCniNamespace             = common.IstioCniNamespace