diff --git a/tests/e2e/multicluster/common.go b/tests/e2e/multicluster/common.go index 501ca7cb0c..416ee2c0ec 100644 --- a/tests/e2e/multicluster/common.go +++ b/tests/e2e/multicluster/common.go @@ -24,6 +24,7 @@ import ( "time" "github.com/istio-ecosystem/sail-operator/pkg/kube" + . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/certs" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" @@ -32,6 +33,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +var controlPlaneNamespace = common.ControlPlaneNamespace + // ClusterDeployment represents a cluster along with its sample app version. type ClusterDeployment struct { Kubectl kubectl.Kubectl @@ -66,15 +69,18 @@ func deploySampleAppToClusters(ns, profile string, clusters []ClusterDeployment) } // verifyResponsesAreReceivedFromBothClusters checks that when the sleep pod in the sample namespace -// sends a request to the helloworld service, it receives responses from expectedVersions, -// which can be either "v1" or "v2" on on different clusters. +// sends requests to the helloworld service, it receives responses from expectedVersions, +// which can be either "v1" or "v2" on different clusters. func verifyResponsesAreReceivedFromExpectedVersions(k kubectl.Kubectl, expectedVersions ...string) { + Step(fmt.Sprintf("Checking app connectivity from %s", k.ClusterName)) if len(expectedVersions) == 0 { expectedVersions = []string{"v1", "v2"} } for _, v := range expectedVersions { + // Each curl will fetch the URL multiple times, minimizing time wasted waiting for different responses. + // This is because running the exec is much more expensive than having curl fetch the URL several times. Eventually(k.WithNamespace("sample").Exec, 10*time.Minute, 2*time.Second). - WithArguments("deploy/sleep", "sleep", "curl -sS helloworld.sample:5000/hello"). + WithArguments("deploy/sleep", "sleep", "curl -sS -Z helloworld.sample:5000/hello{,,,,,,,,,}"). Should(ContainSubstring(fmt.Sprintf("Hello version: %s", v)), fmt.Sprintf("sleep pod in %s did not receive any response from %s", k.ClusterName, v)) } @@ -154,3 +160,21 @@ func awaitSecretCreation(cluster string, cl client.Client) { return err }).ShouldNot(HaveOccurred(), fmt.Sprintf("Secret is not created on %s cluster", cluster)) } + +// expectLoadBalancerAddress to be assigned. +// If the address is not assigned, cross-cluster traffic will not be sent since Istio doesn't know where to send it for the remote cluster(s). +func expectLoadBalancerAddress(ctx context.Context, k kubectl.Kubectl, cl client.Client, name string) { + address := common.GetSVCLoadBalancerAddress(ctx, cl, controlPlaneNamespace, name) + Expect(address).ToNot(BeEmpty(), fmt.Sprintf("Gateway service %q does not have an external address on %s", name, k.ClusterName)) + Success(fmt.Sprintf("Gateway service %q has an external address %s on %s", name, address, k.ClusterName)) +} + +// eventuallyLoadBalancerIsReachable to make sure that cross cluster communication is working on the infrastructure level +func eventuallyLoadBalancerIsReachable(ctx context.Context, kSrc kubectl.Kubectl, kDest kubectl.Kubectl, clDest client.Client, name string) { + address := common.GetSVCLoadBalancerAddress(ctx, clDest, controlPlaneNamespace, name) + Eventually(ctx, kSrc.WithNamespace("sample").Exec).WithArguments( + "deploy/sleep", "sleep", fmt.Sprintf("curl -sS -o /dev/null -w '%%{http_code}' %s:15021/healthz/ready", address)). + Should(Equal("200"), fmt.Sprintf("Gateway %q(%s) on %s is not reachable from %s, this might indicate a problem with the underlying infrastructure", + name, address, kDest.ClusterName, kSrc.ClusterName)) + Success(fmt.Sprintf("Gateway %q(%s) on %s is reachable from %s", name, address, kDest.ClusterName, kSrc.ClusterName)) +} diff --git a/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go b/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go index 47b8cc0f1b..37cef4c985 100644 --- a/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go +++ b/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go @@ -92,6 +92,10 @@ values: It("updates Gateway status to Available", func(ctx SpecContext) { common.AwaitDeployment(ctx, "istio-ingressgateway", k1, clPrimary) }) + + It("has an external IP assigned", func(ctx SpecContext) { + expectLoadBalancerAddress(ctx, k1, clPrimary, "istio-ingressgateway") + }) }) When("Istio external is installed in Cluster #2", func() { @@ -354,6 +358,10 @@ spec: Success("Sample pods has expected annotation") }) + It("can reach the ingress gateway from remote cluster", func(ctx SpecContext) { + eventuallyLoadBalancerIsReachable(ctx, k2, k1, clPrimary, "istio-ingressgateway") + }) + It("can access the sample app from the local service", func(ctx SpecContext) { verifyResponsesAreReceivedFromExpectedVersions(k2, "v1") Success("Sample app is accessible from hello service in Cluster #2") diff --git a/tests/e2e/multicluster/multicluster_multiprimary_test.go b/tests/e2e/multicluster/multicluster_multiprimary_test.go index c63dd4c980..c5f35728b7 100644 --- a/tests/e2e/multicluster/multicluster_multiprimary_test.go +++ b/tests/e2e/multicluster/multicluster_multiprimary_test.go @@ -126,6 +126,11 @@ func generateMultiPrimaryTestCases(profile string) { common.AwaitDeployment(ctx, "istio-eastwestgateway", k2, clRemote) Success("Gateway is created and available in both clusters") }) + + It("has external IPs assigned", func(ctx SpecContext) { + expectLoadBalancerAddress(ctx, k1, clPrimary, "istio-eastwestgateway") + expectLoadBalancerAddress(ctx, k2, clRemote, "istio-eastwestgateway") + }) }) When("are installed remote secrets on each cluster", func() { @@ -177,6 +182,11 @@ func generateMultiPrimaryTestCases(profile string) { Success("Sample app is created in both clusters and Running") }) + It("can reach target east-west gateway from each cluster", func(ctx SpecContext) { + eventuallyLoadBalancerIsReachable(ctx, k1, k2, clRemote, "istio-eastwestgateway") + eventuallyLoadBalancerIsReachable(ctx, k2, k1, clPrimary, "istio-eastwestgateway") + }) + It("can access the sample app from both clusters", func(ctx SpecContext) { verifyResponsesAreReceivedFromExpectedVersions(k1) verifyResponsesAreReceivedFromExpectedVersions(k2) diff --git a/tests/e2e/multicluster/multicluster_primaryremote_test.go b/tests/e2e/multicluster/multicluster_primaryremote_test.go index b80d00ad23..b4d9828ac1 100644 --- a/tests/e2e/multicluster/multicluster_primaryremote_test.go +++ b/tests/e2e/multicluster/multicluster_primaryremote_test.go @@ -109,6 +109,10 @@ pilot: It("updates Gateway status to Available", func(ctx SpecContext) { common.AwaitDeployment(ctx, "istio-eastwestgateway", k1, clPrimary) }) + + It("has an external IP assigned", func(ctx SpecContext) { + expectLoadBalancerAddress(ctx, k1, clPrimary, "istio-eastwestgateway") + }) }) When("Istio and IstioCNI are created in Remote cluster", func() { @@ -179,6 +183,10 @@ values: It("updates Gateway status to Available", func(ctx SpecContext) { common.AwaitDeployment(ctx, "istio-eastwestgateway", k2, clRemote) }) + + It("has an external IP assigned", func(ctx SpecContext) { + expectLoadBalancerAddress(ctx, k2, clRemote, "istio-eastwestgateway") + }) }) When("sample apps are deployed in both clusters", func() { @@ -196,6 +204,11 @@ values: Success("Sample app is created in both clusters and Running") }) + It("can reach target east-west gateway from each cluster", func(ctx SpecContext) { + eventuallyLoadBalancerIsReachable(ctx, k1, k2, clRemote, "istio-eastwestgateway") + eventuallyLoadBalancerIsReachable(ctx, k2, k1, clPrimary, "istio-eastwestgateway") + }) + It("can access the sample app from both clusters", func(ctx SpecContext) { verifyResponsesAreReceivedFromExpectedVersions(k1) verifyResponsesAreReceivedFromExpectedVersions(k2) diff --git a/tests/e2e/multicluster/multicluster_suite_test.go b/tests/e2e/multicluster/multicluster_suite_test.go index 8661703be9..eef7b7695d 100644 --- a/tests/e2e/multicluster/multicluster_suite_test.go +++ b/tests/e2e/multicluster/multicluster_suite_test.go @@ -37,7 +37,6 @@ var ( clRemote client.Client err error debugInfoLogged bool - controlPlaneNamespace = common.ControlPlaneNamespace externalControlPlaneNamespace = env.Get("EXTERNAL_CONTROL_PLANE_NS", "external-istiod") istioName = env.Get("ISTIO_NAME", "default") istioCniNamespace = common.IstioCniNamespace