From 80925049dd6a449238238dea6349fdff92686fe5 Mon Sep 17 00:00:00 2001 From: Mike Kolesnik Date: Tue, 2 Dec 2025 14:50:19 +0200 Subject: [PATCH 1/3] E2E code consolidations (#1397) * E2E: Extract common function to await conditions This code is very repetitive and verbose, extracting it to a common function allows the tests to be much more concise and readable, without affecting functionality. The main benefit of this is making the tests easier to read and maintain, and avoid repeating code (and making mistakes). Signed-off-by: Mike Kolesnik * E2E: Add specific AwaitDeployment function Most of the tests that await for the DeploymentAvailable condition do it on the control plane namespace for a Deployment. Extracting this specific case makes the tests even easier to read and understand. Signed-off-by: Mike Kolesnik * E2E: Add CheckSamplePodsReady function Reuse CheckPodsReady and add a specific function CheckSamplePodsReady since most of the calls are on the sample namespace. Both checks are now in the `checks.go` file for logical consistency. Signed-off-by: Mike Kolesnik * E2E: Extract waiting for CNI DaemonSet to be ready This is repetitive and verbose, extracting this makes the tests easier to read and maintain. Signed-off-by: Mike Kolesnik --------- Signed-off-by: Mike Kolesnik --- tests/e2e/ambient/ambient_test.go | 14 +-- tests/e2e/controlplane/control_plane_test.go | 24 ++--- .../controlplane/control_plane_update_test.go | 22 +---- tests/e2e/dualstack/dualstack_test.go | 14 +-- .../multicluster_externalcontrolplane_test.go | 41 ++------ .../multicluster_multiprimary_test.go | 85 +++------------- .../multicluster_primaryremote_test.go | 63 ++---------- .../multi_control_plane_test.go | 20 +--- tests/e2e/operator/operator_install_test.go | 3 +- tests/e2e/util/common/checks.go | 98 +++++++++++++++++++ tests/e2e/util/common/e2e_utils.go | 28 +----- 11 files changed, 150 insertions(+), 262 deletions(-) create mode 100644 tests/e2e/util/common/checks.go diff --git a/tests/e2e/ambient/ambient_test.go b/tests/e2e/ambient/ambient_test.go index 07f97cc9f3..288533ce9f 100644 --- a/tests/e2e/ambient/ambient_test.go +++ b/tests/e2e/ambient/ambient_test.go @@ -27,12 +27,10 @@ import ( . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" - . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -119,22 +117,16 @@ profile: ambient`) }) It("updates the Istio CR status to Reconciled", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReconciled, metav1.ConditionTrue), "Istio is not Reconciled; unexpected Condition") - Success("Istio CR is Reconciled") + common.AwaitCondition(ctx, v1.IstioConditionReconciled, kube.Key(istioName), &v1.Istio{}, k, cl) }) It("updates the Istio CR status to Ready", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready; unexpected Condition") - Success("Istio CR is Ready") + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key(istioName), &v1.Istio{}, k, cl) }) It("deploys istiod", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key("istiod", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Istiod is not Available; unexpected Condition") + common.AwaitDeployment(ctx, "istiod", k, cl) Expect(common.GetVersionFromIstiod()).To(Equal(version.Version), "Unexpected istiod version") - Success("Istiod is deployed in the namespace and Running") }) It("uses the correct image", func(ctx SpecContext) { diff --git a/tests/e2e/controlplane/control_plane_test.go b/tests/e2e/controlplane/control_plane_test.go index 413052edd7..0339222ef7 100644 --- a/tests/e2e/controlplane/control_plane_test.go +++ b/tests/e2e/controlplane/control_plane_test.go @@ -29,13 +29,11 @@ import ( . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" - . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "istio.io/istio/pkg/ptr" @@ -128,15 +126,11 @@ metadata: }) It("updates the status to Reconciled", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioCniName), &v1.IstioCNI{}). - Should(HaveConditionStatus(v1.IstioCNIConditionReconciled, metav1.ConditionTrue), "IstioCNI is not Reconciled; unexpected Condition") - Success("IstioCNI is Reconciled") + common.AwaitCondition(ctx, v1.IstioCNIConditionReconciled, kube.Key(istioCniName), &v1.IstioCNI{}, k, cl) }) It("updates the status to Ready", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioCniName), &v1.IstioCNI{}). - Should(HaveConditionStatus(v1.IstioCNIConditionReady, metav1.ConditionTrue), "IstioCNI is not Ready; unexpected Condition") - Success("IstioCNI is Ready") + common.AwaitCondition(ctx, v1.IstioCNIConditionReady, kube.Key(istioCniName), &v1.IstioCNI{}, k, cl) }) It("doesn't continuously reconcile the IstioCNI CR", func() { @@ -152,22 +146,16 @@ metadata: }) It("updates the Istio CR status to Reconciled", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReconciled, metav1.ConditionTrue), "Istio is not Reconciled; unexpected Condition") - Success("Istio CR is Reconciled") + common.AwaitCondition(ctx, v1.IstioConditionReconciled, kube.Key(istioName), &v1.Istio{}, k, cl) }) It("updates the Istio CR status to Ready", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready; unexpected Condition") - Success("Istio CR is Ready") + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key(istioName), &v1.Istio{}, k, cl) }) It("deploys istiod", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key("istiod", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Istiod is not Available; unexpected Condition") + common.AwaitDeployment(ctx, "istiod", k, cl) Expect(common.GetVersionFromIstiod()).To(Equal(version.Version), "Unexpected istiod version") - Success("Istiod is deployed in the namespace and Running") }) It("uses the correct image", func(ctx SpecContext) { @@ -194,7 +182,7 @@ metadata: samplePods := &corev1.PodList{} It("updates the pods status to Running", func(ctx SpecContext) { - Eventually(common.CheckPodsReady).WithArguments(ctx, cl, sampleNamespace).Should(Succeed(), "Error checking status of sample pods") + Eventually(common.CheckSamplePodsReady).WithArguments(ctx, cl).Should(Succeed(), "Error checking status of sample pods") Expect(cl.List(ctx, samplePods, client.InNamespace(sampleNamespace))).To(Succeed(), "Error getting the pods in sample namespace") Success("sample pods are ready") diff --git a/tests/e2e/controlplane/control_plane_update_test.go b/tests/e2e/controlplane/control_plane_update_test.go index 7c3818c86a..e5ab3b4c99 100644 --- a/tests/e2e/controlplane/control_plane_update_test.go +++ b/tests/e2e/controlplane/control_plane_update_test.go @@ -61,9 +61,7 @@ var _ = Describe("Control Plane updates", Label("control-plane", "slow"), Ordere Expect(k.CreateNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI namespace failed to be created") common.CreateIstioCNI(k, istioversion.Base) - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioCniName), &v1.IstioCNI{}). - Should(HaveConditionStatus(v1.IstioCNIConditionReady, metav1.ConditionTrue), "IstioCNI is not Ready; unexpected Condition") - Success("IstioCNI is Ready") + common.AwaitCondition(ctx, v1.IstioCNIConditionReady, kube.Key(istioCniName), &v1.IstioCNI{}, k, cl) }) When(fmt.Sprintf("the Istio CR is created with RevisionBased updateStrategy for base version %s", istioversion.Base), func() { @@ -75,9 +73,7 @@ updateStrategy: }) It("deploys istiod and pod is Ready", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key("default"), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), "Istiod is not Available; unexpected Condition") - Success("Istiod is deployed in the namespace and Running") + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key("default"), &v1.Istio{}, k, cl) }) }) @@ -124,7 +120,7 @@ spec: Success("sample deployed") samplePods := &corev1.PodList{} - Eventually(common.CheckPodsReady).WithArguments(ctx, cl, sampleNamespace).Should(Succeed(), "Error checking status of sample pods") + Eventually(common.CheckSamplePodsReady).WithArguments(ctx, cl).Should(Succeed(), "Error checking status of sample pods") Expect(cl.List(ctx, samplePods, client.InNamespace(sampleNamespace))).To(Succeed(), "Error getting the pods in sample namespace") Success("sample pods are ready") @@ -138,9 +134,7 @@ spec: }) It("IstioRevisionTag state change to inUse true", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key("default"), &v1.IstioRevisionTag{}). - Should(HaveConditionStatus(v1.IstioRevisionTagConditionInUse, metav1.ConditionTrue), "unexpected Condition; expected InUse true") - Success("IstioRevisionTag is in use by the sample pods") + common.AwaitCondition(ctx, v1.IstioRevisionTagConditionInUse, kube.Key("default"), &v1.IstioRevisionTag{}, k, cl) }) }) @@ -222,13 +216,7 @@ spec: cl.Delete(ctx, &pod) } - Expect(cl.List(ctx, samplePods, client.InNamespace(sampleNamespace))).To(Succeed()) - Expect(samplePods.Items).ToNot(BeEmpty(), "No pods found in sample namespace") - for _, pod := range samplePods.Items { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(pod.Name, sampleNamespace), &corev1.Pod{}). - Should(HaveConditionStatus(corev1.PodReady, metav1.ConditionTrue), "Pod is not Ready") - } - + Eventually(common.CheckSamplePodsReady).WithArguments(ctx, cl).Should(Succeed(), "Error checking status of sample pods") Success("sample pods restarted and are ready") }) diff --git a/tests/e2e/dualstack/dualstack_test.go b/tests/e2e/dualstack/dualstack_test.go index 35b8b5512f..a37e7e36a6 100644 --- a/tests/e2e/dualstack/dualstack_test.go +++ b/tests/e2e/dualstack/dualstack_test.go @@ -27,13 +27,11 @@ import ( . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" - . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/onsi/gomega/types" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -99,22 +97,16 @@ values: }) It("updates the Istio CR status to Reconciled", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReconciled, metav1.ConditionTrue), "Istio is not Reconciled; unexpected Condition") - Success("Istio CR is Reconciled") + common.AwaitCondition(ctx, v1.IstioConditionReconciled, kube.Key(istioName), &v1.Istio{}, k, cl) }) It("updates the Istio CR status to Ready", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready; unexpected Condition") - Success("Istio CR is Ready") + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key(istioName), &v1.Istio{}, k, cl) }) It("deploys istiod", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key("istiod", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Istiod is not Available; unexpected Condition") + common.AwaitDeployment(ctx, "istiod", k, cl) Expect(common.GetVersionFromIstiod()).To(Equal(version.Version), "Unexpected istiod version") - Success("Istiod is deployed in the namespace and Running") }) It("uses the correct image", func(ctx SpecContext) { diff --git a/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go b/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go index bbadb18b45..b33bc3389e 100644 --- a/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go +++ b/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go @@ -27,14 +27,12 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/version" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" - . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -78,18 +76,12 @@ values: }) It("updates the default Istio CR status to Ready", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key(istioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready on Cluster #1; unexpected Condition") - Success("Istio CR is Ready on Cluster #1") + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key(istioName), &v1.Istio{}, k1, clPrimary) }) It("deploys istiod", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key("istiod", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Istiod is not Available on Cluster #1; unexpected Condition") + common.AwaitDeployment(ctx, "istiod", k1, clPrimary) Expect(common.GetVersionFromIstiod()).To(Equal(v.Version), "Unexpected istiod version") - Success("Istiod is deployed in the namespace and Running on Exernal Cluster") }) }) @@ -99,11 +91,7 @@ values: }) It("updates Gateway status to Available", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key("istio-ingressgateway", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Gateway is not Ready on Cluster #1; unexpected Condition") - - Success("Gateway is created and available in both clusters") + common.AwaitDeployment(ctx, "istio-ingressgateway", k1, clPrimary) }) }) @@ -265,11 +253,8 @@ spec: Expect(k1.CreateFromString(externalControlPlaneYAML)).To(Succeed(), "Istio Resource creation failed on Cluster #1") }) - It("updates both Istio CR status to Ready", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key("external-istiod"), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready on Cluster #1; unexpected Condition") - Success("Istio CR is Ready on Cluster #1") + It("updates external Istio CR status to Ready", func(ctx SpecContext) { + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key("external-istiod"), &v1.Istio{}, k1, clPrimary) }) }) @@ -340,10 +325,7 @@ spec: }) It("updates remote Istio CR status to Ready on Cluster #2", func(ctx SpecContext) { - Eventually(common.GetObject, 10*time.Minute). - WithArguments(ctx, clRemote, kube.Key(externalIstioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready on Remote; unexpected Condition") - Success("Remote Istio CR is Ready on Cluster #2") + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key(externalIstioName), &v1.Istio{}, k2, clRemote, 10*time.Minute) }) }) @@ -357,20 +339,13 @@ spec: Success("Sample app is deployed in Cluster #2") }) - samplePodsCluster2 := &corev1.PodList{} It("updates the pods status to Ready", func(ctx SpecContext) { - Expect(clRemote.List(ctx, samplePodsCluster2, client.InNamespace(sampleNamespace))).To(Succeed()) - Expect(samplePodsCluster2.Items).ToNot(BeEmpty(), "No pods found in sample namespace") - - for _, pod := range samplePodsCluster2.Items { - Eventually(common.GetObject). - WithArguments(ctx, clRemote, kube.Key(pod.Name, sampleNamespace), &corev1.Pod{}). - Should(HaveConditionStatus(corev1.PodReady, metav1.ConditionTrue), "Pod is not Ready on Cluster #2; unexpected Condition") - } + Eventually(common.CheckSamplePodsReady).WithArguments(ctx, clRemote).Should(Succeed(), "Error checking status of sample pods on Cluster #2") Success("Sample app is Running") }) It("has istio.io/rev annotation external-istiod", func(ctx SpecContext) { + samplePodsCluster2 := &corev1.PodList{} Expect(clRemote.List(ctx, samplePodsCluster2, client.InNamespace(sampleNamespace))).To(Succeed()) Expect(samplePodsCluster2.Items).ToNot(BeEmpty(), "No pods found in sample namespace") diff --git a/tests/e2e/multicluster/multicluster_multiprimary_test.go b/tests/e2e/multicluster/multicluster_multiprimary_test.go index 2994fc3ab9..9365ec2c6b 100644 --- a/tests/e2e/multicluster/multicluster_multiprimary_test.go +++ b/tests/e2e/multicluster/multicluster_multiprimary_test.go @@ -28,14 +28,11 @@ import ( "github.com/istio-ecosystem/sail-operator/tests/e2e/util/certs" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" - . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" ) var _ = Describe("Multicluster deployment models", Label("multicluster", "multicluster-multiprimary"), Ordered, func() { @@ -91,61 +88,26 @@ values: }) It("updates both Istio CR status to Ready", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key(istioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready on Cluster #1; unexpected Condition") - Success("Istio CR is Ready on Cluster #1") - - Eventually(common.GetObject). - WithArguments(ctx, clRemote, kube.Key(istioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready on Cluster #2; unexpected Condition") - Success("Istio CR is Ready on Cluster #2") + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key(istioName), &v1.Istio{}, k1, clPrimary) + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key(istioName), &v1.Istio{}, k2, clRemote) }) It("updates both IstioCNI CR status to Ready", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key(istioCniName), &v1.IstioCNI{}). - Should(HaveConditionStatus(v1.IstioCNIConditionReady, metav1.ConditionTrue), "Istio CNI is not Ready on Cluster #1; unexpected Condition") - Success("Istio CNI CR is Ready on Cluster #1") - - Eventually(common.GetObject). - WithArguments(ctx, clRemote, kube.Key(istioCniName), &v1.IstioCNI{}). - Should(HaveConditionStatus(v1.IstioCNIConditionReady, metav1.ConditionTrue), "Istio CNI is not Ready on Cluster #2; unexpected Condition") - Success("Istio CNI CR is Ready on Cluster #2") + common.AwaitCondition(ctx, v1.IstioCNIConditionReady, kube.Key(istioCniName), &v1.IstioCNI{}, k1, clPrimary) + common.AwaitCondition(ctx, v1.IstioCNIConditionReady, kube.Key(istioCniName), &v1.IstioCNI{}, k2, clRemote) }) It("deploys istiod", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key("istiod", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Istiod is not Available on Cluster #1; unexpected Condition") + common.AwaitDeployment(ctx, "istiod", k1, clPrimary) Expect(common.GetVersionFromIstiod()).To(Equal(version.Version), "Unexpected istiod version") - Success("Istiod is deployed in the namespace and Running on Cluster #1") - Eventually(common.GetObject). - WithArguments(ctx, clRemote, kube.Key("istiod", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Istiod is not Available on Cluster #2; unexpected Condition") + common.AwaitDeployment(ctx, "istiod", k2, clRemote) Expect(common.GetVersionFromIstiod()).To(Equal(version.Version), "Unexpected istiod version") - Success("Istiod is deployed in the namespace and Running on Cluster #2") }) It("deploys istio-cni-node", func(ctx SpecContext) { - Eventually(func() bool { - daemonset := &appsv1.DaemonSet{} - if err := clPrimary.Get(ctx, kube.Key("istio-cni-node", istioCniNamespace), daemonset); err != nil { - return false - } - return daemonset.Status.NumberAvailable == daemonset.Status.CurrentNumberScheduled - }).Should(BeTrue(), "CNI DaemonSet Pods are not Available on Cluster #1") - Success("CNI DaemonSet is deployed in the namespace and Running on Cluster #1") - - Eventually(func() bool { - daemonset := &appsv1.DaemonSet{} - if err := clRemote.Get(ctx, kube.Key("istio-cni-node", istioCniNamespace), daemonset); err != nil { - return false - } - return daemonset.Status.NumberAvailable == daemonset.Status.CurrentNumberScheduled - }).Should(BeTrue(), "IstioCNI DaemonSet Pods are not Available on Cluster #2") - Success("IstioCNI DaemonSet is deployed in the namespace and Running on Cluster #2") + common.AwaitCniDaemonSet(ctx, k1, clPrimary) + common.AwaitCniDaemonSet(ctx, k2, clRemote) }) }) @@ -160,13 +122,8 @@ values: }) It("updates both Gateway status to Available", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key("istio-eastwestgateway", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Gateway is not Ready on Cluster #1; unexpected Condition") - - Eventually(common.GetObject). - WithArguments(ctx, clRemote, kube.Key("istio-eastwestgateway", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Gateway is not Ready on Cluster #2; unexpected Condition") + common.AwaitDeployment(ctx, "istio-eastwestgateway", k1, clPrimary) + common.AwaitDeployment(ctx, "istio-eastwestgateway", k2, clRemote) Success("Gateway is created and available in both clusters") }) }) @@ -224,26 +181,8 @@ values: }) It("updates the pods status to Ready", func(ctx SpecContext) { - samplePodsCluster1 := &corev1.PodList{} - - Expect(clPrimary.List(ctx, samplePodsCluster1, client.InNamespace(sampleNamespace))).To(Succeed()) - Expect(samplePodsCluster1.Items).ToNot(BeEmpty(), "No pods found in sample namespace") - - for _, pod := range samplePodsCluster1.Items { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key(pod.Name, sampleNamespace), &corev1.Pod{}). - Should(HaveConditionStatus(corev1.PodReady, metav1.ConditionTrue), "Pod is not Ready on Cluster #1; unexpected Condition") - } - - samplePodsCluster2 := &corev1.PodList{} - Expect(clRemote.List(ctx, samplePodsCluster2, client.InNamespace(sampleNamespace))).To(Succeed()) - Expect(samplePodsCluster2.Items).ToNot(BeEmpty(), "No pods found in sample namespace") - - for _, pod := range samplePodsCluster2.Items { - Eventually(common.GetObject). - WithArguments(ctx, clRemote, kube.Key(pod.Name, sampleNamespace), &corev1.Pod{}). - Should(HaveConditionStatus(corev1.PodReady, metav1.ConditionTrue), "Pod is not Ready on Cluster #2; unexpected Condition") - } + Eventually(common.CheckSamplePodsReady).WithArguments(ctx, clPrimary).Should(Succeed(), "Error checking status of sample pods on Cluster #1") + Eventually(common.CheckSamplePodsReady).WithArguments(ctx, clRemote).Should(Succeed(), "Error checking status of sample pods on Cluster #2") Success("Sample app is created in both clusters and Running") }) diff --git a/tests/e2e/multicluster/multicluster_primaryremote_test.go b/tests/e2e/multicluster/multicluster_primaryremote_test.go index f0f15a978b..cb97c784e7 100644 --- a/tests/e2e/multicluster/multicluster_primaryremote_test.go +++ b/tests/e2e/multicluster/multicluster_primaryremote_test.go @@ -29,14 +29,11 @@ import ( "github.com/istio-ecosystem/sail-operator/tests/e2e/util/certs" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" - . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" ) var _ = Describe("Multicluster deployment models", Label("multicluster", "multicluster-primaryremote"), Ordered, func() { @@ -101,36 +98,20 @@ values: }) It("updates Istio CR on Primary cluster status to Ready", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key(istioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready on Primary; unexpected Condition") - Success("Istio CR is Ready on Primary Cluster") + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key(istioName), &v1.Istio{}, k1, clPrimary) }) It("updates IstioCNI CR on Primary cluster status to Ready", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key(istioCniName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioCNIConditionReady, metav1.ConditionTrue), "IstioCNI is not Ready on Primary; unexpected Condition") - Success("IstioCNI CR is Ready on Primary Cluster") + common.AwaitCondition(ctx, v1.IstioCNIConditionReady, kube.Key(istioCniName), &v1.IstioCNI{}, k1, clPrimary) }) It("deploys istiod", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key("istiod", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Istiod is not Available on Primary; unexpected Condition") + common.AwaitDeployment(ctx, "istiod", k1, clPrimary) Expect(common.GetVersionFromIstiod()).To(Equal(v.Version), "Unexpected istiod version") - Success("Istiod is deployed in the namespace and Running on Primary Cluster") }) It("deploys istio-cni-node", func(ctx SpecContext) { - Eventually(func() bool { - daemonset := &appsv1.DaemonSet{} - if err := clPrimary.Get(ctx, kube.Key("istio-cni-node", istioCniNamespace), daemonset); err != nil { - return false - } - return daemonset.Status.NumberAvailable == daemonset.Status.CurrentNumberScheduled - }).Should(BeTrue(), "IstioCNI DaemonSet Pods are not Available on Primary Cluster") - Success("IstioCNI DaemonSet is deployed in the namespace and Running on Primary Cluster") + common.AwaitCniDaemonSet(ctx, k1, clPrimary) }) }) @@ -146,9 +127,7 @@ values: }) It("updates Gateway status to Available", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key("istio-eastwestgateway", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Gateway is not Ready on Primary; unexpected Condition") + common.AwaitDeployment(ctx, "istio-eastwestgateway", k1, clPrimary) }) }) @@ -207,10 +186,7 @@ values: }) It("updates remote Istio CR status to Ready", func(ctx SpecContext) { - Eventually(common.GetObject, 10*time.Minute). - WithArguments(ctx, clRemote, kube.Key(istioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready on Remote; unexpected Condition") - Success("Istio CR is Ready on Remote Cluster") + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key(istioName), &v1.Istio{}, k2, clRemote, 10*time.Minute) }) }) @@ -221,10 +197,7 @@ values: }) It("updates Gateway status to Available", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clRemote, kube.Key("istio-eastwestgateway", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Gateway is not Ready on Remote; unexpected Condition") - Success("Gateway is created and available in Remote cluster") + common.AwaitDeployment(ctx, "istio-eastwestgateway", k2, clRemote) }) }) @@ -247,26 +220,8 @@ values: }) It("updates the pods status to Ready", func(ctx SpecContext) { - samplePodsPrimary := &corev1.PodList{} - - Expect(clPrimary.List(ctx, samplePodsPrimary, client.InNamespace(sampleNamespace))).To(Succeed()) - Expect(samplePodsPrimary.Items).ToNot(BeEmpty(), "No pods found in sample namespace") - - for _, pod := range samplePodsPrimary.Items { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key(pod.Name, sampleNamespace), &corev1.Pod{}). - Should(HaveConditionStatus(corev1.PodReady, metav1.ConditionTrue), "Pod is not Ready on Primary; unexpected Condition") - } - - samplePodsRemote := &corev1.PodList{} - Expect(clRemote.List(ctx, samplePodsRemote, client.InNamespace(sampleNamespace))).To(Succeed()) - Expect(samplePodsRemote.Items).ToNot(BeEmpty(), "No pods found in sample namespace") - - for _, pod := range samplePodsRemote.Items { - Eventually(common.GetObject). - WithArguments(ctx, clRemote, kube.Key(pod.Name, sampleNamespace), &corev1.Pod{}). - Should(HaveConditionStatus(corev1.PodReady, metav1.ConditionTrue), "Pod is not Ready on Remote; unexpected Condition") - } + Eventually(common.CheckSamplePodsReady).WithArguments(ctx, clPrimary).Should(Succeed(), "Error checking status of sample pods on Primary cluster") + Eventually(common.CheckSamplePodsReady).WithArguments(ctx, clRemote).Should(Succeed(), "Error checking status of sample pods on Remote cluster") Success("Sample app is created in both clusters and Running") }) diff --git a/tests/e2e/multicontrolplane/multi_control_plane_test.go b/tests/e2e/multicontrolplane/multi_control_plane_test.go index 18fb8d94cf..7e50bddb26 100644 --- a/tests/e2e/multicontrolplane/multi_control_plane_test.go +++ b/tests/e2e/multicontrolplane/multi_control_plane_test.go @@ -26,11 +26,9 @@ import ( . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" - . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var _ = Describe("Multi control plane deployment model", Label("smoke", "multicontrol-plane"), Ordered, func() { @@ -60,10 +58,7 @@ var _ = Describe("Multi control plane deployment model", Label("smoke", "multico It("Installs IstioCNI", func(ctx SpecContext) { common.CreateIstioCNI(k, latestVersion.Name) - - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioCniName), &v1.IstioCNI{}). - Should(HaveConditionStatus(v1.IstioCNIConditionReady, metav1.ConditionTrue), "IstioCNI is not Ready; unexpected Condition") - Success("IstioCNI is Ready") + common.AwaitCondition(ctx, v1.IstioCNIConditionReady, kube.Key(istioCniName), &v1.IstioCNI{}, k, cl) }) DescribeTable("Installs Istios", @@ -100,13 +95,8 @@ spec: Entry("Mesh 1", istioName1), Entry("Mesh 2", istioName2), func(ctx SpecContext, name string) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(name), &v1.Istio{}). - Should( - And( - HaveConditionStatus(v1.IstioConditionReconciled, metav1.ConditionTrue), - HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), - ), "Istio is not Reconciled and Ready; unexpected Condition") - Success(fmt.Sprintf("Istio %s ready", name)) + common.AwaitCondition(ctx, v1.IstioConditionReconciled, kube.Key(name), &v1.Istio{}, k, cl) + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key(name), &v1.Istio{}, k, cl) }) DescribeTable("Deploys applications", @@ -134,10 +124,8 @@ spec: Entry("App 2b", appNamespace2b), func(ctx SpecContext, ns string) { for _, deployment := range []string{"sleep", "httpbin"} { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(deployment, ns), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error waiting for deployment to be available") + common.AwaitCondition(ctx, appsv1.DeploymentAvailable, kube.Key(deployment, ns), &appsv1.Deployment{}, k, cl) } - Success(fmt.Sprintf("Applications in namespace %s ready", ns)) }) }) diff --git a/tests/e2e/operator/operator_install_test.go b/tests/e2e/operator/operator_install_test.go index 9899437c26..03f3000191 100644 --- a/tests/e2e/operator/operator_install_test.go +++ b/tests/e2e/operator/operator_install_test.go @@ -75,8 +75,7 @@ var _ = Describe("Operator", Label("smoke", "operator"), Ordered, func() { It("updates the CRDs status to Established", func(ctx SpecContext) { for _, crdName := range sailCRDs { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(crdName), &apiextensionsv1.CustomResourceDefinition{}). - Should(HaveConditionStatus(apiextensionsv1.Established, metav1.ConditionTrue), "Error getting Istio CRD") + common.AwaitCondition(ctx, apiextensionsv1.Established, kube.Key(crdName), &apiextensionsv1.CustomResourceDefinition{}, k, cl) } Success("CRDs are Established") }) diff --git a/tests/e2e/util/common/checks.go b/tests/e2e/util/common/checks.go new file mode 100644 index 0000000000..a3df3ca9a7 --- /dev/null +++ b/tests/e2e/util/common/checks.go @@ -0,0 +1,98 @@ +//go:build e2e + +// Copyright Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "context" + "fmt" + "reflect" + + "github.com/istio-ecosystem/sail-operator/pkg/kube" + . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" + . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// AwaitCondition to be True. A key and a pointer to the object struct must be supplied. Extra arguments to pass to `Eventually` can be optionally supplied. +func AwaitCondition[T ~string](ctx context.Context, condition T, key client.ObjectKey, obj client.Object, k kubectl.Kubectl, cl client.Client, args ...any) { + kind := reflect.TypeOf(obj).Elem().Name() + cluster := "cluster" + if k.ClusterName != "" { + cluster = k.ClusterName + } + + Eventually(GetObject, args...). + WithArguments(ctx, cl, key, obj). + Should(HaveConditionStatus(condition, metav1.ConditionTrue), + fmt.Sprintf("%s %q is not %s on %s; unexpected Condition", kind, key.Name, condition, cluster)) + Success(fmt.Sprintf("%s %q is %s on %s", kind, key.Name, condition, cluster)) +} + +// AwaitDeployment to reach the Available state. +func AwaitDeployment(ctx context.Context, name string, k kubectl.Kubectl, cl client.Client) { + AwaitCondition(ctx, appsv1.DeploymentAvailable, kube.Key(name, controlPlaneNamespace), &appsv1.Deployment{}, k, cl) +} + +func isPodReady(pod *corev1.Pod) bool { + for _, cond := range pod.Status.Conditions { + if cond.Type == corev1.PodReady && cond.Status == corev1.ConditionTrue { + return true + } + } + return false +} + +// CheckPodsReady in the given namespace. +func CheckPodsReady(ctx context.Context, cl client.Client, namespace string) error { + podList := &corev1.PodList{} + if err := cl.List(ctx, podList, client.InNamespace(namespace)); err != nil { + return fmt.Errorf("Failed to list pods: %w", err) + } + if len(podList.Items) == 0 { + return fmt.Errorf("No pods found in namespace %q", namespace) + } + + for _, pod := range podList.Items { + if !isPodReady(&pod) { + return fmt.Errorf("Pod %q in namespace %q is not ready", pod.Name, namespace) + } + } + + return nil +} + +func CheckSamplePodsReady(ctx context.Context, cl client.Client) error { + return CheckPodsReady(ctx, cl, sampleNamespace) +} + +// AwaitCniDaemonSet to be deployed and reach the scheduled number of pods. +func AwaitCniDaemonSet(ctx context.Context, k kubectl.Kubectl, cl client.Client) { + key := kube.Key("istio-cni-node", istioCniNamespace) + Eventually(func() bool { + daemonset := &appsv1.DaemonSet{} + if err := cl.Get(ctx, key, daemonset); err != nil { + return false + } + return daemonset.Status.NumberAvailable == daemonset.Status.CurrentNumberScheduled + }).Should(BeTrue(), fmt.Sprintf("DaemonSet '%s' is not Available in the '%s' namespace on %s cluster", key.Name, key.Namespace, k.ClusterName)) + Success(fmt.Sprintf("DaemonSet '%s' is deployed and running in the '%s' namespace on %s cluster", key.Name, key.Namespace, k.ClusterName)) +} diff --git a/tests/e2e/util/common/e2e_utils.go b/tests/e2e/util/common/e2e_utils.go index ff5e289d16..00a4e79585 100644 --- a/tests/e2e/util/common/e2e_utils.go +++ b/tests/e2e/util/common/e2e_utils.go @@ -64,6 +64,7 @@ var ( istioName = env.Get("ISTIO_NAME", "default") istioCniName = env.Get("ISTIOCNI_NAME", "default") istioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") + sampleNamespace = env.Get("SAMPLE_NAMESPACE", "sample") ztunnelNamespace = env.Get("ZTUNNEL_NAMESPACE", "ztunnel") // version can have one of the following formats: @@ -284,33 +285,6 @@ func GetVersionFromIstiod() (*semver.Version, error) { return nil, fmt.Errorf("error getting version from istiod: version not found in output: %s", output) } -func isPodReady(pod *corev1.Pod) bool { - for _, cond := range pod.Status.Conditions { - if cond.Type == corev1.PodReady && cond.Status == corev1.ConditionTrue { - return true - } - } - return false -} - -func CheckPodsReady(ctx context.Context, cl client.Client, namespace string) error { - podList := &corev1.PodList{} - if err := cl.List(ctx, podList, client.InNamespace(namespace)); err != nil { - return fmt.Errorf("Failed to list pods: %w", err) - } - if len(podList.Items) == 0 { - return fmt.Errorf("No pods found in namespace %q", namespace) - } - - for _, pod := range podList.Items { - if !isPodReady(&pod) { - return fmt.Errorf("pod %q in namespace %q is not ready", pod.Name, namespace) - } - } - - return nil -} - // Resolve domain name and return ip address. // By default, return ipv4 address and if missing, return ipv6. func ResolveHostDomainToIP(hostDomain string) (string, error) { From 46d957ee4e3b58b04432131eef8b6f5bb361ce93 Mon Sep 17 00:00:00 2001 From: Mike Kolesnik Date: Thu, 4 Dec 2025 13:21:35 +0200 Subject: [PATCH 2/3] Add Ambient multi cluster E2E tests for Multi Primary deployments (#1353) * Expose common namespaces in E2E tests To be able to use directly without redefining or passing them all over. The namespaces are reused in the test suite files (instead of redefined) in order to minimize the amount of changes, but we can further refactor this in the future. Signed-off-by: Mike Kolesnik * Extract multicluster resource creation This males tests easier to maintain while avoiding code duplication Signed-off-by: Mike Kolesnik * Add Ambient tests to E2E Multi Primary Istio 1.27+ supports running Multi-Primary deployments in Abmient mode, so tests were adapted to run both on Sidecar and Ambient mode. Both modes run the same tests, sequentialy & independently of each other. Signed-off-by: Mike Kolesnik --------- Signed-off-by: Mike Kolesnik --- tests/e2e/ambient/ambient_suite_test.go | 7 +- .../controlplane/control_plane_suite_test.go | 4 +- tests/e2e/dualstack/dualstack_suite_test.go | 5 +- tests/e2e/multicluster/common.go | 85 +++++++++++- .../multicluster_externalcontrolplane_test.go | 5 +- .../multicluster_multiprimary_test.go | 93 ++++++------- .../multicluster_primaryremote_test.go | 55 ++------ .../multicluster/multicluster_suite_test.go | 4 +- .../multi_control_plane_suite_test.go | 3 +- tests/e2e/setup/setup-kind.sh | 6 + tests/e2e/util/common/checks.go | 4 +- tests/e2e/util/common/e2e_utils.go | 123 ++++++++++++------ tests/e2e/util/kubectl/kubectl.go | 6 + tools/update_deps.sh | 11 ++ 14 files changed, 257 insertions(+), 154 deletions(-) diff --git a/tests/e2e/ambient/ambient_suite_test.go b/tests/e2e/ambient/ambient_suite_test.go index 393c9971be..824fa18509 100644 --- a/tests/e2e/ambient/ambient_suite_test.go +++ b/tests/e2e/ambient/ambient_suite_test.go @@ -21,6 +21,7 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/env" k8sclient "github.com/istio-ecosystem/sail-operator/tests/e2e/util/client" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -30,10 +31,10 @@ import ( var ( cl client.Client err error - controlPlaneNamespace = env.Get("CONTROL_PLANE_NS", "istio-system") + controlPlaneNamespace = common.ControlPlaneNamespace istioName = env.Get("ISTIO_NAME", "default") - istioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") - ztunnelNamespace = env.Get("ZTUNNEL_NAMESPACE", "ztunnel") + istioCniNamespace = common.IstioCniNamespace + ztunnelNamespace = common.ZtunnelNamespace istioCniName = env.Get("ISTIOCNI_NAME", "default") expectedRegistry = env.Get("EXPECTED_REGISTRY", "^docker\\.io|^gcr\\.io") multicluster = env.GetBool("MULTICLUSTER", false) diff --git a/tests/e2e/controlplane/control_plane_suite_test.go b/tests/e2e/controlplane/control_plane_suite_test.go index a7e73942ef..57ef2eb60d 100644 --- a/tests/e2e/controlplane/control_plane_suite_test.go +++ b/tests/e2e/controlplane/control_plane_suite_test.go @@ -33,9 +33,9 @@ var ( err error namespace = common.OperatorNamespace deploymentName = env.Get("DEPLOYMENT_NAME", "sail-operator") - controlPlaneNamespace = env.Get("CONTROL_PLANE_NS", "istio-system") + controlPlaneNamespace = common.ControlPlaneNamespace istioName = env.Get("ISTIO_NAME", "default") - istioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") + istioCniNamespace = common.IstioCniNamespace istioCniName = env.Get("ISTIOCNI_NAME", "default") expectedRegistry = env.Get("EXPECTED_REGISTRY", "^docker\\.io|^gcr\\.io") sampleNamespace = env.Get("SAMPLE_NAMESPACE", "sample") diff --git a/tests/e2e/dualstack/dualstack_suite_test.go b/tests/e2e/dualstack/dualstack_suite_test.go index f2dd6c1c32..3ce8e00770 100644 --- a/tests/e2e/dualstack/dualstack_suite_test.go +++ b/tests/e2e/dualstack/dualstack_suite_test.go @@ -21,6 +21,7 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/env" k8sclient "github.com/istio-ecosystem/sail-operator/tests/e2e/util/client" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -30,9 +31,9 @@ import ( var ( cl client.Client err error - controlPlaneNamespace = env.Get("CONTROL_PLANE_NS", "istio-system") + controlPlaneNamespace = common.ControlPlaneNamespace istioName = env.Get("ISTIO_NAME", "default") - istioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") + istioCniNamespace = common.IstioCniNamespace istioCniName = env.Get("ISTIOCNI_NAME", "default") expectedRegistry = env.Get("EXPECTED_REGISTRY", "^docker\\.io|^gcr\\.io") multicluster = env.GetBool("MULTICLUSTER", false) diff --git a/tests/e2e/multicluster/common.go b/tests/e2e/multicluster/common.go index 4d792e6b79..501ca7cb0c 100644 --- a/tests/e2e/multicluster/common.go +++ b/tests/e2e/multicluster/common.go @@ -17,13 +17,19 @@ package multicluster import ( + "context" "fmt" "strings" "text/template" "time" + "github.com/istio-ecosystem/sail-operator/pkg/kube" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/certs" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" ) // ClusterDeployment represents a cluster along with its sample app version. @@ -33,16 +39,29 @@ type ClusterDeployment struct { } // deploySampleApp deploys the sample apps (helloworld and sleep) in the given cluster. -func deploySampleApp(k kubectl.Kubectl, ns string, appVersion string) { +func deploySampleApp(k kubectl.Kubectl, ns, appVersion, profile string) { Expect(k.WithNamespace(ns).ApplyKustomize("helloworld", "service=helloworld")).To(Succeed(), "Sample service deploy failed on Cluster") Expect(k.WithNamespace(ns).ApplyKustomize("helloworld", "version="+appVersion)).To(Succeed(), "Sample service deploy failed on Cluster") Expect(k.WithNamespace(ns).ApplyKustomize("sleep")).To(Succeed(), "Sample sleep deploy failed on Cluster") + + // In Ambient mode, services need to be marked as "global" in order to load balance requests among clusters + if profile == "ambient" { + Expect(k.LabelNamespaced("service", ns, "helloworld", "istio.io/global", "true")).To(Succeed(), "Error labeling sample namespace") + } } // deploySampleAppToClusters deploys the sample app to all provided clusters. -func deploySampleAppToClusters(ns string, clusters []ClusterDeployment) { +func deploySampleAppToClusters(ns, profile string, clusters []ClusterDeployment) { for _, cd := range clusters { - deploySampleApp(cd.Kubectl, ns, cd.AppVersion) + k := cd.Kubectl + Expect(k.CreateNamespace(ns)).To(Succeed(), fmt.Sprintf("Namespace failed to be created on Cluster %s", k.ClusterName)) + if profile == "ambient" { + Expect(k.Label("namespace", ns, "istio.io/dataplane-mode", "ambient")).To(Succeed(), "Error labeling sample namespace") + } else { + Expect(k.Label("namespace", ns, "istio-injection", "enabled")).To(Succeed(), "Error labeling sample namespace") + } + + deploySampleApp(k, ns, cd.AppVersion, profile) } } @@ -75,3 +94,63 @@ func genTemplate(manifestTmpl string, values any) string { Expect(tmpl.Execute(&b, values)).To(Succeed()) return b.String() } + +func createIstioNamespaces(k kubectl.Kubectl, network, profile string) { + Expect(k.CreateNamespace(common.ControlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") + Expect(k.CreateNamespace(common.IstioCniNamespace)).To(Succeed(), "Istio CNI namespace failed to be created") + + if profile == "ambient" { + Expect(k.Label("namespace", common.ControlPlaneNamespace, "topology.istio.io/network", network)).To(Succeed(), "Error labeling istio namespace") + Expect(k.CreateNamespace(common.ZtunnelNamespace)).To(Succeed(), "Ztunnel namespace failed to be created") + } +} + +func createIstioResources(k kubectl.Kubectl, version, cluster, network, profile string, values ...string) { + cniSpec := fmt.Sprintf(` +profile: %s`, profile) + common.CreateIstioCNI(k, version, cniSpec) + + if profile == "ambient" { + spec := fmt.Sprintf(` +values: + ztunnel: + multiCluster: + clusterName: %s + network: %s`, cluster, network) + common.CreateZTunnel(k, version, spec) + } + + spec := fmt.Sprintf(` +profile: %s +values: + global: + meshID: mesh1 + multiCluster: + clusterName: %s + network: %s`, profile, cluster, network) + for _, value := range values { + spec += common.Indent(value) + } + + if profile == "ambient" { + spec += fmt.Sprintf(` + pilot: + trustedZtunnelNamespace: %s + env: + AMBIENT_ENABLE_MULTI_NETWORK: "true"`, common.ZtunnelNamespace) + } + + common.CreateIstio(k, version, spec) +} + +func createIntermediateCA(k kubectl.Kubectl, zone, network, artifacts string, cl client.Client) { + Expect(certs.PushIntermediateCA(k, common.ControlPlaneNamespace, zone, network, artifacts, cl)). + To(Succeed(), fmt.Sprintf("Error pushing intermediate CA to %s Cluster", k.ClusterName)) +} + +func awaitSecretCreation(cluster string, cl client.Client) { + Eventually(func() error { + _, err := common.GetObject(context.Background(), cl, kube.Key("cacerts", common.ControlPlaneNamespace), &corev1.Secret{}) + return err + }).ShouldNot(HaveOccurred(), fmt.Sprintf("Secret is not created on %s cluster", cluster)) +} diff --git a/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go b/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go index b33bc3389e..47b8cc0f1b 100644 --- a/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go +++ b/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go @@ -65,8 +65,7 @@ var _ = Describe("Multicluster deployment models", Label("multicluster", "multic When("default Istio is created in Cluster #1 to handle ingress to External Control Plane", func() { BeforeAll(func(ctx SpecContext) { - Expect(k1.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be created") - Expect(k1.CreateNamespace(istioCniNamespace)).To(Succeed(), "Istio CNI namespace failed to be created") + createIstioNamespaces(k1, "network1", "default") common.CreateIstioCNI(k1, v.Name) common.CreateIstio(k1, v.Name, ` @@ -335,7 +334,7 @@ spec: // Label the namespace with the istio revision name Expect(k2.Label("namespace", sampleNamespace, "istio.io/rev", "external-istiod")).To(Succeed(), "Labeling failed on Cluster #2") - deploySampleApp(k2, sampleNamespace, "v1") + deploySampleApp(k2, sampleNamespace, "v1", "default") Success("Sample app is deployed in Cluster #2") }) diff --git a/tests/e2e/multicluster/multicluster_multiprimary_test.go b/tests/e2e/multicluster/multicluster_multiprimary_test.go index 9365ec2c6b..c63dd4c980 100644 --- a/tests/e2e/multicluster/multicluster_multiprimary_test.go +++ b/tests/e2e/multicluster/multicluster_multiprimary_test.go @@ -17,7 +17,6 @@ package multicluster import ( - "context" "fmt" "time" @@ -25,7 +24,7 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/istioversion" "github.com/istio-ecosystem/sail-operator/pkg/kube" . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" - "github.com/istio-ecosystem/sail-operator/tests/e2e/util/certs" + "github.com/istio-ecosystem/sail-operator/pkg/version" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl" @@ -39,10 +38,25 @@ var _ = Describe("Multicluster deployment models", Label("multicluster", "multic SetDefaultEventuallyTimeout(180 * time.Second) SetDefaultEventuallyPollingInterval(time.Second) + Context("Sidecar", func() { + generateMultiPrimaryTestCases("default") + }) + Context("Ambient", Label("ambient"), func() { + generateMultiPrimaryTestCases("ambient") + }) +}) + +func generateMultiPrimaryTestCases(profile string) { Describe("Multi-Primary Multi-Network configuration", func() { // Test the Multi-Primary Multi-Network configuration for each supported Istio version - for _, version := range istioversion.GetLatestPatchVersions() { - Context(fmt.Sprintf("Istio version %s", version.Version), func() { + for _, v := range istioversion.GetLatestPatchVersions() { + // Ambient multi-cluster is supported only since 1.27 + if profile == "ambient" && version.Constraint("<1.27").Check(v.Version) { + Log(fmt.Sprintf("Skipping test, because Istio version %s does not support Ambient Multi-Cluster configuration", v.Version)) + continue + } + + Context(fmt.Sprintf("Istio version %s", v.Version), func() { clr1 := cleaner.New(clPrimary, "cluster=primary") clr2 := cleaner.New(clRemote, "cluster=remote") @@ -53,38 +67,19 @@ var _ = Describe("Multicluster deployment models", Label("multicluster", "multic When("Istio and IstioCNI resources are created in both clusters", func() { BeforeAll(func(ctx SpecContext) { - Expect(k1.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") - Expect(k2.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") - Expect(k1.CreateNamespace(istioCniNamespace)).To(Succeed(), "Istio CNI namespace failed to be created") - Expect(k2.CreateNamespace(istioCniNamespace)).To(Succeed(), "Istio CNI namespace failed to be created") + createIstioNamespaces(k1, "network1", profile) + createIstioNamespaces(k2, "network2", profile) // Push the intermediate CA to both clusters - Expect(certs.PushIntermediateCA(k1, controlPlaneNamespace, "east", "network1", artifacts, clPrimary)).To(Succeed()) - Expect(certs.PushIntermediateCA(k2, controlPlaneNamespace, "west", "network2", artifacts, clRemote)).To(Succeed()) + createIntermediateCA(k1, "east", "network1", artifacts, clPrimary) + createIntermediateCA(k2, "west", "network2", artifacts, clRemote) // Wait for the secret to be created in both clusters - Eventually(func() error { - _, err := common.GetObject(context.Background(), clPrimary, kube.Key("cacerts", controlPlaneNamespace), &corev1.Secret{}) - return err - }).ShouldNot(HaveOccurred(), "Secret is not created on Cluster #1") - - Eventually(func() error { - _, err := common.GetObject(context.Background(), clRemote, kube.Key("cacerts", controlPlaneNamespace), &corev1.Secret{}) - return err - }).ShouldNot(HaveOccurred(), "Secret is not created on Cluster #1") - - common.CreateIstioCNI(k1, version.Name) - common.CreateIstioCNI(k2, version.Name) - - spec := ` -values: - global: - meshID: mesh1 - multiCluster: - clusterName: %s - network: %s` - common.CreateIstio(k1, version.Name, fmt.Sprintf(spec, "cluster1", "network1")) - common.CreateIstio(k2, version.Name, fmt.Sprintf(spec, "cluster2", "network2")) + awaitSecretCreation(k1.ClusterName, clPrimary) + awaitSecretCreation(k2.ClusterName, clRemote) + + createIstioResources(k1, v.Name, "cluster1", "network1", profile) + createIstioResources(k2, v.Name, "cluster2", "network2", profile) }) It("updates both Istio CR status to Ready", func(ctx SpecContext) { @@ -99,10 +94,10 @@ values: It("deploys istiod", func(ctx SpecContext) { common.AwaitDeployment(ctx, "istiod", k1, clPrimary) - Expect(common.GetVersionFromIstiod()).To(Equal(version.Version), "Unexpected istiod version") + Expect(common.GetVersionFromIstiod()).To(Equal(v.Version), "Unexpected istiod version") common.AwaitDeployment(ctx, "istiod", k2, clRemote) - Expect(common.GetVersionFromIstiod()).To(Equal(version.Version), "Unexpected istiod version") + Expect(common.GetVersionFromIstiod()).To(Equal(v.Version), "Unexpected istiod version") }) It("deploys istio-cni-node", func(ctx SpecContext) { @@ -113,12 +108,17 @@ values: When("Gateway is created in both clusters", func() { BeforeAll(func(ctx SpecContext) { - Expect(k1.WithNamespace(controlPlaneNamespace).Apply(eastGatewayYAML)).To(Succeed(), "Gateway creation failed on Cluster #1") - Expect(k2.WithNamespace(controlPlaneNamespace).Apply(westGatewayYAML)).To(Succeed(), "Gateway creation failed on Cluster #2") - - // Expose the Gateway service in both clusters - Expect(k1.WithNamespace(controlPlaneNamespace).Apply(exposeServiceYAML)).To(Succeed(), "Expose Service creation failed on Cluster #1") - Expect(k2.WithNamespace(controlPlaneNamespace).Apply(exposeServiceYAML)).To(Succeed(), "Expose Service creation failed on Cluster #2") + if profile == "ambient" { + common.CreateAmbientGateway(k1, controlPlaneNamespace, "network1") + common.CreateAmbientGateway(k2, controlPlaneNamespace, "network2") + } else { + Expect(k1.WithNamespace(controlPlaneNamespace).Apply(eastGatewayYAML)).To(Succeed(), "Gateway creation failed on Cluster #1") + Expect(k2.WithNamespace(controlPlaneNamespace).Apply(westGatewayYAML)).To(Succeed(), "Gateway creation failed on Cluster #2") + + // Expose the Gateway service in both clusters + Expect(k1.WithNamespace(controlPlaneNamespace).Apply(exposeServiceYAML)).To(Succeed(), "Expose Service creation failed on Cluster #1") + Expect(k2.WithNamespace(controlPlaneNamespace).Apply(exposeServiceYAML)).To(Succeed(), "Expose Service creation failed on Cluster #2") + } }) It("updates both Gateway status to Available", func(ctx SpecContext) { @@ -164,16 +164,7 @@ values: When("sample apps are deployed in both clusters", func() { BeforeAll(func(ctx SpecContext) { - // Create namespace - Expect(k1.CreateNamespace(sampleNamespace)).To(Succeed(), "Namespace failed to be created on Cluster #1") - Expect(k2.CreateNamespace(sampleNamespace)).To(Succeed(), "Namespace failed to be created on Cluster #2") - - // Label the namespace - Expect(k1.Label("namespace", sampleNamespace, "istio-injection", "enabled")).To(Succeed(), "Error labeling sample namespace") - Expect(k2.Label("namespace", sampleNamespace, "istio-injection", "enabled")).To(Succeed(), "Error labeling sample namespace") - - // Deploy the sample app in both clusters - deploySampleAppToClusters(sampleNamespace, []ClusterDeployment{ + deploySampleAppToClusters(sampleNamespace, profile, []ClusterDeployment{ {Kubectl: k1, AppVersion: "v1"}, {Kubectl: k2, AppVersion: "v2"}, }) @@ -245,4 +236,4 @@ values: }) } }) -}) +} diff --git a/tests/e2e/multicluster/multicluster_primaryremote_test.go b/tests/e2e/multicluster/multicluster_primaryremote_test.go index cb97c784e7..b80d00ad23 100644 --- a/tests/e2e/multicluster/multicluster_primaryremote_test.go +++ b/tests/e2e/multicluster/multicluster_primaryremote_test.go @@ -17,7 +17,6 @@ package multicluster import ( - "context" "fmt" "time" @@ -26,7 +25,6 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/kube" . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" "github.com/istio-ecosystem/sail-operator/pkg/version" - "github.com/istio-ecosystem/sail-operator/tests/e2e/util/certs" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl" @@ -37,6 +35,7 @@ import ( ) var _ = Describe("Multicluster deployment models", Label("multicluster", "multicluster-primaryremote"), Ordered, func() { + profile := "default" SetDefaultEventuallyTimeout(180 * time.Second) SetDefaultEventuallyPollingInterval(time.Second) @@ -60,41 +59,22 @@ var _ = Describe("Multicluster deployment models", Label("multicluster", "multic When("Istio and IstioCNI resources are created in both clusters", func() { BeforeAll(func(ctx SpecContext) { - Expect(k1.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") - Expect(k2.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") - Expect(k1.CreateNamespace(istioCniNamespace)).To(Succeed(), "Istio CNI Namespace failed to be created") - Expect(k2.CreateNamespace(istioCniNamespace)).To(Succeed(), "Istio CNI Namespace failed to be created") + createIstioNamespaces(k1, "network1", profile) + createIstioNamespaces(k2, "network2", profile) // Push the intermediate CA to both clusters - Expect(certs.PushIntermediateCA(k1, controlPlaneNamespace, "east", "network1", artifacts, clPrimary)). - To(Succeed(), "Error pushing intermediate CA to Primary Cluster") - Expect(certs.PushIntermediateCA(k2, controlPlaneNamespace, "west", "network2", artifacts, clRemote)). - To(Succeed(), "Error pushing intermediate CA to Remote Cluster") + createIntermediateCA(k1, "east", "network1", artifacts, clPrimary) + createIntermediateCA(k2, "west", "network2", artifacts, clRemote) // Wait for the secret to be created in both clusters - Eventually(func() error { - _, err := common.GetObject(context.Background(), clPrimary, kube.Key("cacerts", controlPlaneNamespace), &corev1.Secret{}) - return err - }).ShouldNot(HaveOccurred(), "Secret is not created on Primary Cluster") + awaitSecretCreation(k1.ClusterName, clPrimary) + awaitSecretCreation(k2.ClusterName, clRemote) - Eventually(func() error { - _, err := common.GetObject(context.Background(), clRemote, kube.Key("cacerts", controlPlaneNamespace), &corev1.Secret{}) - return err - }).ShouldNot(HaveOccurred(), "Secret is not created on Primary Cluster") - - common.CreateIstioCNI(k1, v.Name) - - spec := ` -values: - pilot: - env: - EXTERNAL_ISTIOD: "true" - global: - meshID: mesh1 - multiCluster: - clusterName: cluster1 - network: network1` - common.CreateIstio(k1, v.Name, spec) + pilot := ` +pilot: + env: + EXTERNAL_ISTIOD: "true"` + createIstioResources(k1, v.Name, "cluster1", "network1", profile, pilot) }) It("updates Istio CR on Primary cluster status to Ready", func(ctx SpecContext) { @@ -203,16 +183,7 @@ values: When("sample apps are deployed in both clusters", func() { BeforeAll(func(ctx SpecContext) { - // Create namespace - Expect(k1.CreateNamespace(sampleNamespace)).To(Succeed(), "Namespace failed to be created on Cluster #1") - Expect(k2.CreateNamespace(sampleNamespace)).To(Succeed(), "Namespace failed to be created on Cluster #2") - - // Label the namespace - Expect(k1.Label("namespace", sampleNamespace, "istio-injection", "enabled")).To(Succeed(), "Error labeling sample namespace") - Expect(k2.Label("namespace", sampleNamespace, "istio-injection", "enabled")).To(Succeed(), "Error labeling sample namespace") - - // Deploy the sample app in both clusters - deploySampleAppToClusters(sampleNamespace, []ClusterDeployment{ + deploySampleAppToClusters(sampleNamespace, profile, []ClusterDeployment{ {Kubectl: k1, AppVersion: "v1"}, {Kubectl: k2, AppVersion: "v2"}, }) diff --git a/tests/e2e/multicluster/multicluster_suite_test.go b/tests/e2e/multicluster/multicluster_suite_test.go index d754387058..8661703be9 100644 --- a/tests/e2e/multicluster/multicluster_suite_test.go +++ b/tests/e2e/multicluster/multicluster_suite_test.go @@ -37,10 +37,10 @@ var ( clRemote client.Client err error debugInfoLogged bool - controlPlaneNamespace = env.Get("CONTROL_PLANE_NS", "istio-system") + controlPlaneNamespace = common.ControlPlaneNamespace externalControlPlaneNamespace = env.Get("EXTERNAL_CONTROL_PLANE_NS", "external-istiod") istioName = env.Get("ISTIO_NAME", "default") - istioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") + istioCniNamespace = common.IstioCniNamespace istioCniName = env.Get("ISTIOCNI_NAME", "default") multicluster = env.GetBool("MULTICLUSTER", false) keepOnFailure = env.GetBool("KEEP_ON_FAILURE", false) diff --git a/tests/e2e/multicontrolplane/multi_control_plane_suite_test.go b/tests/e2e/multicontrolplane/multi_control_plane_suite_test.go index 9485cb193c..749d1fdc40 100644 --- a/tests/e2e/multicontrolplane/multi_control_plane_suite_test.go +++ b/tests/e2e/multicontrolplane/multi_control_plane_suite_test.go @@ -21,6 +21,7 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/env" k8sclient "github.com/istio-ecosystem/sail-operator/tests/e2e/util/client" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -35,7 +36,7 @@ var ( controlPlaneNamespace2 = env.Get("CONTROL_PLANE_NS2", "istio-system2") istioName1 = env.Get("ISTIO_NAME1", "mesh1") istioName2 = env.Get("ISTIO_NAME2", "mesh2") - istioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") + istioCniNamespace = common.IstioCniNamespace istioCniName = env.Get("ISTIOCNI_NAME", "default") appNamespace1 = env.Get("APP_NAMESPACE1", "app1") appNamespace2a = env.Get("APP_NAMESPACE2A", "app2a") diff --git a/tests/e2e/setup/setup-kind.sh b/tests/e2e/setup/setup-kind.sh index 02671c90b4..4d00e8a2c1 100755 --- a/tests/e2e/setup/setup-kind.sh +++ b/tests/e2e/setup/setup-kind.sh @@ -18,6 +18,7 @@ set -eux -o pipefail SCRIPTPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" ROOT=$(cd "${SCRIPTPATH}/../../.." && pwd) +GW_API_VERSION=${GW_API_VERSION:-v1.4.0} # shellcheck source=common/scripts/kind_provisioner.sh source "${ROOT}/common/scripts/kind_provisioner.sh" @@ -70,6 +71,11 @@ if [ "${MULTICLUSTER}" == "true" ]; then setup_kind_clusters "${KIND_IMAGE}" "" setup_kind_registry "${CLUSTER_NAMES[@]}" + # Apply Gateway API CRDs which are needed for Multi-Cluster Ambient testing + for config in "${KUBECONFIGS[@]}"; do + kubectl apply --kubeconfig="$config" --server-side -f "https://github.com/kubernetes-sigs/gateway-api/releases/download/${GW_API_VERSION}/experimental-install.yaml" + done + export KUBECONFIG="${KUBECONFIGS[0]}" export KUBECONFIG2="${KUBECONFIGS[1]}" echo "Your KinD environment is ready, to use it: export KUBECONFIG=$(IFS=:; echo "${KUBECONFIGS[*]}")" diff --git a/tests/e2e/util/common/checks.go b/tests/e2e/util/common/checks.go index a3df3ca9a7..ef5c49d8ca 100644 --- a/tests/e2e/util/common/checks.go +++ b/tests/e2e/util/common/checks.go @@ -49,7 +49,7 @@ func AwaitCondition[T ~string](ctx context.Context, condition T, key client.Obje // AwaitDeployment to reach the Available state. func AwaitDeployment(ctx context.Context, name string, k kubectl.Kubectl, cl client.Client) { - AwaitCondition(ctx, appsv1.DeploymentAvailable, kube.Key(name, controlPlaneNamespace), &appsv1.Deployment{}, k, cl) + AwaitCondition(ctx, appsv1.DeploymentAvailable, kube.Key(name, ControlPlaneNamespace), &appsv1.Deployment{}, k, cl) } func isPodReady(pod *corev1.Pod) bool { @@ -86,7 +86,7 @@ func CheckSamplePodsReady(ctx context.Context, cl client.Client) error { // AwaitCniDaemonSet to be deployed and reach the scheduled number of pods. func AwaitCniDaemonSet(ctx context.Context, k kubectl.Kubectl, cl client.Client) { - key := kube.Key("istio-cni-node", istioCniNamespace) + key := kube.Key("istio-cni-node", IstioCniNamespace) Eventually(func() bool { daemonset := &appsv1.DaemonSet{} if err := cl.Get(ctx, key, daemonset); err != nil { diff --git a/tests/e2e/util/common/e2e_utils.go b/tests/e2e/util/common/e2e_utils.go index 00a4e79585..9a3735f5b7 100644 --- a/tests/e2e/util/common/e2e_utils.go +++ b/tests/e2e/util/common/e2e_utils.go @@ -56,16 +56,16 @@ const ( ) var ( - OperatorImage = env.Get("IMAGE", "quay.io/sail-dev/sail-operator:latest") - OperatorNamespace = env.Get("NAMESPACE", "sail-operator") + ControlPlaneNamespace = env.Get("CONTROL_PLANE_NS", "istio-system") + IstioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") + OperatorImage = env.Get("IMAGE", "quay.io/sail-dev/sail-operator:latest") + OperatorNamespace = env.Get("NAMESPACE", "sail-operator") + ZtunnelNamespace = env.Get("ZTUNNEL_NAMESPACE", "ztunnel") - deploymentName = env.Get("DEPLOYMENT_NAME", "sail-operator") - controlPlaneNamespace = env.Get("CONTROL_PLANE_NS", "istio-system") - istioName = env.Get("ISTIO_NAME", "default") - istioCniName = env.Get("ISTIOCNI_NAME", "default") - istioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") - sampleNamespace = env.Get("SAMPLE_NAMESPACE", "sample") - ztunnelNamespace = env.Get("ZTUNNEL_NAMESPACE", "ztunnel") + deploymentName = env.Get("DEPLOYMENT_NAME", "sail-operator") + istioName = env.Get("ISTIO_NAME", "default") + istioCniName = env.Get("ISTIOCNI_NAME", "default") + sampleNamespace = env.Get("SAMPLE_NAMESPACE", "sample") // version can have one of the following formats: // - 1.22.2 @@ -205,14 +205,14 @@ func logIstioDebugInfo(k kubectl.Kubectl) { resource, err := k.GetYAML("istio", istioName) logDebugElement("=====Istio YAML=====", resource, err) - output, err := k.WithNamespace(controlPlaneNamespace).GetPods("", "-o wide") - logDebugElement("=====Pods in "+controlPlaneNamespace+"=====", output, err) + output, err := k.WithNamespace(ControlPlaneNamespace).GetPods("", "-o wide") + logDebugElement("=====Pods in "+ControlPlaneNamespace+"=====", output, err) - logs, err := k.WithNamespace(controlPlaneNamespace).Logs("deploy/istiod", ptr.Of(120*time.Second)) + logs, err := k.WithNamespace(ControlPlaneNamespace).Logs("deploy/istiod", ptr.Of(120*time.Second)) logDebugElement("=====Istiod logs=====", logs, err) - events, err := k.WithNamespace(controlPlaneNamespace).GetEvents() - logDebugElement("=====Events in "+controlPlaneNamespace+"=====", events, err) + events, err := k.WithNamespace(ControlPlaneNamespace).GetEvents() + logDebugElement("=====Events in "+ControlPlaneNamespace+"=====", events, err) // Running istioctl proxy-status to get the status of the proxies. proxyStatus, err := istioctl.GetProxyStatus() @@ -223,20 +223,20 @@ func logCNIDebugInfo(k kubectl.Kubectl) { resource, err := k.GetYAML("istiocni", istioCniName) logDebugElement("=====IstioCNI YAML=====", resource, err) - ds, err := k.WithNamespace(istioCniNamespace).GetYAML("daemonset", "istio-cni-node") + ds, err := k.WithNamespace(IstioCniNamespace).GetYAML("daemonset", "istio-cni-node") logDebugElement("=====Istio CNI DaemonSet YAML=====", ds, err) - events, err := k.WithNamespace(istioCniNamespace).GetEvents() - logDebugElement("=====Events in "+istioCniNamespace+"=====", events, err) + events, err := k.WithNamespace(IstioCniNamespace).GetEvents() + logDebugElement("=====Events in "+IstioCniNamespace+"=====", events, err) // Temporary information to gather more details about failure - pods, err := k.WithNamespace(istioCniNamespace).GetPods("", "-o wide") - logDebugElement("=====Pods in "+istioCniNamespace+"=====", pods, err) + pods, err := k.WithNamespace(IstioCniNamespace).GetPods("", "-o wide") + logDebugElement("=====Pods in "+IstioCniNamespace+"=====", pods, err) - describe, err := k.WithNamespace(istioCniNamespace).Describe("daemonset", "istio-cni-node") + describe, err := k.WithNamespace(IstioCniNamespace).Describe("daemonset", "istio-cni-node") logDebugElement("=====Istio CNI DaemonSet describe=====", describe, err) - logs, err := k.WithNamespace(istioCniNamespace).Logs("daemonset/istio-cni-node", ptr.Of(120*time.Second)) + logs, err := k.WithNamespace(IstioCniNamespace).Logs("daemonset/istio-cni-node", ptr.Of(120*time.Second)) logDebugElement("=====Istio CNI logs=====", logs, err) } @@ -244,22 +244,22 @@ func logZtunnelDebugInfo(k kubectl.Kubectl) { resource, err := k.GetYAML("ztunnel", "default") logDebugElement("=====ZTunnel YAML=====", resource, err) - ds, err := k.WithNamespace(ztunnelNamespace).GetYAML("daemonset", "ztunnel") + ds, err := k.WithNamespace(ZtunnelNamespace).GetYAML("daemonset", "ztunnel") logDebugElement("=====ZTunnel DaemonSet YAML=====", ds, err) - events, err := k.WithNamespace(ztunnelNamespace).GetEvents() - logDebugElement("=====Events in "+ztunnelNamespace+"=====", events, err) + events, err := k.WithNamespace(ZtunnelNamespace).GetEvents() + logDebugElement("=====Events in "+ZtunnelNamespace+"=====", events, err) - describe, err := k.WithNamespace(ztunnelNamespace).Describe("daemonset", "ztunnel") + describe, err := k.WithNamespace(ZtunnelNamespace).Describe("daemonset", "ztunnel") logDebugElement("=====ZTunnel DaemonSet describe=====", describe, err) - logs, err := k.WithNamespace(ztunnelNamespace).Logs("daemonset/ztunnel", ptr.Of(120*time.Second)) + logs, err := k.WithNamespace(ZtunnelNamespace).Logs("daemonset/ztunnel", ptr.Of(120*time.Second)) logDebugElement("=====ztunnel logs=====", logs, err) } func logCertsDebugInfo(k kubectl.Kubectl) { - certs, err := k.WithNamespace(controlPlaneNamespace).GetSecret("cacerts") - logDebugElement("=====CA certs in "+controlPlaneNamespace+"=====", certs, err) + certs, err := k.WithNamespace(ControlPlaneNamespace).GetSecret("cacerts") + logDebugElement("=====CA certs in "+ControlPlaneNamespace+"=====", certs, err) } func logDebugElement(caption string, info string, err error) { @@ -273,7 +273,7 @@ func logDebugElement(caption string, info string, err error) { func GetVersionFromIstiod() (*semver.Version, error) { k := kubectl.New() - output, err := k.WithNamespace(controlPlaneNamespace).Exec("deploy/istiod", "", "pilot-discovery version") + output, err := k.WithNamespace(ControlPlaneNamespace).Exec("deploy/istiod", "", "pilot-discovery version") if err != nil { return nil, fmt.Errorf("error getting version from istiod: %w", err) } @@ -329,19 +329,12 @@ metadata: spec: version: %s namespace: %s` - yaml = fmt.Sprintf(yaml, istioName, version, controlPlaneNamespace) - for _, spec := range specs { - yaml += Indent(spec) - } - - Log("Istio YAML:", Indent(yaml)) - Expect(k.CreateFromString(yaml)). - To(Succeed(), withClusterName("Istio CR failed to be created", k)) - Success(withClusterName("Istio CR created", k)) + yaml = fmt.Sprintf(yaml, istioName, version, ControlPlaneNamespace) + createResource(k, "Istio", yaml, specs...) } // CreateIstioCNI custom resource using a given `kubectl` client and with the specified version. -func CreateIstioCNI(k kubectl.Kubectl, version string) { +func CreateIstioCNI(k kubectl.Kubectl, version string, specs ...string) { yaml := ` apiVersion: sailoperator.io/v1 kind: IstioCNI @@ -350,10 +343,54 @@ metadata: spec: version: %s namespace: %s` - yaml = fmt.Sprintf(yaml, istioCniName, version, istioCniNamespace) - Log("IstioCNI YAML:", Indent(yaml)) - Expect(k.CreateFromString(yaml)).To(Succeed(), withClusterName("IstioCNI creation failed", k)) - Success(withClusterName("IstioCNI created", k)) + yaml = fmt.Sprintf(yaml, istioCniName, version, IstioCniNamespace) + createResource(k, "IstioCNI", yaml, specs...) +} + +func CreateZTunnel(k kubectl.Kubectl, version string, specs ...string) { + yaml := ` +apiVersion: sailoperator.io/v1alpha1 +kind: ZTunnel +metadata: + name: default +spec: + profile: ambient + version: %s + namespace: %s` + yaml = fmt.Sprintf(yaml, version, ZtunnelNamespace) + createResource(k, "ZTunnel", yaml, specs...) +} + +func CreateAmbientGateway(k kubectl.Kubectl, namespace, network string) { + yaml := `kind: Gateway +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: istio-eastwestgateway + namespace: %s + labels: + topology.istio.io/network: %s +spec: + gatewayClassName: istio-east-west + listeners: + - name: mesh + port: 15008 + protocol: HBONE + tls: + mode: Terminate + options: + gateway.istio.io/tls-terminate-mode: ISTIO_MUTUAL` + yaml = fmt.Sprintf(yaml, namespace, network) + createResource(k, "Gateway", yaml) +} + +func createResource(k kubectl.Kubectl, kind, yaml string, specs ...string) { + for _, spec := range specs { + yaml += Indent(spec) + } + + Log(fmt.Sprintf("%s YAML:", kind), Indent(yaml)) + Expect(k.CreateFromString(yaml)).To(Succeed(), withClusterName(fmt.Sprintf("%s creation failed:", kind), k)) + Success(withClusterName(fmt.Sprintf("%s created", kind), k)) } func Indent(str string) string { diff --git a/tests/e2e/util/kubectl/kubectl.go b/tests/e2e/util/kubectl/kubectl.go index 95949a89a9..b10ac948c6 100644 --- a/tests/e2e/util/kubectl/kubectl.go +++ b/tests/e2e/util/kubectl/kubectl.go @@ -305,6 +305,12 @@ func (k Kubectl) Label(kind, name, labelKey, labelValue string) error { return err } +// LabelNamespaced adds a label to the specified resource in the specified namespace +func (k Kubectl) LabelNamespaced(kind, namespace, name, labelKey, labelValue string) error { + _, err := k.executeCommand(k.build(fmt.Sprintf(" label %s -n %s %s %s=%s", kind, namespace, name, labelKey, labelValue))) + return err +} + // executeCommand handles running the command and then resets the namespace automatically func (k Kubectl) executeCommand(cmd string) (string, error) { return shell.ExecuteCommand(cmd) diff --git a/tools/update_deps.sh b/tools/update_deps.sh index d1addb88d0..9de1ed4c0e 100755 --- a/tools/update_deps.sh +++ b/tools/update_deps.sh @@ -160,6 +160,17 @@ OPM_LATEST_VERSION=$(getVersionForUpdate operator-framework/operator-registry "$ OLM_LATEST_VERSION=$(getVersionForUpdate operator-framework/operator-lifecycle-manager "${OLM_VERSION}") "$SED_CMD" -i "s|OLM_VERSION ?= .*|OLM_VERSION ?= ${OLM_LATEST_VERSION}|" "${ROOTDIR}/Makefile.core.mk" +# Update gateway-api +GW_API_LATEST_VERSION=$(getLatestVersion kubernetes-sigs/gateway-api) +"$SED_CMD" -i "s|GW_API_VERSION=.*|GW_API_VERSION=\${GW_API_VERSION:-${GW_API_LATEST_VERSION}}|" "${ROOTDIR}/tests/e2e/setup/setup-kind.sh" + +# Update kube-rbac-proxy +RBAC_PROXY_LATEST_VERSION=$(getLatestVersion brancz/kube-rbac-proxy | cut -d/ -f1) +# Only update it if the newer image is available in the registry +if docker manifest inspect "gcr.io/kubebuilder/kube-rbac-proxy:${RBAC_PROXY_LATEST_VERSION}" >/dev/null 2>/dev/null; then + "$SED_CMD" -i "s|gcr.io/kubebuilder/kube-rbac-proxy:.*|gcr.io/kubebuilder/kube-rbac-proxy:${RBAC_PROXY_LATEST_VERSION}|" "${ROOTDIR}/chart/values.yaml" +fi + # Update gitleaks GITLEAKS_LATEST_VERSION=$(getVersionForUpdate gitleaks/gitleaks "${GITLEAKS_VERSION}") "$SED_CMD" -i "s|GITLEAKS_VERSION ?= .*|GITLEAKS_VERSION ?= ${GITLEAKS_LATEST_VERSION}|" "${ROOTDIR}/Makefile.core.mk" From 61000a82f2c1784531ccffdc9b21c42bbfca67b2 Mon Sep 17 00:00:00 2001 From: Mike Kolesnik Date: Thu, 8 Jan 2026 09:58:52 +0200 Subject: [PATCH 3/3] E2E: Fixes to stabilize the multicluster tests further (#1472) * E2E: Add a validation for E/W GW to get LB IP This might take a while, expecially on a public cloud. Make sure the tests wait for the IP to be assigned before proceeding, otherwise they'll get stuck later waiting for the communication to proceed (and sometimes fail). Signed-off-by: Mike Kolesnik * E2E: Add check that multicluster gateway rechable To rule out infrastructure problems, a sanity check is added to try and reach the gateway on the other side. If this check fails, this could indicate an infrastructure problem. Signed-off-by: Mike Kolesnik * E2E: Improve connectivity checking in multicluster Having curl fetch the URL several times saves time waiting on responses since it's much cheaper than each `exec` call. This makes successful tests finish much faster instead of waiting for the response to randomly land on the expected version. Signed-off-by: Mike Kolesnik --------- Signed-off-by: Mike Kolesnik --- tests/e2e/multicluster/common.go | 30 +++++++++++++++++-- .../multicluster_externalcontrolplane_test.go | 8 +++++ .../multicluster_multiprimary_test.go | 10 +++++++ .../multicluster_primaryremote_test.go | 13 ++++++++ .../multicluster/multicluster_suite_test.go | 1 - 5 files changed, 58 insertions(+), 4 deletions(-) diff --git a/tests/e2e/multicluster/common.go b/tests/e2e/multicluster/common.go index 501ca7cb0c..416ee2c0ec 100644 --- a/tests/e2e/multicluster/common.go +++ b/tests/e2e/multicluster/common.go @@ -24,6 +24,7 @@ import ( "time" "github.com/istio-ecosystem/sail-operator/pkg/kube" + . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/certs" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" @@ -32,6 +33,8 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +var controlPlaneNamespace = common.ControlPlaneNamespace + // ClusterDeployment represents a cluster along with its sample app version. type ClusterDeployment struct { Kubectl kubectl.Kubectl @@ -66,15 +69,18 @@ func deploySampleAppToClusters(ns, profile string, clusters []ClusterDeployment) } // verifyResponsesAreReceivedFromBothClusters checks that when the sleep pod in the sample namespace -// sends a request to the helloworld service, it receives responses from expectedVersions, -// which can be either "v1" or "v2" on on different clusters. +// sends requests to the helloworld service, it receives responses from expectedVersions, +// which can be either "v1" or "v2" on different clusters. func verifyResponsesAreReceivedFromExpectedVersions(k kubectl.Kubectl, expectedVersions ...string) { + Step(fmt.Sprintf("Checking app connectivity from %s", k.ClusterName)) if len(expectedVersions) == 0 { expectedVersions = []string{"v1", "v2"} } for _, v := range expectedVersions { + // Each curl will fetch the URL multiple times, minimizing time wasted waiting for different responses. + // This is because running the exec is much more expensive than having curl fetch the URL several times. Eventually(k.WithNamespace("sample").Exec, 10*time.Minute, 2*time.Second). - WithArguments("deploy/sleep", "sleep", "curl -sS helloworld.sample:5000/hello"). + WithArguments("deploy/sleep", "sleep", "curl -sS -Z helloworld.sample:5000/hello{,,,,,,,,,}"). Should(ContainSubstring(fmt.Sprintf("Hello version: %s", v)), fmt.Sprintf("sleep pod in %s did not receive any response from %s", k.ClusterName, v)) } @@ -154,3 +160,21 @@ func awaitSecretCreation(cluster string, cl client.Client) { return err }).ShouldNot(HaveOccurred(), fmt.Sprintf("Secret is not created on %s cluster", cluster)) } + +// expectLoadBalancerAddress to be assigned. +// If the address is not assigned, cross-cluster traffic will not be sent since Istio doesn't know where to send it for the remote cluster(s). +func expectLoadBalancerAddress(ctx context.Context, k kubectl.Kubectl, cl client.Client, name string) { + address := common.GetSVCLoadBalancerAddress(ctx, cl, controlPlaneNamespace, name) + Expect(address).ToNot(BeEmpty(), fmt.Sprintf("Gateway service %q does not have an external address on %s", name, k.ClusterName)) + Success(fmt.Sprintf("Gateway service %q has an external address %s on %s", name, address, k.ClusterName)) +} + +// eventuallyLoadBalancerIsReachable to make sure that cross cluster communication is working on the infrastructure level +func eventuallyLoadBalancerIsReachable(ctx context.Context, kSrc kubectl.Kubectl, kDest kubectl.Kubectl, clDest client.Client, name string) { + address := common.GetSVCLoadBalancerAddress(ctx, clDest, controlPlaneNamespace, name) + Eventually(ctx, kSrc.WithNamespace("sample").Exec).WithArguments( + "deploy/sleep", "sleep", fmt.Sprintf("curl -sS -o /dev/null -w '%%{http_code}' %s:15021/healthz/ready", address)). + Should(Equal("200"), fmt.Sprintf("Gateway %q(%s) on %s is not reachable from %s, this might indicate a problem with the underlying infrastructure", + name, address, kDest.ClusterName, kSrc.ClusterName)) + Success(fmt.Sprintf("Gateway %q(%s) on %s is reachable from %s", name, address, kDest.ClusterName, kSrc.ClusterName)) +} diff --git a/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go b/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go index 47b8cc0f1b..37cef4c985 100644 --- a/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go +++ b/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go @@ -92,6 +92,10 @@ values: It("updates Gateway status to Available", func(ctx SpecContext) { common.AwaitDeployment(ctx, "istio-ingressgateway", k1, clPrimary) }) + + It("has an external IP assigned", func(ctx SpecContext) { + expectLoadBalancerAddress(ctx, k1, clPrimary, "istio-ingressgateway") + }) }) When("Istio external is installed in Cluster #2", func() { @@ -354,6 +358,10 @@ spec: Success("Sample pods has expected annotation") }) + It("can reach the ingress gateway from remote cluster", func(ctx SpecContext) { + eventuallyLoadBalancerIsReachable(ctx, k2, k1, clPrimary, "istio-ingressgateway") + }) + It("can access the sample app from the local service", func(ctx SpecContext) { verifyResponsesAreReceivedFromExpectedVersions(k2, "v1") Success("Sample app is accessible from hello service in Cluster #2") diff --git a/tests/e2e/multicluster/multicluster_multiprimary_test.go b/tests/e2e/multicluster/multicluster_multiprimary_test.go index c63dd4c980..c5f35728b7 100644 --- a/tests/e2e/multicluster/multicluster_multiprimary_test.go +++ b/tests/e2e/multicluster/multicluster_multiprimary_test.go @@ -126,6 +126,11 @@ func generateMultiPrimaryTestCases(profile string) { common.AwaitDeployment(ctx, "istio-eastwestgateway", k2, clRemote) Success("Gateway is created and available in both clusters") }) + + It("has external IPs assigned", func(ctx SpecContext) { + expectLoadBalancerAddress(ctx, k1, clPrimary, "istio-eastwestgateway") + expectLoadBalancerAddress(ctx, k2, clRemote, "istio-eastwestgateway") + }) }) When("are installed remote secrets on each cluster", func() { @@ -177,6 +182,11 @@ func generateMultiPrimaryTestCases(profile string) { Success("Sample app is created in both clusters and Running") }) + It("can reach target east-west gateway from each cluster", func(ctx SpecContext) { + eventuallyLoadBalancerIsReachable(ctx, k1, k2, clRemote, "istio-eastwestgateway") + eventuallyLoadBalancerIsReachable(ctx, k2, k1, clPrimary, "istio-eastwestgateway") + }) + It("can access the sample app from both clusters", func(ctx SpecContext) { verifyResponsesAreReceivedFromExpectedVersions(k1) verifyResponsesAreReceivedFromExpectedVersions(k2) diff --git a/tests/e2e/multicluster/multicluster_primaryremote_test.go b/tests/e2e/multicluster/multicluster_primaryremote_test.go index b80d00ad23..b4d9828ac1 100644 --- a/tests/e2e/multicluster/multicluster_primaryremote_test.go +++ b/tests/e2e/multicluster/multicluster_primaryremote_test.go @@ -109,6 +109,10 @@ pilot: It("updates Gateway status to Available", func(ctx SpecContext) { common.AwaitDeployment(ctx, "istio-eastwestgateway", k1, clPrimary) }) + + It("has an external IP assigned", func(ctx SpecContext) { + expectLoadBalancerAddress(ctx, k1, clPrimary, "istio-eastwestgateway") + }) }) When("Istio and IstioCNI are created in Remote cluster", func() { @@ -179,6 +183,10 @@ values: It("updates Gateway status to Available", func(ctx SpecContext) { common.AwaitDeployment(ctx, "istio-eastwestgateway", k2, clRemote) }) + + It("has an external IP assigned", func(ctx SpecContext) { + expectLoadBalancerAddress(ctx, k2, clRemote, "istio-eastwestgateway") + }) }) When("sample apps are deployed in both clusters", func() { @@ -196,6 +204,11 @@ values: Success("Sample app is created in both clusters and Running") }) + It("can reach target east-west gateway from each cluster", func(ctx SpecContext) { + eventuallyLoadBalancerIsReachable(ctx, k1, k2, clRemote, "istio-eastwestgateway") + eventuallyLoadBalancerIsReachable(ctx, k2, k1, clPrimary, "istio-eastwestgateway") + }) + It("can access the sample app from both clusters", func(ctx SpecContext) { verifyResponsesAreReceivedFromExpectedVersions(k1) verifyResponsesAreReceivedFromExpectedVersions(k2) diff --git a/tests/e2e/multicluster/multicluster_suite_test.go b/tests/e2e/multicluster/multicluster_suite_test.go index 8661703be9..eef7b7695d 100644 --- a/tests/e2e/multicluster/multicluster_suite_test.go +++ b/tests/e2e/multicluster/multicluster_suite_test.go @@ -37,7 +37,6 @@ var ( clRemote client.Client err error debugInfoLogged bool - controlPlaneNamespace = common.ControlPlaneNamespace externalControlPlaneNamespace = env.Get("EXTERNAL_CONTROL_PLANE_NS", "external-istiod") istioName = env.Get("ISTIO_NAME", "default") istioCniNamespace = common.IstioCniNamespace