diff --git a/tests/e2e/ambient/ambient_suite_test.go b/tests/e2e/ambient/ambient_suite_test.go index 3c5ff0f73..376cb81da 100644 --- a/tests/e2e/ambient/ambient_suite_test.go +++ b/tests/e2e/ambient/ambient_suite_test.go @@ -21,6 +21,7 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/env" k8sclient "github.com/istio-ecosystem/sail-operator/tests/e2e/util/client" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -30,10 +31,10 @@ import ( var ( cl client.Client err error - controlPlaneNamespace = env.Get("CONTROL_PLANE_NS", "istio-system") + controlPlaneNamespace = common.ControlPlaneNamespace istioName = env.Get("ISTIO_NAME", "default") - istioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") - ztunnelNamespace = env.Get("ZTUNNEL_NAMESPACE", "ztunnel") + istioCniNamespace = common.IstioCniNamespace + ztunnelNamespace = common.ZtunnelNamespace istioCniName = env.Get("ISTIOCNI_NAME", "default") expectedRegistry = env.Get("EXPECTED_REGISTRY", "^docker\\.io|^gcr\\.io|^registry\\.istio\\.io") multicluster = env.GetBool("MULTICLUSTER", false) diff --git a/tests/e2e/ambient/ambient_test.go b/tests/e2e/ambient/ambient_test.go index 6c721e302..6753cb948 100644 --- a/tests/e2e/ambient/ambient_test.go +++ b/tests/e2e/ambient/ambient_test.go @@ -28,12 +28,10 @@ import ( . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" - . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -123,22 +121,16 @@ profile: ambient`) }) It("updates the Istio CR status to Reconciled", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReconciled, metav1.ConditionTrue), "Istio is not Reconciled; unexpected Condition") - Success("Istio CR is Reconciled") + common.AwaitCondition(ctx, v1.IstioConditionReconciled, kube.Key(istioName), &v1.Istio{}, k, cl) }) It("updates the Istio CR status to Ready", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready; unexpected Condition") - Success("Istio CR is Ready") + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key(istioName), &v1.Istio{}, k, cl) }) It("deploys istiod", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key("istiod", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Istiod is not Available; unexpected Condition") + common.AwaitDeployment(ctx, "istiod", k, cl) Expect(common.GetVersionFromIstiod()).To(Equal(version.Version), "Unexpected istiod version") - Success("Istiod is deployed in the namespace and Running") }) It("uses the correct image", func(ctx SpecContext) { diff --git a/tests/e2e/controlplane/control_plane_suite_test.go b/tests/e2e/controlplane/control_plane_suite_test.go index 3c16b54f3..9c073be00 100644 --- a/tests/e2e/controlplane/control_plane_suite_test.go +++ b/tests/e2e/controlplane/control_plane_suite_test.go @@ -33,9 +33,9 @@ var ( err error namespace = common.OperatorNamespace deploymentName = env.Get("DEPLOYMENT_NAME", "sail-operator") - controlPlaneNamespace = env.Get("CONTROL_PLANE_NS", "istio-system") + controlPlaneNamespace = common.ControlPlaneNamespace istioName = env.Get("ISTIO_NAME", "default") - istioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") + istioCniNamespace = common.IstioCniNamespace istioCniName = env.Get("ISTIOCNI_NAME", "default") expectedRegistry = env.Get("EXPECTED_REGISTRY", "^docker\\.io|^gcr\\.io|^registry\\.istio\\.io") sampleNamespace = env.Get("SAMPLE_NAMESPACE", "sample") diff --git a/tests/e2e/controlplane/control_plane_test.go b/tests/e2e/controlplane/control_plane_test.go index 04e729287..d13aa41a7 100644 --- a/tests/e2e/controlplane/control_plane_test.go +++ b/tests/e2e/controlplane/control_plane_test.go @@ -30,13 +30,11 @@ import ( . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" - . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "istio.io/istio/pkg/ptr" @@ -129,15 +127,11 @@ metadata: }) It("updates the status to Reconciled", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioCniName), &v1.IstioCNI{}). - Should(HaveConditionStatus(v1.IstioCNIConditionReconciled, metav1.ConditionTrue), "IstioCNI is not Reconciled; unexpected Condition") - Success("IstioCNI is Reconciled") + common.AwaitCondition(ctx, v1.IstioCNIConditionReconciled, kube.Key(istioCniName), &v1.IstioCNI{}, k, cl) }) It("updates the status to Ready", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioCniName), &v1.IstioCNI{}). - Should(HaveConditionStatus(v1.IstioCNIConditionReady, metav1.ConditionTrue), "IstioCNI is not Ready; unexpected Condition") - Success("IstioCNI is Ready") + common.AwaitCondition(ctx, v1.IstioCNIConditionReady, kube.Key(istioCniName), &v1.IstioCNI{}, k, cl) }) It("doesn't continuously reconcile the IstioCNI CR", func() { @@ -153,22 +147,16 @@ metadata: }) It("updates the Istio CR status to Reconciled", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReconciled, metav1.ConditionTrue), "Istio is not Reconciled; unexpected Condition") - Success("Istio CR is Reconciled") + common.AwaitCondition(ctx, v1.IstioConditionReconciled, kube.Key(istioName), &v1.Istio{}, k, cl) }) It("updates the Istio CR status to Ready", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready; unexpected Condition") - Success("Istio CR is Ready") + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key(istioName), &v1.Istio{}, k, cl) }) It("deploys istiod", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key("istiod", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Istiod is not Available; unexpected Condition") + common.AwaitDeployment(ctx, "istiod", k, cl) Expect(common.GetVersionFromIstiod()).To(Equal(version.Version), "Unexpected istiod version") - Success("Istiod is deployed in the namespace and Running") }) It("uses the correct image", func(ctx SpecContext) { @@ -195,7 +183,7 @@ metadata: samplePods := &corev1.PodList{} It("updates the pods status to Running", func(ctx SpecContext) { - Eventually(common.CheckPodsReady).WithArguments(ctx, cl, sampleNamespace).Should(Succeed(), "Error checking status of sample pods") + Eventually(common.CheckSamplePodsReady).WithArguments(ctx, cl).Should(Succeed(), "Error checking status of sample pods") Expect(cl.List(ctx, samplePods, client.InNamespace(sampleNamespace))).To(Succeed(), "Error getting the pods in sample namespace") Success("sample pods are ready") diff --git a/tests/e2e/controlplane/control_plane_update_test.go b/tests/e2e/controlplane/control_plane_update_test.go index d0fca84c5..4e60cf621 100644 --- a/tests/e2e/controlplane/control_plane_update_test.go +++ b/tests/e2e/controlplane/control_plane_update_test.go @@ -62,9 +62,7 @@ var _ = Describe("Control Plane updates", Label("control-plane", "slow"), Ordere Expect(k.CreateNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI namespace failed to be created") common.CreateIstioCNI(k, istioversion.Base) - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioCniName), &v1.IstioCNI{}). - Should(HaveConditionStatus(v1.IstioCNIConditionReady, metav1.ConditionTrue), "IstioCNI is not Ready; unexpected Condition") - Success("IstioCNI is Ready") + common.AwaitCondition(ctx, v1.IstioCNIConditionReady, kube.Key(istioCniName), &v1.IstioCNI{}, k, cl) }) When(fmt.Sprintf("the Istio CR is created with RevisionBased updateStrategy for base version %s", istioversion.Base), func() { @@ -76,9 +74,7 @@ updateStrategy: }) It("deploys istiod and pod is Ready", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key("default"), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), "Istiod is not Available; unexpected Condition") - Success("Istiod is deployed in the namespace and Running") + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key("default"), &v1.Istio{}, k, cl) }) }) @@ -125,7 +121,7 @@ spec: Success("sample deployed") samplePods := &corev1.PodList{} - Eventually(common.CheckPodsReady).WithArguments(ctx, cl, sampleNamespace).Should(Succeed(), "Error checking status of sample pods") + Eventually(common.CheckSamplePodsReady).WithArguments(ctx, cl).Should(Succeed(), "Error checking status of sample pods") Expect(cl.List(ctx, samplePods, client.InNamespace(sampleNamespace))).To(Succeed(), "Error getting the pods in sample namespace") Success("sample pods are ready") @@ -139,9 +135,7 @@ spec: }) It("IstioRevisionTag state change to inUse true", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key("default"), &v1.IstioRevisionTag{}). - Should(HaveConditionStatus(v1.IstioRevisionTagConditionInUse, metav1.ConditionTrue), "unexpected Condition; expected InUse true") - Success("IstioRevisionTag is in use by the sample pods") + common.AwaitCondition(ctx, v1.IstioRevisionTagConditionInUse, kube.Key("default"), &v1.IstioRevisionTag{}, k, cl) }) }) @@ -223,8 +217,7 @@ spec: cl.Delete(ctx, &pod) } - Eventually(common.CheckPodsReady).WithArguments(ctx, cl, sampleNamespace).Should(Succeed(), "Sample pods are not ready after restart") - + Eventually(common.CheckSamplePodsReady).WithArguments(ctx, cl).Should(Succeed(), "Error checking status of sample pods") Success("sample pods restarted and are ready") }) diff --git a/tests/e2e/dualstack/dualstack_suite_test.go b/tests/e2e/dualstack/dualstack_suite_test.go index 92cc88020..011d2c118 100644 --- a/tests/e2e/dualstack/dualstack_suite_test.go +++ b/tests/e2e/dualstack/dualstack_suite_test.go @@ -21,6 +21,7 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/env" k8sclient "github.com/istio-ecosystem/sail-operator/tests/e2e/util/client" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -30,9 +31,9 @@ import ( var ( cl client.Client err error - controlPlaneNamespace = env.Get("CONTROL_PLANE_NS", "istio-system") + controlPlaneNamespace = common.ControlPlaneNamespace istioName = env.Get("ISTIO_NAME", "default") - istioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") + istioCniNamespace = common.IstioCniNamespace istioCniName = env.Get("ISTIOCNI_NAME", "default") expectedRegistry = env.Get("EXPECTED_REGISTRY", "^docker\\.io|^gcr\\.io|^registry\\.istio\\.io") multicluster = env.GetBool("MULTICLUSTER", false) diff --git a/tests/e2e/dualstack/dualstack_test.go b/tests/e2e/dualstack/dualstack_test.go index 174e34fb6..cc036958e 100644 --- a/tests/e2e/dualstack/dualstack_test.go +++ b/tests/e2e/dualstack/dualstack_test.go @@ -28,13 +28,11 @@ import ( . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" - . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/onsi/gomega/types" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -100,22 +98,16 @@ values: }) It("updates the Istio CR status to Reconciled", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReconciled, metav1.ConditionTrue), "Istio is not Reconciled; unexpected Condition") - Success("Istio CR is Reconciled") + common.AwaitCondition(ctx, v1.IstioConditionReconciled, kube.Key(istioName), &v1.Istio{}, k, cl) }) It("updates the Istio CR status to Ready", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready; unexpected Condition") - Success("Istio CR is Ready") + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key(istioName), &v1.Istio{}, k, cl) }) It("deploys istiod", func(ctx SpecContext) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key("istiod", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Istiod is not Available; unexpected Condition") + common.AwaitDeployment(ctx, "istiod", k, cl) Expect(common.GetVersionFromIstiod()).To(Equal(version.Version), "Unexpected istiod version") - Success("Istiod is deployed in the namespace and Running") }) It("uses the correct image", func(ctx SpecContext) { diff --git a/tests/e2e/multicluster/common.go b/tests/e2e/multicluster/common.go index 4d792e6b7..416ee2c0e 100644 --- a/tests/e2e/multicluster/common.go +++ b/tests/e2e/multicluster/common.go @@ -17,15 +17,24 @@ package multicluster import ( + "context" "fmt" "strings" "text/template" "time" + "github.com/istio-ecosystem/sail-operator/pkg/kube" + . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/certs" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" ) +var controlPlaneNamespace = common.ControlPlaneNamespace + // ClusterDeployment represents a cluster along with its sample app version. type ClusterDeployment struct { Kubectl kubectl.Kubectl @@ -33,29 +42,45 @@ type ClusterDeployment struct { } // deploySampleApp deploys the sample apps (helloworld and sleep) in the given cluster. -func deploySampleApp(k kubectl.Kubectl, ns string, appVersion string) { +func deploySampleApp(k kubectl.Kubectl, ns, appVersion, profile string) { Expect(k.WithNamespace(ns).ApplyKustomize("helloworld", "service=helloworld")).To(Succeed(), "Sample service deploy failed on Cluster") Expect(k.WithNamespace(ns).ApplyKustomize("helloworld", "version="+appVersion)).To(Succeed(), "Sample service deploy failed on Cluster") Expect(k.WithNamespace(ns).ApplyKustomize("sleep")).To(Succeed(), "Sample sleep deploy failed on Cluster") + + // In Ambient mode, services need to be marked as "global" in order to load balance requests among clusters + if profile == "ambient" { + Expect(k.LabelNamespaced("service", ns, "helloworld", "istio.io/global", "true")).To(Succeed(), "Error labeling sample namespace") + } } // deploySampleAppToClusters deploys the sample app to all provided clusters. -func deploySampleAppToClusters(ns string, clusters []ClusterDeployment) { +func deploySampleAppToClusters(ns, profile string, clusters []ClusterDeployment) { for _, cd := range clusters { - deploySampleApp(cd.Kubectl, ns, cd.AppVersion) + k := cd.Kubectl + Expect(k.CreateNamespace(ns)).To(Succeed(), fmt.Sprintf("Namespace failed to be created on Cluster %s", k.ClusterName)) + if profile == "ambient" { + Expect(k.Label("namespace", ns, "istio.io/dataplane-mode", "ambient")).To(Succeed(), "Error labeling sample namespace") + } else { + Expect(k.Label("namespace", ns, "istio-injection", "enabled")).To(Succeed(), "Error labeling sample namespace") + } + + deploySampleApp(k, ns, cd.AppVersion, profile) } } // verifyResponsesAreReceivedFromBothClusters checks that when the sleep pod in the sample namespace -// sends a request to the helloworld service, it receives responses from expectedVersions, -// which can be either "v1" or "v2" on on different clusters. +// sends requests to the helloworld service, it receives responses from expectedVersions, +// which can be either "v1" or "v2" on different clusters. func verifyResponsesAreReceivedFromExpectedVersions(k kubectl.Kubectl, expectedVersions ...string) { + Step(fmt.Sprintf("Checking app connectivity from %s", k.ClusterName)) if len(expectedVersions) == 0 { expectedVersions = []string{"v1", "v2"} } for _, v := range expectedVersions { + // Each curl will fetch the URL multiple times, minimizing time wasted waiting for different responses. + // This is because running the exec is much more expensive than having curl fetch the URL several times. Eventually(k.WithNamespace("sample").Exec, 10*time.Minute, 2*time.Second). - WithArguments("deploy/sleep", "sleep", "curl -sS helloworld.sample:5000/hello"). + WithArguments("deploy/sleep", "sleep", "curl -sS -Z helloworld.sample:5000/hello{,,,,,,,,,}"). Should(ContainSubstring(fmt.Sprintf("Hello version: %s", v)), fmt.Sprintf("sleep pod in %s did not receive any response from %s", k.ClusterName, v)) } @@ -75,3 +100,81 @@ func genTemplate(manifestTmpl string, values any) string { Expect(tmpl.Execute(&b, values)).To(Succeed()) return b.String() } + +func createIstioNamespaces(k kubectl.Kubectl, network, profile string) { + Expect(k.CreateNamespace(common.ControlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") + Expect(k.CreateNamespace(common.IstioCniNamespace)).To(Succeed(), "Istio CNI namespace failed to be created") + + if profile == "ambient" { + Expect(k.Label("namespace", common.ControlPlaneNamespace, "topology.istio.io/network", network)).To(Succeed(), "Error labeling istio namespace") + Expect(k.CreateNamespace(common.ZtunnelNamespace)).To(Succeed(), "Ztunnel namespace failed to be created") + } +} + +func createIstioResources(k kubectl.Kubectl, version, cluster, network, profile string, values ...string) { + cniSpec := fmt.Sprintf(` +profile: %s`, profile) + common.CreateIstioCNI(k, version, cniSpec) + + if profile == "ambient" { + spec := fmt.Sprintf(` +values: + ztunnel: + multiCluster: + clusterName: %s + network: %s`, cluster, network) + common.CreateZTunnel(k, version, spec) + } + + spec := fmt.Sprintf(` +profile: %s +values: + global: + meshID: mesh1 + multiCluster: + clusterName: %s + network: %s`, profile, cluster, network) + for _, value := range values { + spec += common.Indent(value) + } + + if profile == "ambient" { + spec += fmt.Sprintf(` + pilot: + trustedZtunnelNamespace: %s + env: + AMBIENT_ENABLE_MULTI_NETWORK: "true"`, common.ZtunnelNamespace) + } + + common.CreateIstio(k, version, spec) +} + +func createIntermediateCA(k kubectl.Kubectl, zone, network, artifacts string, cl client.Client) { + Expect(certs.PushIntermediateCA(k, common.ControlPlaneNamespace, zone, network, artifacts, cl)). + To(Succeed(), fmt.Sprintf("Error pushing intermediate CA to %s Cluster", k.ClusterName)) +} + +func awaitSecretCreation(cluster string, cl client.Client) { + Eventually(func() error { + _, err := common.GetObject(context.Background(), cl, kube.Key("cacerts", common.ControlPlaneNamespace), &corev1.Secret{}) + return err + }).ShouldNot(HaveOccurred(), fmt.Sprintf("Secret is not created on %s cluster", cluster)) +} + +// expectLoadBalancerAddress to be assigned. +// If the address is not assigned, cross-cluster traffic will not be sent since Istio doesn't know where to send it for the remote cluster(s). +func expectLoadBalancerAddress(ctx context.Context, k kubectl.Kubectl, cl client.Client, name string) { + address := common.GetSVCLoadBalancerAddress(ctx, cl, controlPlaneNamespace, name) + Expect(address).ToNot(BeEmpty(), fmt.Sprintf("Gateway service %q does not have an external address on %s", name, k.ClusterName)) + Success(fmt.Sprintf("Gateway service %q has an external address %s on %s", name, address, k.ClusterName)) +} + +// eventuallyLoadBalancerIsReachable to make sure that cross cluster communication is working on the infrastructure level +func eventuallyLoadBalancerIsReachable(ctx context.Context, kSrc kubectl.Kubectl, kDest kubectl.Kubectl, clDest client.Client, name string) { + address := common.GetSVCLoadBalancerAddress(ctx, clDest, controlPlaneNamespace, name) + Eventually(ctx, kSrc.WithNamespace("sample").Exec).WithArguments( + "deploy/sleep", "sleep", fmt.Sprintf("curl -sS -o /dev/null -w '%%{http_code}' %s:15021/healthz/ready", address)). + Should(Equal("200"), fmt.Sprintf("Gateway %q(%s) on %s is not reachable from %s, this might indicate a problem with the underlying infrastructure", + name, address, kDest.ClusterName, kSrc.ClusterName)) + Success(fmt.Sprintf("Gateway %q(%s) on %s is reachable from %s", name, address, kDest.ClusterName, kSrc.ClusterName)) +} diff --git a/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go b/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go index 5d49ca6fc..4779b7ac6 100644 --- a/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go +++ b/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go @@ -28,14 +28,12 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/version" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" - . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" admissionregistrationv1 "k8s.io/api/admissionregistration/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" ) @@ -68,8 +66,7 @@ var _ = Describe("Multicluster deployment models", Label("multicluster", "multic When("default Istio is created in Cluster #1 to handle ingress to External Control Plane", func() { BeforeAll(func(ctx SpecContext) { - Expect(k1.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be created") - Expect(k1.CreateNamespace(istioCniNamespace)).To(Succeed(), "Istio CNI namespace failed to be created") + createIstioNamespaces(k1, "network1", "default") common.CreateIstioCNI(k1, v.Name) common.CreateIstio(k1, v.Name, ` @@ -79,18 +76,12 @@ values: }) It("updates the default Istio CR status to Ready", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key(istioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready on Cluster #1; unexpected Condition") - Success("Istio CR is Ready on Cluster #1") + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key(istioName), &v1.Istio{}, k1, clPrimary) }) It("deploys istiod", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key("istiod", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Istiod is not Available on Cluster #1; unexpected Condition") + common.AwaitDeployment(ctx, "istiod", k1, clPrimary) Expect(common.GetVersionFromIstiod()).To(Equal(v.Version), "Unexpected istiod version") - Success("Istiod is deployed in the namespace and Running on Exernal Cluster") }) }) @@ -100,11 +91,11 @@ values: }) It("updates Gateway status to Available", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key("istio-ingressgateway", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Gateway is not Ready on Cluster #1; unexpected Condition") + common.AwaitDeployment(ctx, "istio-ingressgateway", k1, clPrimary) + }) - Success("Gateway is created and available in both clusters") + It("has an external IP assigned", func(ctx SpecContext) { + expectLoadBalancerAddress(ctx, k1, clPrimary, "istio-ingressgateway") }) }) @@ -266,11 +257,8 @@ spec: Expect(k1.CreateFromString(externalControlPlaneYAML)).To(Succeed(), "Istio Resource creation failed on Cluster #1") }) - It("updates both Istio CR status to Ready", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key("external-istiod"), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready on Cluster #1; unexpected Condition") - Success("Istio CR is Ready on Cluster #1") + It("updates external Istio CR status to Ready", func(ctx SpecContext) { + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key("external-istiod"), &v1.Istio{}, k1, clPrimary) }) }) @@ -341,10 +329,7 @@ spec: }) It("updates remote Istio CR status to Ready on Cluster #2", func(ctx SpecContext) { - Eventually(common.GetObject, 10*time.Minute). - WithArguments(ctx, clRemote, kube.Key(externalIstioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready on Remote; unexpected Condition") - Success("Remote Istio CR is Ready on Cluster #2") + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key(externalIstioName), &v1.Istio{}, k2, clRemote, 10*time.Minute) }) }) @@ -354,24 +339,17 @@ spec: // Label the namespace with the istio revision name Expect(k2.Label("namespace", sampleNamespace, "istio.io/rev", "external-istiod")).To(Succeed(), "Labeling failed on Cluster #2") - deploySampleApp(k2, sampleNamespace, "v1") + deploySampleApp(k2, sampleNamespace, "v1", "default") Success("Sample app is deployed in Cluster #2") }) - samplePodsCluster2 := &corev1.PodList{} It("updates the pods status to Ready", func(ctx SpecContext) { - Expect(clRemote.List(ctx, samplePodsCluster2, client.InNamespace(sampleNamespace))).To(Succeed()) - Expect(samplePodsCluster2.Items).ToNot(BeEmpty(), "No pods found in sample namespace") - - for _, pod := range samplePodsCluster2.Items { - Eventually(common.GetObject). - WithArguments(ctx, clRemote, kube.Key(pod.Name, sampleNamespace), &corev1.Pod{}). - Should(HaveConditionStatus(corev1.PodReady, metav1.ConditionTrue), "Pod is not Ready on Cluster #2; unexpected Condition") - } + Eventually(common.CheckSamplePodsReady).WithArguments(ctx, clRemote).Should(Succeed(), "Error checking status of sample pods on Cluster #2") Success("Sample app is Running") }) It("has istio.io/rev annotation external-istiod", func(ctx SpecContext) { + samplePodsCluster2 := &corev1.PodList{} Expect(clRemote.List(ctx, samplePodsCluster2, client.InNamespace(sampleNamespace))).To(Succeed()) Expect(samplePodsCluster2.Items).ToNot(BeEmpty(), "No pods found in sample namespace") @@ -381,6 +359,10 @@ spec: Success("Sample pods has expected annotation") }) + It("can reach the ingress gateway from remote cluster", func(ctx SpecContext) { + eventuallyLoadBalancerIsReachable(ctx, k2, k1, clPrimary, "istio-ingressgateway") + }) + It("can access the sample app from the local service", func(ctx SpecContext) { verifyResponsesAreReceivedFromExpectedVersions(k2, "v1") Success("Sample app is accessible from hello service in Cluster #2") diff --git a/tests/e2e/multicluster/multicluster_multiprimary_test.go b/tests/e2e/multicluster/multicluster_multiprimary_test.go index aa0116932..8c31de028 100644 --- a/tests/e2e/multicluster/multicluster_multiprimary_test.go +++ b/tests/e2e/multicluster/multicluster_multiprimary_test.go @@ -17,7 +17,6 @@ package multicluster import ( - "context" "fmt" "time" @@ -26,27 +25,39 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/istioversion" "github.com/istio-ecosystem/sail-operator/pkg/kube" . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" - "github.com/istio-ecosystem/sail-operator/tests/e2e/util/certs" + "github.com/istio-ecosystem/sail-operator/pkg/version" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" - . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" ) var _ = Describe("Multicluster deployment models", Label("multicluster", "multicluster-multiprimary"), Ordered, func() { SetDefaultEventuallyTimeout(time.Duration(env.GetInt("DEFAULT_TEST_TIMEOUT", 180)) * time.Second) SetDefaultEventuallyPollingInterval(time.Second) + Context("Sidecar", func() { + generateMultiPrimaryTestCases("default") + }) + Context("Ambient", Label("ambient"), func() { + generateMultiPrimaryTestCases("ambient") + }) +}) + +func generateMultiPrimaryTestCases(profile string) { Describe("Multi-Primary Multi-Network configuration", func() { // Test the Multi-Primary Multi-Network configuration for each supported Istio version - for _, version := range istioversion.GetLatestPatchVersions() { - Context(fmt.Sprintf("Istio version %s", version.Version), func() { + for _, v := range istioversion.GetLatestPatchVersions() { + // Ambient multi-cluster is supported only since 1.27 + if profile == "ambient" && version.Constraint("<1.27").Check(v.Version) { + Log(fmt.Sprintf("Skipping test, because Istio version %s does not support Ambient Multi-Cluster configuration", v.Version)) + continue + } + + Context(fmt.Sprintf("Istio version %s", v.Version), func() { clr1 := cleaner.New(clPrimary, "cluster=primary") clr2 := cleaner.New(clRemote, "cluster=remote") @@ -57,119 +68,70 @@ var _ = Describe("Multicluster deployment models", Label("multicluster", "multic When("Istio and IstioCNI resources are created in both clusters", func() { BeforeAll(func(ctx SpecContext) { - Expect(k1.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") - Expect(k2.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") - Expect(k1.CreateNamespace(istioCniNamespace)).To(Succeed(), "Istio CNI namespace failed to be created") - Expect(k2.CreateNamespace(istioCniNamespace)).To(Succeed(), "Istio CNI namespace failed to be created") + createIstioNamespaces(k1, "network1", profile) + createIstioNamespaces(k2, "network2", profile) // Push the intermediate CA to both clusters - Expect(certs.PushIntermediateCA(k1, controlPlaneNamespace, "east", "network1", artifacts, clPrimary)).To(Succeed()) - Expect(certs.PushIntermediateCA(k2, controlPlaneNamespace, "west", "network2", artifacts, clRemote)).To(Succeed()) + createIntermediateCA(k1, "east", "network1", artifacts, clPrimary) + createIntermediateCA(k2, "west", "network2", artifacts, clRemote) // Wait for the secret to be created in both clusters - Eventually(func() error { - _, err := common.GetObject(context.Background(), clPrimary, kube.Key("cacerts", controlPlaneNamespace), &corev1.Secret{}) - return err - }).ShouldNot(HaveOccurred(), "Secret is not created on Cluster #1") - - Eventually(func() error { - _, err := common.GetObject(context.Background(), clRemote, kube.Key("cacerts", controlPlaneNamespace), &corev1.Secret{}) - return err - }).ShouldNot(HaveOccurred(), "Secret is not created on Cluster #1") - - common.CreateIstioCNI(k1, version.Name) - common.CreateIstioCNI(k2, version.Name) - - spec := ` -values: - global: - meshID: mesh1 - multiCluster: - clusterName: %s - network: %s` - common.CreateIstio(k1, version.Name, fmt.Sprintf(spec, "cluster1", "network1")) - common.CreateIstio(k2, version.Name, fmt.Sprintf(spec, "cluster2", "network2")) + awaitSecretCreation(k1.ClusterName, clPrimary) + awaitSecretCreation(k2.ClusterName, clRemote) + + createIstioResources(k1, v.Name, "cluster1", "network1", profile) + createIstioResources(k2, v.Name, "cluster2", "network2", profile) }) It("updates both Istio CR status to Ready", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key(istioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready on Cluster #1; unexpected Condition") - Success("Istio CR is Ready on Cluster #1") - - Eventually(common.GetObject). - WithArguments(ctx, clRemote, kube.Key(istioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready on Cluster #2; unexpected Condition") - Success("Istio CR is Ready on Cluster #2") + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key(istioName), &v1.Istio{}, k1, clPrimary) + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key(istioName), &v1.Istio{}, k2, clRemote) }) It("updates both IstioCNI CR status to Ready", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key(istioCniName), &v1.IstioCNI{}). - Should(HaveConditionStatus(v1.IstioCNIConditionReady, metav1.ConditionTrue), "Istio CNI is not Ready on Cluster #1; unexpected Condition") - Success("Istio CNI CR is Ready on Cluster #1") - - Eventually(common.GetObject). - WithArguments(ctx, clRemote, kube.Key(istioCniName), &v1.IstioCNI{}). - Should(HaveConditionStatus(v1.IstioCNIConditionReady, metav1.ConditionTrue), "Istio CNI is not Ready on Cluster #2; unexpected Condition") - Success("Istio CNI CR is Ready on Cluster #2") + common.AwaitCondition(ctx, v1.IstioCNIConditionReady, kube.Key(istioCniName), &v1.IstioCNI{}, k1, clPrimary) + common.AwaitCondition(ctx, v1.IstioCNIConditionReady, kube.Key(istioCniName), &v1.IstioCNI{}, k2, clRemote) }) It("deploys istiod", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key("istiod", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Istiod is not Available on Cluster #1; unexpected Condition") - Expect(common.GetVersionFromIstiod()).To(Equal(version.Version), "Unexpected istiod version") - Success("Istiod is deployed in the namespace and Running on Cluster #1") - - Eventually(common.GetObject). - WithArguments(ctx, clRemote, kube.Key("istiod", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Istiod is not Available on Cluster #2; unexpected Condition") - Expect(common.GetVersionFromIstiod()).To(Equal(version.Version), "Unexpected istiod version") - Success("Istiod is deployed in the namespace and Running on Cluster #2") + common.AwaitDeployment(ctx, "istiod", k1, clPrimary) + Expect(common.GetVersionFromIstiod()).To(Equal(v.Version), "Unexpected istiod version") + + common.AwaitDeployment(ctx, "istiod", k2, clRemote) + Expect(common.GetVersionFromIstiod()).To(Equal(v.Version), "Unexpected istiod version") }) It("deploys istio-cni-node", func(ctx SpecContext) { - Eventually(func() bool { - daemonset := &appsv1.DaemonSet{} - if err := clPrimary.Get(ctx, kube.Key("istio-cni-node", istioCniNamespace), daemonset); err != nil { - return false - } - return daemonset.Status.NumberAvailable == daemonset.Status.CurrentNumberScheduled - }).Should(BeTrue(), "CNI DaemonSet Pods are not Available on Cluster #1") - Success("CNI DaemonSet is deployed in the namespace and Running on Cluster #1") - - Eventually(func() bool { - daemonset := &appsv1.DaemonSet{} - if err := clRemote.Get(ctx, kube.Key("istio-cni-node", istioCniNamespace), daemonset); err != nil { - return false - } - return daemonset.Status.NumberAvailable == daemonset.Status.CurrentNumberScheduled - }).Should(BeTrue(), "IstioCNI DaemonSet Pods are not Available on Cluster #2") - Success("IstioCNI DaemonSet is deployed in the namespace and Running on Cluster #2") + common.AwaitCniDaemonSet(ctx, k1, clPrimary) + common.AwaitCniDaemonSet(ctx, k2, clRemote) }) }) When("Gateway is created in both clusters", func() { BeforeAll(func(ctx SpecContext) { - Expect(k1.WithNamespace(controlPlaneNamespace).Apply(eastGatewayYAML)).To(Succeed(), "Gateway creation failed on Cluster #1") - Expect(k2.WithNamespace(controlPlaneNamespace).Apply(westGatewayYAML)).To(Succeed(), "Gateway creation failed on Cluster #2") - - // Expose the Gateway service in both clusters - Expect(k1.WithNamespace(controlPlaneNamespace).Apply(exposeServiceYAML)).To(Succeed(), "Expose Service creation failed on Cluster #1") - Expect(k2.WithNamespace(controlPlaneNamespace).Apply(exposeServiceYAML)).To(Succeed(), "Expose Service creation failed on Cluster #2") + if profile == "ambient" { + common.CreateAmbientGateway(k1, controlPlaneNamespace, "network1") + common.CreateAmbientGateway(k2, controlPlaneNamespace, "network2") + } else { + Expect(k1.WithNamespace(controlPlaneNamespace).Apply(eastGatewayYAML)).To(Succeed(), "Gateway creation failed on Cluster #1") + Expect(k2.WithNamespace(controlPlaneNamespace).Apply(westGatewayYAML)).To(Succeed(), "Gateway creation failed on Cluster #2") + + // Expose the Gateway service in both clusters + Expect(k1.WithNamespace(controlPlaneNamespace).Apply(exposeServiceYAML)).To(Succeed(), "Expose Service creation failed on Cluster #1") + Expect(k2.WithNamespace(controlPlaneNamespace).Apply(exposeServiceYAML)).To(Succeed(), "Expose Service creation failed on Cluster #2") + } }) It("updates both Gateway status to Available", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key("istio-eastwestgateway", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Gateway is not Ready on Cluster #1; unexpected Condition") - - Eventually(common.GetObject). - WithArguments(ctx, clRemote, kube.Key("istio-eastwestgateway", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Gateway is not Ready on Cluster #2; unexpected Condition") + common.AwaitDeployment(ctx, "istio-eastwestgateway", k1, clPrimary) + common.AwaitDeployment(ctx, "istio-eastwestgateway", k2, clRemote) Success("Gateway is created and available in both clusters") }) + + It("has external IPs assigned", func(ctx SpecContext) { + expectLoadBalancerAddress(ctx, k1, clPrimary, "istio-eastwestgateway") + expectLoadBalancerAddress(ctx, k2, clRemote, "istio-eastwestgateway") + }) }) When("are installed remote secrets on each cluster", func() { @@ -208,16 +170,7 @@ values: When("sample apps are deployed in both clusters", func() { BeforeAll(func(ctx SpecContext) { - // Create namespace - Expect(k1.CreateNamespace(sampleNamespace)).To(Succeed(), "Namespace failed to be created on Cluster #1") - Expect(k2.CreateNamespace(sampleNamespace)).To(Succeed(), "Namespace failed to be created on Cluster #2") - - // Label the namespace - Expect(k1.Label("namespace", sampleNamespace, "istio-injection", "enabled")).To(Succeed(), "Error labeling sample namespace") - Expect(k2.Label("namespace", sampleNamespace, "istio-injection", "enabled")).To(Succeed(), "Error labeling sample namespace") - - // Deploy the sample app in both clusters - deploySampleAppToClusters(sampleNamespace, []ClusterDeployment{ + deploySampleAppToClusters(sampleNamespace, profile, []ClusterDeployment{ {Kubectl: k1, AppVersion: "v1"}, {Kubectl: k2, AppVersion: "v2"}, }) @@ -225,29 +178,16 @@ values: }) It("updates the pods status to Ready", func(ctx SpecContext) { - samplePodsCluster1 := &corev1.PodList{} - - Expect(clPrimary.List(ctx, samplePodsCluster1, client.InNamespace(sampleNamespace))).To(Succeed()) - Expect(samplePodsCluster1.Items).ToNot(BeEmpty(), "No pods found in sample namespace") - - for _, pod := range samplePodsCluster1.Items { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key(pod.Name, sampleNamespace), &corev1.Pod{}). - Should(HaveConditionStatus(corev1.PodReady, metav1.ConditionTrue), "Pod is not Ready on Cluster #1; unexpected Condition") - } - - samplePodsCluster2 := &corev1.PodList{} - Expect(clRemote.List(ctx, samplePodsCluster2, client.InNamespace(sampleNamespace))).To(Succeed()) - Expect(samplePodsCluster2.Items).ToNot(BeEmpty(), "No pods found in sample namespace") - - for _, pod := range samplePodsCluster2.Items { - Eventually(common.GetObject). - WithArguments(ctx, clRemote, kube.Key(pod.Name, sampleNamespace), &corev1.Pod{}). - Should(HaveConditionStatus(corev1.PodReady, metav1.ConditionTrue), "Pod is not Ready on Cluster #2; unexpected Condition") - } + Eventually(common.CheckSamplePodsReady).WithArguments(ctx, clPrimary).Should(Succeed(), "Error checking status of sample pods on Cluster #1") + Eventually(common.CheckSamplePodsReady).WithArguments(ctx, clRemote).Should(Succeed(), "Error checking status of sample pods on Cluster #2") Success("Sample app is created in both clusters and Running") }) + It("can reach target east-west gateway from each cluster", func(ctx SpecContext) { + eventuallyLoadBalancerIsReachable(ctx, k1, k2, clRemote, "istio-eastwestgateway") + eventuallyLoadBalancerIsReachable(ctx, k2, k1, clPrimary, "istio-eastwestgateway") + }) + It("can access the sample app from both clusters", func(ctx SpecContext) { verifyResponsesAreReceivedFromExpectedVersions(k1) verifyResponsesAreReceivedFromExpectedVersions(k2) @@ -307,4 +247,4 @@ values: }) } }) -}) +} diff --git a/tests/e2e/multicluster/multicluster_primaryremote_test.go b/tests/e2e/multicluster/multicluster_primaryremote_test.go index c7b8548d5..98ccb7f34 100644 --- a/tests/e2e/multicluster/multicluster_primaryremote_test.go +++ b/tests/e2e/multicluster/multicluster_primaryremote_test.go @@ -17,7 +17,6 @@ package multicluster import ( - "context" "fmt" "time" @@ -27,20 +26,17 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/kube" . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" "github.com/istio-ecosystem/sail-operator/pkg/version" - "github.com/istio-ecosystem/sail-operator/tests/e2e/util/certs" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" - . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" ) var _ = Describe("Multicluster deployment models", Label("multicluster", "multicluster-primaryremote"), Ordered, func() { + profile := "default" SetDefaultEventuallyTimeout(time.Duration(env.GetInt("DEFAULT_TEST_TIMEOUT", 180)) * time.Second) SetDefaultEventuallyPollingInterval(time.Second) @@ -64,74 +60,39 @@ var _ = Describe("Multicluster deployment models", Label("multicluster", "multic When("Istio and IstioCNI resources are created in both clusters", func() { BeforeAll(func(ctx SpecContext) { - Expect(k1.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") - Expect(k2.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") - Expect(k1.CreateNamespace(istioCniNamespace)).To(Succeed(), "Istio CNI Namespace failed to be created") - Expect(k2.CreateNamespace(istioCniNamespace)).To(Succeed(), "Istio CNI Namespace failed to be created") + createIstioNamespaces(k1, "network1", profile) + createIstioNamespaces(k2, "network2", profile) // Push the intermediate CA to both clusters - Expect(certs.PushIntermediateCA(k1, controlPlaneNamespace, "east", "network1", artifacts, clPrimary)). - To(Succeed(), "Error pushing intermediate CA to Primary Cluster") - Expect(certs.PushIntermediateCA(k2, controlPlaneNamespace, "west", "network2", artifacts, clRemote)). - To(Succeed(), "Error pushing intermediate CA to Remote Cluster") + createIntermediateCA(k1, "east", "network1", artifacts, clPrimary) + createIntermediateCA(k2, "west", "network2", artifacts, clRemote) // Wait for the secret to be created in both clusters - Eventually(func() error { - _, err := common.GetObject(context.Background(), clPrimary, kube.Key("cacerts", controlPlaneNamespace), &corev1.Secret{}) - return err - }).ShouldNot(HaveOccurred(), "Secret is not created on Primary Cluster") - - Eventually(func() error { - _, err := common.GetObject(context.Background(), clRemote, kube.Key("cacerts", controlPlaneNamespace), &corev1.Secret{}) - return err - }).ShouldNot(HaveOccurred(), "Secret is not created on Primary Cluster") - - common.CreateIstioCNI(k1, v.Name) - - spec := ` -values: - pilot: - env: - EXTERNAL_ISTIOD: "true" - global: - meshID: mesh1 - multiCluster: - clusterName: cluster1 - network: network1` - common.CreateIstio(k1, v.Name, spec) + awaitSecretCreation(k1.ClusterName, clPrimary) + awaitSecretCreation(k2.ClusterName, clRemote) + + pilot := ` +pilot: + env: + EXTERNAL_ISTIOD: "true"` + createIstioResources(k1, v.Name, "cluster1", "network1", profile, pilot) }) It("updates Istio CR on Primary cluster status to Ready", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key(istioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready on Primary; unexpected Condition") - Success("Istio CR is Ready on Primary Cluster") + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key(istioName), &v1.Istio{}, k1, clPrimary) }) It("updates IstioCNI CR on Primary cluster status to Ready", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key(istioCniName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioCNIConditionReady, metav1.ConditionTrue), "IstioCNI is not Ready on Primary; unexpected Condition") - Success("IstioCNI CR is Ready on Primary Cluster") + common.AwaitCondition(ctx, v1.IstioCNIConditionReady, kube.Key(istioCniName), &v1.IstioCNI{}, k1, clPrimary) }) It("deploys istiod", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key("istiod", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Istiod is not Available on Primary; unexpected Condition") + common.AwaitDeployment(ctx, "istiod", k1, clPrimary) Expect(common.GetVersionFromIstiod()).To(Equal(v.Version), "Unexpected istiod version") - Success("Istiod is deployed in the namespace and Running on Primary Cluster") }) It("deploys istio-cni-node", func(ctx SpecContext) { - Eventually(func() bool { - daemonset := &appsv1.DaemonSet{} - if err := clPrimary.Get(ctx, kube.Key("istio-cni-node", istioCniNamespace), daemonset); err != nil { - return false - } - return daemonset.Status.NumberAvailable == daemonset.Status.CurrentNumberScheduled - }).Should(BeTrue(), "IstioCNI DaemonSet Pods are not Available on Primary Cluster") - Success("IstioCNI DaemonSet is deployed in the namespace and Running on Primary Cluster") + common.AwaitCniDaemonSet(ctx, k1, clPrimary) }) }) @@ -147,9 +108,11 @@ values: }) It("updates Gateway status to Available", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key("istio-eastwestgateway", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Gateway is not Ready on Primary; unexpected Condition") + common.AwaitDeployment(ctx, "istio-eastwestgateway", k1, clPrimary) + }) + + It("has an external IP assigned", func(ctx SpecContext) { + expectLoadBalancerAddress(ctx, k1, clPrimary, "istio-eastwestgateway") }) }) @@ -208,10 +171,7 @@ values: }) It("updates remote Istio CR status to Ready", func(ctx SpecContext) { - Eventually(common.GetObject, 10*time.Minute). - WithArguments(ctx, clRemote, kube.Key(istioName), &v1.Istio{}). - Should(HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready on Remote; unexpected Condition") - Success("Istio CR is Ready on Remote Cluster") + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key(istioName), &v1.Istio{}, k2, clRemote, 10*time.Minute) }) }) @@ -222,25 +182,17 @@ values: }) It("updates Gateway status to Available", func(ctx SpecContext) { - Eventually(common.GetObject). - WithArguments(ctx, clRemote, kube.Key("istio-eastwestgateway", controlPlaneNamespace), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Gateway is not Ready on Remote; unexpected Condition") - Success("Gateway is created and available in Remote cluster") + common.AwaitDeployment(ctx, "istio-eastwestgateway", k2, clRemote) + }) + + It("has an external IP assigned", func(ctx SpecContext) { + expectLoadBalancerAddress(ctx, k2, clRemote, "istio-eastwestgateway") }) }) When("sample apps are deployed in both clusters", func() { BeforeAll(func(ctx SpecContext) { - // Create namespace - Expect(k1.CreateNamespace(sampleNamespace)).To(Succeed(), "Namespace failed to be created on Cluster #1") - Expect(k2.CreateNamespace(sampleNamespace)).To(Succeed(), "Namespace failed to be created on Cluster #2") - - // Label the namespace - Expect(k1.Label("namespace", sampleNamespace, "istio-injection", "enabled")).To(Succeed(), "Error labeling sample namespace") - Expect(k2.Label("namespace", sampleNamespace, "istio-injection", "enabled")).To(Succeed(), "Error labeling sample namespace") - - // Deploy the sample app in both clusters - deploySampleAppToClusters(sampleNamespace, []ClusterDeployment{ + deploySampleAppToClusters(sampleNamespace, profile, []ClusterDeployment{ {Kubectl: k1, AppVersion: "v1"}, {Kubectl: k2, AppVersion: "v2"}, }) @@ -248,29 +200,16 @@ values: }) It("updates the pods status to Ready", func(ctx SpecContext) { - samplePodsPrimary := &corev1.PodList{} - - Expect(clPrimary.List(ctx, samplePodsPrimary, client.InNamespace(sampleNamespace))).To(Succeed()) - Expect(samplePodsPrimary.Items).ToNot(BeEmpty(), "No pods found in sample namespace") - - for _, pod := range samplePodsPrimary.Items { - Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key(pod.Name, sampleNamespace), &corev1.Pod{}). - Should(HaveConditionStatus(corev1.PodReady, metav1.ConditionTrue), "Pod is not Ready on Primary; unexpected Condition") - } - - samplePodsRemote := &corev1.PodList{} - Expect(clRemote.List(ctx, samplePodsRemote, client.InNamespace(sampleNamespace))).To(Succeed()) - Expect(samplePodsRemote.Items).ToNot(BeEmpty(), "No pods found in sample namespace") - - for _, pod := range samplePodsRemote.Items { - Eventually(common.GetObject). - WithArguments(ctx, clRemote, kube.Key(pod.Name, sampleNamespace), &corev1.Pod{}). - Should(HaveConditionStatus(corev1.PodReady, metav1.ConditionTrue), "Pod is not Ready on Remote; unexpected Condition") - } + Eventually(common.CheckSamplePodsReady).WithArguments(ctx, clPrimary).Should(Succeed(), "Error checking status of sample pods on Primary cluster") + Eventually(common.CheckSamplePodsReady).WithArguments(ctx, clRemote).Should(Succeed(), "Error checking status of sample pods on Remote cluster") Success("Sample app is created in both clusters and Running") }) + It("can reach target east-west gateway from each cluster", func(ctx SpecContext) { + eventuallyLoadBalancerIsReachable(ctx, k1, k2, clRemote, "istio-eastwestgateway") + eventuallyLoadBalancerIsReachable(ctx, k2, k1, clPrimary, "istio-eastwestgateway") + }) + It("can access the sample app from both clusters", func(ctx SpecContext) { verifyResponsesAreReceivedFromExpectedVersions(k1) verifyResponsesAreReceivedFromExpectedVersions(k2) diff --git a/tests/e2e/multicluster/multicluster_suite_test.go b/tests/e2e/multicluster/multicluster_suite_test.go index d75438705..eef7b7695 100644 --- a/tests/e2e/multicluster/multicluster_suite_test.go +++ b/tests/e2e/multicluster/multicluster_suite_test.go @@ -37,10 +37,9 @@ var ( clRemote client.Client err error debugInfoLogged bool - controlPlaneNamespace = env.Get("CONTROL_PLANE_NS", "istio-system") externalControlPlaneNamespace = env.Get("EXTERNAL_CONTROL_PLANE_NS", "external-istiod") istioName = env.Get("ISTIO_NAME", "default") - istioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") + istioCniNamespace = common.IstioCniNamespace istioCniName = env.Get("ISTIOCNI_NAME", "default") multicluster = env.GetBool("MULTICLUSTER", false) keepOnFailure = env.GetBool("KEEP_ON_FAILURE", false) diff --git a/tests/e2e/multicontrolplane/multi_control_plane_suite_test.go b/tests/e2e/multicontrolplane/multi_control_plane_suite_test.go index 9485cb193..749d1fdc4 100644 --- a/tests/e2e/multicontrolplane/multi_control_plane_suite_test.go +++ b/tests/e2e/multicontrolplane/multi_control_plane_suite_test.go @@ -21,6 +21,7 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/env" k8sclient "github.com/istio-ecosystem/sail-operator/tests/e2e/util/client" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -35,7 +36,7 @@ var ( controlPlaneNamespace2 = env.Get("CONTROL_PLANE_NS2", "istio-system2") istioName1 = env.Get("ISTIO_NAME1", "mesh1") istioName2 = env.Get("ISTIO_NAME2", "mesh2") - istioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") + istioCniNamespace = common.IstioCniNamespace istioCniName = env.Get("ISTIOCNI_NAME", "default") appNamespace1 = env.Get("APP_NAMESPACE1", "app1") appNamespace2a = env.Get("APP_NAMESPACE2A", "app2a") diff --git a/tests/e2e/multicontrolplane/multi_control_plane_test.go b/tests/e2e/multicontrolplane/multi_control_plane_test.go index 86748aba3..fa8c4e02a 100644 --- a/tests/e2e/multicontrolplane/multi_control_plane_test.go +++ b/tests/e2e/multicontrolplane/multi_control_plane_test.go @@ -27,11 +27,9 @@ import ( . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" - . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" appsv1 "k8s.io/api/apps/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) var _ = Describe("Multi control plane deployment model", Label("smoke", "multicontrol-plane"), Ordered, func() { @@ -61,10 +59,7 @@ var _ = Describe("Multi control plane deployment model", Label("smoke", "multico It("Installs IstioCNI", func(ctx SpecContext) { common.CreateIstioCNI(k, latestVersion.Name) - - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioCniName), &v1.IstioCNI{}). - Should(HaveConditionStatus(v1.IstioCNIConditionReady, metav1.ConditionTrue), "IstioCNI is not Ready; unexpected Condition") - Success("IstioCNI is Ready") + common.AwaitCondition(ctx, v1.IstioCNIConditionReady, kube.Key(istioCniName), &v1.IstioCNI{}, k, cl) }) DescribeTable("Installs Istios", @@ -101,13 +96,8 @@ spec: Entry("Mesh 1", istioName1), Entry("Mesh 2", istioName2), func(ctx SpecContext, name string) { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(name), &v1.Istio{}). - Should( - And( - HaveConditionStatus(v1.IstioConditionReconciled, metav1.ConditionTrue), - HaveConditionStatus(v1.IstioConditionReady, metav1.ConditionTrue), - ), "Istio is not Reconciled and Ready; unexpected Condition") - Success(fmt.Sprintf("Istio %s ready", name)) + common.AwaitCondition(ctx, v1.IstioConditionReconciled, kube.Key(name), &v1.Istio{}, k, cl) + common.AwaitCondition(ctx, v1.IstioConditionReady, kube.Key(name), &v1.Istio{}, k, cl) }) DescribeTable("Deploys applications", @@ -135,10 +125,8 @@ spec: Entry("App 2b", appNamespace2b), func(ctx SpecContext, ns string) { for _, deployment := range []string{"sleep", "httpbin"} { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(deployment, ns), &appsv1.Deployment{}). - Should(HaveConditionStatus(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error waiting for deployment to be available") + common.AwaitCondition(ctx, appsv1.DeploymentAvailable, kube.Key(deployment, ns), &appsv1.Deployment{}, k, cl) } - Success(fmt.Sprintf("Applications in namespace %s ready", ns)) }) }) diff --git a/tests/e2e/operator/operator_install_test.go b/tests/e2e/operator/operator_install_test.go index 63a0f2a22..07805a464 100644 --- a/tests/e2e/operator/operator_install_test.go +++ b/tests/e2e/operator/operator_install_test.go @@ -76,8 +76,7 @@ var _ = Describe("Operator", Label("smoke", "operator"), Ordered, func() { It("updates the CRDs status to Established", func(ctx SpecContext) { for _, crdName := range sailCRDs { - Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(crdName), &apiextensionsv1.CustomResourceDefinition{}). - Should(HaveConditionStatus(apiextensionsv1.Established, metav1.ConditionTrue), "Error getting Istio CRD") + common.AwaitCondition(ctx, apiextensionsv1.Established, kube.Key(crdName), &apiextensionsv1.CustomResourceDefinition{}, k, cl) } Success("CRDs are Established") }) diff --git a/tests/e2e/setup/setup-kind.sh b/tests/e2e/setup/setup-kind.sh index 4912025ce..c88f6fd1a 100755 --- a/tests/e2e/setup/setup-kind.sh +++ b/tests/e2e/setup/setup-kind.sh @@ -18,6 +18,7 @@ set -eux -o pipefail SCRIPTPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" ROOT=$(cd "${SCRIPTPATH}/../../.." && pwd) +GW_API_VERSION=${GW_API_VERSION:-v1.4.0} # shellcheck source=common/scripts/kind_provisioner.sh source "${ROOT}/common/scripts/kind_provisioner.sh" @@ -70,6 +71,11 @@ if [ "${MULTICLUSTER}" == "true" ]; then setup_kind_clusters "${KIND_IMAGE}" "" setup_kind_registry "${CLUSTER_NAMES[@]}" + # Apply Gateway API CRDs which are needed for Multi-Cluster Ambient testing + for config in "${KUBECONFIGS[@]}"; do + kubectl apply --kubeconfig="$config" --server-side -f "https://github.com/kubernetes-sigs/gateway-api/releases/download/${GW_API_VERSION}/experimental-install.yaml" + done + export KUBECONFIG="${KUBECONFIGS[0]}" export KUBECONFIG2="${KUBECONFIGS[1]}" echo "Your KinD environment is ready, to use it: export KUBECONFIG=$(IFS=:; echo "${KUBECONFIGS[*]}")" diff --git a/tests/e2e/util/common/checks.go b/tests/e2e/util/common/checks.go new file mode 100644 index 000000000..ef5c49d8c --- /dev/null +++ b/tests/e2e/util/common/checks.go @@ -0,0 +1,98 @@ +//go:build e2e + +// Copyright Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "context" + "fmt" + "reflect" + + "github.com/istio-ecosystem/sail-operator/pkg/kube" + . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" + . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// AwaitCondition to be True. A key and a pointer to the object struct must be supplied. Extra arguments to pass to `Eventually` can be optionally supplied. +func AwaitCondition[T ~string](ctx context.Context, condition T, key client.ObjectKey, obj client.Object, k kubectl.Kubectl, cl client.Client, args ...any) { + kind := reflect.TypeOf(obj).Elem().Name() + cluster := "cluster" + if k.ClusterName != "" { + cluster = k.ClusterName + } + + Eventually(GetObject, args...). + WithArguments(ctx, cl, key, obj). + Should(HaveConditionStatus(condition, metav1.ConditionTrue), + fmt.Sprintf("%s %q is not %s on %s; unexpected Condition", kind, key.Name, condition, cluster)) + Success(fmt.Sprintf("%s %q is %s on %s", kind, key.Name, condition, cluster)) +} + +// AwaitDeployment to reach the Available state. +func AwaitDeployment(ctx context.Context, name string, k kubectl.Kubectl, cl client.Client) { + AwaitCondition(ctx, appsv1.DeploymentAvailable, kube.Key(name, ControlPlaneNamespace), &appsv1.Deployment{}, k, cl) +} + +func isPodReady(pod *corev1.Pod) bool { + for _, cond := range pod.Status.Conditions { + if cond.Type == corev1.PodReady && cond.Status == corev1.ConditionTrue { + return true + } + } + return false +} + +// CheckPodsReady in the given namespace. +func CheckPodsReady(ctx context.Context, cl client.Client, namespace string) error { + podList := &corev1.PodList{} + if err := cl.List(ctx, podList, client.InNamespace(namespace)); err != nil { + return fmt.Errorf("Failed to list pods: %w", err) + } + if len(podList.Items) == 0 { + return fmt.Errorf("No pods found in namespace %q", namespace) + } + + for _, pod := range podList.Items { + if !isPodReady(&pod) { + return fmt.Errorf("Pod %q in namespace %q is not ready", pod.Name, namespace) + } + } + + return nil +} + +func CheckSamplePodsReady(ctx context.Context, cl client.Client) error { + return CheckPodsReady(ctx, cl, sampleNamespace) +} + +// AwaitCniDaemonSet to be deployed and reach the scheduled number of pods. +func AwaitCniDaemonSet(ctx context.Context, k kubectl.Kubectl, cl client.Client) { + key := kube.Key("istio-cni-node", IstioCniNamespace) + Eventually(func() bool { + daemonset := &appsv1.DaemonSet{} + if err := cl.Get(ctx, key, daemonset); err != nil { + return false + } + return daemonset.Status.NumberAvailable == daemonset.Status.CurrentNumberScheduled + }).Should(BeTrue(), fmt.Sprintf("DaemonSet '%s' is not Available in the '%s' namespace on %s cluster", key.Name, key.Namespace, k.ClusterName)) + Success(fmt.Sprintf("DaemonSet '%s' is deployed and running in the '%s' namespace on %s cluster", key.Name, key.Namespace, k.ClusterName)) +} diff --git a/tests/e2e/util/common/e2e_utils.go b/tests/e2e/util/common/e2e_utils.go index ff5e289d1..9a3735f5b 100644 --- a/tests/e2e/util/common/e2e_utils.go +++ b/tests/e2e/util/common/e2e_utils.go @@ -56,15 +56,16 @@ const ( ) var ( - OperatorImage = env.Get("IMAGE", "quay.io/sail-dev/sail-operator:latest") - OperatorNamespace = env.Get("NAMESPACE", "sail-operator") + ControlPlaneNamespace = env.Get("CONTROL_PLANE_NS", "istio-system") + IstioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") + OperatorImage = env.Get("IMAGE", "quay.io/sail-dev/sail-operator:latest") + OperatorNamespace = env.Get("NAMESPACE", "sail-operator") + ZtunnelNamespace = env.Get("ZTUNNEL_NAMESPACE", "ztunnel") - deploymentName = env.Get("DEPLOYMENT_NAME", "sail-operator") - controlPlaneNamespace = env.Get("CONTROL_PLANE_NS", "istio-system") - istioName = env.Get("ISTIO_NAME", "default") - istioCniName = env.Get("ISTIOCNI_NAME", "default") - istioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") - ztunnelNamespace = env.Get("ZTUNNEL_NAMESPACE", "ztunnel") + deploymentName = env.Get("DEPLOYMENT_NAME", "sail-operator") + istioName = env.Get("ISTIO_NAME", "default") + istioCniName = env.Get("ISTIOCNI_NAME", "default") + sampleNamespace = env.Get("SAMPLE_NAMESPACE", "sample") // version can have one of the following formats: // - 1.22.2 @@ -204,14 +205,14 @@ func logIstioDebugInfo(k kubectl.Kubectl) { resource, err := k.GetYAML("istio", istioName) logDebugElement("=====Istio YAML=====", resource, err) - output, err := k.WithNamespace(controlPlaneNamespace).GetPods("", "-o wide") - logDebugElement("=====Pods in "+controlPlaneNamespace+"=====", output, err) + output, err := k.WithNamespace(ControlPlaneNamespace).GetPods("", "-o wide") + logDebugElement("=====Pods in "+ControlPlaneNamespace+"=====", output, err) - logs, err := k.WithNamespace(controlPlaneNamespace).Logs("deploy/istiod", ptr.Of(120*time.Second)) + logs, err := k.WithNamespace(ControlPlaneNamespace).Logs("deploy/istiod", ptr.Of(120*time.Second)) logDebugElement("=====Istiod logs=====", logs, err) - events, err := k.WithNamespace(controlPlaneNamespace).GetEvents() - logDebugElement("=====Events in "+controlPlaneNamespace+"=====", events, err) + events, err := k.WithNamespace(ControlPlaneNamespace).GetEvents() + logDebugElement("=====Events in "+ControlPlaneNamespace+"=====", events, err) // Running istioctl proxy-status to get the status of the proxies. proxyStatus, err := istioctl.GetProxyStatus() @@ -222,20 +223,20 @@ func logCNIDebugInfo(k kubectl.Kubectl) { resource, err := k.GetYAML("istiocni", istioCniName) logDebugElement("=====IstioCNI YAML=====", resource, err) - ds, err := k.WithNamespace(istioCniNamespace).GetYAML("daemonset", "istio-cni-node") + ds, err := k.WithNamespace(IstioCniNamespace).GetYAML("daemonset", "istio-cni-node") logDebugElement("=====Istio CNI DaemonSet YAML=====", ds, err) - events, err := k.WithNamespace(istioCniNamespace).GetEvents() - logDebugElement("=====Events in "+istioCniNamespace+"=====", events, err) + events, err := k.WithNamespace(IstioCniNamespace).GetEvents() + logDebugElement("=====Events in "+IstioCniNamespace+"=====", events, err) // Temporary information to gather more details about failure - pods, err := k.WithNamespace(istioCniNamespace).GetPods("", "-o wide") - logDebugElement("=====Pods in "+istioCniNamespace+"=====", pods, err) + pods, err := k.WithNamespace(IstioCniNamespace).GetPods("", "-o wide") + logDebugElement("=====Pods in "+IstioCniNamespace+"=====", pods, err) - describe, err := k.WithNamespace(istioCniNamespace).Describe("daemonset", "istio-cni-node") + describe, err := k.WithNamespace(IstioCniNamespace).Describe("daemonset", "istio-cni-node") logDebugElement("=====Istio CNI DaemonSet describe=====", describe, err) - logs, err := k.WithNamespace(istioCniNamespace).Logs("daemonset/istio-cni-node", ptr.Of(120*time.Second)) + logs, err := k.WithNamespace(IstioCniNamespace).Logs("daemonset/istio-cni-node", ptr.Of(120*time.Second)) logDebugElement("=====Istio CNI logs=====", logs, err) } @@ -243,22 +244,22 @@ func logZtunnelDebugInfo(k kubectl.Kubectl) { resource, err := k.GetYAML("ztunnel", "default") logDebugElement("=====ZTunnel YAML=====", resource, err) - ds, err := k.WithNamespace(ztunnelNamespace).GetYAML("daemonset", "ztunnel") + ds, err := k.WithNamespace(ZtunnelNamespace).GetYAML("daemonset", "ztunnel") logDebugElement("=====ZTunnel DaemonSet YAML=====", ds, err) - events, err := k.WithNamespace(ztunnelNamespace).GetEvents() - logDebugElement("=====Events in "+ztunnelNamespace+"=====", events, err) + events, err := k.WithNamespace(ZtunnelNamespace).GetEvents() + logDebugElement("=====Events in "+ZtunnelNamespace+"=====", events, err) - describe, err := k.WithNamespace(ztunnelNamespace).Describe("daemonset", "ztunnel") + describe, err := k.WithNamespace(ZtunnelNamespace).Describe("daemonset", "ztunnel") logDebugElement("=====ZTunnel DaemonSet describe=====", describe, err) - logs, err := k.WithNamespace(ztunnelNamespace).Logs("daemonset/ztunnel", ptr.Of(120*time.Second)) + logs, err := k.WithNamespace(ZtunnelNamespace).Logs("daemonset/ztunnel", ptr.Of(120*time.Second)) logDebugElement("=====ztunnel logs=====", logs, err) } func logCertsDebugInfo(k kubectl.Kubectl) { - certs, err := k.WithNamespace(controlPlaneNamespace).GetSecret("cacerts") - logDebugElement("=====CA certs in "+controlPlaneNamespace+"=====", certs, err) + certs, err := k.WithNamespace(ControlPlaneNamespace).GetSecret("cacerts") + logDebugElement("=====CA certs in "+ControlPlaneNamespace+"=====", certs, err) } func logDebugElement(caption string, info string, err error) { @@ -272,7 +273,7 @@ func logDebugElement(caption string, info string, err error) { func GetVersionFromIstiod() (*semver.Version, error) { k := kubectl.New() - output, err := k.WithNamespace(controlPlaneNamespace).Exec("deploy/istiod", "", "pilot-discovery version") + output, err := k.WithNamespace(ControlPlaneNamespace).Exec("deploy/istiod", "", "pilot-discovery version") if err != nil { return nil, fmt.Errorf("error getting version from istiod: %w", err) } @@ -284,33 +285,6 @@ func GetVersionFromIstiod() (*semver.Version, error) { return nil, fmt.Errorf("error getting version from istiod: version not found in output: %s", output) } -func isPodReady(pod *corev1.Pod) bool { - for _, cond := range pod.Status.Conditions { - if cond.Type == corev1.PodReady && cond.Status == corev1.ConditionTrue { - return true - } - } - return false -} - -func CheckPodsReady(ctx context.Context, cl client.Client, namespace string) error { - podList := &corev1.PodList{} - if err := cl.List(ctx, podList, client.InNamespace(namespace)); err != nil { - return fmt.Errorf("Failed to list pods: %w", err) - } - if len(podList.Items) == 0 { - return fmt.Errorf("No pods found in namespace %q", namespace) - } - - for _, pod := range podList.Items { - if !isPodReady(&pod) { - return fmt.Errorf("pod %q in namespace %q is not ready", pod.Name, namespace) - } - } - - return nil -} - // Resolve domain name and return ip address. // By default, return ipv4 address and if missing, return ipv6. func ResolveHostDomainToIP(hostDomain string) (string, error) { @@ -355,19 +329,12 @@ metadata: spec: version: %s namespace: %s` - yaml = fmt.Sprintf(yaml, istioName, version, controlPlaneNamespace) - for _, spec := range specs { - yaml += Indent(spec) - } - - Log("Istio YAML:", Indent(yaml)) - Expect(k.CreateFromString(yaml)). - To(Succeed(), withClusterName("Istio CR failed to be created", k)) - Success(withClusterName("Istio CR created", k)) + yaml = fmt.Sprintf(yaml, istioName, version, ControlPlaneNamespace) + createResource(k, "Istio", yaml, specs...) } // CreateIstioCNI custom resource using a given `kubectl` client and with the specified version. -func CreateIstioCNI(k kubectl.Kubectl, version string) { +func CreateIstioCNI(k kubectl.Kubectl, version string, specs ...string) { yaml := ` apiVersion: sailoperator.io/v1 kind: IstioCNI @@ -376,10 +343,54 @@ metadata: spec: version: %s namespace: %s` - yaml = fmt.Sprintf(yaml, istioCniName, version, istioCniNamespace) - Log("IstioCNI YAML:", Indent(yaml)) - Expect(k.CreateFromString(yaml)).To(Succeed(), withClusterName("IstioCNI creation failed", k)) - Success(withClusterName("IstioCNI created", k)) + yaml = fmt.Sprintf(yaml, istioCniName, version, IstioCniNamespace) + createResource(k, "IstioCNI", yaml, specs...) +} + +func CreateZTunnel(k kubectl.Kubectl, version string, specs ...string) { + yaml := ` +apiVersion: sailoperator.io/v1alpha1 +kind: ZTunnel +metadata: + name: default +spec: + profile: ambient + version: %s + namespace: %s` + yaml = fmt.Sprintf(yaml, version, ZtunnelNamespace) + createResource(k, "ZTunnel", yaml, specs...) +} + +func CreateAmbientGateway(k kubectl.Kubectl, namespace, network string) { + yaml := `kind: Gateway +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: istio-eastwestgateway + namespace: %s + labels: + topology.istio.io/network: %s +spec: + gatewayClassName: istio-east-west + listeners: + - name: mesh + port: 15008 + protocol: HBONE + tls: + mode: Terminate + options: + gateway.istio.io/tls-terminate-mode: ISTIO_MUTUAL` + yaml = fmt.Sprintf(yaml, namespace, network) + createResource(k, "Gateway", yaml) +} + +func createResource(k kubectl.Kubectl, kind, yaml string, specs ...string) { + for _, spec := range specs { + yaml += Indent(spec) + } + + Log(fmt.Sprintf("%s YAML:", kind), Indent(yaml)) + Expect(k.CreateFromString(yaml)).To(Succeed(), withClusterName(fmt.Sprintf("%s creation failed:", kind), k)) + Success(withClusterName(fmt.Sprintf("%s created", kind), k)) } func Indent(str string) string { diff --git a/tests/e2e/util/kubectl/kubectl.go b/tests/e2e/util/kubectl/kubectl.go index 95949a89a..b10ac948c 100644 --- a/tests/e2e/util/kubectl/kubectl.go +++ b/tests/e2e/util/kubectl/kubectl.go @@ -305,6 +305,12 @@ func (k Kubectl) Label(kind, name, labelKey, labelValue string) error { return err } +// LabelNamespaced adds a label to the specified resource in the specified namespace +func (k Kubectl) LabelNamespaced(kind, namespace, name, labelKey, labelValue string) error { + _, err := k.executeCommand(k.build(fmt.Sprintf(" label %s -n %s %s %s=%s", kind, namespace, name, labelKey, labelValue))) + return err +} + // executeCommand handles running the command and then resets the namespace automatically func (k Kubectl) executeCommand(cmd string) (string, error) { return shell.ExecuteCommand(cmd) diff --git a/tools/update_deps.sh b/tools/update_deps.sh index 57c765411..54102794b 100755 --- a/tools/update_deps.sh +++ b/tools/update_deps.sh @@ -160,6 +160,18 @@ OPM_LATEST_VERSION=$(getVersionForUpdate operator-framework/operator-registry "$ OLM_LATEST_VERSION=$(getVersionForUpdate operator-framework/operator-lifecycle-manager "${OLM_VERSION}") "$SED_CMD" -i "s|OLM_VERSION ?= .*|OLM_VERSION ?= ${OLM_LATEST_VERSION}|" "${ROOTDIR}/Makefile.core.mk" +# Update gateway-api +GW_API_LATEST_VERSION=$(getLatestVersion kubernetes-sigs/gateway-api) +"$SED_CMD" -i "s|GW_API_VERSION=.*|GW_API_VERSION=\${GW_API_VERSION:-${GW_API_LATEST_VERSION}}|" "${ROOTDIR}/tests/e2e/setup/setup-kind.sh" + +# Update kube-rbac-proxy +RBAC_PROXY_LATEST_VERSION=$(getLatestVersion brancz/kube-rbac-proxy | cut -d/ -f1) +# Only update it if the newer image is available in the registry +if docker manifest inspect "gcr.io/kubebuilder/kube-rbac-proxy:${RBAC_PROXY_LATEST_VERSION}" >/dev/null 2>/dev/null; then + "$SED_CMD" -i "s|gcr.io/kubebuilder/kube-rbac-proxy:.*|gcr.io/kubebuilder/kube-rbac-proxy:${RBAC_PROXY_LATEST_VERSION}|" "${ROOTDIR}/chart/values.yaml" +fi + + # Update gitleaks GITLEAKS_LATEST_VERSION=$(getVersionForUpdate gitleaks/gitleaks "${GITLEAKS_VERSION}") "$SED_CMD" -i "s|GITLEAKS_VERSION ?= .*|GITLEAKS_VERSION ?= ${GITLEAKS_LATEST_VERSION}|" "${ROOTDIR}/Makefile.core.mk"