From 7d82978aba1dd8f846840fcbfe9d6196f51bafd3 Mon Sep 17 00:00:00 2001 From: Francisco Herrera Date: Thu, 27 Feb 2025 16:30:29 +0100 Subject: [PATCH] External controlplane e2e test (#676) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * e2e multicluster test improve Signed-off-by: Francisco H Clean up multicluister test Signed-off-by: Francisco H Adding primary remote multicluster e2e test Signed-off-by: Francisco H Ensure e2e tests use the correct versions yaml file (#667) We use `-ldflags` to set the `istioversion.versionsFilename` variable, but this only works when building the binary. It doesn't work when running ginkgo-based e2e tests. When the `versionsFilename` is not set via ldflags, it will now use the environment variable `VERSIONS_YAML_FILE`. If this env var isn't set, we default to `versions.yaml`. Signed-off-by: Marko Lukša Signed-off-by: Francisco H e2e multicluster test improve Signed-off-by: Francisco H Fix lint errors Signed-off-by: Francisco H Delete references to bookinfo on multicluster test Signed-off-by: Francisco H Fix change package on external multicluster test Signed-off-by: Francisco H * Apply suggestions from code review Co-authored-by: Nick Fox <6226732+nrfox@users.noreply.github.com> Signed-off-by: Francisco H * Changes from review Signed-off-by: Francisco H * More changes from review Signed-off-by: Francisco H * Add tag to common file in multicluster test Signed-off-by: Francisco H * Delete duplicated Label Signed-off-by: Francisco H --------- Signed-off-by: Francisco H Co-authored-by: Nick Fox <6226732+nrfox@users.noreply.github.com> --- docs/README.md | 2 +- tests/e2e/multicluster/common.go | 56 ++- .../multicluster_externalcontrolplane_test.go | 465 ++++++++++++++++++ .../multicluster_multiprimary_test.go | 57 +-- .../multicluster_primaryremote_test.go | 52 +- .../multicluster/multicluster_suite_test.go | 44 +- tests/e2e/util/common/e2e_utils.go | 8 +- tests/e2e/util/istioctl/istioctl.go | 7 +- 8 files changed, 598 insertions(+), 93 deletions(-) create mode 100644 tests/e2e/multicluster/multicluster_externalcontrolplane_test.go diff --git a/docs/README.md b/docs/README.md index b6a31086ef..1edf9392a2 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1618,7 +1618,7 @@ In this setup there is an external control plane cluster (`cluster1`) and a remo kubectl wait --context "${CTX_CLUSTER1}" --for=condition=Ready istios/external-istiod --timeout=3m ``` -8. Create the Istio resources to route traffic from the ingress gateway to the external control plane. +8. Create the `Gateway` and `VirtualService` resources to route traffic from the ingress gateway to the external control plane. ```sh kubectl apply --context "${CTX_CLUSTER1}" -f - < version: Version: "1.2.3" +// Any errors will fail the test. +func genTemplate(manifestTmpl string, values any) string { + tmpl, err := template.New("manifest-template").Parse(manifestTmpl) + Expect(err).ToNot(HaveOccurred(), + "template is likely either malformed YAML or the values do not match what is expected") + + var b strings.Builder + Expect(tmpl.Execute(&b, values)).To(Succeed()) + return b.String() +} diff --git a/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go b/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go new file mode 100644 index 0000000000..3c28dfd7b0 --- /dev/null +++ b/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go @@ -0,0 +1,465 @@ +//go:build e2e + +// Copyright Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package multicluster + +import ( + "fmt" + "path/filepath" + "time" + + v1 "github.com/istio-ecosystem/sail-operator/api/v1" + "github.com/istio-ecosystem/sail-operator/pkg/istioversion" + "github.com/istio-ecosystem/sail-operator/pkg/kube" + "github.com/istio-ecosystem/sail-operator/pkg/test/project" + . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" + "github.com/istio-ecosystem/sail-operator/pkg/version" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" + . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/helm" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + admissionregistrationv1 "k8s.io/api/admissionregistration/v1" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +const ( + externalIstioName = "external-istiod" +) + +var _ = Describe("Multicluster deployment models", Label("multicluster-external"), Ordered, func() { + SetDefaultEventuallyTimeout(180 * time.Second) + SetDefaultEventuallyPollingInterval(time.Second) + debugInfoLogged := false + + BeforeAll(func(ctx SpecContext) { + if !skipDeploy { + Expect(k1.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created on Cluster #1") + Expect(k2.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created on Cluster #2") + + Expect(helm.Install("sail-operator", filepath.Join(project.RootDir, "chart"), "--namespace "+namespace, "--set=image="+image, "--kubeconfig "+kubeconfig)). + To(Succeed(), "Operator failed to be deployed in Cluster #1") + + Expect(helm.Install("sail-operator", filepath.Join(project.RootDir, "chart"), "--namespace "+namespace, "--set=image="+image, "--kubeconfig "+kubeconfig2)). + To(Succeed(), "Operator failed to be deployed in Cluster #2") + + Eventually(common.GetObject). + WithArguments(ctx, clPrimary, kube.Key(deploymentName, namespace), &appsv1.Deployment{}). + Should(HaveCondition(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error getting Istio CRD") + Success("Operator is deployed in the Cluster #1 namespace and Running") + + Eventually(common.GetObject). + WithArguments(ctx, clRemote, kube.Key(deploymentName, namespace), &appsv1.Deployment{}). + Should(HaveCondition(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error getting Istio CRD") + Success("Operator is deployed in the Cluster #2 namespace and Running") + } + }) + + Describe("External Control Plane Multi-Network configuration", func() { + // Test the External Control Plane Multi-Network configuration for each supported Istio version + for _, v := range istioversion.List { + // The configuration is only supported in Istio 1.24+. + if version.Constraint("<1.24").Check(v.Version) { + Log(fmt.Sprintf("Skipping test, because Istio version %s does not support External Control Plane Multi-Network configuration", v.Version)) + continue + } + + Context(fmt.Sprintf("Istio version %s", v.Version), func() { + When("default Istio is created in Cluster #1 to handle ingress to External Control Plane", func() { + BeforeAll(func(ctx SpecContext) { + Expect(k1.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be created") + + multiclusterYAML := ` +apiVersion: sailoperator.io/v1 +kind: Istio +metadata: + name: {{ .Name }} +spec: + version: {{ .Version }} + namespace: {{ .Namespace }} + values: + global: + network: {{ .Network }}` + multiclusterYAML = genTemplate(multiclusterYAML, map[string]any{ + "Name": istioName, + "Namespace": controlPlaneNamespace, + "Network": "network1", + "Version": v.Name, + }) + Log("Istio CR Cluster #1: ", multiclusterYAML) + Expect(k1.CreateFromString(multiclusterYAML)).To(Succeed(), "Istio Resource creation failed on Cluster #1") + }) + + It("updates the default Istio CR status to Ready", func(ctx SpecContext) { + Eventually(common.GetObject). + WithArguments(ctx, clPrimary, kube.Key(istioName), &v1.Istio{}). + Should(HaveCondition(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready on Cluster #1; unexpected Condition") + Success("Istio CR is Ready on Cluster #1") + }) + + It("deploys istiod", func(ctx SpecContext) { + Eventually(common.GetObject). + WithArguments(ctx, clPrimary, kube.Key("istiod", controlPlaneNamespace), &appsv1.Deployment{}). + Should(HaveCondition(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Istiod is not Available on Cluster #1; unexpected Condition") + Expect(common.GetVersionFromIstiod()).To(Equal(v.Version), "Unexpected istiod version") + Success("Istiod is deployed in the namespace and Running on Exernal Cluster") + }) + }) + + When("Gateway is created in Cluster #1", func() { + BeforeAll(func(ctx SpecContext) { + Expect(k1.WithNamespace(controlPlaneNamespace).Apply(controlPlaneGatewayYAML)).To(Succeed(), "Gateway creation failed on Cluster #1") + }) + + It("updates Gateway status to Available", func(ctx SpecContext) { + Eventually(common.GetObject). + WithArguments(ctx, clPrimary, kube.Key("istio-ingressgateway", controlPlaneNamespace), &appsv1.Deployment{}). + Should(HaveCondition(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Gateway is not Ready on Cluster #1; unexpected Condition") + + Success("Gateway is created and available in both clusters") + }) + }) + + When("Istio external is installed in Cluster #2", func() { + BeforeAll(func(ctx SpecContext) { + Expect(k2.CreateNamespace(externalControlPlaneNamespace)).To(Succeed(), "Namespace failed to be created") + Expect(clRemote.Get(ctx, client.ObjectKey{Name: externalControlPlaneNamespace}, &corev1.Namespace{})).To(Succeed()) + + remotePilotAddress := common.GetSVCLoadBalancerAddress(ctx, clPrimary, controlPlaneNamespace, "istio-ingressgateway") + remoteIstioYAML := ` +apiVersion: sailoperator.io/v1 +kind: Istio +metadata: + name: {{ .Name }} +spec: + version: {{ .Version }} + namespace: {{ .Namespace }} + profile: remote + values: + defaultRevision: {{ .Name }} + global: + istioNamespace: {{ .Namespace }} + remotePilotAddress: {{ .RemotePilotAddress }} + configCluster: true + pilot: + configMap: true + istiodRemote: + injectionPath: /inject/cluster/cluster2/net/network1 +` + remoteIstioYAML = genTemplate(remoteIstioYAML, map[string]any{ + "Name": externalIstioName, + "Namespace": externalControlPlaneNamespace, + "RemotePilotAddress": remotePilotAddress, + "Version": v.Name, + }) + Log("Istio external-istiod CR: ", remoteIstioYAML) + By("Creating Istio external-istiod CR on Cluster #2") + Expect(k2.CreateFromString(remoteIstioYAML)).To(Succeed(), "Istio external-istiod Resource creation failed on Cluster #2") + }) + + // This is needed for istioctl create-remote-secret in a later test but we can't check + // the Istio external-istiod status for Ready because the webhook readiness check will fail + // since the External Control Plane cluster doesn't exist yet. + It("has a service account for the remote profile", func(ctx SpecContext) { + Eventually(func() error { + _, err := common.GetObject(ctx, clRemote, kube.Key("istiod-"+externalIstioName, externalControlPlaneNamespace), &corev1.ServiceAccount{}) + return err + }).ShouldNot(HaveOccurred(), "Service Account is not created on Cluster #2") + }) + }) + + When("a remote secret is installed on the External Control Plane cluster", func() { + BeforeAll(func(ctx SpecContext) { + Expect(k1.CreateNamespace(externalControlPlaneNamespace)).To(Succeed(), "Namespace failed to be created") + _, err := common.GetObject(ctx, clPrimary, types.NamespacedName{Name: externalControlPlaneNamespace}, &corev1.Namespace{}) + Expect(err).NotTo(HaveOccurred()) + + externalSVCAccountYAML := ` +apiVersion: v1 +kind: ServiceAccount +metadata: + name: istiod-service-account +` + k1.WithNamespace(externalControlPlaneNamespace).CreateFromString(externalSVCAccountYAML) + + internalIPCluster2, err := k2.GetInternalIP("node-role.kubernetes.io/control-plane") + Expect(internalIPCluster2).NotTo(BeEmpty(), "Internal IP is empty for the Cluster #2") + Expect(err).NotTo(HaveOccurred()) + + secret, err := istioctl.CreateRemoteSecret( + kubeconfig2, + "cluster2", + internalIPCluster2, + "--type=config", + "--namespace="+externalControlPlaneNamespace, + "--service-account=istiod-"+externalIstioName, + "--create-service-account=false", + ) + Expect(err).NotTo(HaveOccurred()) + Expect(k1.ApplyString(secret)).To(Succeed(), "Remote secret creation failed on Cluster #1") + }) + + It("has a remote secret in the External Control Plane namespace", func(ctx SpecContext) { + secret, err := common.GetObject(ctx, clPrimary, kube.Key("istio-kubeconfig", externalControlPlaneNamespace), &corev1.Secret{}) + Expect(err).NotTo(HaveOccurred()) + Expect(secret).NotTo(BeNil(), "Secret is not created on the Cluster #1") + + Success("Remote secrets is created in the External Control Plane namespace") + }) + }) + + When("the External Control Plane Istio is installed on the Cluster #1", func() { + BeforeAll(func(ctx SpecContext) { + externalControlPlaneYAML := ` +apiVersion: sailoperator.io/v1 +kind: Istio +metadata: + name: {{ .Name }} +spec: + version: {{ .Version }} + namespace: {{ .Namespace }} + profile: empty + values: + meshConfig: + rootNamespace: {{ .Namespace }} + defaultConfig: + discoveryAddress: {{ .ExternalIstiodAddr }}:15012 + pilot: + enabled: true + volumes: + - name: config-volume + configMap: + name: istio-{{ .Name }} + - name: inject-volume + configMap: + name: istio-sidecar-injector-{{ .Name }} + volumeMounts: + - name: config-volume + mountPath: /etc/istio/config + - name: inject-volume + mountPath: /var/lib/istio/inject + env: + INJECTION_WEBHOOK_CONFIG_NAME: "istio-sidecar-injector-{{ .Name }}-{{ .Namespace }}" + VALIDATION_WEBHOOK_CONFIG_NAME: "istio-validator-{{ .Name }}-{{ .Namespace }}" + EXTERNAL_ISTIOD: "true" + LOCAL_CLUSTER_SECRET_WATCHER: "true" + CLUSTER_ID: cluster2 + SHARED_MESH_CONFIG: istio + global: + caAddress: {{ .ExternalIstiodAddr }}:15012 + istioNamespace: {{ .Namespace }} + operatorManageWebhooks: true + configValidation: false + meshID: mesh1 + multiCluster: + clusterName: cluster2 + network: network1 +` + externalIstiodAddr := common.GetSVCLoadBalancerAddress(ctx, clPrimary, controlPlaneNamespace, "istio-ingressgateway") + externalControlPlaneYAML = genTemplate(externalControlPlaneYAML, map[string]any{ + "ExternalIstiodAddr": externalIstiodAddr, + "Namespace": externalControlPlaneNamespace, + "Name": externalIstioName, + "Version": v.Name, + }) + Log("Istio CR Cluster #1: ", externalControlPlaneYAML) + Expect(k1.CreateFromString(externalControlPlaneYAML)).To(Succeed(), "Istio Resource creation failed on Cluster #1") + }) + + It("updates both Istio CR status to Ready", func(ctx SpecContext) { + Eventually(common.GetObject). + WithArguments(ctx, clPrimary, kube.Key("external-istiod"), &v1.Istio{}). + Should(HaveCondition(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready on Cluster #1; unexpected Condition") + Success("Istio CR is Ready on Cluster #1") + }) + }) + + When("Gateway and VirtualService resources are created to route traffic from the ingress gateway to the external contorlplane", func() { + BeforeAll(func(ctx SpecContext) { + routingResourcesYAML := ` +apiVersion: networking.istio.io/v1 +kind: Gateway +metadata: + name: {{ .Name }}-gw + namespace: {{ .Namespace }} +spec: + selector: + istio: ingressgateway + servers: + - port: + number: 15012 + protocol: tls + name: tls-XDS + tls: + mode: PASSTHROUGH + hosts: + - "*" + - port: + number: 15017 + protocol: tls + name: tls-WEBHOOK + tls: + mode: PASSTHROUGH + hosts: + - "*" +--- +apiVersion: networking.istio.io/v1 +kind: VirtualService +metadata: + name: {{ .Name }}-vs + namespace: {{ .Namespace }} +spec: + hosts: + - "*" + gateways: + - {{ .Name }}-gw + tls: + - match: + - port: 15012 + sniHosts: + - "*" + route: + - destination: + host: istiod-{{ .Name }}.{{ .Namespace }}.svc.cluster.local + port: + number: 15012 + - match: + - port: 15017 + sniHosts: + - "*" + route: + - destination: + host: istiod-{{ .Name }}.{{ .Namespace }}.svc.cluster.local + port: + number: 443 +` + routingResourcesYAML = genTemplate(routingResourcesYAML, map[string]any{ + "Name": externalIstioName, + "Namespace": externalControlPlaneNamespace, + }) + Expect(k1.ApplyString(routingResourcesYAML)).To(Succeed()) + }) + + It("updates remote Istio CR status to Ready on Cluster #2", func(ctx SpecContext) { + Eventually(common.GetObject). + WithArguments(ctx, clRemote, kube.Key(externalIstioName), &v1.Istio{}). + Should(HaveCondition(v1.IstioConditionReady, metav1.ConditionTrue), "Istio is not Ready on Remote; unexpected Condition") + Success("Remote Istio CR is Ready on Cluster #2") + }) + }) + + When("sample app deployed in Cluster #2", func() { + BeforeAll(func(ctx SpecContext) { + Expect(k2.CreateNamespace(sampleNamespace)).To(Succeed(), "Namespace failed to be created") + // Label the namespace with the istio revision name + Expect(k2.Label("namespace", sampleNamespace, "istio.io/rev", "external-istiod")).To(Succeed(), "Labeling failed on Cluster #2") + + deploySampleApp(k2, sampleNamespace, v, "v1") + Success("Sample app is deployed in Cluster #2") + }) + + samplePodsCluster2 := &corev1.PodList{} + It("updates the pods status to Ready", func(ctx SpecContext) { + Expect(clRemote.List(ctx, samplePodsCluster2, client.InNamespace(sampleNamespace))).To(Succeed()) + Expect(samplePodsCluster2.Items).ToNot(BeEmpty(), "No pods found in sample namespace") + + for _, pod := range samplePodsCluster2.Items { + Eventually(common.GetObject). + WithArguments(ctx, clRemote, kube.Key(pod.Name, sampleNamespace), &corev1.Pod{}). + Should(HaveCondition(corev1.PodReady, metav1.ConditionTrue), "Pod is not Ready on Cluster #2; unexpected Condition") + } + Success("Sample app is Running") + }) + + It("has istio.io/rev annotation external-istiod", func(ctx SpecContext) { + Expect(clRemote.List(ctx, samplePodsCluster2, client.InNamespace(sampleNamespace))).To(Succeed()) + Expect(samplePodsCluster2.Items).ToNot(BeEmpty(), "No pods found in sample namespace") + + for _, pod := range samplePodsCluster2.Items { + Expect(pod.Annotations).To(HaveKeyWithValue("istio.io/rev", "external-istiod"), "The pod dom't have expected annotation") + } + Success("Sample pods has expected annotation") + }) + + It("can access the sample app from the local service", func(ctx SpecContext) { + verifyResponsesAreReceivedFromExpectedVersions(k2, "v1") + Success("Sample app is accessible from hello service in Cluster #2") + }) + }) + + When("istio CR is deleted in both clusters", func() { + BeforeAll(func() { + // Delete the Istio and remote Istio CRs in both clusters + Expect(k1.Delete("istio", istioName)).To(Succeed(), "Istio CR failed to be deleted") + Expect(k1.Delete("istio", externalIstioName)).To(Succeed(), "Istio CR failed to be deleted") + Expect(k2.Delete("istio", externalIstioName)).To(Succeed(), "remote Istio CR failed to be deleted") + Success("Istio resources are deleted in both clusters") + }) + + It("removes istiod pod", func(ctx SpecContext) { + // Check istiod pod is deleted in both clusters + Eventually(clPrimary.Get).WithArguments(ctx, kube.Key("istiod", controlPlaneNamespace), &appsv1.Deployment{}). + Should(ReturnNotFoundError(), "Istiod should not exist anymore on Cluster #1") + }) + + It("removes mutating webhook from Remote cluster", func(ctx SpecContext) { + Eventually(clRemote.Get).WithArguments(ctx, kube.Key("istiod-"+externalIstioName), &admissionregistrationv1.MutatingWebhookConfiguration{}). + Should(ReturnNotFoundError(), "Remote webhook should not exist anymore on Cluster #2") + }) + }) + + AfterAll(func(ctx SpecContext) { + if CurrentSpecReport().Failed() { + common.LogDebugInfo(common.MultiCluster, k1, k2) + debugInfoLogged = true + } + + // Delete namespaces to ensure clean up for new tests iteration + Expect(k1.DeleteNamespaceNoWait(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #1") + Expect(k1.DeleteNamespaceNoWait(externalControlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #1") + Expect(k2.DeleteNamespaceNoWait(externalControlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #2") + Expect(k2.DeleteNamespaceNoWait(sampleNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #2") + + Expect(k1.WaitNamespaceDeleted(controlPlaneNamespace)).To(Succeed()) + Expect(k1.WaitNamespaceDeleted(externalControlPlaneNamespace)).To(Succeed()) + Expect(k2.WaitNamespaceDeleted(externalControlPlaneNamespace)).To(Succeed()) + Success("ControlPlane Namespaces are empty") + + Expect(k2.WaitNamespaceDeleted(sampleNamespace)).To(Succeed()) + Success("Sample app is deleted in Cluster #2") + }) + }) + } + }) + + AfterAll(func(ctx SpecContext) { + if CurrentSpecReport().Failed() && !debugInfoLogged { + common.LogDebugInfo(common.MultiCluster, k1, k2) + debugInfoLogged = true + } + + // Delete the Sail Operator from both clusters + Expect(k1.DeleteNamespaceNoWait(namespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #1") + Expect(k2.DeleteNamespaceNoWait(namespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #2") + Expect(k1.WaitNamespaceDeleted(namespace)).To(Succeed()) + Expect(k2.WaitNamespaceDeleted(namespace)).To(Succeed()) + }) +}) diff --git a/tests/e2e/multicluster/multicluster_multiprimary_test.go b/tests/e2e/multicluster/multicluster_multiprimary_test.go index 80ebd5055e..c571f876bb 100644 --- a/tests/e2e/multicluster/multicluster_multiprimary_test.go +++ b/tests/e2e/multicluster/multicluster_multiprimary_test.go @@ -19,15 +19,18 @@ package multicluster import ( "context" "fmt" + "path/filepath" "time" v1 "github.com/istio-ecosystem/sail-operator/api/v1" "github.com/istio-ecosystem/sail-operator/pkg/istioversion" "github.com/istio-ecosystem/sail-operator/pkg/kube" + "github.com/istio-ecosystem/sail-operator/pkg/test/project" . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/certs" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/helm" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -37,7 +40,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -var _ = Describe("Multicluster deployment models", Ordered, func() { +var _ = Describe("Multicluster deployment models", Label("multicluster-multiprimary"), Ordered, func() { SetDefaultEventuallyTimeout(180 * time.Second) SetDefaultEventuallyPollingInterval(time.Second) debugInfoLogged := false @@ -48,10 +51,10 @@ var _ = Describe("Multicluster deployment models", Ordered, func() { Expect(k1.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created on Cluster #1") Expect(k2.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created on Cluster #2") - Expect(common.InstallOperatorViaHelm("--kubeconfig", kubeconfig)). + Expect(helm.Install("sail-operator", filepath.Join(project.RootDir, "chart"), "--namespace "+namespace, "--set=image="+image, "--kubeconfig "+kubeconfig)). To(Succeed(), "Operator failed to be deployed in Cluster #1") - Expect(common.InstallOperatorViaHelm("--kubeconfig "+kubeconfig2)). + Expect(helm.Install("sail-operator", filepath.Join(project.RootDir, "chart"), "--namespace "+namespace, "--set=image="+image, "--kubeconfig "+kubeconfig2)). To(Succeed(), "Operator failed to be deployed in Cluster #2") Eventually(common.GetObject). @@ -198,59 +201,51 @@ spec: When("sample apps are deployed in both clusters", func() { BeforeAll(func(ctx SpecContext) { - // Create the namespace - Expect(k1.CreateNamespace("sample")).To(Succeed(), "Namespace failed to be created") - Expect(k2.CreateNamespace("sample")).To(Succeed(), "Namespace failed to be created") + // Create namespace + Expect(k1.CreateNamespace(sampleNamespace)).To(Succeed(), "Namespace failed to be created on Cluster #1") + Expect(k2.CreateNamespace(sampleNamespace)).To(Succeed(), "Namespace failed to be created on Cluster #2") - // Label the sample namespace - Expect(k1.Patch("namespace", "sample", "merge", `{"metadata":{"labels":{"istio-injection":"enabled"}}}`)). + // Label the namespace + Expect(k1.Patch("namespace", sampleNamespace, "merge", `{"metadata":{"labels":{"istio-injection":"enabled"}}}`)). To(Succeed(), "Error patching sample namespace") - Expect(k2.Patch("namespace", "sample", "merge", `{"metadata":{"labels":{"istio-injection":"enabled"}}}`)). + Expect(k2.Patch("namespace", sampleNamespace, "merge", `{"metadata":{"labels":{"istio-injection":"enabled"}}}`)). To(Succeed(), "Error patching sample namespace") // Deploy the sample app in both clusters - helloWorldURL := common.GetSampleYAML(version, "helloworld") - sleepURL := common.GetSampleYAML(version, "sleep") - - // On Cluster 0, create a service for the helloworld app v1 - Expect(k1.WithNamespace("sample").ApplyWithLabels(helloWorldURL, "service=helloworld")).To(Succeed(), "Failed to deploy helloworld service") - Expect(k1.WithNamespace("sample").ApplyWithLabels(helloWorldURL, "version=v1")).To(Succeed(), "Failed to deploy helloworld service") - Expect(k1.WithNamespace("sample").Apply(sleepURL)).To(Succeed(), "Failed to deploy sleep service") - - // On Cluster 1, create a service for the helloworld app v2 - Expect(k2.WithNamespace("sample").ApplyWithLabels(helloWorldURL, "service=helloworld")).To(Succeed(), "Failed to deploy helloworld service") - Expect(k2.WithNamespace("sample").ApplyWithLabels(helloWorldURL, "version=v2")).To(Succeed(), "Failed to deploy helloworld service") - Expect(k2.WithNamespace("sample").Apply(sleepURL)).To(Succeed(), "Failed to deploy sleep service") + deploySampleAppToClusters(sampleNamespace, version, []ClusterDeployment{ + {Kubectl: k1, AppVersion: "v1"}, + {Kubectl: k2, AppVersion: "v2"}, + }) Success("Sample app is deployed in both clusters") }) It("updates the pods status to Ready", func(ctx SpecContext) { samplePodsCluster1 := &corev1.PodList{} - Expect(clPrimary.List(ctx, samplePodsCluster1, client.InNamespace("sample"))).To(Succeed()) + Expect(clPrimary.List(ctx, samplePodsCluster1, client.InNamespace(sampleNamespace))).To(Succeed()) Expect(samplePodsCluster1.Items).ToNot(BeEmpty(), "No pods found in sample namespace") for _, pod := range samplePodsCluster1.Items { Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key(pod.Name, "sample"), &corev1.Pod{}). + WithArguments(ctx, clPrimary, kube.Key(pod.Name, sampleNamespace), &corev1.Pod{}). Should(HaveCondition(corev1.PodReady, metav1.ConditionTrue), "Pod is not Ready on Cluster #1; unexpected Condition") } samplePodsCluster2 := &corev1.PodList{} - Expect(clRemote.List(ctx, samplePodsCluster2, client.InNamespace("sample"))).To(Succeed()) + Expect(clRemote.List(ctx, samplePodsCluster2, client.InNamespace(sampleNamespace))).To(Succeed()) Expect(samplePodsCluster2.Items).ToNot(BeEmpty(), "No pods found in sample namespace") for _, pod := range samplePodsCluster2.Items { Eventually(common.GetObject). - WithArguments(ctx, clRemote, kube.Key(pod.Name, "sample"), &corev1.Pod{}). + WithArguments(ctx, clRemote, kube.Key(pod.Name, sampleNamespace), &corev1.Pod{}). Should(HaveCondition(corev1.PodReady, metav1.ConditionTrue), "Pod is not Ready on Cluster #2; unexpected Condition") } Success("Sample app is created in both clusters and Running") }) It("can access the sample app from both clusters", func(ctx SpecContext) { - verifyResponsesAreReceivedFromBothClusters(k1, "Cluster #1") - verifyResponsesAreReceivedFromBothClusters(k2, "Cluster #2") + verifyResponsesAreReceivedFromExpectedVersions(k1) + verifyResponsesAreReceivedFromExpectedVersions(k2) Success("Sample app is accessible from both clusters") }) }) @@ -281,15 +276,15 @@ spec: // Delete namespaces to ensure clean up for new tests iteration Expect(k1.DeleteNamespaceNoWait(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #1") Expect(k2.DeleteNamespaceNoWait(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #2") - Expect(k1.DeleteNamespaceNoWait("sample")).To(Succeed(), "Namespace failed to be deleted on Cluster #1") - Expect(k2.DeleteNamespaceNoWait("sample")).To(Succeed(), "Namespace failed to be deleted on Cluster #2") + Expect(k1.DeleteNamespaceNoWait(sampleNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #1") + Expect(k2.DeleteNamespaceNoWait(sampleNamespace)).To(Succeed(), "Namespace failed to be deleted on Cluster #2") Expect(k1.WaitNamespaceDeleted(controlPlaneNamespace)).To(Succeed()) Expect(k2.WaitNamespaceDeleted(controlPlaneNamespace)).To(Succeed()) Success("ControlPlane Namespaces are empty") - Expect(k1.WaitNamespaceDeleted("sample")).To(Succeed()) - Expect(k2.WaitNamespaceDeleted("sample")).To(Succeed()) + Expect(k1.WaitNamespaceDeleted(sampleNamespace)).To(Succeed()) + Expect(k2.WaitNamespaceDeleted(sampleNamespace)).To(Succeed()) Success("Sample app is deleted in both clusters") }) }) diff --git a/tests/e2e/multicluster/multicluster_primaryremote_test.go b/tests/e2e/multicluster/multicluster_primaryremote_test.go index f009edf608..f55b0aa216 100644 --- a/tests/e2e/multicluster/multicluster_primaryremote_test.go +++ b/tests/e2e/multicluster/multicluster_primaryremote_test.go @@ -38,7 +38,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -var _ = Describe("Multicluster deployment models", Ordered, func() { +var _ = Describe("Multicluster deployment models", Label("multicluster-primaryremote"), Ordered, func() { SetDefaultEventuallyTimeout(180 * time.Second) SetDefaultEventuallyPollingInterval(time.Second) debugInfoLogged := false @@ -172,7 +172,7 @@ spec: global: remotePilotAddress: %s` - remotePilotAddress, err := common.GetSVCLoadBalancerAddress(ctx, clPrimary, controlPlaneNamespace, "istio-eastwestgateway") + remotePilotAddress := common.GetSVCLoadBalancerAddress(ctx, clPrimary, controlPlaneNamespace, "istio-eastwestgateway") Expect(remotePilotAddress).NotTo(BeEmpty(), "Remote Pilot Address is empty") Expect(err).NotTo(HaveOccurred(), "Error getting Remote Pilot Address") istioYAML := fmt.Sprintf(istioYAMLTemplate, v.Name, remotePilotAddress) @@ -245,59 +245,51 @@ spec: When("sample apps are deployed in both clusters", func() { BeforeAll(func(ctx SpecContext) { - // Create the namespace - Expect(k1.CreateNamespace("sample")).To(Succeed(), "Namespace failed to be created") - Expect(k2.CreateNamespace("sample")).To(Succeed(), "Namespace failed to be created") + // Create namespace + Expect(k1.CreateNamespace(sampleNamespace)).To(Succeed(), "Namespace failed to be created on Cluster #1") + Expect(k2.CreateNamespace(sampleNamespace)).To(Succeed(), "Namespace failed to be created on Cluster #2") - // Label the sample namespace - Expect(k1.Patch("namespace", "sample", "merge", `{"metadata":{"labels":{"istio-injection":"enabled"}}}`)). + // Label the namespace + Expect(k1.Patch("namespace", sampleNamespace, "merge", `{"metadata":{"labels":{"istio-injection":"enabled"}}}`)). To(Succeed(), "Error patching sample namespace") - Expect(k2.Patch("namespace", "sample", "merge", `{"metadata":{"labels":{"istio-injection":"enabled"}}}`)). + Expect(k2.Patch("namespace", sampleNamespace, "merge", `{"metadata":{"labels":{"istio-injection":"enabled"}}}`)). To(Succeed(), "Error patching sample namespace") // Deploy the sample app in both clusters - helloWorldURL := common.GetSampleYAML(v, "helloworld") - sleepURL := common.GetSampleYAML(v, "sleep") - - // On Cluster 0, create a service for the helloworld app v1 - Expect(k1.WithNamespace("sample").ApplyWithLabels(helloWorldURL, "service=helloworld")).To(Succeed(), "Failed to deploy helloworld service") - Expect(k1.WithNamespace("sample").ApplyWithLabels(helloWorldURL, "version=v1")).To(Succeed(), "Failed to deploy helloworld service") - Expect(k1.WithNamespace("sample").Apply(sleepURL)).To(Succeed(), "Failed to deploy sleep service") - - // On Cluster 1, create a service for the helloworld app v2 - Expect(k2.WithNamespace("sample").ApplyWithLabels(helloWorldURL, "service=helloworld")).To(Succeed(), "Failed to deploy helloworld service") - Expect(k2.WithNamespace("sample").ApplyWithLabels(helloWorldURL, "version=v2")).To(Succeed(), "Failed to deploy helloworld service") - Expect(k2.WithNamespace("sample").Apply(sleepURL)).To(Succeed(), "Failed to deploy sleep service") + deploySampleAppToClusters(sampleNamespace, v, []ClusterDeployment{ + {Kubectl: k1, AppVersion: "v1"}, + {Kubectl: k2, AppVersion: "v2"}, + }) Success("Sample app is deployed in both clusters") }) It("updates the pods status to Ready", func(ctx SpecContext) { samplePodsPrimary := &corev1.PodList{} - Expect(clPrimary.List(ctx, samplePodsPrimary, client.InNamespace("sample"))).To(Succeed()) + Expect(clPrimary.List(ctx, samplePodsPrimary, client.InNamespace(sampleNamespace))).To(Succeed()) Expect(samplePodsPrimary.Items).ToNot(BeEmpty(), "No pods found in sample namespace") for _, pod := range samplePodsPrimary.Items { Eventually(common.GetObject). - WithArguments(ctx, clPrimary, kube.Key(pod.Name, "sample"), &corev1.Pod{}). + WithArguments(ctx, clPrimary, kube.Key(pod.Name, sampleNamespace), &corev1.Pod{}). Should(HaveCondition(corev1.PodReady, metav1.ConditionTrue), "Pod is not Ready on Primary; unexpected Condition") } samplePodsRemote := &corev1.PodList{} - Expect(clRemote.List(ctx, samplePodsRemote, client.InNamespace("sample"))).To(Succeed()) + Expect(clRemote.List(ctx, samplePodsRemote, client.InNamespace(sampleNamespace))).To(Succeed()) Expect(samplePodsRemote.Items).ToNot(BeEmpty(), "No pods found in sample namespace") for _, pod := range samplePodsRemote.Items { Eventually(common.GetObject). - WithArguments(ctx, clRemote, kube.Key(pod.Name, "sample"), &corev1.Pod{}). + WithArguments(ctx, clRemote, kube.Key(pod.Name, sampleNamespace), &corev1.Pod{}). Should(HaveCondition(corev1.PodReady, metav1.ConditionTrue), "Pod is not Ready on Remote; unexpected Condition") } Success("Sample app is created in both clusters and Running") }) It("can access the sample app from both clusters", func(ctx SpecContext) { - verifyResponsesAreReceivedFromBothClusters(k1, "Cluster #1") - verifyResponsesAreReceivedFromBothClusters(k2, "Cluster #2") + verifyResponsesAreReceivedFromExpectedVersions(k1) + verifyResponsesAreReceivedFromExpectedVersions(k2) Success("Sample app is accessible from both clusters") }) }) @@ -325,15 +317,15 @@ spec: // Delete namespaces to ensure clean up for new tests iteration Expect(k1.DeleteNamespaceNoWait(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Primary Cluster") Expect(k2.DeleteNamespaceNoWait(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be deleted on Remote Cluster") - Expect(k1.DeleteNamespaceNoWait("sample")).To(Succeed(), "Namespace failed to be deleted on Primary Cluster") - Expect(k2.DeleteNamespaceNoWait("sample")).To(Succeed(), "Namespace failed to be deleted on Remote Cluster") + Expect(k1.DeleteNamespaceNoWait(sampleNamespace)).To(Succeed(), "Namespace failed to be deleted on Primary Cluster") + Expect(k2.DeleteNamespaceNoWait(sampleNamespace)).To(Succeed(), "Namespace failed to be deleted on Remote Cluster") Expect(k1.WaitNamespaceDeleted(controlPlaneNamespace)).To(Succeed()) Expect(k2.WaitNamespaceDeleted(controlPlaneNamespace)).To(Succeed()) Success("ControlPlane Namespaces were deleted") - Expect(k1.WaitNamespaceDeleted("sample")).To(Succeed()) - Expect(k2.WaitNamespaceDeleted("sample")).To(Succeed()) + Expect(k1.WaitNamespaceDeleted(sampleNamespace)).To(Succeed()) + Expect(k2.WaitNamespaceDeleted(sampleNamespace)).To(Succeed()) Success("Sample app is deleted in both clusters") // Delete the resources created by istioctl create-remote-secret diff --git a/tests/e2e/multicluster/multicluster_suite_test.go b/tests/e2e/multicluster/multicluster_suite_test.go index 5e03a38821..d87a2dcc46 100644 --- a/tests/e2e/multicluster/multicluster_suite_test.go +++ b/tests/e2e/multicluster/multicluster_suite_test.go @@ -25,7 +25,6 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/env" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/certs" k8sclient "github.com/istio-ecosystem/sail-operator/tests/e2e/util/client" - "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -33,24 +32,28 @@ import ( ) var ( - clPrimary client.Client - clRemote client.Client - err error - ocp = env.GetBool("OCP", false) - namespace = common.OperatorNamespace - deploymentName = env.Get("DEPLOYMENT_NAME", "sail-operator") - controlPlaneNamespace = env.Get("CONTROL_PLANE_NS", "istio-system") - istioName = env.Get("ISTIO_NAME", "default") - skipDeploy = env.GetBool("SKIP_DEPLOY", false) - multicluster = env.GetBool("MULTICLUSTER", false) - kubeconfig = env.Get("KUBECONFIG", "") - kubeconfig2 = env.Get("KUBECONFIG2", "") - artifacts = env.Get("ARTIFACTS", "/tmp/artifacts") + clPrimary client.Client + clRemote client.Client + err error + ocp = env.GetBool("OCP", false) + namespace = env.Get("NAMESPACE", "sail-operator") + deploymentName = env.Get("DEPLOYMENT_NAME", "sail-operator") + controlPlaneNamespace = env.Get("CONTROL_PLANE_NS", "istio-system") + externalControlPlaneNamespace = env.Get("EXTERNAL_CONTROL_PLANE_NS", "external-istiod") + istioName = env.Get("ISTIO_NAME", "default") + image = env.Get("IMAGE", "quay.io/maistra-dev/sail-operator:latest") + skipDeploy = env.GetBool("SKIP_DEPLOY", false) + multicluster = env.GetBool("MULTICLUSTER", false) + kubeconfig = env.Get("KUBECONFIG", "") + kubeconfig2 = env.Get("KUBECONFIG2", "") + artifacts = env.Get("ARTIFACTS", "/tmp/artifacts") + sampleNamespace = env.Get("SAMPLE_NAMESPACE", "sample") - eastGatewayYAML string - westGatewayYAML string - exposeServiceYAML string - exposeIstiodYAML string + controlPlaneGatewayYAML string + eastGatewayYAML string + westGatewayYAML string + exposeServiceYAML string + exposeIstiodYAML string k1 kubectl.Kubectl k2 kubectl.Kubectl @@ -93,12 +96,13 @@ func setup(t *testing.T) { // Set base path baseRepoDir := filepath.Join(workDir, "../../..") + controlPlaneGatewayYAML = fmt.Sprintf("%s/docs/multicluster/controlplane-gateway.yaml", baseRepoDir) eastGatewayYAML = fmt.Sprintf("%s/docs/multicluster/east-west-gateway-net1.yaml", baseRepoDir) westGatewayYAML = fmt.Sprintf("%s/docs/multicluster/east-west-gateway-net2.yaml", baseRepoDir) exposeServiceYAML = fmt.Sprintf("%s/docs/multicluster/expose-services.yaml", baseRepoDir) exposeIstiodYAML = fmt.Sprintf("%s/docs/multicluster/expose-istiod.yaml", baseRepoDir) // Initialize kubectl utilities, one for each cluster - k1 = kubectl.New().WithClusterName("primary").WithKubeconfig(kubeconfig) - k2 = kubectl.New().WithClusterName("remote").WithKubeconfig(kubeconfig2) + k1 = kubectl.New().WithKubeconfig(kubeconfig) + k2 = kubectl.New().WithKubeconfig(kubeconfig2) } diff --git a/tests/e2e/util/common/e2e_utils.go b/tests/e2e/util/common/e2e_utils.go index 55d14f5cac..ef3100c870 100644 --- a/tests/e2e/util/common/e2e_utils.go +++ b/tests/e2e/util/common/e2e_utils.go @@ -104,12 +104,10 @@ func GetPodNameByLabel(ctx context.Context, cl client.Client, ns, labelKey, labe } // GetSVCLoadBalancerAddress returns the address of the service with the given name -func GetSVCLoadBalancerAddress(ctx context.Context, cl client.Client, ns, svcName string) (string, error) { +func GetSVCLoadBalancerAddress(ctx context.Context, cl client.Client, ns, svcName string) string { svc := &corev1.Service{} err := cl.Get(ctx, client.ObjectKey{Namespace: ns, Name: svcName}, svc) - if err != nil { - return "", err - } + Expect(err).NotTo(HaveOccurred(), fmt.Sprintf("Error getting LoadBalancer Service '%s/%s'", ns, svcName)) // To avoid flakiness, wait for the LoadBalancer to be ready Eventually(func() ([]corev1.LoadBalancerIngress, error) { @@ -117,7 +115,7 @@ func GetSVCLoadBalancerAddress(ctx context.Context, cl client.Client, ns, svcNam return svc.Status.LoadBalancer.Ingress, err }, "1m", "1s").ShouldNot(BeEmpty(), "LoadBalancer should be ready") - return svc.Status.LoadBalancer.Ingress[0].IP, nil + return svc.Status.LoadBalancer.Ingress[0].IP } // CheckNamespaceEmpty checks if the given namespace is empty diff --git a/tests/e2e/util/istioctl/istioctl.go b/tests/e2e/util/istioctl/istioctl.go index c880c819f2..c4b6c60992 100644 --- a/tests/e2e/util/istioctl/istioctl.go +++ b/tests/e2e/util/istioctl/istioctl.go @@ -18,6 +18,7 @@ package istioctl import ( "fmt" + "strings" "github.com/istio-ecosystem/sail-operator/pkg/env" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/shell" @@ -47,8 +48,12 @@ func istioctl(format string, args ...interface{}) string { // - remoteKubeconfig: kubeconfig of the remote cluster // - secretName: name of the secret // - internalIP: internal IP of the remote cluster -func CreateRemoteSecret(remoteKubeconfig string, secretName string, internalIP string) (string, error) { +func CreateRemoteSecret(remoteKubeconfig string, secretName string, internalIP string, additionalFlags ...string) (string, error) { cmd := istioctl("create-remote-secret --kubeconfig %s --name %s --server=https://%s:6443", remoteKubeconfig, secretName, internalIP) + if len(additionalFlags) != 0 { + cmd += (" " + strings.Join(additionalFlags, " ")) + } + yaml, err := shell.ExecuteCommand(cmd) return yaml, err