diff --git a/tests/e2e/ambient/ambient_suite_test.go b/tests/e2e/ambient/ambient_suite_test.go index 393c9971be..824fa18509 100644 --- a/tests/e2e/ambient/ambient_suite_test.go +++ b/tests/e2e/ambient/ambient_suite_test.go @@ -21,6 +21,7 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/env" k8sclient "github.com/istio-ecosystem/sail-operator/tests/e2e/util/client" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -30,10 +31,10 @@ import ( var ( cl client.Client err error - controlPlaneNamespace = env.Get("CONTROL_PLANE_NS", "istio-system") + controlPlaneNamespace = common.ControlPlaneNamespace istioName = env.Get("ISTIO_NAME", "default") - istioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") - ztunnelNamespace = env.Get("ZTUNNEL_NAMESPACE", "ztunnel") + istioCniNamespace = common.IstioCniNamespace + ztunnelNamespace = common.ZtunnelNamespace istioCniName = env.Get("ISTIOCNI_NAME", "default") expectedRegistry = env.Get("EXPECTED_REGISTRY", "^docker\\.io|^gcr\\.io") multicluster = env.GetBool("MULTICLUSTER", false) diff --git a/tests/e2e/controlplane/control_plane_suite_test.go b/tests/e2e/controlplane/control_plane_suite_test.go index a7e73942ef..57ef2eb60d 100644 --- a/tests/e2e/controlplane/control_plane_suite_test.go +++ b/tests/e2e/controlplane/control_plane_suite_test.go @@ -33,9 +33,9 @@ var ( err error namespace = common.OperatorNamespace deploymentName = env.Get("DEPLOYMENT_NAME", "sail-operator") - controlPlaneNamespace = env.Get("CONTROL_PLANE_NS", "istio-system") + controlPlaneNamespace = common.ControlPlaneNamespace istioName = env.Get("ISTIO_NAME", "default") - istioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") + istioCniNamespace = common.IstioCniNamespace istioCniName = env.Get("ISTIOCNI_NAME", "default") expectedRegistry = env.Get("EXPECTED_REGISTRY", "^docker\\.io|^gcr\\.io") sampleNamespace = env.Get("SAMPLE_NAMESPACE", "sample") diff --git a/tests/e2e/dualstack/dualstack_suite_test.go b/tests/e2e/dualstack/dualstack_suite_test.go index f2dd6c1c32..3ce8e00770 100644 --- a/tests/e2e/dualstack/dualstack_suite_test.go +++ b/tests/e2e/dualstack/dualstack_suite_test.go @@ -21,6 +21,7 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/env" k8sclient "github.com/istio-ecosystem/sail-operator/tests/e2e/util/client" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -30,9 +31,9 @@ import ( var ( cl client.Client err error - controlPlaneNamespace = env.Get("CONTROL_PLANE_NS", "istio-system") + controlPlaneNamespace = common.ControlPlaneNamespace istioName = env.Get("ISTIO_NAME", "default") - istioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") + istioCniNamespace = common.IstioCniNamespace istioCniName = env.Get("ISTIOCNI_NAME", "default") expectedRegistry = env.Get("EXPECTED_REGISTRY", "^docker\\.io|^gcr\\.io") multicluster = env.GetBool("MULTICLUSTER", false) diff --git a/tests/e2e/multicluster/common.go b/tests/e2e/multicluster/common.go index 4d792e6b79..501ca7cb0c 100644 --- a/tests/e2e/multicluster/common.go +++ b/tests/e2e/multicluster/common.go @@ -17,13 +17,19 @@ package multicluster import ( + "context" "fmt" "strings" "text/template" "time" + "github.com/istio-ecosystem/sail-operator/pkg/kube" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/certs" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" ) // ClusterDeployment represents a cluster along with its sample app version. @@ -33,16 +39,29 @@ type ClusterDeployment struct { } // deploySampleApp deploys the sample apps (helloworld and sleep) in the given cluster. -func deploySampleApp(k kubectl.Kubectl, ns string, appVersion string) { +func deploySampleApp(k kubectl.Kubectl, ns, appVersion, profile string) { Expect(k.WithNamespace(ns).ApplyKustomize("helloworld", "service=helloworld")).To(Succeed(), "Sample service deploy failed on Cluster") Expect(k.WithNamespace(ns).ApplyKustomize("helloworld", "version="+appVersion)).To(Succeed(), "Sample service deploy failed on Cluster") Expect(k.WithNamespace(ns).ApplyKustomize("sleep")).To(Succeed(), "Sample sleep deploy failed on Cluster") + + // In Ambient mode, services need to be marked as "global" in order to load balance requests among clusters + if profile == "ambient" { + Expect(k.LabelNamespaced("service", ns, "helloworld", "istio.io/global", "true")).To(Succeed(), "Error labeling sample namespace") + } } // deploySampleAppToClusters deploys the sample app to all provided clusters. -func deploySampleAppToClusters(ns string, clusters []ClusterDeployment) { +func deploySampleAppToClusters(ns, profile string, clusters []ClusterDeployment) { for _, cd := range clusters { - deploySampleApp(cd.Kubectl, ns, cd.AppVersion) + k := cd.Kubectl + Expect(k.CreateNamespace(ns)).To(Succeed(), fmt.Sprintf("Namespace failed to be created on Cluster %s", k.ClusterName)) + if profile == "ambient" { + Expect(k.Label("namespace", ns, "istio.io/dataplane-mode", "ambient")).To(Succeed(), "Error labeling sample namespace") + } else { + Expect(k.Label("namespace", ns, "istio-injection", "enabled")).To(Succeed(), "Error labeling sample namespace") + } + + deploySampleApp(k, ns, cd.AppVersion, profile) } } @@ -75,3 +94,63 @@ func genTemplate(manifestTmpl string, values any) string { Expect(tmpl.Execute(&b, values)).To(Succeed()) return b.String() } + +func createIstioNamespaces(k kubectl.Kubectl, network, profile string) { + Expect(k.CreateNamespace(common.ControlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") + Expect(k.CreateNamespace(common.IstioCniNamespace)).To(Succeed(), "Istio CNI namespace failed to be created") + + if profile == "ambient" { + Expect(k.Label("namespace", common.ControlPlaneNamespace, "topology.istio.io/network", network)).To(Succeed(), "Error labeling istio namespace") + Expect(k.CreateNamespace(common.ZtunnelNamespace)).To(Succeed(), "Ztunnel namespace failed to be created") + } +} + +func createIstioResources(k kubectl.Kubectl, version, cluster, network, profile string, values ...string) { + cniSpec := fmt.Sprintf(` +profile: %s`, profile) + common.CreateIstioCNI(k, version, cniSpec) + + if profile == "ambient" { + spec := fmt.Sprintf(` +values: + ztunnel: + multiCluster: + clusterName: %s + network: %s`, cluster, network) + common.CreateZTunnel(k, version, spec) + } + + spec := fmt.Sprintf(` +profile: %s +values: + global: + meshID: mesh1 + multiCluster: + clusterName: %s + network: %s`, profile, cluster, network) + for _, value := range values { + spec += common.Indent(value) + } + + if profile == "ambient" { + spec += fmt.Sprintf(` + pilot: + trustedZtunnelNamespace: %s + env: + AMBIENT_ENABLE_MULTI_NETWORK: "true"`, common.ZtunnelNamespace) + } + + common.CreateIstio(k, version, spec) +} + +func createIntermediateCA(k kubectl.Kubectl, zone, network, artifacts string, cl client.Client) { + Expect(certs.PushIntermediateCA(k, common.ControlPlaneNamespace, zone, network, artifacts, cl)). + To(Succeed(), fmt.Sprintf("Error pushing intermediate CA to %s Cluster", k.ClusterName)) +} + +func awaitSecretCreation(cluster string, cl client.Client) { + Eventually(func() error { + _, err := common.GetObject(context.Background(), cl, kube.Key("cacerts", common.ControlPlaneNamespace), &corev1.Secret{}) + return err + }).ShouldNot(HaveOccurred(), fmt.Sprintf("Secret is not created on %s cluster", cluster)) +} diff --git a/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go b/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go index b33bc3389e..47b8cc0f1b 100644 --- a/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go +++ b/tests/e2e/multicluster/multicluster_externalcontrolplane_test.go @@ -65,8 +65,7 @@ var _ = Describe("Multicluster deployment models", Label("multicluster", "multic When("default Istio is created in Cluster #1 to handle ingress to External Control Plane", func() { BeforeAll(func(ctx SpecContext) { - Expect(k1.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Namespace failed to be created") - Expect(k1.CreateNamespace(istioCniNamespace)).To(Succeed(), "Istio CNI namespace failed to be created") + createIstioNamespaces(k1, "network1", "default") common.CreateIstioCNI(k1, v.Name) common.CreateIstio(k1, v.Name, ` @@ -335,7 +334,7 @@ spec: // Label the namespace with the istio revision name Expect(k2.Label("namespace", sampleNamespace, "istio.io/rev", "external-istiod")).To(Succeed(), "Labeling failed on Cluster #2") - deploySampleApp(k2, sampleNamespace, "v1") + deploySampleApp(k2, sampleNamespace, "v1", "default") Success("Sample app is deployed in Cluster #2") }) diff --git a/tests/e2e/multicluster/multicluster_multiprimary_test.go b/tests/e2e/multicluster/multicluster_multiprimary_test.go index 9365ec2c6b..c63dd4c980 100644 --- a/tests/e2e/multicluster/multicluster_multiprimary_test.go +++ b/tests/e2e/multicluster/multicluster_multiprimary_test.go @@ -17,7 +17,6 @@ package multicluster import ( - "context" "fmt" "time" @@ -25,7 +24,7 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/istioversion" "github.com/istio-ecosystem/sail-operator/pkg/kube" . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" - "github.com/istio-ecosystem/sail-operator/tests/e2e/util/certs" + "github.com/istio-ecosystem/sail-operator/pkg/version" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl" @@ -39,10 +38,25 @@ var _ = Describe("Multicluster deployment models", Label("multicluster", "multic SetDefaultEventuallyTimeout(180 * time.Second) SetDefaultEventuallyPollingInterval(time.Second) + Context("Sidecar", func() { + generateMultiPrimaryTestCases("default") + }) + Context("Ambient", Label("ambient"), func() { + generateMultiPrimaryTestCases("ambient") + }) +}) + +func generateMultiPrimaryTestCases(profile string) { Describe("Multi-Primary Multi-Network configuration", func() { // Test the Multi-Primary Multi-Network configuration for each supported Istio version - for _, version := range istioversion.GetLatestPatchVersions() { - Context(fmt.Sprintf("Istio version %s", version.Version), func() { + for _, v := range istioversion.GetLatestPatchVersions() { + // Ambient multi-cluster is supported only since 1.27 + if profile == "ambient" && version.Constraint("<1.27").Check(v.Version) { + Log(fmt.Sprintf("Skipping test, because Istio version %s does not support Ambient Multi-Cluster configuration", v.Version)) + continue + } + + Context(fmt.Sprintf("Istio version %s", v.Version), func() { clr1 := cleaner.New(clPrimary, "cluster=primary") clr2 := cleaner.New(clRemote, "cluster=remote") @@ -53,38 +67,19 @@ var _ = Describe("Multicluster deployment models", Label("multicluster", "multic When("Istio and IstioCNI resources are created in both clusters", func() { BeforeAll(func(ctx SpecContext) { - Expect(k1.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") - Expect(k2.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") - Expect(k1.CreateNamespace(istioCniNamespace)).To(Succeed(), "Istio CNI namespace failed to be created") - Expect(k2.CreateNamespace(istioCniNamespace)).To(Succeed(), "Istio CNI namespace failed to be created") + createIstioNamespaces(k1, "network1", profile) + createIstioNamespaces(k2, "network2", profile) // Push the intermediate CA to both clusters - Expect(certs.PushIntermediateCA(k1, controlPlaneNamespace, "east", "network1", artifacts, clPrimary)).To(Succeed()) - Expect(certs.PushIntermediateCA(k2, controlPlaneNamespace, "west", "network2", artifacts, clRemote)).To(Succeed()) + createIntermediateCA(k1, "east", "network1", artifacts, clPrimary) + createIntermediateCA(k2, "west", "network2", artifacts, clRemote) // Wait for the secret to be created in both clusters - Eventually(func() error { - _, err := common.GetObject(context.Background(), clPrimary, kube.Key("cacerts", controlPlaneNamespace), &corev1.Secret{}) - return err - }).ShouldNot(HaveOccurred(), "Secret is not created on Cluster #1") - - Eventually(func() error { - _, err := common.GetObject(context.Background(), clRemote, kube.Key("cacerts", controlPlaneNamespace), &corev1.Secret{}) - return err - }).ShouldNot(HaveOccurred(), "Secret is not created on Cluster #1") - - common.CreateIstioCNI(k1, version.Name) - common.CreateIstioCNI(k2, version.Name) - - spec := ` -values: - global: - meshID: mesh1 - multiCluster: - clusterName: %s - network: %s` - common.CreateIstio(k1, version.Name, fmt.Sprintf(spec, "cluster1", "network1")) - common.CreateIstio(k2, version.Name, fmt.Sprintf(spec, "cluster2", "network2")) + awaitSecretCreation(k1.ClusterName, clPrimary) + awaitSecretCreation(k2.ClusterName, clRemote) + + createIstioResources(k1, v.Name, "cluster1", "network1", profile) + createIstioResources(k2, v.Name, "cluster2", "network2", profile) }) It("updates both Istio CR status to Ready", func(ctx SpecContext) { @@ -99,10 +94,10 @@ values: It("deploys istiod", func(ctx SpecContext) { common.AwaitDeployment(ctx, "istiod", k1, clPrimary) - Expect(common.GetVersionFromIstiod()).To(Equal(version.Version), "Unexpected istiod version") + Expect(common.GetVersionFromIstiod()).To(Equal(v.Version), "Unexpected istiod version") common.AwaitDeployment(ctx, "istiod", k2, clRemote) - Expect(common.GetVersionFromIstiod()).To(Equal(version.Version), "Unexpected istiod version") + Expect(common.GetVersionFromIstiod()).To(Equal(v.Version), "Unexpected istiod version") }) It("deploys istio-cni-node", func(ctx SpecContext) { @@ -113,12 +108,17 @@ values: When("Gateway is created in both clusters", func() { BeforeAll(func(ctx SpecContext) { - Expect(k1.WithNamespace(controlPlaneNamespace).Apply(eastGatewayYAML)).To(Succeed(), "Gateway creation failed on Cluster #1") - Expect(k2.WithNamespace(controlPlaneNamespace).Apply(westGatewayYAML)).To(Succeed(), "Gateway creation failed on Cluster #2") - - // Expose the Gateway service in both clusters - Expect(k1.WithNamespace(controlPlaneNamespace).Apply(exposeServiceYAML)).To(Succeed(), "Expose Service creation failed on Cluster #1") - Expect(k2.WithNamespace(controlPlaneNamespace).Apply(exposeServiceYAML)).To(Succeed(), "Expose Service creation failed on Cluster #2") + if profile == "ambient" { + common.CreateAmbientGateway(k1, controlPlaneNamespace, "network1") + common.CreateAmbientGateway(k2, controlPlaneNamespace, "network2") + } else { + Expect(k1.WithNamespace(controlPlaneNamespace).Apply(eastGatewayYAML)).To(Succeed(), "Gateway creation failed on Cluster #1") + Expect(k2.WithNamespace(controlPlaneNamespace).Apply(westGatewayYAML)).To(Succeed(), "Gateway creation failed on Cluster #2") + + // Expose the Gateway service in both clusters + Expect(k1.WithNamespace(controlPlaneNamespace).Apply(exposeServiceYAML)).To(Succeed(), "Expose Service creation failed on Cluster #1") + Expect(k2.WithNamespace(controlPlaneNamespace).Apply(exposeServiceYAML)).To(Succeed(), "Expose Service creation failed on Cluster #2") + } }) It("updates both Gateway status to Available", func(ctx SpecContext) { @@ -164,16 +164,7 @@ values: When("sample apps are deployed in both clusters", func() { BeforeAll(func(ctx SpecContext) { - // Create namespace - Expect(k1.CreateNamespace(sampleNamespace)).To(Succeed(), "Namespace failed to be created on Cluster #1") - Expect(k2.CreateNamespace(sampleNamespace)).To(Succeed(), "Namespace failed to be created on Cluster #2") - - // Label the namespace - Expect(k1.Label("namespace", sampleNamespace, "istio-injection", "enabled")).To(Succeed(), "Error labeling sample namespace") - Expect(k2.Label("namespace", sampleNamespace, "istio-injection", "enabled")).To(Succeed(), "Error labeling sample namespace") - - // Deploy the sample app in both clusters - deploySampleAppToClusters(sampleNamespace, []ClusterDeployment{ + deploySampleAppToClusters(sampleNamespace, profile, []ClusterDeployment{ {Kubectl: k1, AppVersion: "v1"}, {Kubectl: k2, AppVersion: "v2"}, }) @@ -245,4 +236,4 @@ values: }) } }) -}) +} diff --git a/tests/e2e/multicluster/multicluster_primaryremote_test.go b/tests/e2e/multicluster/multicluster_primaryremote_test.go index cb97c784e7..b80d00ad23 100644 --- a/tests/e2e/multicluster/multicluster_primaryremote_test.go +++ b/tests/e2e/multicluster/multicluster_primaryremote_test.go @@ -17,7 +17,6 @@ package multicluster import ( - "context" "fmt" "time" @@ -26,7 +25,6 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/kube" . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" "github.com/istio-ecosystem/sail-operator/pkg/version" - "github.com/istio-ecosystem/sail-operator/tests/e2e/util/certs" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl" @@ -37,6 +35,7 @@ import ( ) var _ = Describe("Multicluster deployment models", Label("multicluster", "multicluster-primaryremote"), Ordered, func() { + profile := "default" SetDefaultEventuallyTimeout(180 * time.Second) SetDefaultEventuallyPollingInterval(time.Second) @@ -60,41 +59,22 @@ var _ = Describe("Multicluster deployment models", Label("multicluster", "multic When("Istio and IstioCNI resources are created in both clusters", func() { BeforeAll(func(ctx SpecContext) { - Expect(k1.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") - Expect(k2.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created") - Expect(k1.CreateNamespace(istioCniNamespace)).To(Succeed(), "Istio CNI Namespace failed to be created") - Expect(k2.CreateNamespace(istioCniNamespace)).To(Succeed(), "Istio CNI Namespace failed to be created") + createIstioNamespaces(k1, "network1", profile) + createIstioNamespaces(k2, "network2", profile) // Push the intermediate CA to both clusters - Expect(certs.PushIntermediateCA(k1, controlPlaneNamespace, "east", "network1", artifacts, clPrimary)). - To(Succeed(), "Error pushing intermediate CA to Primary Cluster") - Expect(certs.PushIntermediateCA(k2, controlPlaneNamespace, "west", "network2", artifacts, clRemote)). - To(Succeed(), "Error pushing intermediate CA to Remote Cluster") + createIntermediateCA(k1, "east", "network1", artifacts, clPrimary) + createIntermediateCA(k2, "west", "network2", artifacts, clRemote) // Wait for the secret to be created in both clusters - Eventually(func() error { - _, err := common.GetObject(context.Background(), clPrimary, kube.Key("cacerts", controlPlaneNamespace), &corev1.Secret{}) - return err - }).ShouldNot(HaveOccurred(), "Secret is not created on Primary Cluster") + awaitSecretCreation(k1.ClusterName, clPrimary) + awaitSecretCreation(k2.ClusterName, clRemote) - Eventually(func() error { - _, err := common.GetObject(context.Background(), clRemote, kube.Key("cacerts", controlPlaneNamespace), &corev1.Secret{}) - return err - }).ShouldNot(HaveOccurred(), "Secret is not created on Primary Cluster") - - common.CreateIstioCNI(k1, v.Name) - - spec := ` -values: - pilot: - env: - EXTERNAL_ISTIOD: "true" - global: - meshID: mesh1 - multiCluster: - clusterName: cluster1 - network: network1` - common.CreateIstio(k1, v.Name, spec) + pilot := ` +pilot: + env: + EXTERNAL_ISTIOD: "true"` + createIstioResources(k1, v.Name, "cluster1", "network1", profile, pilot) }) It("updates Istio CR on Primary cluster status to Ready", func(ctx SpecContext) { @@ -203,16 +183,7 @@ values: When("sample apps are deployed in both clusters", func() { BeforeAll(func(ctx SpecContext) { - // Create namespace - Expect(k1.CreateNamespace(sampleNamespace)).To(Succeed(), "Namespace failed to be created on Cluster #1") - Expect(k2.CreateNamespace(sampleNamespace)).To(Succeed(), "Namespace failed to be created on Cluster #2") - - // Label the namespace - Expect(k1.Label("namespace", sampleNamespace, "istio-injection", "enabled")).To(Succeed(), "Error labeling sample namespace") - Expect(k2.Label("namespace", sampleNamespace, "istio-injection", "enabled")).To(Succeed(), "Error labeling sample namespace") - - // Deploy the sample app in both clusters - deploySampleAppToClusters(sampleNamespace, []ClusterDeployment{ + deploySampleAppToClusters(sampleNamespace, profile, []ClusterDeployment{ {Kubectl: k1, AppVersion: "v1"}, {Kubectl: k2, AppVersion: "v2"}, }) diff --git a/tests/e2e/multicluster/multicluster_suite_test.go b/tests/e2e/multicluster/multicluster_suite_test.go index d754387058..8661703be9 100644 --- a/tests/e2e/multicluster/multicluster_suite_test.go +++ b/tests/e2e/multicluster/multicluster_suite_test.go @@ -37,10 +37,10 @@ var ( clRemote client.Client err error debugInfoLogged bool - controlPlaneNamespace = env.Get("CONTROL_PLANE_NS", "istio-system") + controlPlaneNamespace = common.ControlPlaneNamespace externalControlPlaneNamespace = env.Get("EXTERNAL_CONTROL_PLANE_NS", "external-istiod") istioName = env.Get("ISTIO_NAME", "default") - istioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") + istioCniNamespace = common.IstioCniNamespace istioCniName = env.Get("ISTIOCNI_NAME", "default") multicluster = env.GetBool("MULTICLUSTER", false) keepOnFailure = env.GetBool("KEEP_ON_FAILURE", false) diff --git a/tests/e2e/multicontrolplane/multi_control_plane_suite_test.go b/tests/e2e/multicontrolplane/multi_control_plane_suite_test.go index 9485cb193c..749d1fdc40 100644 --- a/tests/e2e/multicontrolplane/multi_control_plane_suite_test.go +++ b/tests/e2e/multicontrolplane/multi_control_plane_suite_test.go @@ -21,6 +21,7 @@ import ( "github.com/istio-ecosystem/sail-operator/pkg/env" k8sclient "github.com/istio-ecosystem/sail-operator/tests/e2e/util/client" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -35,7 +36,7 @@ var ( controlPlaneNamespace2 = env.Get("CONTROL_PLANE_NS2", "istio-system2") istioName1 = env.Get("ISTIO_NAME1", "mesh1") istioName2 = env.Get("ISTIO_NAME2", "mesh2") - istioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") + istioCniNamespace = common.IstioCniNamespace istioCniName = env.Get("ISTIOCNI_NAME", "default") appNamespace1 = env.Get("APP_NAMESPACE1", "app1") appNamespace2a = env.Get("APP_NAMESPACE2A", "app2a") diff --git a/tests/e2e/setup/setup-kind.sh b/tests/e2e/setup/setup-kind.sh index 02671c90b4..4d00e8a2c1 100755 --- a/tests/e2e/setup/setup-kind.sh +++ b/tests/e2e/setup/setup-kind.sh @@ -18,6 +18,7 @@ set -eux -o pipefail SCRIPTPATH="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" ROOT=$(cd "${SCRIPTPATH}/../../.." && pwd) +GW_API_VERSION=${GW_API_VERSION:-v1.4.0} # shellcheck source=common/scripts/kind_provisioner.sh source "${ROOT}/common/scripts/kind_provisioner.sh" @@ -70,6 +71,11 @@ if [ "${MULTICLUSTER}" == "true" ]; then setup_kind_clusters "${KIND_IMAGE}" "" setup_kind_registry "${CLUSTER_NAMES[@]}" + # Apply Gateway API CRDs which are needed for Multi-Cluster Ambient testing + for config in "${KUBECONFIGS[@]}"; do + kubectl apply --kubeconfig="$config" --server-side -f "https://github.com/kubernetes-sigs/gateway-api/releases/download/${GW_API_VERSION}/experimental-install.yaml" + done + export KUBECONFIG="${KUBECONFIGS[0]}" export KUBECONFIG2="${KUBECONFIGS[1]}" echo "Your KinD environment is ready, to use it: export KUBECONFIG=$(IFS=:; echo "${KUBECONFIGS[*]}")" diff --git a/tests/e2e/util/common/checks.go b/tests/e2e/util/common/checks.go index a3df3ca9a7..ef5c49d8ca 100644 --- a/tests/e2e/util/common/checks.go +++ b/tests/e2e/util/common/checks.go @@ -49,7 +49,7 @@ func AwaitCondition[T ~string](ctx context.Context, condition T, key client.Obje // AwaitDeployment to reach the Available state. func AwaitDeployment(ctx context.Context, name string, k kubectl.Kubectl, cl client.Client) { - AwaitCondition(ctx, appsv1.DeploymentAvailable, kube.Key(name, controlPlaneNamespace), &appsv1.Deployment{}, k, cl) + AwaitCondition(ctx, appsv1.DeploymentAvailable, kube.Key(name, ControlPlaneNamespace), &appsv1.Deployment{}, k, cl) } func isPodReady(pod *corev1.Pod) bool { @@ -86,7 +86,7 @@ func CheckSamplePodsReady(ctx context.Context, cl client.Client) error { // AwaitCniDaemonSet to be deployed and reach the scheduled number of pods. func AwaitCniDaemonSet(ctx context.Context, k kubectl.Kubectl, cl client.Client) { - key := kube.Key("istio-cni-node", istioCniNamespace) + key := kube.Key("istio-cni-node", IstioCniNamespace) Eventually(func() bool { daemonset := &appsv1.DaemonSet{} if err := cl.Get(ctx, key, daemonset); err != nil { diff --git a/tests/e2e/util/common/e2e_utils.go b/tests/e2e/util/common/e2e_utils.go index 641cf7e5c4..9a6f03c474 100644 --- a/tests/e2e/util/common/e2e_utils.go +++ b/tests/e2e/util/common/e2e_utils.go @@ -56,16 +56,16 @@ const ( ) var ( - OperatorImage = env.Get("IMAGE", "quay.io/sail-dev/sail-operator:latest") - OperatorNamespace = env.Get("NAMESPACE", "sail-operator") + ControlPlaneNamespace = env.Get("CONTROL_PLANE_NS", "istio-system") + IstioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") + OperatorImage = env.Get("IMAGE", "quay.io/sail-dev/sail-operator:latest") + OperatorNamespace = env.Get("NAMESPACE", "sail-operator") + ZtunnelNamespace = env.Get("ZTUNNEL_NAMESPACE", "ztunnel") - deploymentName = env.Get("DEPLOYMENT_NAME", "sail-operator") - controlPlaneNamespace = env.Get("CONTROL_PLANE_NS", "istio-system") - istioName = env.Get("ISTIO_NAME", "default") - istioCniName = env.Get("ISTIOCNI_NAME", "default") - istioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") - sampleNamespace = env.Get("SAMPLE_NAMESPACE", "sample") - ztunnelNamespace = env.Get("ZTUNNEL_NAMESPACE", "ztunnel") + deploymentName = env.Get("DEPLOYMENT_NAME", "sail-operator") + istioName = env.Get("ISTIO_NAME", "default") + istioCniName = env.Get("ISTIOCNI_NAME", "default") + sampleNamespace = env.Get("SAMPLE_NAMESPACE", "sample") // version can have one of the following formats: // - 1.22.2 @@ -207,14 +207,14 @@ func logIstioDebugInfo(k kubectl.Kubectl) { resource, err := k.GetYAML("istio", istioName) logDebugElement("=====Istio YAML=====", resource, err) - output, err := k.WithNamespace(controlPlaneNamespace).GetPods("", "-o wide") - logDebugElement("=====Pods in "+controlPlaneNamespace+"=====", output, err) + output, err := k.WithNamespace(ControlPlaneNamespace).GetPods("", "-o wide") + logDebugElement("=====Pods in "+ControlPlaneNamespace+"=====", output, err) - logs, err := k.WithNamespace(controlPlaneNamespace).Logs("deploy/istiod", ptr.Of(120*time.Second)) + logs, err := k.WithNamespace(ControlPlaneNamespace).Logs("deploy/istiod", ptr.Of(120*time.Second)) logDebugElement("=====Istiod logs=====", logs, err) - events, err := k.WithNamespace(controlPlaneNamespace).GetEvents() - logDebugElement("=====Events in "+controlPlaneNamespace+"=====", events, err) + events, err := k.WithNamespace(ControlPlaneNamespace).GetEvents() + logDebugElement("=====Events in "+ControlPlaneNamespace+"=====", events, err) // Running istioctl proxy-status to get the status of the proxies. proxyStatus, err := istioctl.GetProxyStatus() @@ -225,20 +225,20 @@ func logCNIDebugInfo(k kubectl.Kubectl) { resource, err := k.GetYAML("istiocni", istioCniName) logDebugElement("=====IstioCNI YAML=====", resource, err) - ds, err := k.WithNamespace(istioCniNamespace).GetYAML("daemonset", "istio-cni-node") + ds, err := k.WithNamespace(IstioCniNamespace).GetYAML("daemonset", "istio-cni-node") logDebugElement("=====Istio CNI DaemonSet YAML=====", ds, err) - events, err := k.WithNamespace(istioCniNamespace).GetEvents() - logDebugElement("=====Events in "+istioCniNamespace+"=====", events, err) + events, err := k.WithNamespace(IstioCniNamespace).GetEvents() + logDebugElement("=====Events in "+IstioCniNamespace+"=====", events, err) // Temporary information to gather more details about failure - pods, err := k.WithNamespace(istioCniNamespace).GetPods("", "-o wide") - logDebugElement("=====Pods in "+istioCniNamespace+"=====", pods, err) + pods, err := k.WithNamespace(IstioCniNamespace).GetPods("", "-o wide") + logDebugElement("=====Pods in "+IstioCniNamespace+"=====", pods, err) - describe, err := k.WithNamespace(istioCniNamespace).Describe("daemonset", "istio-cni-node") + describe, err := k.WithNamespace(IstioCniNamespace).Describe("daemonset", "istio-cni-node") logDebugElement("=====Istio CNI DaemonSet describe=====", describe, err) - logs, err := k.WithNamespace(istioCniNamespace).Logs("daemonset/istio-cni-node", ptr.Of(120*time.Second)) + logs, err := k.WithNamespace(IstioCniNamespace).Logs("daemonset/istio-cni-node", ptr.Of(120*time.Second)) logDebugElement("=====Istio CNI logs=====", logs, err) } @@ -246,22 +246,22 @@ func logZtunnelDebugInfo(k kubectl.Kubectl) { resource, err := k.GetYAML("ztunnel", "default") logDebugElement("=====ZTunnel YAML=====", resource, err) - ds, err := k.WithNamespace(ztunnelNamespace).GetYAML("daemonset", "ztunnel") + ds, err := k.WithNamespace(ZtunnelNamespace).GetYAML("daemonset", "ztunnel") logDebugElement("=====ZTunnel DaemonSet YAML=====", ds, err) - events, err := k.WithNamespace(ztunnelNamespace).GetEvents() - logDebugElement("=====Events in "+ztunnelNamespace+"=====", events, err) + events, err := k.WithNamespace(ZtunnelNamespace).GetEvents() + logDebugElement("=====Events in "+ZtunnelNamespace+"=====", events, err) - describe, err := k.WithNamespace(ztunnelNamespace).Describe("daemonset", "ztunnel") + describe, err := k.WithNamespace(ZtunnelNamespace).Describe("daemonset", "ztunnel") logDebugElement("=====ZTunnel DaemonSet describe=====", describe, err) - logs, err := k.WithNamespace(ztunnelNamespace).Logs("daemonset/ztunnel", ptr.Of(120*time.Second)) + logs, err := k.WithNamespace(ZtunnelNamespace).Logs("daemonset/ztunnel", ptr.Of(120*time.Second)) logDebugElement("=====ztunnel logs=====", logs, err) } func logCertsDebugInfo(k kubectl.Kubectl) { - certs, err := k.WithNamespace(controlPlaneNamespace).GetSecret("cacerts") - logDebugElement("=====CA certs in "+controlPlaneNamespace+"=====", certs, err) + certs, err := k.WithNamespace(ControlPlaneNamespace).GetSecret("cacerts") + logDebugElement("=====CA certs in "+ControlPlaneNamespace+"=====", certs, err) } func logSampleNamespacesDebugInfo(k kubectl.Kubectl, suite testSuite) { @@ -329,7 +329,7 @@ func logDebugElement(caption string, info string, err error) { func GetVersionFromIstiod() (*semver.Version, error) { k := kubectl.New() - output, err := k.WithNamespace(controlPlaneNamespace).Exec("deploy/istiod", "", "pilot-discovery version") + output, err := k.WithNamespace(ControlPlaneNamespace).Exec("deploy/istiod", "", "pilot-discovery version") if err != nil { return nil, fmt.Errorf("error getting version from istiod: %w", err) } @@ -385,19 +385,12 @@ metadata: spec: version: %s namespace: %s` - yaml = fmt.Sprintf(yaml, istioName, version, controlPlaneNamespace) - for _, spec := range specs { - yaml += Indent(spec) - } - - Log("Istio YAML:", Indent(yaml)) - Expect(k.CreateFromString(yaml)). - To(Succeed(), withClusterName("Istio CR failed to be created", k)) - Success(withClusterName("Istio CR created", k)) + yaml = fmt.Sprintf(yaml, istioName, version, ControlPlaneNamespace) + createResource(k, "Istio", yaml, specs...) } // CreateIstioCNI custom resource using a given `kubectl` client and with the specified version. -func CreateIstioCNI(k kubectl.Kubectl, version string) { +func CreateIstioCNI(k kubectl.Kubectl, version string, specs ...string) { yaml := ` apiVersion: sailoperator.io/v1 kind: IstioCNI @@ -406,10 +399,54 @@ metadata: spec: version: %s namespace: %s` - yaml = fmt.Sprintf(yaml, istioCniName, version, istioCniNamespace) - Log("IstioCNI YAML:", Indent(yaml)) - Expect(k.CreateFromString(yaml)).To(Succeed(), withClusterName("IstioCNI creation failed", k)) - Success(withClusterName("IstioCNI created", k)) + yaml = fmt.Sprintf(yaml, istioCniName, version, IstioCniNamespace) + createResource(k, "IstioCNI", yaml, specs...) +} + +func CreateZTunnel(k kubectl.Kubectl, version string, specs ...string) { + yaml := ` +apiVersion: sailoperator.io/v1alpha1 +kind: ZTunnel +metadata: + name: default +spec: + profile: ambient + version: %s + namespace: %s` + yaml = fmt.Sprintf(yaml, version, ZtunnelNamespace) + createResource(k, "ZTunnel", yaml, specs...) +} + +func CreateAmbientGateway(k kubectl.Kubectl, namespace, network string) { + yaml := `kind: Gateway +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: istio-eastwestgateway + namespace: %s + labels: + topology.istio.io/network: %s +spec: + gatewayClassName: istio-east-west + listeners: + - name: mesh + port: 15008 + protocol: HBONE + tls: + mode: Terminate + options: + gateway.istio.io/tls-terminate-mode: ISTIO_MUTUAL` + yaml = fmt.Sprintf(yaml, namespace, network) + createResource(k, "Gateway", yaml) +} + +func createResource(k kubectl.Kubectl, kind, yaml string, specs ...string) { + for _, spec := range specs { + yaml += Indent(spec) + } + + Log(fmt.Sprintf("%s YAML:", kind), Indent(yaml)) + Expect(k.CreateFromString(yaml)).To(Succeed(), withClusterName(fmt.Sprintf("%s creation failed:", kind), k)) + Success(withClusterName(fmt.Sprintf("%s created", kind), k)) } func Indent(str string) string { diff --git a/tests/e2e/util/kubectl/kubectl.go b/tests/e2e/util/kubectl/kubectl.go index 95949a89a9..b10ac948c6 100644 --- a/tests/e2e/util/kubectl/kubectl.go +++ b/tests/e2e/util/kubectl/kubectl.go @@ -305,6 +305,12 @@ func (k Kubectl) Label(kind, name, labelKey, labelValue string) error { return err } +// LabelNamespaced adds a label to the specified resource in the specified namespace +func (k Kubectl) LabelNamespaced(kind, namespace, name, labelKey, labelValue string) error { + _, err := k.executeCommand(k.build(fmt.Sprintf(" label %s -n %s %s %s=%s", kind, namespace, name, labelKey, labelValue))) + return err +} + // executeCommand handles running the command and then resets the namespace automatically func (k Kubectl) executeCommand(cmd string) (string, error) { return shell.ExecuteCommand(cmd) diff --git a/tools/update_deps.sh b/tools/update_deps.sh index 351e0bab7e..afe529d43c 100755 --- a/tools/update_deps.sh +++ b/tools/update_deps.sh @@ -95,6 +95,10 @@ OPM_LATEST_VERSION=$(getLatestVersion operator-framework/operator-registry) OLM_LATEST_VERSION=$(getLatestVersion operator-framework/operator-lifecycle-manager) "$SED_CMD" -i "s|OLM_VERSION ?= .*|OLM_VERSION ?= ${OLM_LATEST_VERSION}|" "${ROOTDIR}/Makefile.core.mk" +# Update gateway-api +GW_API_LATEST_VERSION=$(getLatestVersion kubernetes-sigs/gateway-api) +"$SED_CMD" -i "s|GW_API_VERSION=.*|GW_API_VERSION=\${GW_API_VERSION:-${GW_API_LATEST_VERSION}}|" "${ROOTDIR}/tests/e2e/setup/setup-kind.sh" + # Update kube-rbac-proxy RBAC_PROXY_LATEST_VERSION=$(getLatestVersion brancz/kube-rbac-proxy | cut -d/ -f1) # Only update it if the newer image is available in the registry