diff --git a/tests/e2e/ambient/ambient_suite_test.go b/tests/e2e/ambient/ambient_suite_test.go index eecfccab4b..ec3f17dcbf 100644 --- a/tests/e2e/ambient/ambient_suite_test.go +++ b/tests/e2e/ambient/ambient_suite_test.go @@ -62,5 +62,5 @@ func setup() { cl, err = k8sclient.InitK8sClient("") Expect(err).NotTo(HaveOccurred()) - k = kubectl.New("clAmbient") + k = kubectl.New() } diff --git a/tests/e2e/controlplane/control_plane_suite_test.go b/tests/e2e/controlplane/control_plane_suite_test.go index e73d01a114..ce355e11d9 100644 --- a/tests/e2e/controlplane/control_plane_suite_test.go +++ b/tests/e2e/controlplane/control_plane_suite_test.go @@ -62,5 +62,5 @@ func setup() { cl, err = k8sclient.InitK8sClient("") Expect(err).NotTo(HaveOccurred()) - k = kubectl.New("clControlPlane") + k = kubectl.New() } diff --git a/tests/e2e/dualstack/dualstack_suite_test.go b/tests/e2e/dualstack/dualstack_suite_test.go index a18166cc46..dd581b58ad 100644 --- a/tests/e2e/dualstack/dualstack_suite_test.go +++ b/tests/e2e/dualstack/dualstack_suite_test.go @@ -62,5 +62,5 @@ func setup() { cl, err = k8sclient.InitK8sClient("") Expect(err).NotTo(HaveOccurred()) - k = kubectl.New("clDualStack") + k = kubectl.New() } diff --git a/tests/e2e/multicluster/multicluster_multiprimary_test.go b/tests/e2e/multicluster/multicluster_multiprimary_test.go index aeb202ffef..b4ebc1d930 100644 --- a/tests/e2e/multicluster/multicluster_multiprimary_test.go +++ b/tests/e2e/multicluster/multicluster_multiprimary_test.go @@ -274,8 +274,7 @@ spec: AfterAll(func(ctx SpecContext) { if CurrentSpecReport().Failed() { - common.LogDebugInfo(k1) - common.LogDebugInfo(k2) + common.LogDebugInfo(k1, k2) debugInfoLogged = true } @@ -299,8 +298,7 @@ spec: AfterAll(func(ctx SpecContext) { if CurrentSpecReport().Failed() && !debugInfoLogged { - common.LogDebugInfo(k1) - common.LogDebugInfo(k2) + common.LogDebugInfo(k1, k2) debugInfoLogged = true } diff --git a/tests/e2e/multicluster/multicluster_primaryremote_test.go b/tests/e2e/multicluster/multicluster_primaryremote_test.go index 54b861c7a5..b10438cb32 100644 --- a/tests/e2e/multicluster/multicluster_primaryremote_test.go +++ b/tests/e2e/multicluster/multicluster_primaryremote_test.go @@ -318,8 +318,7 @@ spec: AfterAll(func(ctx SpecContext) { if CurrentSpecReport().Failed() { - common.LogDebugInfo(k1) - common.LogDebugInfo(k2) + common.LogDebugInfo(k1, k2) debugInfoLogged = true } @@ -349,8 +348,7 @@ spec: AfterAll(func(ctx SpecContext) { if CurrentSpecReport().Failed() && !debugInfoLogged { - common.LogDebugInfo(k1) - common.LogDebugInfo(k2) + common.LogDebugInfo(k1, k2) debugInfoLogged = true } diff --git a/tests/e2e/multicluster/multicluster_suite_test.go b/tests/e2e/multicluster/multicluster_suite_test.go index d00b9c64eb..0865775f5b 100644 --- a/tests/e2e/multicluster/multicluster_suite_test.go +++ b/tests/e2e/multicluster/multicluster_suite_test.go @@ -99,6 +99,6 @@ func setup(t *testing.T) { exposeIstiodYAML = fmt.Sprintf("%s/docs/multicluster/expose-istiod.yaml", baseRepoDir) // Initialize kubectl utilities, one for each cluster - k1 = kubectl.New("clPrimary").WithKubeconfig(kubeconfig) - k2 = kubectl.New("clRemote").WithKubeconfig(kubeconfig2) + k1 = kubectl.New().WithClusterName("primary").WithKubeconfig(kubeconfig) + k2 = kubectl.New().WithClusterName("remote").WithKubeconfig(kubeconfig2) } diff --git a/tests/e2e/multicontrolplane/multi_control_plane_suite_test.go b/tests/e2e/multicontrolplane/multi_control_plane_suite_test.go new file mode 100644 index 0000000000..25bb144f0d --- /dev/null +++ b/tests/e2e/multicontrolplane/multi_control_plane_suite_test.go @@ -0,0 +1,71 @@ +//go:build e2e + +// Copyright Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane + +import ( + "testing" + + "github.com/istio-ecosystem/sail-operator/pkg/env" + "github.com/istio-ecosystem/sail-operator/pkg/istioversion" + k8sclient "github.com/istio-ecosystem/sail-operator/tests/e2e/util/client" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/kubectl" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + cl client.Client + err error + version = istioversion.New + namespace = common.OperatorNamespace + deploymentName = env.Get("DEPLOYMENT_NAME", "sail-operator") + controlPlaneNamespace1 = env.Get("CONTROL_PLANE_NS1", "istio-system1") + controlPlaneNamespace2 = env.Get("CONTROL_PLANE_NS2", "istio-system2") + istioName1 = env.Get("ISTIO_NAME1", "mesh1") + istioName2 = env.Get("ISTIO_NAME2", "mesh2") + istioCniNamespace = env.Get("ISTIOCNI_NAMESPACE", "istio-cni") + istioCniName = env.Get("ISTIOCNI_NAME", "default") + skipDeploy = env.GetBool("SKIP_DEPLOY", false) + appNamespace1 = env.Get("APP_NAMESPACE1", "app1") + appNamespace2a = env.Get("APP_NAMESPACE2A", "app2a") + appNamespace2b = env.Get("APP_NAMESPACE2B", "app2b") + multicluster = env.GetBool("MULTICLUSTER", false) + ipFamily = env.Get("IP_FAMILY", "ipv4") + + k kubectl.Kubectl +) + +func TestInstall(t *testing.T) { + if ipFamily == "dual" || multicluster { + t.Skip("Skipping the multi control plane tests") + } + RegisterFailHandler(Fail) + setup() + RunSpecs(t, "Multiple Control Planes Test Suite") +} + +func setup() { + GinkgoWriter.Println("************ Running Setup ************") + + GinkgoWriter.Println("Initializing k8s client") + cl, err = k8sclient.InitK8sClient("") + Expect(err).NotTo(HaveOccurred()) + + k = kubectl.New() +} diff --git a/tests/e2e/multicontrolplane/multi_control_plane_test.go b/tests/e2e/multicontrolplane/multi_control_plane_test.go new file mode 100644 index 0000000000..e185fe9f16 --- /dev/null +++ b/tests/e2e/multicontrolplane/multi_control_plane_test.go @@ -0,0 +1,209 @@ +//go:build e2e + +// Copyright Istio Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR Condition OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controlplane + +import ( + "fmt" + "time" + + v1 "github.com/istio-ecosystem/sail-operator/api/v1" + "github.com/istio-ecosystem/sail-operator/pkg/istioversion" + "github.com/istio-ecosystem/sail-operator/pkg/kube" + . "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo" + "github.com/istio-ecosystem/sail-operator/tests/e2e/util/common" + . "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = Describe("Multi control plane deployment model", Ordered, func() { + SetDefaultEventuallyTimeout(180 * time.Second) + SetDefaultEventuallyPollingInterval(time.Second) + debugInfoLogged := false + + BeforeAll(func(ctx SpecContext) { + Expect(k.CreateNamespace(namespace)).To(Succeed(), "Namespace failed to be created") + + if skipDeploy { + Success("Skipping operator installation because it was deployed externally") + } else { + Expect(common.InstallOperatorViaHelm()). + To(Succeed(), "Operator failed to be deployed") + } + + Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(deploymentName, namespace), &appsv1.Deployment{}). + Should(HaveCondition(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error getting Istio CRD") + Success("Operator is deployed in the namespace and Running") + }) + + Describe("Installation", func() { + It("Sets up namespaces", func(ctx SpecContext) { + Expect(k.CreateNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI namespace failed to be created") + Expect(k.CreateNamespace(controlPlaneNamespace1)).To(Succeed(), "Istio namespace failed to be created") + Expect(k.CreateNamespace(controlPlaneNamespace2)).To(Succeed(), "Istio namespace failed to be created") + + Expect(k.Label("namespace", controlPlaneNamespace1, "mesh", istioName1)).To(Succeed(), "Failed to label namespace") + Expect(k.Label("namespace", controlPlaneNamespace2, "mesh", istioName2)).To(Succeed(), "Failed to label namespace") + }) + + It("Installs IstioCNI", func(ctx SpecContext) { + yaml := ` +apiVersion: sailoperator.io/v1 +kind: IstioCNI +metadata: + name: default +spec: + version: %s + namespace: %s` + yaml = fmt.Sprintf(yaml, version, istioCniNamespace) + Expect(k.CreateFromString(yaml)).To(Succeed(), "failed to create IstioCNI") + Success("IstioCNI created") + + Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(istioCniName), &v1.IstioCNI{}). + Should(HaveCondition(v1.IstioCNIConditionReady, metav1.ConditionTrue), "IstioCNI is not Ready; unexpected Condition") + Success("IstioCNI is Ready") + }) + + DescribeTable("Installs Istios", + Entry("Mesh 1", istioName1, controlPlaneNamespace1), + Entry("Mesh 2", istioName2, controlPlaneNamespace2), + func(ctx SpecContext, name, ns string) { + Expect(k.CreateFromString(` +apiVersion: sailoperator.io/v1 +kind: Istio +metadata: + name: `+name+` +spec: + version: `+version+` + namespace: `+ns+` + values: + meshConfig: + discoverySelectors: + - matchLabels: + mesh: `+name)).To(Succeed(), "failed to create Istio CR") + + Expect(k.CreateFromString(` +apiVersion: security.istio.io/v1 +kind: PeerAuthentication +metadata: + name: default + namespace: `+ns+` +spec: + mtls: + mode: STRICT`)).To(Succeed(), "failed to create PeerAuthentication") + }) + + DescribeTable("Waits for Istios", + Entry("Mesh 1", istioName1), + Entry("Mesh 2", istioName2), + func(ctx SpecContext, name string) { + Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(name), &v1.Istio{}). + Should( + And( + HaveCondition(v1.IstioConditionReconciled, metav1.ConditionTrue), + HaveCondition(v1.IstioConditionReady, metav1.ConditionTrue), + ), "Istio is not Reconciled and Ready; unexpected Condition") + Success(fmt.Sprintf("Istio %s ready", name)) + }) + + DescribeTable("Deploys applications", + Entry("App 1", appNamespace1, istioName1), + Entry("App 2a", appNamespace2a, istioName2), + Entry("App 2b", appNamespace2b, istioName2), + func(ns, mesh string) { + Expect(k.CreateNamespace(ns)).To(Succeed(), "Failed to create namespace") + Expect(k.Label("namespace", ns, "mesh", mesh)).To(Succeed(), "Failed to label namespace") + Expect(k.Label("namespace", ns, "istio.io/rev", mesh)).To(Succeed(), "Failed to label namespace") + for _, appName := range []string{"sleep", "httpbin"} { + Expect(k.WithNamespace(ns). + Apply(common.GetSampleYAML(istioversion.Map[version], appName))). + To(Succeed(), "Failed to deploy application") + } + Success(fmt.Sprintf("Applications in namespace %s deployed", ns)) + }) + + DescribeTable("Waits for apps to be ready", + Entry("App 1", appNamespace1), + Entry("App 2a", appNamespace2a), + Entry("App 2b", appNamespace2b), + func(ctx SpecContext, ns string) { + for _, deployment := range []string{"sleep", "httpbin"} { + Eventually(common.GetObject).WithArguments(ctx, cl, kube.Key(deployment, ns), &appsv1.Deployment{}). + Should(HaveCondition(appsv1.DeploymentAvailable, metav1.ConditionTrue), "Error waiting for deployment to be available") + } + Success(fmt.Sprintf("Applications in namespace %s ready", ns)) + }) + }) + + Describe("Verification", func() { + It("Verifies app2a cannot connect to app1", func(ctx SpecContext) { + output, err := k.WithNamespace(appNamespace2a). + Exec("deploy/sleep", "sleep", fmt.Sprintf("curl -sIL http://httpbin.%s:8000", appNamespace1)) + Expect(err).NotTo(HaveOccurred(), "error running curl in sleep pod") + Expect(output).To(ContainSubstring("503 Service Unavailable"), fmt.Sprintf("Unexpected response from sleep pod in namespace %s", appNamespace1)) + Success("As expected, app2a in mesh2 is not allowed to communicate with app1 in mesh1") + }) + + It("Verifies app2a can connect to app2b", func(ctx SpecContext) { + output, err := k.WithNamespace(appNamespace2a). + Exec("deploy/sleep", "sleep", fmt.Sprintf("curl -sIL http://httpbin.%s:8000", appNamespace2b)) + Expect(err).NotTo(HaveOccurred(), "error running curl in sleep pod") + Expect(output).To(ContainSubstring("200 OK"), fmt.Sprintf("Unexpected response from sleep pod in namespace %s", appNamespace2b)) + Success("As expected, app2a in mesh2 can communicate with app2b in the same mesh") + }) + }) + + AfterAll(func() { + By("Cleaning up the application namespaces") + Expect(k.DeleteNamespace(appNamespace1, appNamespace2a, appNamespace2b)).To(Succeed()) + + By("Cleaning up the Istio namespace") + Expect(k.DeleteNamespace(controlPlaneNamespace1, controlPlaneNamespace2)).To(Succeed(), "Istio Namespaces failed to be deleted") + + By("Cleaning up the IstioCNI namespace") + Expect(k.DeleteNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI Namespace failed to be deleted") + + By("Deleting any left-over Istio and IstioRevision resources") + Expect(k.Delete("istio", istioName1)).To(Succeed(), "Failed to delete Istio") + Expect(k.Delete("istio", istioName2)).To(Succeed(), "Failed to delete Istio") + Expect(k.Delete("istiocni", istioCniName)).To(Succeed(), "Failed to delete IstioCNI") + Success("Istio Resources deleted") + Success("Cleanup done") + }) + + AfterAll(func() { + if CurrentSpecReport().Failed() && !debugInfoLogged { + common.LogDebugInfo(k) + debugInfoLogged = true + } + + if skipDeploy { + Success("Skipping operator undeploy because it was deployed externally") + return + } + + By("Deleting operator deployment") + Expect(common.UninstallOperator()). + To(Succeed(), "Operator failed to be deleted") + GinkgoWriter.Println("Operator uninstalled") + + Expect(k.DeleteNamespace(namespace)).To(Succeed(), "Namespace failed to be deleted") + Success("Namespace deleted") + }) +}) diff --git a/tests/e2e/operator/operator_suite_test.go b/tests/e2e/operator/operator_suite_test.go index 11063b486b..700677b5ab 100644 --- a/tests/e2e/operator/operator_suite_test.go +++ b/tests/e2e/operator/operator_suite_test.go @@ -57,5 +57,5 @@ func setup() { cl, err = k8sclient.InitK8sClient("") Expect(err).NotTo(HaveOccurred()) - k = kubectl.New("clOperator") + k = kubectl.New() } diff --git a/tests/e2e/util/common/e2e_utils.go b/tests/e2e/util/common/e2e_utils.go index 5b92915a3a..22d8dc5085 100644 --- a/tests/e2e/util/common/e2e_utils.go +++ b/tests/e2e/util/common/e2e_utils.go @@ -131,21 +131,29 @@ func CheckNamespaceEmpty(ctx SpecContext, cl client.Client, ns string) { }).Should(BeEmpty(), "No Services should be present in the namespace") } -func LogDebugInfo(k kubectl.Kubectl) { +func LogDebugInfo(kubectls ...kubectl.Kubectl) { // General debugging information to help diagnose the failure // TODO: Add the creation of file with this information to be attached to the test report GinkgoWriter.Println() - GinkgoWriter.Printf("The test run has failures and the debug information is as follows from cluster: %q:\n", k.GetClusterName()) - GinkgoWriter.Println("=========================================================") - logOperatorDebugInfo(k) - GinkgoWriter.Println("=========================================================") - logIstioDebugInfo(k) - GinkgoWriter.Println("=========================================================") - logCNIDebugInfo(k) - GinkgoWriter.Println("=========================================================") - logCertsDebugInfo(k) - GinkgoWriter.Println("=========================================================") + GinkgoWriter.Println("The test run has failures and the debug information is as follows:") + GinkgoWriter.Println() + for _, k := range kubectls { + if k.ClusterName != "" { + GinkgoWriter.Println("=========================================================") + GinkgoWriter.Println("CLUSTER:", k.ClusterName) + GinkgoWriter.Println("=========================================================") + } + logOperatorDebugInfo(k) + GinkgoWriter.Println("=========================================================") + logIstioDebugInfo(k) + GinkgoWriter.Println("=========================================================") + logCNIDebugInfo(k) + GinkgoWriter.Println("=========================================================") + logCertsDebugInfo(k) + GinkgoWriter.Println("=========================================================") + GinkgoWriter.Println() + } } func logOperatorDebugInfo(k kubectl.Kubectl) { @@ -218,7 +226,7 @@ func logDebugElement(caption string, info string, err error) { } func GetVersionFromIstiod() (*semver.Version, error) { - k := kubectl.New("testCluster") + k := kubectl.New() output, err := k.WithNamespace(controlPlaneNamespace).Exec("deploy/istiod", "", "pilot-discovery version") if err != nil { return nil, fmt.Errorf("error getting version from istiod: %w", err) diff --git a/tests/e2e/util/kubectl/kubectl.go b/tests/e2e/util/kubectl/kubectl.go index d7addba219..c3b05694ef 100644 --- a/tests/e2e/util/kubectl/kubectl.go +++ b/tests/e2e/util/kubectl/kubectl.go @@ -25,15 +25,15 @@ import ( ) type Kubectl struct { - name string - binary string - namespace string - kubeconfig string + ClusterName string + binary string + namespace string + kubeconfig string } // New creates a new kubectl.Kubectl -func New(name string) Kubectl { - return Kubectl{name: name}.WithBinary(os.Getenv("COMMAND")) +func New() Kubectl { + return Kubectl{}.WithBinary(os.Getenv("COMMAND")) } func (k Kubectl) build(cmd string) string { @@ -55,6 +55,12 @@ func (k Kubectl) build(cmd string) string { return strings.Join(args, " ") } +// WithClusterName sets the cluster clusterName on this Kubectl +func (k Kubectl) WithClusterName(name string) Kubectl { + k.ClusterName = name + return k +} + // WithBinary returns a new Kubectl with the binary set to the given value; if the value is "", the binary is set to "kubectl" func (k Kubectl) WithBinary(binary string) Kubectl { if binary == "" { @@ -218,11 +224,6 @@ func (k Kubectl) ForceDelete(kind, name string) error { return k.Delete(kind, name) } -// Gets cluster name defined during initialization -func (k Kubectl) GetClusterName() string { - return k.name -} - // GetYAML returns the yaml of a resource func (k Kubectl) GetYAML(kind, name string) (string, error) { cmd := k.build(fmt.Sprintf(" get %s %s -o yaml", kind, name)) @@ -309,6 +310,12 @@ func (k Kubectl) Logs(pod string, since *time.Duration) (string, error) { return output, nil } +// Label adds a label to the specified resource +func (k Kubectl) Label(kind, name, labelKey, labelValue string) error { + _, err := k.executeCommand(k.build(fmt.Sprintf(" label %s %s %s=%s", kind, name, labelKey, labelValue))) + return err +} + // executeCommand handles running the command and then resets the namespace automatically func (k Kubectl) executeCommand(cmd string) (string, error) { return shell.ExecuteCommand(cmd)