Enhance e2e debug information with artifact collection

tests/e2e/README.md

@@ -16,7 +16,7 @@ This end-to-end test suite utilizes Ginkgo, a testing framework known for its ex
     1. [How to Run the test](#how-to-run-the-test)
     1. [Running the test locally](#running-the-test-locally)
     1. [Settings for end-to-end test execution](#settings-for-end-to-end-test-execution)
-    1. [Customizing the test run](#customizing-the-test-run)    
+    1. [Customizing the test run](#customizing-the-test-run)
     1. [Get test definitions for the end-to-end test](#get-test-definitions-for-the-end-to-end-test)
 1. [Contributing](#contributing)
 
@@ -69,7 +69,7 @@ var _ = Describe("Operator", Label("labels-for-the-test"), Ordered, func() {
         })
     })
 })
-``` 
+```
 Note: The `Label` function is used to label the test. This is useful when you want to run a specific test or a group of tests. The label can be used to filter the tests when running them. Ordered is used to run the tests in the order they are defined. This is useful when you want to run the tests in a specific order.
 
 
@@ -133,7 +133,7 @@ var _ = Describe("Operator", func() {
         })
     })
 })
-``` 
+```
 * Add Labels to the tests. This is useful when you want to run a specific test or a group of tests. The label can be used to filter the tests when running them. For example:
 ```go
 var _ = Describe("Ambient configuration ", Label("smoke", "ambient"), Ordered, func() {
@@ -192,6 +192,37 @@ var _ = Describe("Testing with cleanup", Ordered, func() {
 ```
     * You can use multiple cleaners, each with its own state. This is useful if the test does some global set up, e.g. sets up the operator, and then specific tests create further resources which you want cleaned.
     * To clean resources without waiting, and waiting for them later, use `CleanupNoWait` followed by `WaitForDeletion`. This is particularly useful when working with more than one cluster.
+* Use `debugcollector` to collect comprehensive debug information when tests fail. The debug collector captures the cluster state and saves it as artifacts for easier debugging. Like the cleaner, it records initial state and collects debug information on test failure. For example:
+```go
+import "github.com/istio-ecosystem/sail-operator/tests/e2e/util/debugcollector"
+
+var _ = Describe("Testing with debug collection", Ordered, func() {
+    collector := debugcollector.New(cl, k, "test-suite-name")
+
+    BeforeAll(func(ctx SpecContext) {
+        collector.Record(ctx)
+        // Any additional set up goes here
+    })
+
+    // Tests go here
+
+    AfterAll(func(ctx SpecContext) {
+        if CurrentSpecReport().Failed() {
+            collector.CollectAndSave(ctx)
+        }
+        // Any cleanup logic goes here
+    })
+})
+```
+    * The debug collector saves artifacts to the `$ARTIFACTS` directory (or `/tmp` if not set) with a timestamped folder structure
+    * Collected information includes: pod YAMLs, logs, events, deployments, daemonsets, services, configmaps, and custom resources
+    * You can control the collection depth with the `DEBUG_COLLECTOR_DEPTH` environment variable (values: `full`, `minimal`, `logs-only`)
+    * For multicluster tests, create separate collectors for each cluster with descriptive names
+    * The artifacts directory structure is organized as:
+      - `debug-<suite-name>-<timestamp>/`
+        - `cluster-scoped/` - cluster-wide resources
+        - `namespaces/<namespace>/` - namespace-specific resources, logs, and events
+        - `istioctl/` - istioctl proxy-status output
 
 ## Running the tests
 The end-to-end test can be run in two different environments: OCP (OpenShift Container Platform) and KinD (Kubernetes in Docker).
@@ -405,7 +436,7 @@ make test.e2e.describe
 ```
 
 When you run this target, the test definitions will be printed to the console with format `indent`. For example:
-    
+
 ```
 Name,Text,Start,End,Spec,Focused,Pending,Labels
 Describe,Operator,882,7688,false,false,false,""

tests/e2e/ambient/ambient_test.go

@@ -27,6 +27,7 @@ import (
 	. "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/common"
+	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/debugcollector"
 	. "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -55,8 +56,10 @@ var _ = Describe("Ambient configuration ", Label("smoke", "ambient"), Ordered, f
 
 			Context(fmt.Sprintf("Istio version %s", version.Version), func() {
 				clr := cleaner.New(cl)
+				collector := debugcollector.New(cl, k, "ambient")
 				BeforeAll(func(ctx SpecContext) {
 					clr.Record(ctx)
+					collector.Record(ctx)
 					Expect(k.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created")
 					Expect(k.CreateNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI namespace failed to be created")
 					Expect(k.CreateNamespace(ztunnelNamespace)).To(Succeed(), "ZTunnel namespace failed to be created")
@@ -296,8 +299,11 @@ spec:
 				})
 
 				AfterAll(func(ctx SpecContext) {
-					if CurrentSpecReport().Failed() && keepOnFailure {
-						return
+					if CurrentSpecReport().Failed() {
+						collector.CollectAndSave(ctx)
+						if keepOnFailure {
+							return
+						}
 					}
 
 					clr.Cleanup(ctx)

tests/e2e/controlplane/control_plane_test.go

@@ -28,6 +28,7 @@ import (
 	. "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/common"
+	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/debugcollector"
 	. "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl"
 	. "github.com/onsi/ginkgo/v2"
@@ -100,8 +101,10 @@ metadata:
 		for _, version := range istioversion.GetLatestPatchVersions() {
 			Context(version.Name, func() {
 				clr := cleaner.New(cl)
+				collector := debugcollector.New(cl, k, "control-plane")
 				BeforeAll(func(ctx SpecContext) {
 					clr.Record(ctx)
+					collector.Record(ctx)
 					Expect(k.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created")
 					Expect(k.CreateNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI namespace failed to be created")
 				})
@@ -239,8 +242,11 @@ metadata:
 				})
 
 				AfterAll(func(ctx SpecContext) {
-					if CurrentSpecReport().Failed() && keepOnFailure {
-						return
+					if CurrentSpecReport().Failed() {
+						collector.CollectAndSave(ctx)
+						if keepOnFailure {
+							return
+						}
 					}
 
 					clr.Cleanup(ctx)

tests/e2e/controlplane/control_plane_update_test.go

@@ -28,6 +28,7 @@ import (
 	. "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/common"
+	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/debugcollector"
 	. "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -50,13 +51,15 @@ var _ = Describe("Control Plane updates", Label("control-plane", "slow"), Ordere
 
 		Context(istioversion.Base, func() {
 			clr := cleaner.New(cl)
+			collector := debugcollector.New(cl, k, "control-plane-update")
 
 			BeforeAll(func(ctx SpecContext) {
 				if len(istioversion.List) < 2 {
 					Skip("Skipping update tests because there are not enough versions in versions.yaml")
 				}
 
 				clr.Record(ctx)
+				collector.Record(ctx)
 				Expect(k.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created")
 				Expect(k.CreateNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI namespace failed to be created")
 
@@ -277,6 +280,7 @@ spec:
 
 			AfterAll(func(ctx SpecContext) {
 				if CurrentSpecReport().Failed() {
+					collector.CollectAndSave(ctx)
 					common.LogDebugInfo(common.ControlPlane, k)
 					debugInfoLogged = true
 					if keepOnFailure {

tests/e2e/dualstack/dualstack_test.go

@@ -27,6 +27,7 @@ import (
 	. "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/common"
+	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/debugcollector"
 	. "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -59,8 +60,10 @@ var _ = Describe("DualStack configuration ", Label("dualstack"), Ordered, func()
 
 			Context(fmt.Sprintf("Istio version %s", version.Version), func() {
 				clr := cleaner.New(cl)
+				collector := debugcollector.New(cl, k, "dualstack")
 				BeforeAll(func(ctx SpecContext) {
 					clr.Record(ctx)
+					collector.Record(ctx)
 					Expect(k.CreateNamespace(controlPlaneNamespace)).To(Succeed(), "Istio namespace failed to be created")
 					Expect(k.CreateNamespace(istioCniNamespace)).To(Succeed(), "IstioCNI namespace failed to be created")
 				})
@@ -229,8 +232,11 @@ values:
 				})
 
 				AfterAll(func(ctx SpecContext) {
-					if CurrentSpecReport().Failed() && keepOnFailure {
-						return
+					if CurrentSpecReport().Failed() {
+						collector.CollectAndSave(ctx)
+						if keepOnFailure {
+							return
+						}
 					}
 
 					clr.Cleanup(ctx)

tests/e2e/multicluster/multicluster_externalcontrolplane_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/istio-ecosystem/sail-operator/pkg/version"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/common"
+	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/debugcollector"
 	. "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl"
 	. "github.com/onsi/ginkgo/v2"
@@ -59,10 +60,14 @@ var _ = Describe("Multicluster deployment models", Label("multicluster", "multic
 			Context(fmt.Sprintf("Istio version %s", v.Version), func() {
 				clr1 := cleaner.New(clPrimary, "cluster=primary")
 				clr2 := cleaner.New(clRemote, "cluster=remote")
+				collector1 := debugcollector.New(clPrimary, k1, "multicluster-primary")
+				collector2 := debugcollector.New(clRemote, k2, "multicluster-remote")
 
 				BeforeAll(func(ctx SpecContext) {
 					clr1.Record(ctx)
 					clr2.Record(ctx)
+					collector1.Record(ctx)
+					collector2.Record(ctx)
 				})
 
 				When("default Istio is created in Cluster #1 to handle ingress to External Control Plane", func() {
@@ -412,6 +417,8 @@ spec:
 
 				AfterAll(func(ctx SpecContext) {
 					if CurrentSpecReport().Failed() {
+						collector1.CollectAndSave(ctx)
+						collector2.CollectAndSave(ctx)
 						common.LogDebugInfo(common.MultiCluster, k1, k2)
 						debugInfoLogged = true
 						if keepOnFailure {

tests/e2e/multicluster/multicluster_multiprimary_test.go

@@ -28,6 +28,7 @@ import (
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/certs"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/common"
+	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/debugcollector"
 	. "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl"
 	. "github.com/onsi/ginkgo/v2"
@@ -48,10 +49,14 @@ var _ = Describe("Multicluster deployment models", Label("multicluster", "multic
 			Context(fmt.Sprintf("Istio version %s", version.Version), func() {
 				clr1 := cleaner.New(clPrimary, "cluster=primary")
 				clr2 := cleaner.New(clRemote, "cluster=remote")
+				collector1 := debugcollector.New(clPrimary, k1, "multicluster-primary")
+				collector2 := debugcollector.New(clRemote, k2, "multicluster-remote")
 
 				BeforeAll(func(ctx SpecContext) {
 					clr1.Record(ctx)
 					clr2.Record(ctx)
+					collector1.Record(ctx)
+					collector2.Record(ctx)
 				})
 
 				When("Istio and IstioCNI resources are created in both clusters", func() {
@@ -291,6 +296,8 @@ values:
 
 				AfterAll(func(ctx SpecContext) {
 					if CurrentSpecReport().Failed() {
+						collector1.CollectAndSave(ctx)
+						collector2.CollectAndSave(ctx)
 						common.LogDebugInfo(common.MultiCluster, k1, k2)
 						debugInfoLogged = true
 						if keepOnFailure {

tests/e2e/multicluster/multicluster_primaryremote_test.go

@@ -29,6 +29,7 @@ import (
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/certs"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/common"
+	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/debugcollector"
 	. "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/istioctl"
 	. "github.com/onsi/ginkgo/v2"
@@ -55,10 +56,14 @@ var _ = Describe("Multicluster deployment models", Label("multicluster", "multic
 			Context(fmt.Sprintf("Istio version %s", v.Version), func() {
 				clr1 := cleaner.New(clPrimary, "cluster=primary")
 				clr2 := cleaner.New(clRemote, "cluster=remote")
+				collector1 := debugcollector.New(clPrimary, k1, "multicluster-primary")
+				collector2 := debugcollector.New(clRemote, k2, "multicluster-remote")
 
 				BeforeAll(func(ctx SpecContext) {
 					clr1.Record(ctx)
 					clr2.Record(ctx)
+					collector1.Record(ctx)
+					collector2.Record(ctx)
 				})
 
 				When("Istio and IstioCNI resources are created in both clusters", func() {
@@ -308,6 +313,8 @@ values:
 
 				AfterAll(func(ctx SpecContext) {
 					if CurrentSpecReport().Failed() {
+						collector1.CollectAndSave(ctx)
+						collector2.CollectAndSave(ctx)
 						common.LogDebugInfo(common.MultiCluster, k1, k2)
 						debugInfoLogged = true
 						if keepOnFailure {

tests/e2e/multicontrolplane/multi_control_plane_test.go

@@ -26,6 +26,7 @@ import (
 	. "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/common"
+	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/debugcollector"
 	. "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -43,9 +44,11 @@ var _ = Describe("Multi control plane deployment model", Label("smoke", "multico
 		for _, version := range istioversion.GetLatestPatchVersions() {
 			Context(fmt.Sprintf("Istio version %s", version.Version), func() {
 				clr := cleaner.New(cl)
+				collector := debugcollector.New(cl, k, "multicontrol-plane")
 
 				BeforeAll(func(ctx SpecContext) {
 					clr.Record(ctx)
+					collector.Record(ctx)
 				})
 
 				Describe("Installation", func() {
@@ -161,6 +164,7 @@ spec:
 
 				AfterAll(func(ctx SpecContext) {
 					if CurrentSpecReport().Failed() {
+						collector.CollectAndSave(ctx)
 						common.LogDebugInfo(common.ControlPlane, k)
 						debugInfoLogged = true
 					}

tests/e2e/operator/operator_install_test.go

@@ -28,6 +28,7 @@ import (
 	. "github.com/istio-ecosystem/sail-operator/pkg/test/util/ginkgo"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/cleaner"
 	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/common"
+	"github.com/istio-ecosystem/sail-operator/tests/e2e/util/debugcollector"
 	. "github.com/istio-ecosystem/sail-operator/tests/e2e/util/gomega"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
@@ -62,8 +63,10 @@ var _ = Describe("Operator", Label("smoke", "operator"), Ordered, func() {
 
 	Describe("installation", func() {
 		clr := cleaner.New(cl)
+		collector := debugcollector.New(cl, k, "operator")
 		BeforeAll(func(ctx SpecContext) {
 			clr.Record(ctx)
+			collector.Record(ctx)
 		})
 
 		It("deploys all the CRDs", func(ctx SpecContext) {
@@ -181,12 +184,12 @@ spec:
 		})
 
 		AfterAll(func(ctx SpecContext) {
-			if CurrentSpecReport().Failed() && keepOnFailure {
-				return
-			}
-
 			if CurrentSpecReport().Failed() {
+				collector.CollectAndSave(ctx)
 				common.LogDebugInfo(common.Operator, k)
+				if keepOnFailure {
+					return
+				}
 			}
 			clr.Cleanup(ctx)
 		})