Refactor e2e test setup

We recently made a change to ensure we clean up resources properly after
we perform e2e tests. Specifically to avoid race conditions that leaves
the compliance operator resources in a bricked state, even when the
tests pass.

This commit uses a similar pattern for test setup by leveraging the
`setUp` method before calling `m.Run()` to make sure the cluster is
ready before we run the actual tests.

One advantage is that we're simplifying the logic to a single layer
before the tests run.

Further changes will make it easier to decouple the framework, context,
and e2eutils into a single set of utilities we can share across
repositories if needed.

tests/e2e/framework/common.go

@@ -6,13 +6,28 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"strings"
 	"time"
 
+	"github.com/ComplianceAsCode/compliance-operator/pkg/apis"
 	compv1alpha1 "github.com/ComplianceAsCode/compliance-operator/pkg/apis/compliance/v1alpha1"
+	ocpapi "github.com/openshift/api"
+	configv1 "github.com/openshift/api/config/v1"
+	imagev1 "github.com/openshift/api/image/v1"
+	core "k8s.io/api/core/v1"
+	v1 "k8s.io/api/rbac/v1"
+	schedulingv1 "k8s.io/api/scheduling/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
-	"sigs.k8s.io/controller-runtime/pkg/client"
+	psapi "k8s.io/pod-security-admission/api"
+	dynclient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/yaml"
+
+	mcfgapi "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io"
+	mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1"
 )
 
 // readFile accepts a file path and returns the file contents.
@@ -51,6 +66,14 @@ func (f *Framework) readYAML(y []byte) ([][]byte, error) {
 	return o, nil
 }
 
+func unmarshalJSON(j []byte) (dynclient.Object, error) {
+	obj := &unstructured.Unstructured{}
+	if err := obj.UnmarshalJSON(j); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal object spec: %w", err)
+	}
+	return obj, nil
+}
+
 func (f *Framework) cleanUpFromYAMLFile(p *string) error {
 	c, err := f.readFile(p)
 	if err != nil {
@@ -62,14 +85,52 @@ func (f *Framework) cleanUpFromYAMLFile(p *string) error {
 	}
 
 	for _, d := range documents {
-		obj := &unstructured.Unstructured{}
-		if err := obj.UnmarshalJSON(d); err != nil {
-			return fmt.Errorf("failed to unmarshal object spec: %w", err)
+		obj, err := unmarshalJSON(d)
+		if err != nil {
+			return err
 		}
 		obj.SetNamespace(f.OperatorNamespace)
-		log.Printf("deleting %s %s", obj.GetKind(), obj.GetName())
+		log.Printf("deleting %s %s", obj.GetObjectKind().GroupVersionKind().Kind, obj.GetName())
 		if err := f.Client.Delete(context.TODO(), obj); err != nil {
-			return fmt.Errorf("failed to delete %s: %w", obj, err)
+			return fmt.Errorf("failed to delete %s: %w", obj.GetObjectKind().GroupVersionKind().Kind, err)
+		}
+	}
+	return nil
+}
+
+func (f *Framework) cleanUpProfileBundle(p string) error {
+	pb := &compv1alpha1.ProfileBundle{}
+	if err := f.Client.Get(context.TODO(), types.NamespacedName{Name: p, Namespace: f.OperatorNamespace}, pb); err != nil {
+		return err
+	}
+	err := f.Client.Delete(context.TODO(), pb)
+	if err != nil {
+		return fmt.Errorf("failed to delete ProfileBunlde %s: %w", p, err)
+	}
+	return nil
+}
+
+func (f *Framework) createFromYAMLFile(p *string) error {
+	c, err := f.readFile(p)
+	if err != nil {
+		return err
+	}
+	documents, err := f.readYAML(c)
+	if err != nil {
+		return err
+	}
+
+	for _, d := range documents {
+		obj, err := unmarshalJSON(d)
+		if err != nil {
+			return err
+		}
+
+		obj.SetNamespace(f.OperatorNamespace)
+		log.Printf("creating %s %s", obj.GetObjectKind().GroupVersionKind().Kind, obj.GetName())
+		err = f.Client.CreateWithoutCleanup(context.TODO(), obj)
+		if err != nil {
+			return err
 		}
 	}
 	return nil
@@ -78,7 +139,7 @@ func (f *Framework) cleanUpFromYAMLFile(p *string) error {
 func (f *Framework) waitForScanCleanup() error {
 	timeouterr := wait.Poll(time.Second*5, time.Minute*2, func() (bool, error) {
 		var scans compv1alpha1.ComplianceScanList
-		f.Client.List(context.TODO(), &scans, &client.ListOptions{})
+		f.Client.List(context.TODO(), &scans, &dynclient.ListOptions{})
 		if len(scans.Items) == 0 {
 			return true, nil
 		}
@@ -95,3 +156,282 @@ func (f *Framework) waitForScanCleanup() error {
 	}
 	return nil
 }
+
+func (f *Framework) addFrameworks() error {
+	// compliance-operator objects
+	coObjs := [3]dynclient.ObjectList{&compv1alpha1.ComplianceScanList{},
+		&compv1alpha1.ComplianceRemediationList{},
+		&compv1alpha1.ComplianceSuiteList{},
+	}
+
+	for _, obj := range coObjs {
+		err := AddToFrameworkScheme(apis.AddToScheme, obj)
+		if err != nil {
+			return fmt.Errorf("failed to add custom resource scheme to framework: %v", err)
+		}
+	}
+
+	// Additional testing objects
+	testObjs := [1]dynclient.ObjectList{
+		&configv1.OAuthList{},
+	}
+	for _, obj := range testObjs {
+		err := AddToFrameworkScheme(configv1.Install, obj)
+		if err != nil {
+			return fmt.Errorf("failed to add configv1 resource scheme to framework: %v", err)
+		}
+	}
+
+	// MCO objects
+	mcoObjs := [2]dynclient.ObjectList{
+		&mcfgv1.MachineConfigPoolList{},
+		&mcfgv1.MachineConfigList{},
+	}
+	for _, obj := range mcoObjs {
+		err := AddToFrameworkScheme(mcfgapi.Install, obj)
+		if err != nil {
+			return fmt.Errorf("failed to add custom resource scheme to framework: %v", err)
+		}
+	}
+
+	// OpenShift objects
+	ocpObjs := [2]dynclient.ObjectList{
+		&imagev1.ImageStreamList{},
+		&imagev1.ImageStreamTagList{},
+	}
+	for _, obj := range ocpObjs {
+		if err := AddToFrameworkScheme(ocpapi.Install, obj); err != nil {
+			return fmt.Errorf("failed to add custom resource scheme to framework: %v", err)
+		}
+	}
+
+	//Schedule objects
+	scObjs := [1]dynclient.ObjectList{
+		&schedulingv1.PriorityClassList{},
+	}
+	for _, obj := range scObjs {
+		if err := AddToFrameworkScheme(schedulingv1.AddToScheme, obj); err != nil {
+			return fmt.Errorf("TEST SETUP: failed to add custom resource scheme to framework: %v", err)
+		}
+	}
+
+	return nil
+}
+
+func (f *Framework) initializeMetricsTestResources() error {
+	if _, err := f.KubeClient.RbacV1().ClusterRoles().Create(context.TODO(), &v1.ClusterRole{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "co-metrics-client",
+		},
+		Rules: []v1.PolicyRule{
+			{
+				NonResourceURLs: []string{
+					"/metrics-co",
+				},
+				Verbs: []string{
+					"get",
+				},
+			},
+		},
+	}, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) {
+		return err
+	}
+
+	if _, err := f.KubeClient.RbacV1().ClusterRoleBindings().Create(context.TODO(), &v1.ClusterRoleBinding{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "co-metrics-client",
+		},
+		Subjects: []v1.Subject{
+			{
+				Kind:      "ServiceAccount",
+				Name:      "default",
+				Namespace: f.OperatorNamespace,
+			},
+		},
+		RoleRef: v1.RoleRef{
+			APIGroup: "rbac.authorization.k8s.io",
+			Kind:     "ClusterRole",
+			Name:     "co-metrics-client",
+		},
+	}, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) {
+		return err
+	}
+
+	if _, err := f.KubeClient.CoreV1().Secrets(f.OperatorNamespace).Create(context.TODO(), &core.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "metrics-token",
+			Namespace: f.OperatorNamespace,
+			Annotations: map[string]string{
+				"kubernetes.io/service-account.name": "default",
+			},
+		},
+		Type: "kubernetes.io/service-account-token",
+	}, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) {
+		return err
+	}
+	return nil
+}
+
+func (f *Framework) replaceNamespaceFromManifest() error {
+	log.Printf("updating manifest %s with namespace %s\n", *f.NamespacedManPath, f.OperatorNamespace)
+	if f.NamespacedManPath == nil {
+		return fmt.Errorf("no namespaced manifest given as test argument (operator-sdk might have changed)")
+	}
+	c, err := f.readFile(f.NamespacedManPath)
+	if err != nil {
+		return err
+	}
+
+	newContents := strings.Replace(string(c), "openshift-compliance", f.OperatorNamespace, -1)
+
+	// #nosec
+	err = os.WriteFile(*f.NamespacedManPath, []byte(newContents), 0644)
+	if err != nil {
+		return fmt.Errorf("error writing namespaced manifest file: %s", err)
+	}
+	return nil
+}
+
+func (f *Framework) waitForDeployment(name string, replicas int, retryInterval, timeout time.Duration) error {
+	err := wait.Poll(retryInterval, timeout, func() (done bool, err error) {
+		deployment, err := f.KubeClient.AppsV1().Deployments(f.OperatorNamespace).Get(context.TODO(), name, metav1.GetOptions{})
+		if err != nil {
+			if apierrors.IsNotFound(err) {
+				log.Printf("Waiting for availability of Deployment: %s in Namespace: %s \n", name, f.OperatorNamespace)
+				return false, nil
+			}
+			return false, err
+		}
+
+		if int(deployment.Status.AvailableReplicas) >= replicas {
+			return true, nil
+		}
+		log.Printf("Waiting for full availability of %s deployment (%d/%d)\n", name,
+			deployment.Status.AvailableReplicas, replicas)
+		return false, nil
+	})
+	if err != nil {
+		return err
+	}
+	log.Printf("Deployment available (%d/%d)\n", replicas, replicas)
+	return nil
+}
+
+func (f *Framework) ensureTestNamespaceExists() error {
+	// create namespace only if it doesn't already exist
+	_, err := f.KubeClient.CoreV1().Namespaces().Get(context.TODO(), f.OperatorNamespace, metav1.GetOptions{})
+	if apierrors.IsNotFound(err) {
+		ns := &core.Namespace{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: f.OperatorNamespace,
+				Labels: map[string]string{
+					psapi.EnforceLevelLabel:                          string(psapi.LevelPrivileged),
+					"security.openshift.io/scc.podSecurityLabelSync": "false",
+				},
+			},
+		}
+
+		log.Printf("creating namespace %s", f.OperatorNamespace)
+		_, err = f.KubeClient.CoreV1().Namespaces().Create(context.TODO(), ns, metav1.CreateOptions{})
+		if apierrors.IsAlreadyExists(err) {
+			return fmt.Errorf("namespace %s already exists: %w", f.OperatorNamespace, err)
+		} else if err != nil {
+			return err
+		}
+		return nil
+	} else if apierrors.IsAlreadyExists(err) {
+		log.Printf("using existing namespace %s", f.OperatorNamespace)
+		return nil
+	} else {
+		return nil
+	}
+
+}
+
+// waitForProfileBundleStatus will poll until the compliancescan that we're
+// lookingfor reaches a certain status, or until a timeout is reached.
+func (f *Framework) waitForProfileBundleStatus(name string) error {
+	pb := &compv1alpha1.ProfileBundle{}
+	var lastErr error
+	// retry and ignore errors until timeout
+	timeouterr := wait.Poll(retryInterval, timeout, func() (bool, error) {
+		lastErr = f.Client.Get(context.TODO(), types.NamespacedName{Name: name, Namespace: f.OperatorNamespace}, pb)
+		if lastErr != nil {
+			if apierrors.IsNotFound(lastErr) {
+				log.Printf("waiting for availability of %s ProfileBundle\n", name)
+				return false, nil
+			}
+			log.Printf("retrying due to error: %s\n", lastErr)
+			return false, nil
+		}
+
+		if pb.Status.DataStreamStatus == compv1alpha1.DataStreamValid {
+			return true, nil
+		}
+		log.Printf("waiting ProfileBundle %s to become %s (%s)\n", name, compv1alpha1.DataStreamValid, pb.Status.DataStreamStatus)
+		return false, nil
+	})
+	if timeouterr != nil {
+		return fmt.Errorf("ProfileBundle %s failed to reach state %s", name, compv1alpha1.DataStreamValid)
+	}
+	log.Printf("ProfileBundle ready (%s)\n", pb.Status.DataStreamStatus)
+	return nil
+}
+
+func (f *Framework) updateScanSettingsForDebug() error {
+	for _, ssName := range []string{"default", "default-auto-apply"} {
+		ss := &compv1alpha1.ScanSetting{}
+		sskey := types.NamespacedName{Name: ssName, Namespace: f.OperatorNamespace}
+		if err := f.Client.Get(context.TODO(), sskey, ss); err != nil {
+			return err
+		}
+
+		ssCopy := ss.DeepCopy()
+		ssCopy.Debug = true
+
+		if err := f.Client.Update(context.TODO(), ssCopy); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (f *Framework) ensureE2EScanSettings() error {
+	for _, ssName := range []string{"default", "default-auto-apply"} {
+		ss := &compv1alpha1.ScanSetting{}
+		sskey := types.NamespacedName{Name: ssName, Namespace: f.OperatorNamespace}
+		if err := f.Client.Get(context.TODO(), sskey, ss); err != nil {
+			return err
+		}
+
+		ssCopy := ss.DeepCopy()
+		ssCopy.ObjectMeta = metav1.ObjectMeta{
+			Name:      "e2e-" + ssName,
+			Namespace: f.OperatorNamespace,
+		}
+		ssCopy.Roles = []string{
+			testPoolName,
+		}
+		ssCopy.Debug = true
+
+		if err := f.Client.Create(context.TODO(), ssCopy, nil); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (f *Framework) deleteScanSettings(name string) error {
+	ss := &compv1alpha1.ScanSetting{}
+	sskey := types.NamespacedName{Name: name, Namespace: f.OperatorNamespace}
+	if err := f.Client.Get(context.TODO(), sskey, ss); err != nil {
+		return err
+	}
+
+	err := f.Client.Delete(context.TODO(), ss)
+	if err != nil {
+		return fmt.Errorf("failed to cleanup scan setting %s: %w", name, err)
+	}
+	return nil
+}