OCS Functional Test Suite

Makefile

@@ -65,7 +65,11 @@ cluster-clean:
 	@echo "Removing ocs install from cluster"
 	REGISTRY_NAMESPACE=$(REGISTRY_NAMESPACE) IMAGE_TAG=$(IMAGE_TAG) IMAGE_REGISTRY=$(IMAGE_REGISTRY) ./hack/cluster-clean.sh
 
-functest:
+build-functest:
+	@echo "Building functional tests"
+	hack/build-functest.sh
+
+functest: build-functest
 	@echo "Running functional test suite"
 	hack/functest.sh
 

README.md

@@ -224,3 +224,60 @@ You may modify or delete any of the operator's initial data. To reset and
 restore that data to its initial state, delete the OCSInitialization resource. It
 will be recreated, and all associated resources will be either recreated or
 restored to their original state.
+
+## Functional Tests
+
+Our functional test suite uses the [ginkgo](https://onsi.github.io/ginkgo/) testing framework.
+
+**Prerequisites for running Functional Tests**
+- ocs must already be installed
+- KUBECONFIG env var must be set
+
+**Running functional test**
+
+```make functest```
+
+Below is some sample output of what to expect.
+
+```
+Building functional tests
+hack/build-functest.sh
+GINKO binary found at /home/dvossel/go/bin/ginkgo
+Compiling functests...
+    compiled functests.test
+Running functional test suite
+hack/functest.sh
+Running Functional Test Suite
+Running Suite: Tests Suite
+==========================
+Random Seed: 1568299067
+Will run 1 of 1 specs
+
+•
+Ran 1 of 1 Specs in 7.961 seconds
+SUCCESS! -- 1 Passed | 0 Failed | 0 Pending | 0 Skipped
+PASS
+```
+
+**Developing Functional Tests**
+
+All the functional test code lives in the ```functests/``` directory. For an
+example of how a functional test is structured, look at the ```functests/pvc_creation_test.go```
+file.
+
+The tests themselves should invoke simple to understand steps. Put any complex
+logic into separate helper files in the functests/ directory so test flows are
+easy to follow.
+
+**Running a single test**
+When developing a test, it's common to just want to run a single functional test
+rather than the whole suite. This can be done using ginkgo's "focus" feature.
+
+All you have to do is put a ```F``` in front of the tests declaration to force
+only that test to run. So, if you have an iteration like ```It("some test")```
+defined, you just need to set that to ```FIt("some test")``` to force the test
+suite to only execute that single test.
+
+Make sure to remove the focus from your test before creating the pull request.
+Otherwise the test suite will fail in CI.
+

deploy/olm-catalog/ocs-operator/0.0.1/ocs-operator.v0.0.1.clusterserviceversion.yaml

@@ -910,6 +910,8 @@ spec:
                   value: noobaa/noobaa-core:5
                 - name: NOOBAA_MONGODB_IMAGE
                   value: centos/mongodb-36-centos7
+                - name: MON_COUNT_OVERRIDE
+                  value: "3"
                 image: quay.io/ocs-dev/ocs-operator:latest
                 imagePullPolicy: Always
                 name: ocs-operator

functests/common.go

@@ -0,0 +1,58 @@
+package functests
+
+import (
+	"github.com/onsi/gomega"
+
+	"fmt"
+	"time"
+
+	k8sv1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/rand"
+)
+
+// WaitForPVCBound waits for a pvc with a given name and namespace to reach BOUND phase
+func (t *TestClient) WaitForPVCBound(pvcName string, pvcNamespace string) {
+	gomega.Eventually(func() error {
+		pvc, err := t.k8sClient.CoreV1().PersistentVolumeClaims(pvcNamespace).Get(pvcName, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		if pvc.Status.Phase == k8sv1.ClaimBound {
+			return nil
+		}
+		return fmt.Errorf("Waiting on pvc %s/%s to reach bound state when it is currently %s", pvcNamespace, pvcName, pvc.Status.Phase)
+	}, 200*time.Second, 1*time.Second).ShouldNot(gomega.HaveOccurred())
+}
+
+// GetRandomPVC returns a pvc with a randomized name
+func GetRandomPVC(storageClass string, quantity string) *k8sv1.PersistentVolumeClaim {
+	storageQuantity, err := resource.ParseQuantity(quantity)
+	gomega.Expect(err).To(gomega.BeNil())
+
+	randomName := "test-pvc-" + rand.String(12)
+
+	pvc := &k8sv1.PersistentVolumeClaim{
+		TypeMeta: metav1.TypeMeta{
+			APIVersion: "v1",
+			Kind:       "PersistentVolumeClaim",
+		},
+		ObjectMeta: metav1.ObjectMeta{
+			Name: randomName,
+		},
+		Spec: k8sv1.PersistentVolumeClaimSpec{
+			StorageClassName: &storageClass,
+			AccessModes:      []k8sv1.PersistentVolumeAccessMode{k8sv1.ReadWriteOnce},
+
+			Resources: k8sv1.ResourceRequirements{
+				Requests: k8sv1.ResourceList{
+					"storage": storageQuantity,
+				},
+			},
+		},
+	}
+
+	return pvc
+}

functests/config.go

@@ -0,0 +1,107 @@
+package functests
+
+import (
+	"os"
+
+	"github.com/onsi/ginkgo"
+	"github.com/onsi/gomega"
+
+	ocsv1 "github.com/openshift/ocs-operator/pkg/apis/ocs/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/kubernetes/scheme"
+	"k8s.io/client-go/rest"
+	restclient "k8s.io/client-go/rest"
+	"k8s.io/client-go/tools/clientcmd"
+)
+
+// InstallNamespace is the namespace ocs is installed into
+const InstallNamespace = "openshift-storage"
+
+// TestNamespace is the namespace we run all the tests in.
+const TestNamespace = "ocs-functest"
+
+// TestStorageCluster is the name of the storage cluster the test suite installs
+const TestStorageCluster = "test-storagecluster"
+
+// StorageClassRBD is the name of the ceph rbd storage class the test suite installs
+const StorageClassRBD = TestStorageCluster + "-ceph-rbd"
+
+// MinOSDsCount represents the minimum number of OSDs required for this testsuite to run.
+const MinOSDsCount = 2
+
+var namespaces = []string{InstallNamespace, TestNamespace}
+
+func init() {
+	ocsv1.SchemeBuilder.AddToScheme(scheme.Scheme)
+}
+
+// ShouldSkip will skip the functional test if the suite isn't invoked by our make target
+// This prevents the unit tests from failing when `go test -v ./...` is invoked
+func ShouldSkip() {
+	defer ginkgo.GinkgoRecover()
+	if !shouldExecute() {
+		ginkgo.Skip("Only executed as part of 'make functest'")
+	}
+}
+
+func shouldExecute() bool {
+	// We use this to keep the functional tests from failing
+	// when the unit tests are being called.
+	shouldExecute := os.Getenv("OCS_EXECUTE_FUNC_TESTS")
+	if shouldExecute == "" {
+		return false
+	}
+	return true
+
+}
+
+// GetK8sClient is the function used to retrieve the kubernetes client
+func (t *TestClient) GetK8sClient() *kubernetes.Clientset {
+	return t.k8sClient
+}
+
+// TestClient is a util tool used by the functional tests
+type TestClient struct {
+	k8sClient      *kubernetes.Clientset
+	restClient     *rest.RESTClient
+	parameterCodec runtime.ParameterCodec
+}
+
+// NewTestClient is the way to create a TestClient struct
+func NewTestClient() *TestClient {
+	codecs := serializer.NewCodecFactory(scheme.Scheme)
+	parameterCodec := runtime.NewParameterCodec(scheme.Scheme)
+
+	kubeconfig := os.Getenv("KUBECONFIG")
+	gomega.Expect(kubeconfig).ToNot(gomega.Equal(""))
+
+	// K8s Core api client
+	config, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
+	gomega.Expect(err).To(gomega.BeNil())
+	config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs}
+	config.APIPath = "/apis"
+	config.ContentType = runtime.ContentTypeJSON
+	k8sClient, err := kubernetes.NewForConfig(config)
+	gomega.Expect(err).To(gomega.BeNil())
+
+	// ocs Operator rest client
+	ocsConfig, err := clientcmd.BuildConfigFromFlags("", kubeconfig)
+	gomega.Expect(err).To(gomega.BeNil())
+	ocsConfig.GroupVersion = &ocsv1.SchemeGroupVersion
+	ocsConfig.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: codecs}
+	ocsConfig.APIPath = "/apis"
+	ocsConfig.ContentType = runtime.ContentTypeJSON
+	if ocsConfig.UserAgent == "" {
+		ocsConfig.UserAgent = restclient.DefaultKubernetesUserAgent()
+	}
+	restClient, err := rest.RESTClientFor(ocsConfig)
+	gomega.Expect(err).To(gomega.BeNil())
+
+	return &TestClient{
+		k8sClient:      k8sClient,
+		restClient:     restClient,
+		parameterCodec: parameterCodec,
+	}
+}

functests/functests.go

@@ -0,0 +1 @@
+package functests

functests/functests_suite_test.go

@@ -0,0 +1,23 @@
+package functests_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	tests "github.com/openshift/ocs-operator/functests"
+)
+
+func TestTests(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Tests Suite")
+}
+
+var _ = BeforeSuite(func() {
+	tests.BeforeTestSuiteSetup()
+})
+
+var _ = AfterSuite(func() {
+	tests.AfterTestSuiteCleanup()
+})

functests/pvc_creation_test.go

@@ -0,0 +1,58 @@
+package functests_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	tests "github.com/openshift/ocs-operator/functests"
+
+	k8sv1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+var _ = Describe("PVC Creation", func() {
+
+	BeforeEach(func() {
+		RegisterFailHandler(Fail)
+		tests.ShouldSkip()
+	})
+
+	Describe("rbd", func() {
+		var testClient *tests.TestClient
+		var k8sClient *kubernetes.Clientset
+		var pvc *k8sv1.PersistentVolumeClaim
+		var namespace string
+
+		BeforeEach(func() {
+			namespace = tests.TestNamespace
+			pvc = tests.GetRandomPVC(tests.StorageClassRBD, "1Gi")
+
+			testClient = tests.NewTestClient()
+			k8sClient = testClient.GetK8sClient()
+		})
+
+		AfterEach(func() {
+			err := k8sClient.CoreV1().PersistentVolumeClaims(namespace).Delete(pvc.Name, &metav1.DeleteOptions{})
+			if err != nil && !errors.IsNotFound(err) {
+				Expect(err).To(BeNil())
+			}
+		})
+
+		Context("create pvc", func() {
+			It("and verify bound status", func() {
+				By("Creating PVC")
+				_, err := k8sClient.CoreV1().PersistentVolumeClaims(namespace).Create(pvc)
+				Expect(err).To(BeNil())
+
+				By("Verifying PVC reaches BOUND phase")
+				testClient.WaitForPVCBound(pvc.Name, namespace)
+
+				By("Deleting PVC")
+				err = k8sClient.CoreV1().PersistentVolumeClaims(namespace).Delete(pvc.Name, &metav1.DeleteOptions{})
+				Expect(err).To(BeNil())
+			})
+		})
+	})
+})