From 855da0cc55d3b428c980c28490ea095a29eeb278 Mon Sep 17 00:00:00 2001 From: Kenny Lee Sin Cheong Date: Tue, 17 Sep 2019 13:07:26 -0400 Subject: [PATCH 1/4] minikube.sh script --- scripts/minikube.sh | 52 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100755 scripts/minikube.sh diff --git a/scripts/minikube.sh b/scripts/minikube.sh new file mode 100755 index 0000000..ad9e7d7 --- /dev/null +++ b/scripts/minikube.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -e +set -o pipefail +set -u + +export MINIKUBE_VERSION=${MINIKUBE_VERSION:-v1.2.0} +export KUBERNETES_VERSION=${KUBERNETES_VERSION:-v1.15.2} +export KUBECONFIG=${KUBECONFIG:-$HOME/.kube/config} + + +kubectl > /dev/null || { curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/$KUBERNETES_VERSION/bin/linux/amd64/kubectl && \ + chmod +x kubectl && \ + sudo mv kubectl /usr/local/bin/ ;} + +minikube > /dev/null || { curl -Lo minikube https://storage.googleapis.com/minikube/releases/$MINIKUBE_VERSION/minikube-linux-amd64 && \ + chmod +x minikube && \ + sudo mv minikube /usr/local/bin/ ;} + + +minikube_create() { + minikube version | awk '{ print $3 }' | grep -q "$MINIKUBE_VERSION" || { echo "Wrong minikube version"; exit 1; } + + pgrep -f "[m]inikube" >/dev/null || minikube start --cpus 2 --memory 2048 --kubernetes-version="$KUBERNETES_VERSION" || { echo 'Cannot start minikube.'; exit 1; } + + kubectl version --output json | jq .serverVersion.gitVersion | grep -q "$KUBERNETES_VERSION" && \ + kubectl version --output json | jq .clientVersion.gitVersion | grep -q "$KUBERNETES_VERSION" || { echo "Kubectl version does not match k8s version"; exit 1; } + + kubectl config use-context minikube +} + +minikube_delete() { + minikube delete +} + +build_image() { + eval "$(minikube docker-env)" || { echo 'Cannot switch to minikube docker'; exit 1; } + make manager + docker build -f Dockerfile.e2e -t quay.io/quay/dba-operator:local . +} + + +if [ $1 == "create-minikube" ]; then + minikube_create +elif [ $1 == "delete-minikube" ]; then + minikube_delete +elif [ $1 == "build-image" ]; then + build_image +else + echo "Specify create-minikube, delete-minikube, or build-image" + exit 1 +fi From 7a04bbaf3fdb3d6c6857da476fcf93f94d403caf Mon Sep 17 00:00:00 2001 From: Kenny Lee Sin Cheong Date: Wed, 25 Sep 2019 13:48:10 -0400 Subject: [PATCH 2/4] kubebuilder markers for namespace scoped crd generation --- api/v1alpha1/databasemigration_types.go | 1 + api/v1alpha1/manageddatabase_types.go | 1 + 2 files changed, 2 insertions(+) diff --git a/api/v1alpha1/databasemigration_types.go b/api/v1alpha1/databasemigration_types.go index f16773d..ec91df1 100644 --- a/api/v1alpha1/databasemigration_types.go +++ b/api/v1alpha1/databasemigration_types.go @@ -43,6 +43,7 @@ type DatabaseMigrationStatus struct { // +kubebuilder:object:root=true // DatabaseMigration is the Schema for the databasemigrations API +// +kubebuilder:resource:scope="Namespaced" type DatabaseMigration struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` diff --git a/api/v1alpha1/manageddatabase_types.go b/api/v1alpha1/manageddatabase_types.go index eb0b111..27fd8b4 100644 --- a/api/v1alpha1/manageddatabase_types.go +++ b/api/v1alpha1/manageddatabase_types.go @@ -53,6 +53,7 @@ type ManagedDatabaseStatus struct { // +kubebuilder:object:root=true // ManagedDatabase is the Schema for the manageddatabases API +// +kubebuilder:resource:scope="Namespaced" type ManagedDatabase struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` From 7e061edffee9df64e446338af1dd919cbeb6e3d7 Mon Sep 17 00:00:00 2001 From: Kenny Lee Sin Cheong Date: Thu, 26 Sep 2019 10:47:10 -0400 Subject: [PATCH 3/4] Create CRDs from manifest --- Makefile | 15 ++++++++++++++- scripts/run-e2e-local.sh | 21 +++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) create mode 100755 scripts/run-e2e-local.sh diff --git a/Makefile b/Makefile index ca1e01e..ca50345 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,8 @@ # Image URL to use all building/pushing image targets -IMG ?= controller:latest +IMG ?= ${REPO}:${TAG} +DB_IMG ?= mysql/mysqlserver:latest + # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) CRD_OPTIONS ?= "crd:trivialVersions=true" @@ -17,6 +19,17 @@ all: manager test: generate fmt vet manifests go test ./api/... ./controllers/... -coverprofile cover.out +# Run go test: e2e tests +.PHONY: test-e2e-local +test-e2e-local: KUBECONFIG?=$(HOME)/.kube/config +test-e2e-local: + go test ./test/e2e/... --kubeconfig=${KUBECONFIG} --operator-image=${REPO}:${TAG} --db-image=${DB_IMG} + +# Run e2e tests: Setup/teardown minikube, build image and go test +.PHONY: run-e2e-local +run-e2e-local: + REPO=${REPO} TAG=${TAG} ./scripts/run-e2e-local.sh + # Build manager binary manager: generate fmt vet go build -o bin/manager main.go diff --git a/scripts/run-e2e-local.sh b/scripts/run-e2e-local.sh new file mode 100755 index 0000000..aa992bf --- /dev/null +++ b/scripts/run-e2e-local.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +set -e +set -o pipefail +set -u +set -x + + +export REPO=${REPO:-quay.io/quay/dba-operator} +export TAG=${TAG:-$(git rev-parse --short HEAD)} + +SCRIPTS_DIR=$(dirname "${BASH_SOURCE[0]}") + + +"${SCRIPTS_DIR}"/minikube.sh create-minikube + +"${SCRIPTS_DIR}"/minikube.sh build-image + +make install # Install the genereated CRDs manifest in the cluster +make test-e2e-local # Run the tests + +"${SCRIPTS_DIR}"/minikube.sh delete-minikube From 85dfd70aef91445c9ae736c56efbc4cacf6670bb Mon Sep 17 00:00:00 2001 From: Kenny Lee Sin Cheong Date: Fri, 20 Sep 2019 17:15:03 -0400 Subject: [PATCH 4/4] Scaffolding for e2e tests --- Makefile | 4 +- scripts/minikube.sh | 10 +- scripts/run-e2e-local.sh | 2 +- test/e2e/framework_test.go | 138 ++++++++++++++++++++++++++++ test/e2e/migration_test.go | 22 +++++ test/e2e/setup_test.go | 37 ++++++++ test/e2e/util_test.go | 183 +++++++++++++++++++++++++++++++++++++ 7 files changed, 392 insertions(+), 4 deletions(-) create mode 100644 test/e2e/framework_test.go create mode 100644 test/e2e/migration_test.go create mode 100644 test/e2e/setup_test.go create mode 100644 test/e2e/util_test.go diff --git a/Makefile b/Makefile index ca50345..a95b29a 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,9 @@ +REPO?=quay.io/quay/dba-operator +TAG?=$(shell git rev-parse --short HEAD) # Image URL to use all building/pushing image targets IMG ?= ${REPO}:${TAG} -DB_IMG ?= mysql/mysqlserver:latest +DB_IMG ?= mysql/mysql-server:latest # Produce CRDs that work back to Kubernetes 1.11 (no version conversion) CRD_OPTIONS ?= "crd:trivialVersions=true" diff --git a/scripts/minikube.sh b/scripts/minikube.sh index ad9e7d7..0d55f19 100755 --- a/scripts/minikube.sh +++ b/scripts/minikube.sh @@ -8,6 +8,9 @@ export MINIKUBE_VERSION=${MINIKUBE_VERSION:-v1.2.0} export KUBERNETES_VERSION=${KUBERNETES_VERSION:-v1.15.2} export KUBECONFIG=${KUBECONFIG:-$HOME/.kube/config} +export REPO=${REPO:-quay.io/quay/dba-operator} +export TAG=${TAG:-$(git rev-parse --short HEAD)} + kubectl > /dev/null || { curl -Lo kubectl https://storage.googleapis.com/kubernetes-release/release/$KUBERNETES_VERSION/bin/linux/amd64/kubectl && \ chmod +x kubectl && \ @@ -18,6 +21,7 @@ minikube > /dev/null || { curl -Lo minikube https://storage.googleapis.com/minik sudo mv minikube /usr/local/bin/ ;} +# Create a minikube instance minikube_create() { minikube version | awk '{ print $3 }' | grep -q "$MINIKUBE_VERSION" || { echo "Wrong minikube version"; exit 1; } @@ -29,14 +33,16 @@ minikube_create() { kubectl config use-context minikube } +# Delete the minikube instance minikube_delete() { minikube delete } +# Build the operator image in the minikube's docker context build_image() { eval "$(minikube docker-env)" || { echo 'Cannot switch to minikube docker'; exit 1; } - make manager - docker build -f Dockerfile.e2e -t quay.io/quay/dba-operator:local . + GOOS=linux GOARCH=amd64 make manager + docker build -f Dockerfile.e2e -t $REPO:$TAG . } diff --git a/scripts/run-e2e-local.sh b/scripts/run-e2e-local.sh index aa992bf..4801d86 100755 --- a/scripts/run-e2e-local.sh +++ b/scripts/run-e2e-local.sh @@ -15,7 +15,7 @@ SCRIPTS_DIR=$(dirname "${BASH_SOURCE[0]}") "${SCRIPTS_DIR}"/minikube.sh build-image -make install # Install the genereated CRDs manifest in the cluster +make install || true # Install the genereated CRDs manifest in the cluster make test-e2e-local # Run the tests "${SCRIPTS_DIR}"/minikube.sh delete-minikube diff --git a/test/e2e/framework_test.go b/test/e2e/framework_test.go new file mode 100644 index 0000000..5e7fcf3 --- /dev/null +++ b/test/e2e/framework_test.go @@ -0,0 +1,138 @@ +package e2e + +import ( + "context" + "fmt" + "strconv" + "strings" + "testing" + "time" + + "golang.org/x/sync/errgroup" + + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/config" + + "github.com/app-sre/dba-operator/pkg/dbadmin" +) + +var () + +type finalizerFn func() error + +type TestFramework struct { + client.Client + admin dbadmin.DbAdmin +} + +func New(operatorImage string) (*TestFramework, error) { + c, err := client.New(config.GetConfigOrDie(), client.Options{}) + if err != nil { + return nil, nil + } + + return &TestFramework{ + Client: c, + }, nil +} + +// Create a new test context with the test's name and the current time as its ID. +// The ID will be used for creating objects, such as creating a namespace for the test. +func (tf *TestFramework) NewTestCtx(t *testing.T) TestCtx { + prefix := strings.TrimPrefix( + strings.ReplaceAll( + strings.ToLower(t.Name()), + "/", + "-", + ), + "test", + ) + + id := prefix + "-" + strconv.FormatInt(time.Now().Unix(), 36) + return TestCtx{ + ID: id, + } +} + +type TestCtx struct { + ID string + cleanupFns []finalizerFn + ctx context.Context +} + +func (tctx *TestCtx) Cleanup(t *testing.T) { + var eg errgroup.Group + + for i := len(tctx.cleanupFns) - 1; i >= 0; i-- { + eg.Go(tctx.cleanupFns[i]) + } + + if err := eg.Wait(); err != nil { + t.Fatal(err) + } +} + +func (tctx *TestCtx) AddFinalizerFn(fn finalizerFn) { + tctx.cleanupFns = append(tctx.cleanupFns, fn) +} + +func (tctx *TestCtx) CreateNamespace(t *testing.T, c client.Client) string { + name := tctx.ID + if err := CreateNamespace(tctx.ctx, c, name); err != nil { + t.Fatal(err) + } + + namespaceCleanupFn := func() error { + return DeleteNamespace(tctx.ctx, c, name) + } + + tctx.AddFinalizerFn(namespaceCleanupFn) + + return name +} + +func (tctx *TestCtx) CreateDbaOperator(c client.Client, operatorImage string) error { + deployment, err := MakeDeployment("../../deploy/dba-operator.yaml") + if err != nil { + return err + } + + if operatorImage != "" { + repoTag := strings.Split(operatorImage, ":") + if len(repoTag) != 2 { + return fmt.Errorf("invalid operator image '%s'", operatorImage) + } + + deployment.Spec.Template.Spec.Containers[0].Image = operatorImage + } + + err = CreateDeployment(tctx.ctx, c, tctx.ID, deployment) + if err != nil { + return err + } + + return nil +} + +func (tctx *TestCtx) CreateDB(c client.Client, image string) error { + deployment, err := MakeDeployment("../../deploy/debug.yaml") + if err != nil { + return err + } + + if image != "" { + repoTag := strings.Split(image, ":") + if len(repoTag) != 2 { + return fmt.Errorf("invalid operator image '%s'", image) + } + + deployment.Spec.Template.Spec.Containers[0].Image = image + } + + err = CreateDeployment(tctx.ctx, c, tctx.ID, deployment) + if err != nil { + return err + } + + return nil +} diff --git a/test/e2e/migration_test.go b/test/e2e/migration_test.go new file mode 100644 index 0000000..e0ef49c --- /dev/null +++ b/test/e2e/migration_test.go @@ -0,0 +1,22 @@ +package e2e + +import "testing" + +func TestBasicMigration(t *testing.T) { + ctx := testFramework.NewTestCtx(t) + // defer ctx.Cleanup(t) + + ctx.CreateNamespace(t, testFramework.Client) + + err := ctx.CreateDB(testFramework.Client, *dbImage) + if err != nil { + t.Fatal(err) + } + + err = ctx.CreateDbaOperator(testFramework.Client, *operatorImage) + if err != nil { + t.Fatal(err) + } + + // TODO +} diff --git a/test/e2e/setup_test.go b/test/e2e/setup_test.go new file mode 100644 index 0000000..920f56e --- /dev/null +++ b/test/e2e/setup_test.go @@ -0,0 +1,37 @@ +package e2e + +import ( + "flag" + "log" + "os" + "testing" +) + +var ( + testFramework *TestFramework + operatorImage *string + dbImage *string +) + +func TestMain(m *testing.M) { + operatorImage = flag.String( + "operator-image", + "", + "operator image, e.g. quay.io/quay/dba-operator:v1.0.0", + ) + dbImage = flag.String( + "db-image", + "", + "db image, e.g. mysql/mysql-server:latest", + ) + flag.Parse() + + var err error + if testFramework, err = New(*operatorImage); err != nil { + log.Printf("failed to setup testFramework: %s", err) + os.Exit(1) + } + + exitCode := m.Run() + os.Exit(exitCode) +} diff --git a/test/e2e/util_test.go b/test/e2e/util_test.go new file mode 100644 index 0000000..b367c1a --- /dev/null +++ b/test/e2e/util_test.go @@ -0,0 +1,183 @@ +package e2e + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/pkg/errors" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/apimachinery/pkg/util/yaml" + + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var ( + ErrPodReadyConditionNotFound = errors.New("pod's ready condition not found") + ErrPodCompletedWithFailure = errors.New("pod completed: Failed") + ErrPodCompletedWithSuccess = errors.New("pod completed: Succeeded") +) + +func CreateNamespace(ctx context.Context, c client.Client, name string) error { + namespace := &corev1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "namespace", + Name: name, + }, + } + + if err := c.Create(ctx, namespace); err != nil { + return fmt.Errorf("failed to create namespace '%s': (%w)", namespace, err) + } + + return nil +} + +func DeleteNamespace(ctx context.Context, c client.Client, name string) error { + namespace := &corev1.Namespace{} + if err := c.Get(ctx, client.ObjectKey{Name: name}, namespace); err != nil { + return fmt.Errorf("failed to get namespace '%s': (%w)", namespace, err) + } + + if err := c.Delete(ctx, namespace); err != nil { + return fmt.Errorf("failed to delete namespace '%s': (%w)", namespace, err) + } + + return nil +} + +func MakeDeployment(pathToYaml string) (*appsv1.Deployment, error) { + manifest, err := AbsPathToFile(pathToYaml) + if err != nil { + return nil, err + } + + deployment := appsv1.Deployment{} + if err := yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&deployment); err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("failed to decode file %s", pathToYaml)) + } + + return &deployment, nil +} + +func CreateDeployment(ctx context.Context, c client.Client, namespace string, deployment *appsv1.Deployment) error { + deployment.Namespace = namespace + err := c.Create(ctx, deployment) + if err != nil { + return fmt.Errorf("failed to create deployment %s: %w", deployment.Name, err) + } + + return nil +} + +func MakeService(pathToYaml string) (*corev1.Service, error) { + manifest, err := AbsPathToFile(pathToYaml) + if err != nil { + return nil, err + } + + service := corev1.Service{} + if err := yaml.NewYAMLOrJSONDecoder(manifest, 100).Decode(&service); err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("failed to decode file %s", pathToYaml)) + } + + return &service, nil +} + +func CreateService(ctx context.Context, c client.Client, namespace string, service *corev1.Service) error { + service.Namespace = namespace + err := c.Create(ctx, service) + if err != nil { + return fmt.Errorf("failed to create deployment %s: %w", service.Name, err) + } + + return nil +} + +func WaitForServiceReady(ctx context.Context, c client.Client, timeout time.Duration, namespace, servicename string) error { + return wait.Poll(time.Second, timeout, func() (bool, error) { + service := &corev1.Service{} + if err := c.Get(ctx, client.ObjectKey{Namespace: namespace, Name: servicename}, service); err != nil { + return false, fmt.Errorf("failed to get service %s: (%w)", namespace+"/"+servicename, err) + } + + if service.Spec.Type == corev1.ServiceTypeExternalName { + return true, nil + } + + if service.Spec.ClusterIP != corev1.ClusterIPNone && service.Spec.ClusterIP == "" { + return false, nil + } + + if service.Spec.Type == corev1.ServiceTypeLoadBalancer && service.Status.LoadBalancer.Ingress == nil { + return false, nil + } + + return true, nil + }) +} + +func WaitForPodsReady(ctx context.Context, c client.Client, timeout time.Duration, namespace string, replicas int, opts *client.ListOptions) error { + return wait.Poll(time.Second, timeout, func() (bool, error) { + podList := &corev1.PodList{} + err := c.List(ctx, podList, opts) + if err != nil { + return false, fmt.Errorf("failed to list pods in namespace %s: %w", namespace, err) + } + + runningAndReady := 0 + for _, pod := range podList.Items { + isRunningAndReady, err := PodRunningAndReady(pod) + if err != nil { + return false, err + } + + if isRunningAndReady { + runningAndReady++ + } + } + + if replicas == runningAndReady { + return true, nil + } + + return false, nil + }) +} + +func PodRunningAndReady(pod corev1.Pod) (bool, error) { + switch pod.Status.Phase { + case corev1.PodFailed: + return false, fmt.Errorf("pod completed: (failed): %s", pod.Name+pod.Namespace) + case corev1.PodSucceeded: + return false, fmt.Errorf("pod completed: (failed): %s", pod.Name+pod.Namespace) + case corev1.PodRunning: + for _, cond := range pod.Status.Conditions { + if cond.Type != corev1.PodReady { + continue + } + return cond.Status == corev1.ConditionTrue, nil + } + return false, fmt.Errorf("pod's ready condition not found: %s", pod.Name+pod.Namespace) + } + return false, nil +} + +func AbsPathToFile(relativPath string) (*os.File, error) { + path, err := filepath.Abs(relativPath) + if err != nil { + return nil, fmt.Errorf("failed generate absolute file path of %s: (%w)", relativPath, err) + } + + manifest, err := os.Open(path) + if err != nil { + return nil, fmt.Errorf("failed to open file %s: (%w)", path, err) + } + + return manifest, nil +}