Implement testEnv for e2e tests

.github/workflows/e2e.yaml

@@ -29,14 +29,20 @@ jobs:
           version: v0.11.1
           image: kindest/node:v1.21.1@sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6
           config: .github/kind/config.yaml # disable KIND-net
+      - name: Setup envtest
+        uses: fluxcd/pkg/actions/envtest@main
+        with:
+          version: "1.21.x"
       - name: Setup Calico for network policy
         run: |
           kubectl apply -f https://docs.projectcalico.org/v3.16/manifests/calico.yaml
           kubectl -n kube-system set env daemonset/calico-node FELIX_IGNORELOOSERPF=true
       - name: Setup Kustomize
         uses: fluxcd/pkg//actions/kustomize@main
-      - name: Run test
+      - name: Run tests
         run: make test
+      - name: Run e2e tests
+        run: TEST_KUBECONFIG=$HOME/.kube/config make e2e
       - name: Check if working tree is dirty
         run: |
           if [[ $(git diff --stat) != '' ]]; then

Makefile

@@ -1,5 +1,6 @@
 VERSION?=$(shell grep 'VERSION' cmd/flux/main.go | awk '{ print $$4 }' | tr -d '"')
 EMBEDDED_MANIFESTS_TARGET=cmd/flux/manifests
+TEST_KUBECONFIG?=/tmp/flux-e2e-test-kubeconfig
 
 rwildcard=$(foreach d,$(wildcard $(addsuffix *,$(1))),$(call rwildcard,$(d)/,$(2)) $(filter $(subst *,%,$(2)),$(d)))
 
@@ -14,9 +15,41 @@ fmt:
 vet:
 	go vet ./...
 
+setup-envtest:
+ifeq (, $(shell which setup-envtest))
+	@{ \
+	set -e ;\
+	SETUP_ENVTEST_TMP_DIR=$$(mktemp -d) ;\
+	cd $$SETUP_ENVTEST_TMP_DIR ;\
+	go mod init tmp ;\
+	go get sigs.k8s.io/controller-runtime/tools/setup-envtest@latest ;\
+	rm -rf $$SETUP_ENVTEST_TMP_DIR ;\
+	}
+SETUP_ENVTEST=$(GOBIN)/setup-envtest
+else
+SETUP_ENVTEST=$(shell which setup-envtest)
+endif
+
+setup-kind:
+	kind create cluster --name=flux-e2e-test --kubeconfig=$(TEST_KUBECONFIG) --config=.github/kind/config.yaml
+	kubectl --kubeconfig=$(TEST_KUBECONFIG) apply -f https://docs.projectcalico.org/v3.16/manifests/calico.yaml
+	kubectl --kubeconfig=$(TEST_KUBECONFIG) -n kube-system set env daemonset/calico-node FELIX_IGNORELOOSERPF=true
+
+cleanup-kind:
+	kind delete cluster --name=flux-e2e-test
+	rm $(TEST_KUBECONFIG)
+
 test: $(EMBEDDED_MANIFESTS_TARGET) tidy fmt vet
 	go test ./... -coverprofile cover.out
 
+e2e: $(EMBEDDED_MANIFESTS_TARGET) tidy fmt vet
+	TEST_KUBECONFIG=$(TEST_KUBECONFIG) go test ./cmd/flux/... -coverprofile cover.out --tags=e2e -parallel=1
+
+test-with-kind: setup-envtest
+	make setup-kind
+	make e2e
+	make cleanup-kind
+
 $(EMBEDDED_MANIFESTS_TARGET): $(call rwildcard,manifests/,*.yaml *.json)
 	./manifests/scripts/bundle.sh
 

cmd/flux/install_test.go

@@ -0,0 +1,54 @@
+// +build e2e
+
+package main
+
+import (
+	"testing"
+	"time"
+)
+
+func TestInstallNoArgs(t *testing.T) {
+	cmd := cmdTestCase{
+		args:            "install",
+		wantError:       false,
+		testClusterMode: ExistingClusterMode,
+		goldenFile:      "testdata/install/install_no_args.golden",
+	}
+	cmd.runTestCmd(t)
+
+	testUninstallSilent(t)
+	time.Sleep(30 * time.Second)
+}
+
+func TestInstallExtraComponents(t *testing.T) {
+	cmd := cmdTestCase{
+		args:            "install --components-extra=image-reflector-controller,image-automation-controller",
+		wantError:       false,
+		testClusterMode: ExistingClusterMode,
+		goldenFile:      "testdata/install/install_extra_components.golden",
+	}
+	cmd.runTestCmd(t)
+
+	testUninstallSilentForExtraComponents(t)
+	time.Sleep(30 * time.Second)
+}
+
+func testUninstallSilent(t *testing.T) {
+	cmd := cmdTestCase{
+		args:            "uninstall -s",
+		wantError:       false,
+		testClusterMode: ExistingClusterMode,
+		goldenFile:      "testdata/uninstall/uninstall.golden",
+	}
+	cmd.runTestCmd(t)
+}
+
+func testUninstallSilentForExtraComponents(t *testing.T) {
+	cmd := cmdTestCase{
+		args:            "uninstall -s",
+		wantError:       false,
+		testClusterMode: ExistingClusterMode,
+		goldenFile:      "testdata/uninstall/uninstall_extra_components.golden",
+	}
+	cmd.runTestCmd(t)
+}

cmd/flux/main_test.go

@@ -4,17 +4,23 @@ import (
 	"bufio"
 	"bytes"
 	"context"
+	"fmt"
 	"io"
+	"io/ioutil"
 	"os"
+	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/fluxcd/flux2/internal/utils"
 	"github.com/google/go-cmp/cmp"
 	"github.com/mattn/go-shellwords"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	k8syaml "k8s.io/apimachinery/pkg/util/yaml"
+	"k8s.io/client-go/tools/clientcmd"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	fakeclient "sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"sigs.k8s.io/controller-runtime/pkg/envtest"
 )
 
 func TestMain(m *testing.M) {
@@ -49,50 +55,111 @@ func readYamlObjects(objectFile string) ([]client.Object, error) {
 }
 
 // A KubeManager that can create objects that are subject to a test.
-type fakeKubeManager struct {
-	fakeClient client.WithWatch
+type testEnvKubeManager struct {
+	client  client.WithWatch
+	testEnv *envtest.Environment
 }
 
-func (m *fakeKubeManager) NewClient(kubeconfig string, kubecontext string) (client.WithWatch, error) {
-	return m.fakeClient, nil
+func (m *testEnvKubeManager) NewClient(kubeconfig string, kubecontext string) (client.WithWatch, error) {
+	return m.client, nil
 }
 
-func (m *fakeKubeManager) CreateObjects(clientObjects []client.Object) error {
+func (m *testEnvKubeManager) CreateObjects(clientObjects []client.Object) error {
 	for _, obj := range clientObjects {
-		err := m.fakeClient.Create(context.Background(), obj)
+		err := m.client.Create(context.Background(), obj)
 		if err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func NewFakeKubeManager() *fakeKubeManager {
-	c := fakeclient.NewClientBuilder().WithScheme(utils.NewScheme()).Build()
-	return &fakeKubeManager{
-		fakeClient: c,
+func (m *testEnvKubeManager) Stop() error {
+	if m.testEnv == nil {
+		return fmt.Errorf("do nothing because testEnv is nil")
 	}
+	return m.testEnv.Stop()
 }
 
-// Run the command and return the captured output.
-func executeCommand(cmd string) (string, error) {
-	args, err := shellwords.Parse(cmd)
-	if err != nil {
-		return "", err
-	}
+func NewTestEnvKubeManager(testClusterMode TestClusterMode) (*testEnvKubeManager, error) {
+	switch testClusterMode {
+	case FakeClusterMode:
+		c := fakeclient.NewClientBuilder().WithScheme(utils.NewScheme()).Build()
+		return &testEnvKubeManager{
+			client: c,
+		}, nil
+	case TestEnvClusterMode:
+		useExistingCluster := false
+		testEnv := &envtest.Environment{
+			UseExistingCluster: &useExistingCluster,
+		}
+		cfg, err := testEnv.Start()
+		if err != nil {
+			return nil, err
+		}
+		user, err := testEnv.ControlPlane.AddUser(envtest.User{
+			Name:   "envtest-admin",
+			Groups: []string{"system:masters"},
+		}, nil)
+		if err != nil {
+			return nil, err
+		}
 
-	buf := new(bytes.Buffer)
+		kubeConfig, err := user.KubeConfig()
+		if err != nil {
+			return nil, err
+		}
 
-	rootCmd.SetOut(buf)
-	rootCmd.SetErr(buf)
-	rootCmd.SetArgs(args)
+		tmpFilename := filepath.Join("/tmp", "kubeconfig-"+time.Nanosecond.String())
+		ioutil.WriteFile(tmpFilename, kubeConfig, 0644)
+		rootArgs.kubeconfig = tmpFilename
+		k8sClient, err := client.NewWithWatch(cfg, client.Options{})
+		if err != nil {
+			return nil, err
+		}
+		return &testEnvKubeManager{
+			testEnv: testEnv,
+			client:  k8sClient,
+		}, nil
+	case ExistingClusterMode:
+		// TEST_KUBECONFIG is mandatory to prevent destroying a current cluster accidentally.
+		testKubeConfig := os.Getenv("TEST_KUBECONFIG")
+		if testKubeConfig == "" {
+			return nil, fmt.Errorf("environment variable TEST_KUBECONFIG is required to run tests against an existing cluster")
+		}
+		rootArgs.kubeconfig = testKubeConfig
 
-	_, err = rootCmd.ExecuteC()
-	result := buf.String()
+		useExistingCluster := true
+		config, err := clientcmd.BuildConfigFromFlags("", testKubeConfig)
+		testEnv := &envtest.Environment{
+			UseExistingCluster: &useExistingCluster,
+			Config:             config,
+		}
+		cfg, err := testEnv.Start()
+		if err != nil {
+			return nil, err
+		}
+		k8sClient, err := client.NewWithWatch(cfg, client.Options{})
+		if err != nil {
+			return nil, err
+		}
+		return &testEnvKubeManager{
+			testEnv: testEnv,
+			client:  k8sClient,
+		}, nil
+	}
 
-	return result, err
+	return nil, nil
 }
 
+type TestClusterMode int
+
+const (
+	FakeClusterMode = TestClusterMode(iota + 1)
+	TestEnvClusterMode
+	ExistingClusterMode
+)
+
 // Structure used for each test to load objects into kubernetes, run
 // commands and assert on the expected output.
 type cmdTestCase struct {
@@ -106,11 +173,20 @@ type cmdTestCase struct {
 	goldenFile string
 	// Filename that contains yaml objects to load into Kubernetes
 	objectFile string
+	// TestClusterMode to bootstrap and testing, default to Fake
+	testClusterMode TestClusterMode
 }
 
 func (cmd *cmdTestCase) runTestCmd(t *testing.T) {
-	km := NewFakeKubeManager()
-	rootCtx.kubeManager = km
+	km, err := NewTestEnvKubeManager(cmd.testClusterMode)
+	if err != nil {
+		t.Fatalf("Error creating kube manager: '%v'", err)
+	}
+
+	if km != nil {
+		rootCtx.kubeManager = km
+		defer km.Stop()
+	}
 
 	if cmd.objectFile != "" {
 		clientObjects, err := readYamlObjects(cmd.objectFile)
@@ -149,6 +225,27 @@ func (cmd *cmdTestCase) runTestCmd(t *testing.T) {
 	}
 }
 
+// Run the command and return the captured output.
+func executeCommand(cmd string) (string, error) {
+	args, err := shellwords.Parse(cmd)
+	if err != nil {
+		return "", err
+	}
+
+	buf := new(bytes.Buffer)
+
+	rootCmd.SetOut(buf)
+	rootCmd.SetErr(buf)
+	rootCmd.SetArgs(args)
+
+	logger.stderr = rootCmd.ErrOrStderr()
+
+	_, err = rootCmd.ExecuteC()
+	result := buf.String()
+
+	return result, err
+}
+
 func TestVersion(t *testing.T) {
 	cmd := cmdTestCase{
 		args:        "--version",

cmd/flux/testdata/install/install_extra_components.golden

@@ -0,0 +1,11 @@
+✚ generating manifests
+✔ manifests build completed
+► installing components in flux-system namespace
+◎ verifying installation
+✔ helm-controller: deployment ready
+✔ image-automation-controller: deployment ready
+✔ image-reflector-controller: deployment ready
+✔ kustomize-controller: deployment ready
+✔ notification-controller: deployment ready
+✔ source-controller: deployment ready
+✔ install finished

cmd/flux/testdata/install/install_no_args.golden

@@ -0,0 +1,9 @@
+✚ generating manifests
+✔ manifests build completed
+► installing components in flux-system namespace
+◎ verifying installation
+✔ helm-controller: deployment ready
+✔ kustomize-controller: deployment ready
+✔ notification-controller: deployment ready
+✔ source-controller: deployment ready
+✔ install finished

cmd/flux/testdata/uninstall/uninstall.golden

@@ -0,0 +1,31 @@
+► deleting components in flux-system namespace
+✔ Deployment/flux-system/helm-controller deleted 
+✔ Deployment/flux-system/kustomize-controller deleted 
+✔ Deployment/flux-system/notification-controller deleted 
+✔ Deployment/flux-system/source-controller deleted 
+✔ Service/flux-system/notification-controller deleted 
+✔ Service/flux-system/source-controller deleted 
+✔ Service/flux-system/webhook-receiver deleted 
+✔ NetworkPolicy/flux-system/allow-egress deleted 
+✔ NetworkPolicy/flux-system/allow-scraping deleted 
+✔ NetworkPolicy/flux-system/allow-webhooks deleted 
+✔ ServiceAccount/flux-system/helm-controller deleted 
+✔ ServiceAccount/flux-system/kustomize-controller deleted 
+✔ ServiceAccount/flux-system/notification-controller deleted 
+✔ ServiceAccount/flux-system/source-controller deleted 
+✔ ClusterRole/crd-controller-flux-system deleted 
+✔ ClusterRoleBinding/cluster-reconciler-flux-system deleted 
+✔ ClusterRoleBinding/crd-controller-flux-system deleted 
+► deleting toolkit.fluxcd.io finalizers in all namespaces
+► deleting toolkit.fluxcd.io custom resource definitions
+✔ CustomResourceDefinition/alerts.notification.toolkit.fluxcd.io deleted 
+✔ CustomResourceDefinition/buckets.source.toolkit.fluxcd.io deleted 
+✔ CustomResourceDefinition/gitrepositories.source.toolkit.fluxcd.io deleted 
+✔ CustomResourceDefinition/helmcharts.source.toolkit.fluxcd.io deleted 
+✔ CustomResourceDefinition/helmreleases.helm.toolkit.fluxcd.io deleted 
+✔ CustomResourceDefinition/helmrepositories.source.toolkit.fluxcd.io deleted 
+✔ CustomResourceDefinition/kustomizations.kustomize.toolkit.fluxcd.io deleted 
+✔ CustomResourceDefinition/providers.notification.toolkit.fluxcd.io deleted 
+✔ CustomResourceDefinition/receivers.notification.toolkit.fluxcd.io deleted 
+✔ Namespace/flux-system deleted 
+✔ uninstall finished