Merge pull request #362 from tinakurian/338-cleanup-workspace-resources-on-failure

cleaning up workspace on failure

Author: tinakurian

controllers/workspace/devworkspace_controller.go

@@ -171,6 +171,11 @@ func (r *DevWorkspaceReconciler) Reconcile(req ctrl.Request) (reconcileResult ct
 	flattenedWorkspace, err := flatten.ResolveDevWorkspace(&workspace.Spec.Template, flattenHelpers)
 	if err != nil {
 		reqLogger.Info("DevWorkspace start failed")
+		// Clean up cluster deployment
+		err := provision.ScaleDeploymentToZero(workspace, r.Client)
+		if err != nil {
+			return reconcile.Result{}, err
+		}
 		reconcileStatus.Phase = devworkspace.WorkspaceStatusFailed
 		// TODO: Handle error more elegantly
 		reconcileStatus.Conditions[devworkspace.WorkspaceFailedStart] = fmt.Sprintf("Error processing devfile: %s", err)
@@ -181,6 +186,11 @@ func (r *DevWorkspaceReconciler) Reconcile(req ctrl.Request) (reconcileResult ct
 	storageProvisioner, err := storage.GetProvisioner(workspace)
 	if err != nil {
 		reqLogger.Info("DevWorkspace start failed")
+		// Clean up cluster deployment
+		err := provision.ScaleDeploymentToZero(workspace, r.Client)
+		if err != nil {
+			return reconcile.Result{}, err
+		}
 		reconcileStatus.Phase = devworkspace.WorkspaceStatusFailed
 		reconcileStatus.Conditions[devworkspace.WorkspaceFailedStart] = fmt.Sprintf("Error provisioning storage: %s", err)
 		return reconcile.Result{}, nil
@@ -197,6 +207,11 @@ func (r *DevWorkspaceReconciler) Reconcile(req ctrl.Request) (reconcileResult ct
 	devfilePodAdditions, err := containerlib.GetKubeContainersFromDevfile(&workspace.Spec.Template)
 	if err != nil {
 		reqLogger.Info("DevWorkspace start failed")
+		// Clean up cluster deployment
+		err := provision.ScaleDeploymentToZero(workspace, r.Client)
+		if err != nil {
+			return reconcile.Result{}, err
+		}
 		reconcileStatus.Phase = devworkspace.WorkspaceStatusFailed
 		reconcileStatus.Conditions[devworkspace.WorkspaceFailedStart] = fmt.Sprintf("Error processing devfile: %s", err)
 		return reconcile.Result{}, nil
@@ -210,6 +225,11 @@ func (r *DevWorkspaceReconciler) Reconcile(req ctrl.Request) (reconcileResult ct
 			return reconcile.Result{Requeue: true, RequeueAfter: storageErr.RequeueAfter}, nil
 		case *storage.ProvisioningError:
 			reqLogger.Info(fmt.Sprintf("DevWorkspace start failed: %s", storageErr))
+			// Clean up cluster deployment
+			err := provision.ScaleDeploymentToZero(workspace, r.Client)
+			if err != nil {
+				return reconcile.Result{}, err
+			}
 			reconcileStatus.Phase = devworkspace.WorkspaceStatusFailed
 			reconcileStatus.Conditions[devworkspace.WorkspaceFailedStart] = fmt.Sprintf("Error provisioning storage: %s", storageErr)
 			return reconcile.Result{}, nil
@@ -234,6 +254,11 @@ func (r *DevWorkspaceReconciler) Reconcile(req ctrl.Request) (reconcileResult ct
 	if !routingStatus.Continue {
 		if routingStatus.FailStartup {
 			reqLogger.Info("DevWorkspace start failed")
+			// Clean up cluster deployment
+			err := provision.ScaleDeploymentToZero(workspace, r.Client)
+			if err != nil {
+				return reconcile.Result{}, err
+			}
 			reconcileStatus.Phase = devworkspace.WorkspaceStatusFailed
 			// TODO: Propagate failure reason from devWorkspaceRouting
 			reconcileStatus.Conditions[devworkspace.WorkspaceFailedStart] = "Failed to install network objects required for devworkspace"
@@ -265,6 +290,11 @@ func (r *DevWorkspaceReconciler) Reconcile(req ctrl.Request) (reconcileResult ct
 			return reconcile.Result{Requeue: true, RequeueAfter: provisionErr.RequeueAfter}, nil
 		case *metadata.ProvisioningError:
 			reqLogger.Info(fmt.Sprintf("DevWorkspace start failed: %s", provisionErr))
+			// Clean up cluster deployment
+			err := provision.ScaleDeploymentToZero(workspace, r.Client)
+			if err != nil {
+				return reconcile.Result{}, err
+			}
 			reconcileStatus.Phase = devworkspace.WorkspaceStatusFailed
 			reconcileStatus.Conditions[devworkspace.WorkspaceFailedStart] = fmt.Sprintf("Error provisioning metadata configmap: %s", provisionErr)
 			return reconcile.Result{}, nil
@@ -307,6 +337,12 @@ func (r *DevWorkspaceReconciler) Reconcile(req ctrl.Request) (reconcileResult ct
 	if !deploymentStatus.Continue {
 		if deploymentStatus.FailStartup {
 			reqLogger.Info("Workspace start failed")
+			// Clean up cluster deployment
+			err := provision.ScaleDeploymentToZero(workspace, r.Client)
+			if err != nil {
+				return reconcile.Result{}, err
+			}
+
 			reconcileStatus.Phase = devworkspace.WorkspaceStatusFailed
 			reconcileStatus.Conditions[devworkspace.WorkspaceFailedStart] = fmt.Sprintf("Devworkspace spec is invalid: %s", deploymentStatus.Err)
 			return reconcile.Result{}, deploymentStatus.Err

controllers/workspace/provision/deployment.go

@@ -18,12 +18,14 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/devfile/devworkspace-operator/apis/controller/v1alpha1"
+	"github.com/devfile/devworkspace-operator/controllers/workspace/env"
 	maputils "github.com/devfile/devworkspace-operator/internal/map"
+	"github.com/devfile/devworkspace-operator/pkg/common"
+	"github.com/devfile/devworkspace-operator/pkg/config"
 	"github.com/devfile/devworkspace-operator/pkg/constants"
 	"github.com/devfile/devworkspace-operator/pkg/infrastructure"
 
-	"github.com/devfile/devworkspace-operator/pkg/common"
-
 	devworkspace "github.com/devfile/api/v2/pkg/apis/workspaces/v1alpha2"
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
@@ -35,10 +37,6 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	runtimeClient "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
-
-	"github.com/devfile/devworkspace-operator/apis/controller/v1alpha1"
-	"github.com/devfile/devworkspace-operator/controllers/workspace/env"
-	"github.com/devfile/devworkspace-operator/pkg/config"
 )
 
 type DeploymentProvisioningStatus struct {
@@ -140,6 +138,23 @@ func DeleteWorkspaceDeployment(ctx context.Context, workspace *devworkspace.DevW
 	return true, nil
 }
 
+// ScaleDeploymentToZero scales the cluster deployment to zero
+func ScaleDeploymentToZero(workspace *devworkspace.DevWorkspace, client runtimeClient.Client) error {
+	patch := []byte(`{"spec":{"replicas": 0}}`)
+	err := client.Patch(context.Background(), &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: workspace.Namespace,
+			Name:      common.DeploymentName(workspace.Status.WorkspaceId),
+		},
+	}, runtimeClient.RawPatch(types.StrategicMergePatchType, patch))
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func GetDevWorkspaceSecurityContext() *corev1.PodSecurityContext {
 	if !infrastructure.IsOpenShift() {
 		uID := int64(1234)