Makes default StorageClass to use disk encryption set, if provided

pkg/cluster/install.go

@@ -134,6 +134,7 @@ func (m *manager) Install(ctx context.Context) error {
 			steps.Action(m.updateClusterData),
 			steps.Action(m.configureIngressCertificate),
 			steps.Condition(m.ingressControllerReady, 30*time.Minute),
+			steps.Action(m.configureDefaultStorageClass),
 			steps.Action(m.finishInstallation),
 		},
 	}

pkg/cluster/storageclass.go

@@ -0,0 +1,73 @@
+package cluster
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+
+	"github.com/Azure/go-autorest/autorest/to"
+	corev1 "k8s.io/api/core/v1"
+	storagev1 "k8s.io/api/storage/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/util/retry"
+)
+
+const (
+	defaultStorageClassName          = "managed-premium"
+	defaultEncryptedStorageClassName = "managed-premium-encrypted-cmk"
+)
+
+// configureDefaultStorageClass replaces default storage class provided by OCP with
+// a new one which uses disk encryption set (if one supplied by a customer).
+func (m *manager) configureDefaultStorageClass(ctx context.Context) error {
+	if m.doc.OpenShiftCluster.Properties.WorkerProfiles[0].DiskEncryptionSetID == "" {
+		return nil
+	}
+
+	return retry.RetryOnConflict(retry.DefaultRetry, func() error {
+		oldSC, err := m.kubernetescli.StorageV1().StorageClasses().Get(ctx, defaultStorageClassName, metav1.GetOptions{})
+		if err != nil {
+			return err
+		}
+
+		if oldSC.Annotations == nil {
+			oldSC.Annotations = map[string]string{}
+		}
+		oldSC.Annotations["storageclass.kubernetes.io/is-default-class"] = "false"
+		_, err = m.kubernetescli.StorageV1().StorageClasses().Update(ctx, oldSC, metav1.UpdateOptions{})
+		if err != nil {
+			return err
+		}
+
+		encryptedSC := newEncryptedStorageClass(m.doc.OpenShiftCluster.Properties.WorkerProfiles[0].DiskEncryptionSetID)
+		_, err = m.kubernetescli.StorageV1().StorageClasses().Create(ctx, encryptedSC, metav1.CreateOptions{})
+		if err != nil {
+			return err
+		}
+
+		return nil
+	})
+}
+
+func newEncryptedStorageClass(diskEncryptionSetID string) *storagev1.StorageClass {
+	volumeBindingMode := storagev1.VolumeBindingWaitForFirstConsumer
+	reclaimPolicy := corev1.PersistentVolumeReclaimDelete
+	return &storagev1.StorageClass{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: defaultEncryptedStorageClassName,
+			Annotations: map[string]string{
+				"storageclass.kubernetes.io/is-default-class": "true",
+			},
+		},
+		Provisioner:          "kubernetes.io/azure-disk",
+		VolumeBindingMode:    &volumeBindingMode,
+		AllowVolumeExpansion: to.BoolPtr(true),
+		ReclaimPolicy:        &reclaimPolicy,
+		Parameters: map[string]string{
+			"kind":                "Managed",
+			"storageaccounttype":  "Premium_LRS",
+			"diskEncryptionSetID": diskEncryptionSetID,
+		},
+	}
+}

pkg/cluster/storageclass_test.go

@@ -0,0 +1,150 @@
+package cluster
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"context"
+	"errors"
+	"testing"
+
+	storagev1 "k8s.io/api/storage/v1"
+	kerrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/client-go/kubernetes/fake"
+	ktesting "k8s.io/client-go/testing"
+
+	"github.com/Azure/ARO-RP/pkg/api"
+)
+
+func TestConfigureStorageClass(t *testing.T) {
+	for _, tt := range []struct {
+		name      string
+		mocks     func(kubernetescli *fake.Clientset)
+		desID     string
+		wantErr   string
+		wantNewSC bool
+	}{
+		{
+			name: "no disk encryption set provided",
+		},
+		{
+			name:      "disk encryption set provided",
+			desID:     "fake-des-id",
+			wantNewSC: true,
+		},
+		{
+			name:  "error getting old default StorageClass",
+			desID: "fake-des-id",
+			mocks: func(kubernetescli *fake.Clientset) {
+				kubernetescli.PrependReactor("get", "storageclasses", func(action ktesting.Action) (handled bool, ret runtime.Object, err error) {
+					if action.(ktesting.GetAction).GetName() != "managed-premium" {
+						return false, nil, nil
+					}
+					return true, nil, errors.New("fake error from get of old StorageClass")
+				})
+			},
+			wantErr: "fake error from get of old StorageClass",
+		},
+		{
+			name:  "error removing default annotation from old StorageClass",
+			desID: "fake-des-id",
+			mocks: func(kubernetescli *fake.Clientset) {
+				kubernetescli.PrependReactor("update", "storageclasses", func(action ktesting.Action) (handled bool, ret runtime.Object, err error) {
+					obj := action.(ktesting.UpdateAction).GetObject().(*storagev1.StorageClass)
+					if obj.Name != "managed-premium" {
+						return false, nil, nil
+					}
+					return true, nil, errors.New("fake error from update of old StorageClass")
+				})
+			},
+			wantErr: "fake error from update of old StorageClass",
+		},
+		{
+			name:  "error creating the new default encrypted StorageClass",
+			desID: "fake-des-id",
+			mocks: func(kubernetescli *fake.Clientset) {
+				kubernetescli.PrependReactor("create", "storageclasses", func(action ktesting.Action) (handled bool, ret runtime.Object, err error) {
+					obj := action.(ktesting.CreateAction).GetObject().(*storagev1.StorageClass)
+					if obj.Name != "managed-premium-encrypted-cmk" {
+						return false, nil, nil
+					}
+					return true, nil, errors.New("fake error while creating encrypted StorageClass")
+				})
+			},
+			wantErr: "fake error while creating encrypted StorageClass",
+		},
+	} {
+		t.Run(tt.name, func(t *testing.T) {
+			ctx := context.Background()
+			kubernetescli := fake.NewSimpleClientset(
+				&storagev1.StorageClass{
+					ObjectMeta: metav1.ObjectMeta{
+						Name: "managed-premium",
+						Annotations: map[string]string{
+							"storageclass.kubernetes.io/is-default-class": "true",
+						},
+					},
+				},
+			)
+
+			if tt.mocks != nil {
+				tt.mocks(kubernetescli)
+			}
+
+			m := &manager{
+				kubernetescli: kubernetescli,
+				doc: &api.OpenShiftClusterDocument{
+					OpenShiftCluster: &api.OpenShiftCluster{
+						Properties: api.OpenShiftClusterProperties{
+							WorkerProfiles: []api.WorkerProfile{
+								{
+									DiskEncryptionSetID: tt.desID,
+								},
+							},
+						},
+					},
+				},
+			}
+
+			err := m.configureDefaultStorageClass(ctx)
+			if err != nil && err.Error() != tt.wantErr ||
+				err == nil && tt.wantErr != "" {
+				t.Error(err)
+			}
+
+			if tt.wantNewSC {
+				oldSC, err := kubernetescli.StorageV1().StorageClasses().Get(ctx, "managed-premium", metav1.GetOptions{})
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				// Old StorageClass is no longer default
+				if oldSC.Annotations["storageclass.kubernetes.io/is-default-class"] != "false" {
+					t.Error(oldSC.Annotations["storageclass.kubernetes.io/is-default-class"])
+				}
+
+				encryptedSC, err := kubernetescli.StorageV1().StorageClasses().Get(ctx, "managed-premium-encrypted-cmk", metav1.GetOptions{})
+				if err != nil {
+					t.Fatal(err)
+				}
+
+				// New StorageClass is default
+				if encryptedSC.Annotations["storageclass.kubernetes.io/is-default-class"] != "true" {
+					t.Error(encryptedSC.Annotations["storageclass.kubernetes.io/is-default-class"])
+				}
+
+				// And has diskEncryptionSetID set to one from worker profile
+				if encryptedSC.Parameters["diskEncryptionSetID"] != tt.desID {
+					t.Error(encryptedSC.Parameters["diskEncryptionSetID"])
+				}
+			} else {
+				_, err := kubernetescli.StorageV1().StorageClasses().Get(ctx, "managed-premium-encrypted-cmk", metav1.GetOptions{})
+				if !kerrors.IsNotFound(err) {
+					t.Error(err)
+				}
+			}
+		})
+	}
+}