diff --git a/pkg/cluster/install.go b/pkg/cluster/install.go index cfbfb68e901..85d31e25145 100644 --- a/pkg/cluster/install.go +++ b/pkg/cluster/install.go @@ -134,6 +134,7 @@ func (m *manager) Install(ctx context.Context) error { steps.Action(m.updateClusterData), steps.Action(m.configureIngressCertificate), steps.Condition(m.ingressControllerReady, 30*time.Minute), + steps.Action(m.configureDefaultStorageClass), steps.Action(m.finishInstallation), }, } diff --git a/pkg/cluster/storageclass.go b/pkg/cluster/storageclass.go new file mode 100644 index 00000000000..44b88513c5d --- /dev/null +++ b/pkg/cluster/storageclass.go @@ -0,0 +1,73 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + + "github.com/Azure/go-autorest/autorest/to" + corev1 "k8s.io/api/core/v1" + storagev1 "k8s.io/api/storage/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/util/retry" +) + +const ( + defaultStorageClassName = "managed-premium" + defaultEncryptedStorageClassName = "managed-premium-encrypted-cmk" +) + +// configureDefaultStorageClass replaces default storage class provided by OCP with +// a new one which uses disk encryption set (if one supplied by a customer). +func (m *manager) configureDefaultStorageClass(ctx context.Context) error { + if m.doc.OpenShiftCluster.Properties.WorkerProfiles[0].DiskEncryptionSetID == "" { + return nil + } + + return retry.RetryOnConflict(retry.DefaultRetry, func() error { + oldSC, err := m.kubernetescli.StorageV1().StorageClasses().Get(ctx, defaultStorageClassName, metav1.GetOptions{}) + if err != nil { + return err + } + + if oldSC.Annotations == nil { + oldSC.Annotations = map[string]string{} + } + oldSC.Annotations["storageclass.kubernetes.io/is-default-class"] = "false" + _, err = m.kubernetescli.StorageV1().StorageClasses().Update(ctx, oldSC, metav1.UpdateOptions{}) + if err != nil { + return err + } + + encryptedSC := newEncryptedStorageClass(m.doc.OpenShiftCluster.Properties.WorkerProfiles[0].DiskEncryptionSetID) + _, err = m.kubernetescli.StorageV1().StorageClasses().Create(ctx, encryptedSC, metav1.CreateOptions{}) + if err != nil { + return err + } + + return nil + }) +} + +func newEncryptedStorageClass(diskEncryptionSetID string) *storagev1.StorageClass { + volumeBindingMode := storagev1.VolumeBindingWaitForFirstConsumer + reclaimPolicy := corev1.PersistentVolumeReclaimDelete + return &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: defaultEncryptedStorageClassName, + Annotations: map[string]string{ + "storageclass.kubernetes.io/is-default-class": "true", + }, + }, + Provisioner: "kubernetes.io/azure-disk", + VolumeBindingMode: &volumeBindingMode, + AllowVolumeExpansion: to.BoolPtr(true), + ReclaimPolicy: &reclaimPolicy, + Parameters: map[string]string{ + "kind": "Managed", + "storageaccounttype": "Premium_LRS", + "diskEncryptionSetID": diskEncryptionSetID, + }, + } +} diff --git a/pkg/cluster/storageclass_test.go b/pkg/cluster/storageclass_test.go new file mode 100644 index 00000000000..3bb827288aa --- /dev/null +++ b/pkg/cluster/storageclass_test.go @@ -0,0 +1,150 @@ +package cluster + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "context" + "errors" + "testing" + + storagev1 "k8s.io/api/storage/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + ktesting "k8s.io/client-go/testing" + + "github.com/Azure/ARO-RP/pkg/api" +) + +func TestConfigureStorageClass(t *testing.T) { + for _, tt := range []struct { + name string + mocks func(kubernetescli *fake.Clientset) + desID string + wantErr string + wantNewSC bool + }{ + { + name: "no disk encryption set provided", + }, + { + name: "disk encryption set provided", + desID: "fake-des-id", + wantNewSC: true, + }, + { + name: "error getting old default StorageClass", + desID: "fake-des-id", + mocks: func(kubernetescli *fake.Clientset) { + kubernetescli.PrependReactor("get", "storageclasses", func(action ktesting.Action) (handled bool, ret runtime.Object, err error) { + if action.(ktesting.GetAction).GetName() != "managed-premium" { + return false, nil, nil + } + return true, nil, errors.New("fake error from get of old StorageClass") + }) + }, + wantErr: "fake error from get of old StorageClass", + }, + { + name: "error removing default annotation from old StorageClass", + desID: "fake-des-id", + mocks: func(kubernetescli *fake.Clientset) { + kubernetescli.PrependReactor("update", "storageclasses", func(action ktesting.Action) (handled bool, ret runtime.Object, err error) { + obj := action.(ktesting.UpdateAction).GetObject().(*storagev1.StorageClass) + if obj.Name != "managed-premium" { + return false, nil, nil + } + return true, nil, errors.New("fake error from update of old StorageClass") + }) + }, + wantErr: "fake error from update of old StorageClass", + }, + { + name: "error creating the new default encrypted StorageClass", + desID: "fake-des-id", + mocks: func(kubernetescli *fake.Clientset) { + kubernetescli.PrependReactor("create", "storageclasses", func(action ktesting.Action) (handled bool, ret runtime.Object, err error) { + obj := action.(ktesting.CreateAction).GetObject().(*storagev1.StorageClass) + if obj.Name != "managed-premium-encrypted-cmk" { + return false, nil, nil + } + return true, nil, errors.New("fake error while creating encrypted StorageClass") + }) + }, + wantErr: "fake error while creating encrypted StorageClass", + }, + } { + t.Run(tt.name, func(t *testing.T) { + ctx := context.Background() + kubernetescli := fake.NewSimpleClientset( + &storagev1.StorageClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: "managed-premium", + Annotations: map[string]string{ + "storageclass.kubernetes.io/is-default-class": "true", + }, + }, + }, + ) + + if tt.mocks != nil { + tt.mocks(kubernetescli) + } + + m := &manager{ + kubernetescli: kubernetescli, + doc: &api.OpenShiftClusterDocument{ + OpenShiftCluster: &api.OpenShiftCluster{ + Properties: api.OpenShiftClusterProperties{ + WorkerProfiles: []api.WorkerProfile{ + { + DiskEncryptionSetID: tt.desID, + }, + }, + }, + }, + }, + } + + err := m.configureDefaultStorageClass(ctx) + if err != nil && err.Error() != tt.wantErr || + err == nil && tt.wantErr != "" { + t.Error(err) + } + + if tt.wantNewSC { + oldSC, err := kubernetescli.StorageV1().StorageClasses().Get(ctx, "managed-premium", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + + // Old StorageClass is no longer default + if oldSC.Annotations["storageclass.kubernetes.io/is-default-class"] != "false" { + t.Error(oldSC.Annotations["storageclass.kubernetes.io/is-default-class"]) + } + + encryptedSC, err := kubernetescli.StorageV1().StorageClasses().Get(ctx, "managed-premium-encrypted-cmk", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + + // New StorageClass is default + if encryptedSC.Annotations["storageclass.kubernetes.io/is-default-class"] != "true" { + t.Error(encryptedSC.Annotations["storageclass.kubernetes.io/is-default-class"]) + } + + // And has diskEncryptionSetID set to one from worker profile + if encryptedSC.Parameters["diskEncryptionSetID"] != tt.desID { + t.Error(encryptedSC.Parameters["diskEncryptionSetID"]) + } + } else { + _, err := kubernetescli.StorageV1().StorageClasses().Get(ctx, "managed-premium-encrypted-cmk", metav1.GetOptions{}) + if !kerrors.IsNotFound(err) { + t.Error(err) + } + } + }) + } +}