diff --git a/oadp-operator/controllers/registry_test.go b/oadp-operator/controllers/registry_test.go index 9a2eea37c3f..e426af6fda8 100644 --- a/oadp-operator/controllers/registry_test.go +++ b/oadp-operator/controllers/registry_test.go @@ -21,31 +21,17 @@ func TestVeleroReconciler_buildRegistryDeployment(t *testing.T) { wantErr bool }{ { - name: "registry without owner reference as well as labels", + name: "given a valid bsl get appropriate registry deployment", registryDeployment: &appsv1.Deployment{ ObjectMeta: metav1.ObjectMeta{ Name: "test-registry", Namespace: "test-ns", }, - }, - bsl: &velerov1.BackupStorageLocation{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-bsl", - Namespace: "test-ns", - }, - }, - }, - { - name: "registry without owner reference but has labels", - registryDeployment: &appsv1.Deployment{ - ObjectMeta: metav1.ObjectMeta{ - Name: "test-registry", - Namespace: "test-ns", - Labels: map[string]string{ - "app.kubernetes.io/name": common.OADPOperatorVelero, - "app.kubernetes.io/instance": "oadp-test-bsl-test-ns-registry", - "app.kubernetes.io/managed-by": common.OADPOperator, - "app.kubernetes.io/component": Registry, + Spec: appsv1.DeploymentSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "component": "oadp-" + "test-bsl" + "-" + "aws" + "-registry", + }, }, }, }, @@ -54,6 +40,20 @@ func TestVeleroReconciler_buildRegistryDeployment(t *testing.T) { Name: "test-bsl", Namespace: "test-ns", }, + Spec: velerov1.BackupStorageLocationSpec{ + Provider: AWSProvider, + StorageType: velerov1.StorageType{ + ObjectStorage: &velerov1.ObjectStorageLocation{ + Bucket: "aws-bucket", + }, + }, + Config: map[string]string{ + Region: "aws-region", + S3URL: "https://sr-url-aws-domain.com", + RootDirectory: "/velero-aws", + InsecureSkipTLSVerify: "false", + }, + }, }, }, } @@ -89,6 +89,11 @@ func TestVeleroReconciler_buildRegistryDeployment(t *testing.T) { }, Spec: appsv1.DeploymentSpec{ Replicas: pointer.Int32(1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "component": "oadp-" + tt.bsl.Name + "-" + tt.bsl.Spec.Provider + "-registry", + }, + }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ @@ -97,6 +102,74 @@ func TestVeleroReconciler_buildRegistryDeployment(t *testing.T) { }, Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyAlways, + Containers: []corev1.Container{ + { + Image: RegistryImage, + Name: "oadp-" + tt.bsl.Name + "-" + tt.bsl.Spec.Provider + "-registry" + "-container", + Ports: []corev1.ContainerPort{ + { + ContainerPort: 5000, + Protocol: corev1.ProtocolTCP, + }, + }, + Env: []corev1.EnvVar{ + { + Name: RegistryStorageEnvVarKey, + Value: S3, + }, + { + Name: RegistryStorageS3AccesskeyEnvVarKey, + Value: "", + }, + { + Name: RegistryStorageS3BucketEnvVarKey, + Value: "aws-bucket", + }, + { + Name: RegistryStorageS3RegionEnvVarKey, + Value: "aws-region", + }, + { + Name: RegistryStorageS3SecretkeyEnvVarKey, + Value: "", + }, + { + Name: RegistryStorageS3RegionendpointEnvVarKey, + Value: "https://sr-url-aws-domain.com", + }, + { + Name: RegistryStorageS3RootdirectoryEnvVarKey, + Value: "/velero-aws", + }, + { + Name: RegistryStorageS3SkipverifyEnvVarKey, + Value: "false", + }, + }, + LivenessProbe: &corev1.Probe{ + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/v2/_catalog?n=5", + Port: intstr.IntOrString{IntVal: 5000}, + }, + }, + PeriodSeconds: 5, + TimeoutSeconds: 3, + InitialDelaySeconds: 15, + }, + ReadinessProbe: &corev1.Probe{ + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/v2/_catalog?n=5", + Port: intstr.IntOrString{IntVal: 5000}, + }, + }, + PeriodSeconds: 5, + TimeoutSeconds: 3, + InitialDelaySeconds: 15, + }, + }, + }, }, }, }, @@ -110,11 +183,8 @@ func TestVeleroReconciler_buildRegistryDeployment(t *testing.T) { if !reflect.DeepEqual(wantRegistryDeployment.Labels, tt.registryDeployment.Labels) { t.Errorf("expected registry deployment labels to be %#v, got %#v", wantRegistryDeployment.Labels, tt.registryDeployment.Labels) } - if !reflect.DeepEqual(wantRegistryDeployment.OwnerReferences, tt.registryDeployment.OwnerReferences) { - t.Errorf("expected registry deployment owner references to be %#v, got %#v", wantRegistryDeployment.OwnerReferences, tt.registryDeployment.OwnerReferences) - } - if !reflect.DeepEqual(wantRegistryDeployment.Spec.Replicas, tt.registryDeployment.Spec.Replicas) { - t.Errorf("expected registry deployment replicas to be %#v, got %#v", wantRegistryDeployment.Spec.Replicas, tt.registryDeployment.Spec.Replicas) + if !reflect.DeepEqual(wantRegistryDeployment.Spec, tt.registryDeployment.Spec) { + t.Errorf("expected registry deployment spec to be %#v, got %#v", wantRegistryDeployment, tt.registryDeployment) } }) } diff --git a/oadp-operator/controllers/velero.go b/oadp-operator/controllers/velero.go index 626afb57946..d729421f150 100644 --- a/oadp-operator/controllers/velero.go +++ b/oadp-operator/controllers/velero.go @@ -2,6 +2,14 @@ package controllers import ( "fmt" + "github.com/openshift/oadp-operator/pkg/credentials" + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "os" + "reflect" + + //"sigs.k8s.io/controller-runtime/pkg/client" "github.com/go-logr/logr" security "github.com/openshift/api/security/v1" @@ -15,6 +23,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/utils/pointer" + //"sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" ) @@ -35,10 +44,9 @@ const ( AWSSharedCredentialsFileEnvKey = "AWS_SHARED_CREDENTIALS_FILE" AzureSharedCredentialsFileEnvKey = "AZURE_SHARED_CREDENTIALS_FILE" GCPSharedCredentialsFileEnvKey = "GCP_SHARED_CREDENTIALS_FILE" - //TODO: Add Proxy env vars - HTTPProxyEnvVar = "HTTP_PROXY" - HTTPSProxyEnvVar = "HTTPS_PROXY" - NoProxyEnvVar = "NO_PROXY" + HTTPProxyEnvVar = "HTTP_PROXY" + HTTPSProxyEnvVar = "HTTPS_PROXY" + NoProxyEnvVar = "NO_PROXY" ) //TODO: Add Image customization options @@ -103,6 +111,64 @@ func (r *VeleroReconciler) ReconcileVeleroServiceAccount(log logr.Logger) (bool, return true, nil } +//TODO: Temporary solution for Non-OLM Operator install +func (r *VeleroReconciler) ReconcileVeleroCRDs(log logr.Logger) (bool, error) { + velero := oadpv1alpha1.Velero{} + if err := r.Get(r.Context, r.NamespacedName, &velero); err != nil { + return false, err + } + + // check for Non-OLM install and proceed with Velero supporting CRD installation + if velero.Spec.OlmManaged != nil && velero.Spec.OlmManaged == pointer.Bool(false) { + err := r.InstallVeleroCRDs(log) + if err != nil { + return false, err + } + } + + return true, nil +} + +func (r *VeleroReconciler) InstallVeleroCRDs(log logr.Logger) error { + var err error + // Install CRDs + for _, unstructuredCrd := range install.AllCRDs("v1").Items { + foundCrd := &v1.CustomResourceDefinition{} + crd := &v1.CustomResourceDefinition{} + if err := runtime.DefaultUnstructuredConverter.FromUnstructured(unstructuredCrd.Object, crd); err != nil { + return err + } + // Add Conversion to the spec, as this will be returned in the foundCrd + crd.Spec.Conversion = &v1.CustomResourceConversion{ + Strategy: v1.NoneConverter, + } + if err = r.Client.Get(r.Context, types.NamespacedName{Name: crd.ObjectMeta.Name}, foundCrd); err != nil { + if errors.IsNotFound(err) { + // Didn't find CRD, we should create it. + log.Info("Creating CRD", "CRD.Name", crd.ObjectMeta.Name) + if err = r.Client.Create(r.Context, crd); err != nil { + return err + } + } else { + // Return other errors + return err + } + } else { + // CRD exists, check if it's updated. + if !reflect.DeepEqual(foundCrd.Spec, crd.Spec) { + // Specs aren't equal, update and fix. + log.Info("Updating CRD", "CRD.Name", crd.ObjectMeta.Name, "foundCrd.Spec", foundCrd.Spec, "crd.Spec", crd.Spec) + foundCrd.Spec = *crd.Spec.DeepCopy() + if err = r.Client.Update(r.Context, foundCrd); err != nil { + return err + } + } + } + } + + return nil +} + func (r *VeleroReconciler) ReconcileVeleroClusterRoleBinding(log logr.Logger) (bool, error) { velero := oadpv1alpha1.Velero{} if err := r.Get(r.Context, r.NamespacedName, &velero); err != nil { @@ -307,6 +373,81 @@ func (r *VeleroReconciler) privilegedSecurityContextConstraints(scc *security.Se // Build VELERO Deployment func (r *VeleroReconciler) buildVeleroDeployment(veleroDeployment *appsv1.Deployment, velero *oadpv1alpha1.Velero) error { + if velero == nil { + return fmt.Errorf("velero CR cannot be nil") + } + if veleroDeployment == nil { + return fmt.Errorf("velero deployment cannot be nil") + } + + // get default values + volumeMounts := []corev1.VolumeMount{ + { + Name: "plugins", + MountPath: "/plugins", + }, + { + Name: "scratch", + MountPath: "/scratch", + }, + { + Name: "certs", + MountPath: "/etc/ssl/certs", + }, + } + + envVars := []corev1.EnvVar{ + { + Name: LDLibraryPathEnvKey, + Value: "/plugins", + }, + { + Name: VeleroNamespaceEnvKey, + Value: velero.Namespace, + }, + { + Name: VeleroScratchDirEnvKey, + Value: "/scratch", + }, + { + Name: HTTPProxyEnvVar, + Value: os.Getenv("HTTP_PROXY"), + }, + { + Name: HTTPSProxyEnvVar, + Value: os.Getenv("HTTPS_PROXY"), + }, + { + Name: NoProxyEnvVar, + Value: os.Getenv("NO_PROXY"), + }, + } + + volumes := []corev1.Volume{ + { + Name: "plugins", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "scratch", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + { + Name: "certs", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + } + + //add any default init containers here if needed eg: setup-certificate-secret + initContainers := []corev1.Container{} + + // assign spec values veleroDeployment.Labels = r.getAppLabels(velero) veleroDeployment.Spec = appsv1.DeploymentSpec{ @@ -330,9 +471,8 @@ func (r *VeleroReconciler) buildVeleroDeployment(veleroDeployment *appsv1.Deploy Tolerations: velero.Spec.VeleroTolerations, Containers: []corev1.Container{ { - Name: common.Velero, - Image: VeleroImage, - //TODO: Make the image policy parametrized + Name: common.Velero, + Image: getVeleroImage(), ImagePullPolicy: corev1.PullAlways, Ports: []corev1.ContainerPort{ { @@ -344,18 +484,29 @@ func (r *VeleroReconciler) buildVeleroDeployment(veleroDeployment *appsv1.Deploy Command: []string{"/velero"}, //TODO: Parametrize restic timeout, Features flag as well as VELERO debug flag Args: []string{"server", "--restic-timeout", "1h"}, - VolumeMounts: r.getVeleroVolumeMounts(velero), - Env: r.getVeleroEnv(velero), + VolumeMounts: volumeMounts, + Env: envVars, }, }, - Volumes: r.getVeleroVolumes(velero), - InitContainers: r.getVeleroInitContainers(velero), + Volumes: volumes, + InitContainers: initContainers, }, }, } + + err := credentials.AppendPluginSpecficSpecs(velero, veleroDeployment) + + if err != nil { + return err + } + return nil } +func getVeleroImage() string { + return fmt.Sprintf("%v/%v/%v:%v", os.Getenv("REGISTRY"), os.Getenv("PROJECT"), os.Getenv("VELERO_REPO"), os.Getenv("VELERO_TAG")) +} + func (r *VeleroReconciler) getAppLabels(velero *oadpv1alpha1.Velero) map[string]string { labels := map[string]string{ "app.kubernetes.io/name": common.Velero, @@ -405,270 +556,3 @@ func (r *VeleroReconciler) getVeleroResourceReqs(velero *oadpv1alpha1.Velero) co } return ResourcesReqs } - -func (r *VeleroReconciler) getVeleroVolumeMounts(velero *oadpv1alpha1.Velero) []corev1.VolumeMount { - - defaultVeleroPluginsList := velero.Spec.DefaultVeleroPlugins - awsPluginVolumeMount := corev1.VolumeMount{ - Name: VeleroAWSSecretName, - MountPath: "/credentials", - } - azurePluginVolumeMount := corev1.VolumeMount{ - Name: VeleroAzureSecretName, - MountPath: "/credentials-azure", - } - gcpPluginVolumeMount := corev1.VolumeMount{ - Name: VeleroGCPSecretName, - MountPath: "/credentials-gcp", - } - - // add default volumemounts - volumeMounts := []corev1.VolumeMount{ - { - Name: "plugins", - MountPath: "/plugins", - }, - { - Name: "scratch", - MountPath: "/scratch", - }, - { - Name: "certs", - MountPath: "/etc/ssl/certs", - }, - } - // add default plugin based volumemounts - if defaultVeleroPluginsList != nil { - for _, plugin := range defaultVeleroPluginsList { - if plugin == oadpv1alpha1.DefaultPluginAWS { - volumeMounts = append(volumeMounts, awsPluginVolumeMount) - } - if plugin == oadpv1alpha1.DefaultPluginMicrosoftAzure { - volumeMounts = append(volumeMounts, azurePluginVolumeMount) - } - if plugin == oadpv1alpha1.DefaultPluginGCP { - volumeMounts = append(volumeMounts, gcpPluginVolumeMount) - } - } - } - return volumeMounts -} - -func (r *VeleroReconciler) getVeleroEnv(velero *oadpv1alpha1.Velero) []corev1.EnvVar { - - // add default Env vars - envVars := []corev1.EnvVar{ - { - Name: LDLibraryPathEnvKey, - Value: "/plugins", - }, - { - Name: VeleroNamespaceEnvKey, - Value: velero.Namespace, - }, - { - Name: VeleroScratchDirEnvKey, - Value: "/scratch", - }, - //TODO: Add the PROXY VARS - } - - awsPluginEnvVar := corev1.EnvVar{ - Name: AWSSharedCredentialsFileEnvKey, - Value: "/credentials/cloud", - } - azurePluginEnvVar := corev1.EnvVar{ - Name: AzureSharedCredentialsFileEnvKey, - Value: "/credentials-azure/cloud", - } - gcpPluginEnvVar := corev1.EnvVar{ - Name: GCPSharedCredentialsFileEnvKey, - Value: "/credentials-gcp/cloud", - } - - // add default plugin based Env vars - defaultVeleroPluginsList := velero.Spec.DefaultVeleroPlugins - - if defaultVeleroPluginsList != nil { - for _, plugin := range defaultVeleroPluginsList { - if plugin == oadpv1alpha1.DefaultPluginAWS { - envVars = append(envVars, awsPluginEnvVar) - } - if plugin == oadpv1alpha1.DefaultPluginMicrosoftAzure { - envVars = append(envVars, azurePluginEnvVar) - } - if plugin == oadpv1alpha1.DefaultPluginGCP { - envVars = append(envVars, gcpPluginEnvVar) - } - } - } - - return envVars -} - -func (r *VeleroReconciler) getVeleroVolumes(velero *oadpv1alpha1.Velero) []corev1.Volume { - // add default volumes - volumes := []corev1.Volume{ - { - Name: "plugins", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - { - Name: "scratch", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - { - Name: "certs", - VolumeSource: corev1.VolumeSource{ - EmptyDir: &corev1.EmptyDirVolumeSource{}, - }, - }, - } - - // add default plugin based volumes - awsPluginVolume := corev1.Volume{ - Name: VeleroAWSSecretName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: VeleroAWSSecretName, - }, - }, - } - azurePluginVolume := corev1.Volume{ - Name: VeleroAzureSecretName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: VeleroAzureSecretName, - }, - }, - } - gcpPluginVolume := corev1.Volume{ - Name: VeleroGCPSecretName, - VolumeSource: corev1.VolumeSource{ - Secret: &corev1.SecretVolumeSource{ - SecretName: VeleroGCPSecretName, - }, - }, - } - - defaultVeleroPluginsList := velero.Spec.DefaultVeleroPlugins - - if defaultVeleroPluginsList != nil { - for _, plugin := range defaultVeleroPluginsList { - if plugin == oadpv1alpha1.DefaultPluginAWS { - volumes = append(volumes, awsPluginVolume) - } - if plugin == oadpv1alpha1.DefaultPluginMicrosoftAzure { - volumes = append(volumes, azurePluginVolume) - } - if plugin == oadpv1alpha1.DefaultPluginGCP { - volumes = append(volumes, gcpPluginVolume) - } - } - } - - return volumes - -} - -func (r *VeleroReconciler) getVeleroInitContainers(velero *oadpv1alpha1.Velero) []corev1.Container { - - // add default volumemounts - volumeMounts := []corev1.VolumeMount{ - { - MountPath: "/certs", - Name: "certs", - }, - } - - defaultVeleroPluginsList := velero.Spec.DefaultVeleroPlugins - - //TODO: Check why this is only done for AWS and not for other providers - if defaultVeleroPluginsList != nil { - for _, plugin := range defaultVeleroPluginsList { - if plugin == oadpv1alpha1.DefaultPluginAWS { - volumeMounts = append(volumeMounts, corev1.VolumeMount{ - MountPath: "/credentials", - Name: VeleroAWSSecretName, - }) - } - } - } - - // add default initcontainers - initContainers := []corev1.Container{ - { - //TODO: Check this image as well as pull policy - Image: VeleroImage, - ImagePullPolicy: corev1.PullAlways, - Name: "setup-certificate-secret", - Command: []string{"sh", "'-ec'", "cp /etc/ssl/certs/* /certs/; ln -sf /credentials/ca_bundle.pem /certs/ca_bundle.pem;"}, - Resources: corev1.ResourceRequirements{}, - TerminationMessagePath: "/dev/termination-log", - TerminationMessagePolicy: "File", - VolumeMounts: volumeMounts, - }, - } - - // add default plugin based initcontainers - if defaultVeleroPluginsList != nil { - for _, plugin := range defaultVeleroPluginsList { - if plugin == oadpv1alpha1.DefaultPluginAWS { - awsInitContainer := buildPluginInitContainer(VeleroPluginForAWS, AWSPluginImage, corev1.PullAlways) - initContainers = append(initContainers, awsInitContainer) - } - if plugin == oadpv1alpha1.DefaultPluginMicrosoftAzure { - azureInitContainer := buildPluginInitContainer(VeleroPluginForAzure, AzurePluginImage, corev1.PullAlways) - initContainers = append(initContainers, azureInitContainer) - } - if plugin == oadpv1alpha1.DefaultPluginGCP { - gcpInitContainer := buildPluginInitContainer(VeleroPluginForGCP, GCPPluginImage, corev1.PullAlways) - initContainers = append(initContainers, gcpInitContainer) - } - if plugin == oadpv1alpha1.DefaultPluginCSI { - csiInitContainer := buildPluginInitContainer(VeleroPluginForCSI, CSIPluginImage, corev1.PullAlways) - initContainers = append(initContainers, csiInitContainer) - } - if plugin == oadpv1alpha1.DefaultPluginOpenShift { - openshiftInitContainer := buildPluginInitContainer(VeleroPluginForOpenshift, OpenshiftPluginImage, corev1.PullAlways) - initContainers = append(initContainers, openshiftInitContainer) - } - //TODO: check if vsphere is needed - } - } - - //add custom plugin based initcontainers - customVeleroPluginList := velero.Spec.CustomVeleroPlugins - - if customVeleroPluginList != nil { - for _, customPlugin := range customVeleroPluginList { - customPluginInitContainer := buildPluginInitContainer(customPlugin.Name, customPlugin.Image, corev1.PullAlways) - initContainers = append(initContainers, customPluginInitContainer) - } - } - - return initContainers -} - -func buildPluginInitContainer(initContainerName string, initContainerImage string, imagePullPolicy corev1.PullPolicy) corev1.Container { - initContainer := corev1.Container{ - Image: initContainerImage, - Name: initContainerName, - ImagePullPolicy: imagePullPolicy, - Resources: corev1.ResourceRequirements{}, - TerminationMessagePath: "/dev/termination-log", - TerminationMessagePolicy: "File", - VolumeMounts: []corev1.VolumeMount{ - { - MountPath: "/target", - Name: "plugins", - }, - }, - } - - return initContainer -} diff --git a/oadp-operator/controllers/velero_controller.go b/oadp-operator/controllers/velero_controller.go index 0e6b494b05a..8d236aa1bde 100644 --- a/oadp-operator/controllers/velero_controller.go +++ b/oadp-operator/controllers/velero_controller.go @@ -82,14 +82,15 @@ func (r *VeleroReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr return result, client.IgnoreNotFound(err) } _, err := ReconcileBatch(r.Log, + r.ReconcileVeleroServiceAccount, + r.ReconcileVeleroClusterRoleBinding, + r.ReconcileVeleroSecurityContextConstraint, + r.ReconcileVeleroCRDs, r.ValidateBackupStorageLocations, r.ReconcileBackupStorageLocations, r.ReconcileRegistries, r.ValidateVolumeSnapshotLocations, r.ReconcileVolumeSnapshotLocations, - r.ReconcileVeleroServiceAccount, - r.ReconcileVeleroClusterRoleBinding, - r.ReconcileVeleroSecurityContextConstraint, r.ReconcileVeleroDeployment, r.ReconcileResticDaemonset, ) diff --git a/oadp-operator/go.mod b/oadp-operator/go.mod index 83f9b8506c3..9817570c8a0 100644 --- a/oadp-operator/go.mod +++ b/oadp-operator/go.mod @@ -8,9 +8,11 @@ require ( github.com/onsi/ginkgo v1.16.4 github.com/onsi/gomega v1.13.0 github.com/openshift/api v0.0.0-20210729133136-d870cea76006 + github.com/prometheus/common v0.26.0 // indirect github.com/vmware-tanzu/velero v1.6.1-0.20210806003158-ed5809b7fc22 golang.org/x/tools v0.1.2 // indirect k8s.io/api v0.21.2 + k8s.io/apiextensions-apiserver v0.21.2 // indirect k8s.io/apimachinery v0.21.2 k8s.io/client-go v0.21.2 k8s.io/utils v0.0.0-20210527160623-6fdb442a123b diff --git a/oadp-operator/go.sum b/oadp-operator/go.sum index b5fe1db7115..1490663e3e9 100644 --- a/oadp-operator/go.sum +++ b/oadp-operator/go.sum @@ -82,9 +82,11 @@ github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdko github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alessio/shellescape v1.2.2/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= @@ -992,6 +994,7 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/oadp-operator/main.go b/oadp-operator/main.go index 61f38596d34..ddbcb76acf6 100644 --- a/oadp-operator/main.go +++ b/oadp-operator/main.go @@ -24,17 +24,18 @@ import ( // to ensure that exec-entrypoint and run can make use of them. _ "k8s.io/client-go/plugin/pkg/client/auth" + security "github.com/openshift/api/security/v1" + oadpv1alpha1 "github.com/openshift/oadp-operator/api/v1alpha1" + "github.com/openshift/oadp-operator/controllers" + velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" + appsv1 "k8s.io/api/apps/v1" + v1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "k8s.io/apimachinery/pkg/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime" clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/healthz" "sigs.k8s.io/controller-runtime/pkg/log/zap" - - security "github.com/openshift/api/security/v1" - oadpv1alpha1 "github.com/openshift/oadp-operator/api/v1alpha1" - "github.com/openshift/oadp-operator/controllers" - velerov1 "github.com/vmware-tanzu/velero/pkg/apis/velero/v1" //+kubebuilder:scaffold:imports ) @@ -90,6 +91,16 @@ func main() { os.Exit(1) } + if err := appsv1.AddToScheme(mgr.GetScheme()); err != nil { + setupLog.Error(err, "unable to add Kubernetes APIs to scheme") + os.Exit(1) + } + + if err := v1.AddToScheme(mgr.GetScheme()); err != nil { + setupLog.Error(err, "unable to add Kubernetes API extensions to scheme") + os.Exit(1) + } + if err = (&controllers.VeleroReconciler{ Client: mgr.GetClient(), Scheme: mgr.GetScheme(), diff --git a/oadp-operator/pkg/credentials/credentials.go b/oadp-operator/pkg/credentials/credentials.go index 105fdbd9723..d4fd9de9466 100644 --- a/oadp-operator/pkg/credentials/credentials.go +++ b/oadp-operator/pkg/credentials/credentials.go @@ -1,16 +1,19 @@ package credentials import ( + "fmt" oadpv1alpha1 "github.com/openshift/oadp-operator/api/v1alpha1" "github.com/openshift/oadp-operator/pkg/common" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "os" ) -type CloudProviderFields struct { +type DefaultPluginFields struct { secretName string mountPath string envCredentialsFile string + pluginImage string } const ( @@ -19,21 +22,31 @@ const ( var ( mountPropagationToHostContainer = corev1.MountPropagationHostToContainer - cloudProvidersPluginFields = map[oadpv1alpha1.DefaultPlugin]CloudProviderFields{ + pluginSpecificFields = map[oadpv1alpha1.DefaultPlugin]DefaultPluginFields{ oadpv1alpha1.DefaultPluginAWS: { secretName: "cloud-credentials", mountPath: "/credentials", envCredentialsFile: "AWS_SHARED_CREDENTIALS_FILE", + pluginImage: fmt.Sprintf("%v/%v/%v:%v", os.Getenv("REGISTRY"), os.Getenv("PROJECT"), os.Getenv("VELERO_AWS_PLUGIN_REPO"), os.Getenv("VELERO_AWS_PLUGIN_TAG")), }, oadpv1alpha1.DefaultPluginGCP: { secretName: "cloud-credentials-gcp", mountPath: "/credentials-gcp", envCredentialsFile: "GOOGLE_APPLICATION_CREDENTIALS", + pluginImage: fmt.Sprintf("%v/%v/%v:%v", os.Getenv("REGISTRY"), os.Getenv("PROJECT"), os.Getenv("VELERO_GCP_PLUGIN_REPO"), os.Getenv("VELERO_GCP_PLUGIN_TAG")), }, oadpv1alpha1.DefaultPluginMicrosoftAzure: { secretName: "cloud-credentials-azure", mountPath: "/credentials-azure", envCredentialsFile: "AZURE_CREDENTIALS_FILE", + pluginImage: fmt.Sprintf("%v/%v/%v:%v", os.Getenv("REGISTRY"), os.Getenv("PROJECT"), os.Getenv("VELERO_AZURE_PLUGIN_REPO"), os.Getenv("VELERO_AZURE_PLUGIN_TAG")), + }, + oadpv1alpha1.DefaultPluginOpenShift: { + pluginImage: fmt.Sprintf("%v/%v/%v:%v", os.Getenv("REGISTRY"), os.Getenv("PROJECT"), os.Getenv("VELERO_OPENSHIFT_PLUGIN_REPO"), os.Getenv("VELERO_OPENSHIFT_PLUGIN_TAG")), + }, + oadpv1alpha1.DefaultPluginCSI: { + //TODO: Check if the Registry needs to an upstream one from CSI + pluginImage: fmt.Sprintf("%v/%v/%v:%v", os.Getenv("REGISTRY"), os.Getenv("PROJECT"), os.Getenv("VELERO_CSI_PLUGIN_REPO"), os.Getenv("VELERO_CSI_PLUGIN_TAG")), }, } ) @@ -41,21 +54,21 @@ var ( func AppendCloudProviderVolumes(velero *oadpv1alpha1.Velero, ds *appsv1.DaemonSet) error { var veleroContainer *corev1.Container // Find Velero container - for _, container := range ds.Spec.Template.Spec.Containers { + for i, container := range ds.Spec.Template.Spec.Containers { if container.Name == common.Velero { - veleroContainer = &container + veleroContainer = &ds.Spec.Template.Spec.Containers[i] } } for _, plugin := range velero.Spec.DefaultVeleroPlugins { // Check that this is a cloud provider plugin in the cloud provider map - // ok is boolean that will be true if `plugin` is a valid key in `cloudProvidersPluginFields` map + // ok is boolean that will be true if `plugin` is a valid key in `pluginSpecificFields` map // pattern from https://golang.org/doc/effective_go#maps - // this replaces the need to iterate through the `cloudProvidersPluginFields` O(n) -> O(1) - if cloudProviderMap, ok := cloudProvidersPluginFields[plugin]; ok { + // this replaces the need to iterate through the `pluginSpecificFields` O(n) -> O(1) + if cloudProviderMap, ok := pluginSpecificFields[plugin]; ok { ds.Spec.Template.Spec.Volumes = append( ds.Spec.Template.Spec.Volumes, corev1.Volume{ - Name: string(plugin), + Name: cloudProviderMap.secretName, VolumeSource: corev1.VolumeSource{ Secret: &corev1.SecretVolumeSource{ SecretName: cloudProviderMap.secretName, @@ -66,8 +79,9 @@ func AppendCloudProviderVolumes(velero *oadpv1alpha1.Velero, ds *appsv1.DaemonSe veleroContainer.VolumeMounts = append( veleroContainer.VolumeMounts, corev1.VolumeMount{ - Name: cloudProviderMap.secretName, - MountPath: cloudProviderMap.mountPath, + Name: cloudProviderMap.secretName, + MountPath: cloudProviderMap.mountPath, + //TODO: Check if MountPropagation is needed for plugin specific volume mounts MountPropagation: &mountPropagationToHostContainer, }, ) @@ -82,3 +96,90 @@ func AppendCloudProviderVolumes(velero *oadpv1alpha1.Velero, ds *appsv1.DaemonSe } return nil } + +// add plugin specific specs to velero deployment +func AppendPluginSpecficSpecs(velero *oadpv1alpha1.Velero, veleroDeployment *appsv1.Deployment) error { + var veleroContainer *corev1.Container + + for i, container := range veleroDeployment.Spec.Template.Spec.Containers { + if container.Name == common.Velero { + veleroContainer = &veleroDeployment.Spec.Template.Spec.Containers[i] + } + } + + for _, plugin := range velero.Spec.DefaultVeleroPlugins { + if pluginSpecificMap, ok := pluginSpecificFields[plugin]; ok { + + // append plugin specific volume mounts + if veleroContainer != nil { + veleroContainer.VolumeMounts = append( + veleroContainer.VolumeMounts, + corev1.VolumeMount{ + Name: pluginSpecificMap.secretName, + MountPath: pluginSpecificMap.mountPath, + }) + + // append plugin specific env vars + veleroContainer.Env = append( + veleroContainer.Env, + corev1.EnvVar{ + Name: pluginSpecificMap.envCredentialsFile, + Value: pluginSpecificMap.mountPath + "/" + cloudFieldPath, + }) + } + + // append plugin specific volumes + veleroDeployment.Spec.Template.Spec.Volumes = append( + veleroDeployment.Spec.Template.Spec.Volumes, + corev1.Volume{ + Name: pluginSpecificMap.secretName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: pluginSpecificMap.secretName, + }, + }, + }) + + // append plugin specifc init containers + veleroDeployment.Spec.Template.Spec.InitContainers = append( + veleroDeployment.Spec.Template.Spec.InitContainers, + corev1.Container{ + Image: pluginSpecificMap.pluginImage, + Name: string(plugin), + ImagePullPolicy: corev1.PullAlways, + Resources: corev1.ResourceRequirements{}, + TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: "File", + VolumeMounts: []corev1.VolumeMount{ + { + MountPath: "/target", + Name: "plugins", + }, + }, + }) + + } + } + // append custom plugin init containers + if velero.Spec.CustomVeleroPlugins != nil { + for _, plugin := range velero.Spec.CustomVeleroPlugins { + veleroDeployment.Spec.Template.Spec.InitContainers = append( + veleroDeployment.Spec.Template.Spec.InitContainers, + corev1.Container{ + Image: plugin.Image, + Name: plugin.Image, + ImagePullPolicy: corev1.PullAlways, + Resources: corev1.ResourceRequirements{}, + TerminationMessagePath: "/dev/termination-log", + TerminationMessagePolicy: "File", + VolumeMounts: []corev1.VolumeMount{ + { + MountPath: "/target", + Name: "plugins", + }, + }, + }) + } + } + return nil +}