K8SPSMDB-1072: add retention section to backup tasks

config/crd/bases/psmdb.percona.com_perconaservermongodbs.yaml

@@ -389,6 +389,22 @@ spec:
                           type: integer
                         name:
                           type: string
+                        retention:
+                          properties:
+                            count:
+                              minimum: 0
+                              type: integer
+                            deleteFromStorage:
+                              default: true
+                              type: boolean
+                            type:
+                              enum:
+                              - count
+                              type: string
+                          required:
+                          - deleteFromStorage
+                          - type
+                          type: object
                         schedule:
                           type: string
                         storageName:

deploy/bundle.yaml

@@ -1108,6 +1108,22 @@ spec:
                           type: integer
                         name:
                           type: string
+                        retention:
+                          properties:
+                            count:
+                              minimum: 0
+                              type: integer
+                            deleteFromStorage:
+                              default: true
+                              type: boolean
+                            type:
+                              enum:
+                              - count
+                              type: string
+                          required:
+                          - deleteFromStorage
+                          - type
+                          type: object
                         schedule:
                           type: string
                         storageName:

deploy/cr.yaml

@@ -770,21 +770,30 @@ spec:
 #      - name: daily-s3-us-west
 #        enabled: true
 #        schedule: "0 0 * * *"
-#        keep: 3
+#        retention:
+#          count: 3
+#          type: count
+#          deleteFromStorage: true
 #        storageName: s3-us-west
 #        compressionType: gzip
 #        compressionLevel: 6
 #      - name: weekly-s3-us-west
 #        enabled: false
 #        schedule: "0 0 * * 0"
-#        keep: 5
+#        retention:
+#          count: 3
+#          type: count
+#          deleteFromStorage: true
 #        storageName: s3-us-west
 #        compressionType: gzip
 #        compressionLevel: 6
 #      - name: weekly-s3-us-west-physical
 #        enabled: false
 #        schedule: "0 5 * * 0"
-#        keep: 5
+#        retention:
+#          count: 3
+#          type: count
+#          deleteFromStorage: true
 #        type: physical
 #        storageName: s3-us-west
 #        compressionType: gzip
@@ -799,7 +808,10 @@ spec:
 #      - name: weekly-s3-us-west-incremental-base
 #        enabled: false
 #        schedule: "0 5 * * 0"
-#        keep: 5
+#        retention:
+#          count: 3
+#          type: count
+#          deleteFromStorage: true
 #        type: incremental-base
 #        storageName: s3-us-west
 #        compressionType: gzip

deploy/crd.yaml

@@ -1108,6 +1108,22 @@ spec:
                           type: integer
                         name:
                           type: string
+                        retention:
+                          properties:
+                            count:
+                              minimum: 0
+                              type: integer
+                            deleteFromStorage:
+                              default: true
+                              type: boolean
+                            type:
+                              enum:
+                              - count
+                              type: string
+                          required:
+                          - deleteFromStorage
+                          - type
+                          type: object
                         schedule:
                           type: string
                         storageName:

deploy/cw-bundle.yaml

@@ -1108,6 +1108,22 @@ spec:
                           type: integer
                         name:
                           type: string
+                        retention:
+                          properties:
+                            count:
+                              minimum: 0
+                              type: integer
+                            deleteFromStorage:
+                              default: true
+                              type: boolean
+                            type:
+                              enum:
+                              - count
+                              type: string
+                          required:
+                          - deleteFromStorage
+                          - type
+                          type: object
                         schedule:
                           type: string
                         storageName:

e2e-tests/scheduled-backup/conf/some-name-rs0-2.yml

@@ -49,31 +49,46 @@ spec:
     tasks:
     - name: every-min-minio
       enabled: true
-      keep: 1
+      retention:
+        type: count
+        count: 1
+        deleteFromStorage: true
       schedule: "* * * * *"
       compressionType: gzip
       storageName: minio
     - name: every-min-aws-s3
       enabled: true
-      keep: 1
+      retention:
+        type: count
+        count: 1
+        deleteFromStorage: true
       schedule: "* * * * *"
       compressionType: gzip
       storageName: aws-s3
     - name: every-min-gcp-cs
       enabled: true
-      keep: 1
+      retention:
+        type: count
+        count: 1
+        deleteFromStorage: true
       schedule: "* * * * *"
       compressionType: gzip
       storageName: gcp-cs
     - name: every-min-azure-blob
       enabled: true
-      keep: 1
+      retention:
+        type: count
+        count: 1
+        deleteFromStorage: true
       schedule: "* * * * *"
       compressionType: gzip
       storageName: azure-blob
     - name: weekly
       enabled: true
-      keep: 1
+      retention:
+        type: count
+        count: 1
+        deleteFromStorage: true
       schedule: "0 0 * * 0"
       compressionType: gzip
       storageName: aws-s3

e2e-tests/scheduled-backup/conf/some-name-rs0-3.yml

@@ -49,13 +49,20 @@ spec:
     tasks:
     - name: every-min-minio
       enabled: true
-      keep: 1
+      retention:
+        type: count
+        count: 1
+        deleteFromStorage: true
       type: physical
       schedule: "* * * * *"
       compressionType: gzip
       storageName: minio
     - name: weekly
       enabled: true
+      retention:
+        type: count
+        count: 1
+        deleteFromStorage: true
       schedule: "0 0 * * 0"
       compressionType: gzip
       storageName: aws-s3

e2e-tests/version-service/conf/crd.yaml

@@ -1108,6 +1108,22 @@ spec:
                           type: integer
                         name:
                           type: string
+                        retention:
+                          properties:
+                            count:
+                              minimum: 0
+                              type: integer
+                            deleteFromStorage:
+                              default: true
+                              type: boolean
+                            type:
+                              enum:
+                              - count
+                              type: string
+                          required:
+                          - deleteFromStorage
+                          - type
+                          type: object
                         schedule:
                           type: string
                         storageName:

pkg/apis/psmdb/v1/psmdb_defaults.go

@@ -518,10 +518,20 @@ func (cr *PerconaServerMongoDB) CheckNSetDefaults(ctx context.Context, platform
 	}
 
 	if cr.Spec.Backup.Enabled {
-		for _, bkpTask := range cr.Spec.Backup.Tasks {
+		for i := range cr.Spec.Backup.Tasks {
+			bkpTask := &cr.Spec.Backup.Tasks[i]
+
+			if bkpTask.Name == "" {
+				return errors.Errorf("backup task %d should have a name", i)
+			}
 			if string(bkpTask.CompressionType) == "" {
 				bkpTask.CompressionType = compress.CompressionTypeGZIP
 			}
+
+			if cr.CompareVersion("1.21.0") >= 0 && bkpTask.Keep > 0 && bkpTask.Retention != nil && bkpTask.Retention.Count == 0 {
+				log.Info(".spec.backup.tasks[].keep will be deprecated in the future. Consider using .spec.backup.tasks[].retention.count instead", "task", bkpTask.Name)
+				continue
+			}
 		}
 		if len(cr.Spec.Backup.ServiceAccountName) == 0 && cr.CompareVersion("1.15.0") < 0 {
 			cr.Spec.Backup.ServiceAccountName = "percona-server-mongodb-operator"

pkg/apis/psmdb/v1/psmdb_types.go

@@ -943,9 +943,12 @@ type MongodSpecInMemory struct {
 }
 
 type BackupTaskSpec struct {
-	Name             string                   `json:"name"`
-	Enabled          bool                     `json:"enabled"`
-	Keep             int                      `json:"keep,omitempty"`
+	Name    string `json:"name"`
+	Enabled bool   `json:"enabled"`
+	// Deprecated: Use Retention instead. This field will be removed in the future
+	Keep int `json:"keep,omitempty"`
+	// +optional
+	Retention        *BackupTaskSpecRetention `json:"retention,omitempty"`
 	Schedule         string                   `json:"schedule,omitempty"`
 	StorageName      string                   `json:"storageName,omitempty"`
 	CompressionType  compress.CompressionType `json:"compressionType,omitempty"`
@@ -955,6 +958,36 @@ type BackupTaskSpec struct {
 	Type defs.BackupType `json:"type,omitempty"`
 }
 
+func (task *BackupTaskSpec) GetRetention(cr *PerconaServerMongoDB) BackupTaskSpecRetention {
+	if task.Retention != nil && cr.CompareVersion("1.21.0") >= 0 {
+		return *task.Retention
+	}
+	return BackupTaskSpecRetention{
+		Type:              BackupTaskSpecRetentionTypeCount,
+		Count:             task.Keep,
+		DeleteFromStorage: true,
+	}
+}
+
+type BackupTaskSpecRetentionType string
+
+const (
+	BackupTaskSpecRetentionTypeCount = "count"
+)
+
+type BackupTaskSpecRetention struct {
+	// +kubebuilder:validation:Minimum=0
+	Count int `json:"count,omitempty"`
+
+	// +kubebuilder:validation:Required
+	// +kubebuilder:validation:Enum={count}
+	Type string `json:"type,omitempty"`
+
+	// +kubebuilder:validation:Required
+	// +kubebuilder:default=true
+	DeleteFromStorage bool `json:"deleteFromStorage,omitempty"`
+}
+
 func (task *BackupTaskSpec) JobName(cr *PerconaServerMongoDB) string {
 	return fmt.Sprintf("%s-backup-%s-%s", cr.Name, task.Name, cr.Namespace)
 }