#1882 - Recovery and backup startegy patroni (#2057)

Recovery and backup startegy patroni

Author: guru-aot

devops/Makefile

@@ -13,9 +13,9 @@ export HOST_PREFIX := $(or $(HOST_PREFIX), $$HOST_PREFIX)
 export NEW_DB := $(or $(NEW_DB), $$NEW_DB)
 export JOB_NAME := $(or $(JOB_NAME), $$JN)
 export SITE_MINDER_LOGOUT_URL := $(or $(SITE_MINDER_LOGOUT_URL), https://logontest7.gov.bc.ca/clp-cgi/logoff.cgi)
-export BACKUP_POSTGRESQL_APP_NAME := $(or $(BACKUP_POSTGRESQL_APP_NAME), postgresql-backup)
-export BACKUP_MONGODB_APP_NAME := $(or $(BACKUP_MONGODB_APP_NAME), mongodb-backup)
-export BACKUP_CONFIGMAP_NAME := $(or $(BACKUP_CONFIGMAP_NAME), backup-conf)
+export PATRONI_SIMSDB_BACKUP_APP_NAME := $(or $(PATRONI_SIMSDB_BACKUP_APP_NAME), patroni-simsdb-backup)
+export PATRONI_SIMSDB_BACKUP_CONFIGMAP_NAME := $(or $(PATRONI_SIMSDB_BACKUP_CONFIGMAP_NAME), patroni-simsdb-backup-conf)
+export PATRONI_SIMSDB_BACKUP_CONFIG_FILE_NAME := $(or $(PATRONI_SIMSDB_BACKUP_CONFIG_FILE_NAME), patroni-simsdb-backup.conf)
 export BYPASS_CRA_INCOME_VERIFICATION := $(or $(BYPASS_CRA_INCOME_VERIFICATION), false)
 export BYPASS_APPLICATION_SUBMIT_VALIDATIONS := $(or $(BYPASS_APPLICATION_SUBMIT_VALIDATIONS), false)
 export SWAGGER_ENABLED := $(or ${SWAGGER_ENABLED}, true)
@@ -143,16 +143,15 @@ oc-build-forms: | print-status build-forms
 oc-deploy-ha-mongo: | print-status deploy-ha-mongo
 oc-deploy-forms: | print-status deploy-forms
 
-# Create/delete backup structure
-# Create/delete config map shared accross postgresql and mongodb.
-oc-db-backup-configmap-init: | print-status db-backup-configmap
-oc-db-backup-configmap-delete: | print-status db-backup-configmap-delete
-# Create/delete structure for postgresql.
-oc-db-backup-init-postgresql: | print-status db-backup-build-postgresql db-backup-deploy-postgresql
-oc-db-backup-delete-postgresql: | print-status db-backup-build-delete-postgresql db-backup-deploy-delete-postgresql
-# Create/delete structure for mongodb.
-oc-db-backup-init-mongodb: | print-status db-backup-build-mongodb db-backup-deploy-mongodb
-oc-db-backup-delete-mongodb: | print-status db-backup-build-delete-mongodb db-backup-deploy-delete-mongodb
+# Build backup structure.
+oc-db-backup-patroni-simsdb-build: | print-status db-backup-build-patroni
+oc-db-backup-patroni-simsdb-build-delete: | print-status db-backup-build-patroni-delete
+
+# Create/delete backup structure.
+# Create/delete structure for patroni.
+oc-db-backup-patroni-simsdb-deploy: | print-status db-backup-configmap db-backup-deploy-patroni
+oc-db-backup-patroni-simsdb-deploy-delete: | print-status db-backup-configmap-delete db-backup-deploy-patroni-delete
+
 
 print-status:
 	@echo " +---------------------------------------------------------+ "
@@ -492,127 +491,72 @@ deploy-forms:
 	$(call rollout_and_wait,dc/$(FORMS_NAME))
 
 # Create database backup structure (build/configmap/deploy)
-
-# Create the config map to be shared by POSTGRES and mongodb.
-db-backup-configmap:
-	test -n "$(NAMESPACE)"
-	test -n "$(BACKUP_POSTGRESQL_APP_NAME)"
-	test -n "$(BACKUP_MONGODB_APP_NAME)"
-	@echo "+\n++ Creating configmap resources for database backups on $(NAMESPACE)\n+"
-	oc -n $(NAMESPACE) create configmap $(BACKUP_CONFIGMAP_NAME) --from-file=./openshift/database-backup/backup.conf
-
-db-backup-configmap-delete:
-	test -n "$(NAMESPACE)"
-	@echo "+\n++ Removing configmap resoures for database backups on $(NAMESPACE)\n+"
-	oc -n $(NAMESPACE) delete configmap/$(BACKUP_CONFIGMAP_NAME)
-
 # Create the POSTGRES structure
-db-backup-build-postgresql:
+db-backup-build-patroni:
 	test -n "$(BUILD_NAMESPACE)"
-	test -n "$(BACKUP_POSTGRESQL_APP_NAME)"
+	test -n "$(PATRONI_SIMSDB_BACKUP_APP_NAME)"
 	@echo "+\n++ Creating build resources for database backups on $(BUILD_NAMESPACE)\n+"
 	oc -n $(BUILD_NAMESPACE) process -f ./openshift/database-backup/backup-build.yaml \
-		-p NAME=$(BACKUP_POSTGRESQL_APP_NAME) OUTPUT_IMAGE_TAG=v1 \
-		-p BASE_IMAGE_FOR_BUILD=artifacts.developer.gov.bc.ca/docker-remote/centos/postgresql-12-centos7:20200917-804ef01 \
+		-p NAME=$(PATRONI_SIMSDB_BACKUP_APP_NAME) OUTPUT_IMAGE_TAG=v1 \
+		-p BASE_IMAGE_FOR_BUILD=quay.io/fedora/postgresql-14:14 \
 		| oc -n $(BUILD_NAMESPACE) apply -f -
 
-db-backup-build-delete-postgresql:
+db-backup-build-patroni-delete:
 	test -n "$(NAMESPACE)"
 	test -n "$(BUILD_NAMESPACE)"
-	test -n "$(BACKUP_POSTGRESQL_APP_NAME)"
+	test -n "$(PATRONI_SIMSDB_BACKUP_APP_NAME)"
 	@echo "+\n++ Removing build resoures for database backups on $(NAMESPACE)\n+"
-	oc -n $(BUILD_NAMESPACE) delete is/$(BACKUP_POSTGRESQL_APP_NAME) bc/$(BACKUP_POSTGRESQL_APP_NAME)
+	oc -n $(BUILD_NAMESPACE) delete is/$(PATRONI_SIMSDB_BACKUP_APP_NAME) bc/$(PATRONI_SIMSDB_BACKUP_APP_NAME)
 
-db-backup-deploy-postgresql:
-	test -n "$(NAMESPACE)"
-	test -n "$(BUILD_NAMESPACE)"
-	test -n "$(BACKUP_POSTGRESQL_APP_NAME)"
-	@echo "+\n++ Creating deploy config resoures for database backups on $(NAMESPACE)\n+"
-	oc -n $(NAMESPACE) process -f ./openshift/database-backup/backup-deploy.yaml \
-  	-p NAME=$(BACKUP_POSTGRESQL_APP_NAME) \
-  	-p IMAGE_NAMESPACE=$(BUILD_NAMESPACE) \
-  	-p SOURCE_IMAGE_NAME=$(BACKUP_POSTGRESQL_APP_NAME) \
-  	-p TAG_NAME=v1 \
-		-p FTP_SECRET_KEY=$(BACKUP_POSTGRESQL_APP_NAME)-ftp-secret \
-  	-p BACKUP_VOLUME_NAME=$(BACKUP_POSTGRESQL_APP_NAME)-pvc \
-		-p BACKUP_VOLUME_SIZE=20Gi \
-  	-p VERIFICATION_VOLUME_SIZE=5Gi \
-  	-p ENVIRONMENT_NAME=$(NAMESPACE) \
-		-p CONFIG_MAP_NAME=$(BACKUP_CONFIGMAP_NAME) \
-		-p VERIFICATION_VOLUME_NAME=$(BACKUP_POSTGRESQL_APP_NAME)-verification-pvc \
-		-p DATABASE_SERVER_NAME=PATRONI_MASTER \
-		-p DATABASE_SECRET_NAME=patroni-creds \
-		-p DATABASE_SECRET_USER_KEY_NAME=superuser-username \
-		-p DATABASE_SECRET_PASSWORD_KEY_NAME=superuser-password \
-  	-p ENVIRONMENT_FRIENDLY_NAME='SIMS $(NAMESPACE) POSTGRESQL DB Backups' \
-		| oc -n $(NAMESPACE) apply -f -	                                                                       
-
-db-backup-deploy-delete-postgresql:
+# Create the config map to be shared by POSTGRES.
+db-backup-configmap:
 	test -n "$(NAMESPACE)"
-	test -n "$(BACKUP_POSTGRESQL_APP_NAME)"
-	@echo "+\n++ Removing deploy resoures for database backups on $(NAMESPACE)\n+"
-	oc -n $(NAMESPACE) delete \
-  	secret/$(BACKUP_POSTGRESQL_APP_NAME) \
-  	secret/$(BACKUP_POSTGRESQL_APP_NAME)-ftp-secret \
-  	dc/$(BACKUP_POSTGRESQL_APP_NAME) \
-  	networkpolicy/$(BACKUP_POSTGRESQL_APP_NAME)
-# The below PVCs are not deleted.
-# pvc/$(BACKUP_POSTGRESQL_APP_NAME)-pvc \
-# pvc/$(BACKUP_POSTGRESQL_APP_NAME)-verification-pvc \
-	
-# Create the MONGODB structure
-db-backup-build-mongodb:
-	test -n "$(BUILD_NAMESPACE)"
-	test -n "$(BACKUP_MONGODB_APP_NAME)"
-	@echo "+\n++ Creating build resources for database backups on $(BUILD_NAMESPACE)\n+"
-	oc -n $(BUILD_NAMESPACE) process -f ./openshift/database-backup/backup-build.yaml \
-		-p NAME=$(BACKUP_MONGODB_APP_NAME) OUTPUT_IMAGE_TAG=v1 \
-		-p BASE_IMAGE_FOR_BUILD=artifacts.developer.gov.bc.ca/redhat-docker-remote/rhscl/mongodb-36-rhel7 \
-		| oc -n $(BUILD_NAMESPACE) apply -f -
+	test -n "$(PATRONI_SIMSDB_BACKUP_APP_NAME)"
+	@echo "+\n++ Creating configmap resources for database backups on $(NAMESPACE)\n+"
+	oc -n $(NAMESPACE) create configmap $(PATRONI_SIMSDB_BACKUP_CONFIGMAP_NAME) --from-file=./openshift/database-backup/$(PATRONI_SIMSDB_BACKUP_CONFIG_FILE_NAME)
 
-db-backup-build-delete-mongodb:
+db-backup-configmap-delete:
 	test -n "$(NAMESPACE)"
-	test -n "$(BUILD_NAMESPACE)"
-	test -n "$(BACKUP_MONGODB_APP_NAME)"
-	@echo "+\n++ Removing build resoures for database backups on $(NAMESPACE)\n+"
-	oc -n $(BUILD_NAMESPACE) delete is/$(BACKUP_MONGODB_APP_NAME) bc/$(BACKUP_MONGODB_APP_NAME)
+	@echo "+\n++ Removing configmap resoures for database backups on $(NAMESPACE)\n+"
+	oc -n $(NAMESPACE) delete configmap/$(PATRONI_SIMSDB_BACKUP_CONFIGMAP_NAME)
 
-db-backup-deploy-mongodb:
+# Deploy POSTGRES backup container.
+db-backup-deploy-patroni:
 	test -n "$(NAMESPACE)"
 	test -n "$(BUILD_NAMESPACE)"
-	test -n "$(BACKUP_MONGODB_APP_NAME)"
+	test -n "$(PATRONI_SIMSDB_BACKUP_APP_NAME)"
 	@echo "+\n++ Creating deploy config resoures for database backups on $(NAMESPACE)\n+"
 	oc -n $(NAMESPACE) process -f ./openshift/database-backup/backup-deploy.yaml \
-  	-p NAME=$(BACKUP_MONGODB_APP_NAME) \
+  	-p NAME=$(PATRONI_SIMSDB_BACKUP_APP_NAME) \
   	-p IMAGE_NAMESPACE=$(BUILD_NAMESPACE) \
-  	-p SOURCE_IMAGE_NAME=$(BACKUP_MONGODB_APP_NAME) \
+  	-p SOURCE_IMAGE_NAME=$(PATRONI_SIMSDB_BACKUP_APP_NAME) \
   	-p TAG_NAME=v1 \
-		-p FTP_SECRET_KEY=$(BACKUP_MONGODB_APP_NAME)-ftp-secret \
-  	-p BACKUP_VOLUME_NAME=$(BACKUP_MONGODB_APP_NAME)-pvc \
-		-p BACKUP_VOLUME_SIZE=5Gi \
-  	-p VERIFICATION_VOLUME_SIZE=1Gi \
+		-p FTP_SECRET_KEY=$(PATRONI_SIMSDB_BACKUP_APP_NAME)-ftp-secret \
+  	-p BACKUP_VOLUME_NAME=$(PATRONI_SIMSDB_BACKUP_APP_NAME)-pvc \
+		-p BACKUP_VOLUME_SIZE=20Gi \
+  	-p VERIFICATION_VOLUME_SIZE=5Gi \
   	-p ENVIRONMENT_NAME=$(NAMESPACE) \
-		-p CONFIG_MAP_NAME=$(BACKUP_CONFIGMAP_NAME) \
-		-p VERIFICATION_VOLUME_NAME=$(BACKUP_MONGODB_APP_NAME)-verification-pvc \
-		-p DATABASE_SERVER_NAME=MONGO \
-		-p DATABASE_SECRET_NAME=mongodb-creds \
-		-p DATABASE_SECRET_USER_KEY_NAME=username \
-		-p DATABASE_SECRET_PASSWORD_KEY_NAME=password \
-  	-p ENVIRONMENT_FRIENDLY_NAME='SIMS $(NAMESPACE) MONGO DB Backups' \
-		| oc -n $(NAMESPACE) apply -f -
+		-p CONFIG_MAP_NAME=$(PATRONI_SIMSDB_BACKUP_CONFIGMAP_NAME) \
+		-p CUSTOM_CONFIG_FILE_NAME=$(PATRONI_SIMSDB_BACKUP_CONFIG_FILE_NAME) \
+		-p VERIFICATION_VOLUME_NAME=$(PATRONI_SIMSDB_BACKUP_APP_NAME)-verification-pvc \
+		-p DATABASE_DEPLOYMENT_NAME=patroni-creds \
+		-p DATABASE_USER_KEY_NAME=superuser-username \
+		-p DATABASE_PASSWORD_KEY_NAME=superuser-password \
+  	-p ENVIRONMENT_FRIENDLY_NAME='SIMS $(NAMESPACE) POSTGRESQL DB Backups' \
+		| oc -n $(NAMESPACE) apply -f -	                                                                       
 
-db-backup-deploy-delete-mongodb:
+db-backup-deploy-patroni-delete:
 	test -n "$(NAMESPACE)"
-	test -n "$(BACKUP_MONGODB_APP_NAME)"
+	test -n "$(PATRONI_SIMSDB_BACKUP_APP_NAME)"
 	@echo "+\n++ Removing deploy resoures for database backups on $(NAMESPACE)\n+"
 	oc -n $(NAMESPACE) delete \
-  	secret/$(BACKUP_MONGODB_APP_NAME) \
-  	secret/$(BACKUP_MONGODB_APP_NAME)-ftp-secret \
-  	dc/$(BACKUP_MONGODB_APP_NAME) \
-  	networkpolicy/$(BACKUP_MONGODB_APP_NAME)
+  	secret/$(PATRONI_SIMSDB_BACKUP_APP_NAME) \
+  	secret/$(PATRONI_SIMSDB_BACKUP_APP_NAME)-ftp-secret \
+  	dc/$(PATRONI_SIMSDB_BACKUP_APP_NAME) \
+  	networkpolicy/$(PATRONI_SIMSDB_BACKUP_APP_NAME)
 # The below PVCs are not deleted.
-# pvc/$(BACKUP_MONGODB_APP_NAME)-pvc
-# pvc/$(BACKUP_MONGODB_APP_NAME)-verification-pvc
+# pvc/$(PATRONI_SIMSDB_BACKUP_APP_NAME)-pvc \
+# pvc/$(PATRONI_SIMSDB_BACKUP_APP_NAME)-verification-pvc \
 
 # Remove redis and resources including secrets from openshift namespace.
 delete-redis:

devops/openshift/database-backup/backup-build.yaml

@@ -28,8 +28,6 @@ objects:
       strategy:
         type: Docker
         dockerStrategy:
-          pullSecret:
-            name: artifactory-secret-credential
           from:
             kind: DockerImage
             name: ${BASE_IMAGE_FOR_BUILD}
@@ -73,4 +71,4 @@ parameters:
     displayName: FROM Image Tag
     description: Base image to build from.  Docker creds or Artificatory setup may be needed to alleviate docker rate-limiting
     required: true
-    value: artifacts.developer.gov.bc.ca/docker-remote/centos/postgresql-12-centos7:20200917-804ef01
+    value: docker.io/centos/postgresql-12-centos7:20200917-804ef01

devops/openshift/database-backup/backup-deploy.yaml

@@ -103,7 +103,6 @@ objects:
         app: ${APP_NAME}
         role: ${ROLE}
         env: ${TAG_NAME}
-        app.kubernetes.io/part-of: ${GROUP_LABEL}
     spec:
       strategy:
         type: Recreate
@@ -141,7 +140,7 @@ objects:
               configMap:
                 name: ${CONFIG_MAP_NAME}
                 items:
-                  - key: ${CONFIG_FILE_NAME}
+                  - key: ${CUSTOM_CONFIG_FILE_NAME}
                     path: ${CONFIG_FILE_NAME}
           containers:
             - name: ${NAME}
@@ -170,16 +169,16 @@ objects:
                   value: ${MONGODB_AUTHENTICATION_DATABASE}
                 - name: TABLE_SCHEMA
                   value: ${TABLE_SCHEMA}
-                - name: ${DATABASE_SERVER_NAME}_USER
+                - name: DATABASE_USER
                   valueFrom:
                     secretKeyRef:
-                      name: ${DATABASE_SECRET_NAME}
-                      key: ${DATABASE_SECRET_USER_KEY_NAME}
-                - name: ${DATABASE_SERVER_NAME}_PASSWORD
+                      name: ${DATABASE_DEPLOYMENT_NAME}
+                      key: ${DATABASE_USER_KEY_NAME}
+                - name: DATABASE_PASSWORD
                   valueFrom:
                     secretKeyRef:
-                      name: ${DATABASE_SECRET_NAME}
-                      key: ${DATABASE_SECRET_PASSWORD_KEY_NAME}
+                      name: ${DATABASE_DEPLOYMENT_NAME}
+                      key: ${DATABASE_PASSWORD_KEY_NAME}
                 - name: FTP_URL
                   valueFrom:
                     secretKeyRef:
@@ -271,22 +270,17 @@ parameters:
     description: This is only required if you are backing up mongo database with a separate authentication database.
     required: false
     value: ""
-  - name: DATABASE_SECRET_NAME
+  - name: DATABASE_DEPLOYMENT_NAME
     displayName: Database Deployment Name
     description: The name associated to the database deployment resources.  In particular, this is used to wire up the credentials associated to the database.
     required: true
     value: postgresql
-  - name: DATABASE_SERVER_NAME
-    displayName: Database server name
-    description: Database server name all uppercase. Should be the same server name from backup.conf file (e.g. PATRONI_MASTER).
-    required: true
-    value: SERVER_NAME
-  - name: DATABASE_SECRET_USER_KEY_NAME
+  - name: DATABASE_USER_KEY_NAME
     displayName: Database User Key Name
     description: The database user key name stored in database deployment resources specified by DATABASE_DEPLOYMENT_NAME.
     required: true
     value: database-user
-  - name: DATABASE_SECRET_PASSWORD_KEY_NAME
+  - name: DATABASE_PASSWORD_KEY_NAME
     displayName: Database Password Key Name
     description: The database password key name stored in database deployment resources specified by DATABASE_DEPLOYMENT_NAME.
     required: true
@@ -380,6 +374,11 @@ parameters:
     description: Used for backward compatibility only.  Ignored when using the recommended `backup.conf` and cron backup strategy.  Period (d,m,s) between backups in a format used by the sleep command
     required: false
     value: ""
+  - name: CUSTOM_CONFIG_FILE_NAME
+    displayName: Custom Config File Name
+    description: The name of the custom configuration file.
+    required: true
+    value: backup.conf
   - name: CONFIG_FILE_NAME
     displayName: Config File Name
     description: The name of the configuration file.
@@ -449,9 +448,4 @@ parameters:
     displayName: Resources Memory Limit
     description: The resources Memory limit (in Mi, Gi, etc) for this build.
     required: true
-    value: 0Mi
-  - name: GROUP_LABEL
-    displayName: Group label
-    description: Group label for backup structure.
-    required: true
-    value: backups
+    value: 0Mi

devops/openshift/database-backup/backup.conf

@@ -0,0 +1,4 @@
+postgres=patroni-master:5432/SIMSDB
+
+# Run a backup at 1am Pacific every day.
+0 1 * * * default ./backup.sh -s