From 83413aceeae31a33ea91fcb7ca3fddd10a8c3a5b Mon Sep 17 00:00:00 2001 From: Fox Date: Fri, 8 Jun 2018 12:13:33 +0200 Subject: [PATCH 1/5] Close #27 Prepost strategies refactored taking in consideration the conversation at issue #27 --- Dockerfile | 4 +-- README.md | 18 +++++++---- imagescripts/create_jobber.sh | 4 +++ imagescripts/create_scripts.sh | 57 ++++++++++++++-------------------- 4 files changed, 42 insertions(+), 41 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5b346ea2..4662db7f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -46,11 +46,11 @@ RUN apk upgrade --update && \ touch /etc/volumerize/remove-all-inc-of-but-n-full /etc/volumerize/remove-all-but-n-full /etc/volumerize/startContainers /etc/volumerize/stopContainers \ /etc/volumerize/backup /etc/volumerize/backupIncremental /etc/volumerize/backupFull /etc/volumerize/restore \ /etc/volumerize/periodicBackup /etc/volumerize/verify /etc/volumerize/cleanup /etc/volumerize/remove-older-than /etc/volumerize/cleanCacheLocks \ - /etc/volumerize/postexecute /etc/volumerize/prexecute && \ + /etc/volumerize/prepoststrategy && \ chmod +x /etc/volumerize/remove-all-inc-of-but-n-full /etc/volumerize/remove-all-but-n-full /etc/volumerize/startContainers /etc/volumerize/stopContainers \ /etc/volumerize/backup /etc/volumerize/backupIncremental /etc/volumerize/backupFull /etc/volumerize/restore \ /etc/volumerize/periodicBackup /etc/volumerize/verify /etc/volumerize/cleanup /etc/volumerize/remove-older-than /etc/volumerize/cleanCacheLocks \ - /etc/volumerize/postexecute /etc/volumerize/prexecute && \ + /etc/volumerize/prepoststrategy && \ # Install Jobber export JOBBER_HOME=/tmp/jobber && \ export JOBBER_LIB=$JOBBER_HOME/lib && \ diff --git a/README.md b/README.md index 1a5a6f5a..a386a4ab 100644 --- a/README.md +++ b/README.md @@ -407,13 +407,20 @@ $ docker run -d \ # Post scripts and pre scripts (prepost strategies) -If you mount a folder in `/prexecute` containing `.sh` files, they will be executed in alphabetical order before the backup and after the docker stop process. -If you mount a folder in `/postexecute` containing `.sh` files, they will be executed in alphabetical order after the backup and before the docker start process. +Pre-scripts must be located at `/preexecute/$duplicity_action/$your_scripts_here`. -In this scripts the global variable `BACKUP_TYPE` contains the type of backup or restore, so it will contain one of this options `backup, backupIncremental, backupFull or restore` so you can customize your scripts to do one thing or another depending of the type of backup or if its a restore. +Post-scripts must be located at `/postexecute/$duplicity_action/$your_scripts_here`. -For an example and implementation for mysqldump and restore look at /prepost_strategies/mysql/README.md +`$duplicity_action` folder must be named `backup`, `restore` or `verify`. + +> Note: `backup` action is the same for the scripts `backup`, `backupFull`, `backupIncremental` and `periodicBackup`. + +All `.sh` files located in the `$duplicity_action` folder will be executed in alphabetical order. + +When using prepost strategies, this will be the execution flow: `pre-scripts -> stop containers -> duplicity action -> start containers -> post-scripts`. + +Some premade strategies are available at [prepost strategies](prepost_strategies). # Container Scripts @@ -431,8 +438,7 @@ This image creates at container startup some convenience scripts. | stopContainers | Stops the specified Docker containers | | remove-older-than | Delete older backups | | cleanCacheLocks | Cleanup of old Cache locks. | -| prexecute | Execute all .sh files in /prexecute folder. | -| postexecute | Execute all .sh files in /postexecute folder. | +| prepoststrategy `$path` | Execute all `.sh` files in `$path` folder in alphabetical order. | Example triggering script inside running container: diff --git a/imagescripts/create_jobber.sh b/imagescripts/create_jobber.sh index 077e7df6..65967654 100755 --- a/imagescripts/create_jobber.sh +++ b/imagescripts/create_jobber.sh @@ -13,9 +13,13 @@ cat > ${JOBBER_SCRIPT_DIR}/periodicBackup <<_EOF_ set -o errexit +PREPOSTSTRATEGY=/preexecute/backup +sourc ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy source ${VOLUMERIZE_SCRIPT_DIR}/stopContainers ${DUPLICITY_COMMAND} ${JOBBER_PARAMETER_PROXY} ${DUPLICITY_MODE} ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_SOURCE} ${VOLUMERIZE_TARGET} source ${VOLUMERIZE_SCRIPT_DIR}/startContainers +PREPOSTSTRATEGY=/postexecute/backup +source ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy _EOF_ JOBBER_CRON_SCHEDULE='0 0 4 * * *' diff --git a/imagescripts/create_scripts.sh b/imagescripts/create_scripts.sh index 2055bb94..82a9108a 100755 --- a/imagescripts/create_scripts.sh +++ b/imagescripts/create_scripts.sh @@ -8,13 +8,13 @@ source $CUR_DIR/base.sh readonly PARAMETER_PROXY='$@' -cat > ${VOLUMERIZE_SCRIPT_DIR}/prexecute << '_EOF_' +cat > ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy << '_EOF_' #!/bin/bash set -o errexit -if [ -d "/prexecute" ]; then - for f in /prexecute/*; do +if [ -d "$PREPOSTSTRATEGY" ]; then + for f in $PREPOSTSTRATEGY/*; do case "$f" in *.sh) echo "running $f"; . "$f" ;; *) echo "ignoring $f" ;; @@ -24,34 +24,18 @@ if [ -d "/prexecute" ]; then fi _EOF_ -cat > ${VOLUMERIZE_SCRIPT_DIR}/postexecute << '_EOF_' -#!/bin/bash - -set -o errexit - -if [ -d "/postexecute" ]; then - for f in /postexecute/*; do - case "$f" in - *.sh) echo "running $f"; . "$f" ;; - *) echo "ignoring $f" ;; - esac - echo - done -fi -_EOF_ - - cat > ${VOLUMERIZE_SCRIPT_DIR}/backup <<_EOF_ #!/bin/bash set -o errexit +PREPOSTSTRATEGY=/preexecute/backup +source ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy source ${VOLUMERIZE_SCRIPT_DIR}/stopContainers -export BACKUP_TYPE=\$(basename -- "\$0") -source ${VOLUMERIZE_SCRIPT_DIR}/prexecute ${DUPLICITY_COMMAND} ${PARAMETER_PROXY} ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_SOURCE} ${VOLUMERIZE_TARGET} -source ${VOLUMERIZE_SCRIPT_DIR}/postexecute source ${VOLUMERIZE_SCRIPT_DIR}/startContainers +PREPOSTSTRATEGY=/postexecute/backup +source ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy _EOF_ cat > ${VOLUMERIZE_SCRIPT_DIR}/backupIncremental <<_EOF_ @@ -59,12 +43,13 @@ cat > ${VOLUMERIZE_SCRIPT_DIR}/backupIncremental <<_EOF_ set -o errexit +PREPOSTSTRATEGY=/preexecute/backup +source ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy source ${VOLUMERIZE_SCRIPT_DIR}/stopContainers -export BACKUP_TYPE=\$(basename -- "\$0") -source ${VOLUMERIZE_SCRIPT_DIR}/prexecute ${DUPLICITY_COMMAND} incremental ${PARAMETER_PROXY} ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_SOURCE} ${VOLUMERIZE_TARGET} -source ${VOLUMERIZE_SCRIPT_DIR}/postexecute source ${VOLUMERIZE_SCRIPT_DIR}/startContainers +PREPOSTSTRATEGY=/postexecute/backup +source ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy _EOF_ cat > ${VOLUMERIZE_SCRIPT_DIR}/backupFull <<_EOF_ @@ -72,12 +57,13 @@ cat > ${VOLUMERIZE_SCRIPT_DIR}/backupFull <<_EOF_ set -o errexit +PREPOSTSTRATEGY=/preexecute/backup +source ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy source ${VOLUMERIZE_SCRIPT_DIR}/stopContainers -export BACKUP_TYPE=\$(basename -- "\$0") -source ${VOLUMERIZE_SCRIPT_DIR}/prexecute ${DUPLICITY_COMMAND} full ${PARAMETER_PROXY} ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_SOURCE} ${VOLUMERIZE_TARGET} -source ${VOLUMERIZE_SCRIPT_DIR}/postexecute source ${VOLUMERIZE_SCRIPT_DIR}/startContainers +PREPOSTSTRATEGY=/postexecute/backup +source ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy _EOF_ cat > ${VOLUMERIZE_SCRIPT_DIR}/restore <<_EOF_ @@ -85,12 +71,13 @@ cat > ${VOLUMERIZE_SCRIPT_DIR}/restore <<_EOF_ set -o errexit +PREPOSTSTRATEGY=/preexecute/restore +source ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy source ${VOLUMERIZE_SCRIPT_DIR}/stopContainers -export BACKUP_TYPE=\$(basename -- "\$0") -source ${VOLUMERIZE_SCRIPT_DIR}/prexecute ${DUPLICITY_COMMAND} restore --force ${PARAMETER_PROXY} ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_TARGET} ${VOLUMERIZE_SOURCE} -source ${VOLUMERIZE_SCRIPT_DIR}/postexecute source ${VOLUMERIZE_SCRIPT_DIR}/startContainers +PREPOSTSTRATEGY=/postexecute/restore +source ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy _EOF_ cat > ${VOLUMERIZE_SCRIPT_DIR}/verify <<_EOF_ @@ -98,7 +85,11 @@ cat > ${VOLUMERIZE_SCRIPT_DIR}/verify <<_EOF_ set -o errexit -exec ${DUPLICITY_COMMAND} verify --compare-data ${PARAMETER_PROXY} ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_TARGET} ${VOLUMERIZE_SOURCE} +PREPOSTSTRATEGY=/preexecute/verify +source ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy +${DUPLICITY_COMMAND} verify --compare-data ${PARAMETER_PROXY} ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_TARGET} ${VOLUMERIZE_SOURCE} +PREPOSTSTRATEGY=/postexecute/verify +source ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy _EOF_ cat > ${VOLUMERIZE_SCRIPT_DIR}/cleanup <<_EOF_ From 21ea207a35e461f0a2238ab4abc2d208a34c461c Mon Sep 17 00:00:00 2001 From: Fox Date: Fri, 8 Jun 2018 13:10:25 +0200 Subject: [PATCH 2/5] Updated MySQL prepost strategy with changes from #27. Also corrected comments from pull #26 --- prepost_strategies/mysql/README.md | 2 +- prepost_strategies/mysql/postexecute/mysqlimport.sh | 9 --------- .../mysql/postexecute/restore/mysqlimport.sh | 7 +++++++ .../mysql/preexecute/backup/mysqldump.sh | 12 ++++++++++++ .../{prexecute => preexecute}/utils/check-env.sh | 10 +++++----- prepost_strategies/mysql/prexecute/mysqldump.sh | 11 ----------- 6 files changed, 25 insertions(+), 26 deletions(-) delete mode 100644 prepost_strategies/mysql/postexecute/mysqlimport.sh create mode 100644 prepost_strategies/mysql/postexecute/restore/mysqlimport.sh create mode 100644 prepost_strategies/mysql/preexecute/backup/mysqldump.sh rename prepost_strategies/mysql/{prexecute => preexecute}/utils/check-env.sh (77%) delete mode 100644 prepost_strategies/mysql/prexecute/mysqldump.sh diff --git a/prepost_strategies/mysql/README.md b/prepost_strategies/mysql/README.md index 6851d211..79de05b4 100644 --- a/prepost_strategies/mysql/README.md +++ b/prepost_strategies/mysql/README.md @@ -42,7 +42,7 @@ services: - MYSQL_DATABASE=somedatabase volumes: - volumerize-cache:/volumerize-cache - - .backup:/backup + - backup:/backup depends_on: - mariadb diff --git a/prepost_strategies/mysql/postexecute/mysqlimport.sh b/prepost_strategies/mysql/postexecute/mysqlimport.sh deleted file mode 100644 index dc80ab4e..00000000 --- a/prepost_strategies/mysql/postexecute/mysqlimport.sh +++ /dev/null @@ -1,9 +0,0 @@ -source /prexecute/utils/check-env.sh - -if [ "$BACKUP_TYPE" = "restore" ]; then - check_env "mysqlimport" "MYSQL_PASSWORD" "MYSQL_USERNAME" "MYSQL_HOST" "MYSQL_DATABASE" - - echo "mysql import starts" - pv ${VOLUMERIZE_SOURCE}/dump-${MYSQL_DATABASE}.sql | mysql -u ${MYSQL_USERNAME} -p${MYSQL_PASSWORD} $MYSQL_DATABASE - echo "Import done" -fi \ No newline at end of file diff --git a/prepost_strategies/mysql/postexecute/restore/mysqlimport.sh b/prepost_strategies/mysql/postexecute/restore/mysqlimport.sh new file mode 100644 index 00000000..6b6cd9d1 --- /dev/null +++ b/prepost_strategies/mysql/postexecute/restore/mysqlimport.sh @@ -0,0 +1,7 @@ +source /preexecute/utils/check-env.sh + +check_env "mysqlimport" "MYSQL_PASSWORD" "MYSQL_USERNAME" "MYSQL_HOST" "MYSQL_DATABASE" + +echo "mysql import starts" +pv ${VOLUMERIZE_SOURCE}/dump-${MYSQL_DATABASE}.sql | mysql -u ${MYSQL_USERNAME} -p${MYSQL_PASSWORD} $MYSQL_DATABASE +echo "Import done" \ No newline at end of file diff --git a/prepost_strategies/mysql/preexecute/backup/mysqldump.sh b/prepost_strategies/mysql/preexecute/backup/mysqldump.sh new file mode 100644 index 00000000..b33a7fa1 --- /dev/null +++ b/prepost_strategies/mysql/preexecute/backup/mysqldump.sh @@ -0,0 +1,12 @@ +source /preexecute/utils/check-env.sh + +check_env "Mysqldump" "MYSQL_PASSWORD" "MYSQL_USERNAME" "MYSQL_HOST" "MYSQL_DATABASE" + +echo "Creating $VOLUMERIZE_SOURCE folder if not exists" +mkdir -p $VOLUMERIZE_SOURCE + +# Based on this answer https://stackoverflow.com/a/32361604 +SIZE_BYTES=$(mysql --skip-column-names -u ${MYSQL_USERNAME} -p${MYSQL_PASSWORD} $MYSQL_DATABASE -e "SELECT ROUND(SUM(data_length * 0.8), 0) FROM information_schema.TABLES WHERE table_schema='${MYSQL_DATABASE}';") + +echo "mysqldump starts (Progress is aproximated)" +mysqldump --databases "${MYSQL_DATABASE}" --single-transaction --add-drop-database --user="${MYSQL_USERNAME}" --password="${MYSQL_PASSWORD}" --host="${MYSQL_HOST}" | pv --progress --size "$SIZE_BYTES" > ${VOLUMERIZE_SOURCE}/dump-${MYSQL_DATABASE}.sql \ No newline at end of file diff --git a/prepost_strategies/mysql/prexecute/utils/check-env.sh b/prepost_strategies/mysql/preexecute/utils/check-env.sh similarity index 77% rename from prepost_strategies/mysql/prexecute/utils/check-env.sh rename to prepost_strategies/mysql/preexecute/utils/check-env.sh index a4d9241c..d6643dcf 100644 --- a/prepost_strategies/mysql/prexecute/utils/check-env.sh +++ b/prepost_strategies/mysql/preexecute/utils/check-env.sh @@ -6,8 +6,8 @@ # @param1 - Name of the failed environment variable ## function _check_env_failed { - echo "[!] ENV VAR $1 IS NOT SET" - printf "\n====ENVIRONMENT VARIABLES FAILED====\n\n" + echo "Environment variable $1 is not set." + echo "Environment variables failed, exit 1" exit 1 } @@ -17,7 +17,7 @@ function _check_env_failed { # @param1 - Name of the environment variable ## function _check_env_ok { - echo "Env var $1 OK" + echo "Env var $1 ok." } ## @@ -26,7 +26,7 @@ function _check_env_ok { # @param2 to ∞ - Environment variables to check ## function check_env { - printf "\n====CHECKING ENVIRONMENT VARIABLES FOR $1====\n\n" + echo "Checking environment variables for $1." for e_var in "$@"; do if [ $e_var = $1 ]; then continue; fi # Jump first arg @@ -39,5 +39,5 @@ function check_env { fi done - printf "\n====ENVIRONMENT VARIABLES OK====\n\n" + echo "Environment variables ok." } \ No newline at end of file diff --git a/prepost_strategies/mysql/prexecute/mysqldump.sh b/prepost_strategies/mysql/prexecute/mysqldump.sh deleted file mode 100644 index dd5cc035..00000000 --- a/prepost_strategies/mysql/prexecute/mysqldump.sh +++ /dev/null @@ -1,11 +0,0 @@ -source /prexecute/utils/check-env.sh - -if [ "$BACKUP_TYPE" = "backup" ] || [ "$BACKUP_TYPE" = "backupIncremental" ] || [ "$BACKUP_TYPE" = "backupFull" ]; then - check_env "Mysqldump" "MYSQL_PASSWORD" "MYSQL_USERNAME" "MYSQL_HOST" "MYSQL_DATABASE" - - echo "Creating VOLUMERIZE_SOURCE folder if not exists" - mkdir -p $VOLUMERIZE_SOURCE - - echo "mysqldump starts" - mysqldump --databases "${MYSQL_DATABASE}" --single-transaction --add-drop-database --user="${MYSQL_USERNAME}" --password="${MYSQL_PASSWORD}" --host="${MYSQL_HOST}" > ${VOLUMERIZE_SOURCE}/dump-${MYSQL_DATABASE}.sql -fi \ No newline at end of file From 12f73391c37f1f8c70aebc93dfa4f9e1ba4b442d Mon Sep 17 00:00:00 2001 From: Steffen Bleul Date: Mon, 11 Jun 2018 23:47:51 +0200 Subject: [PATCH 3/5] Fixes #33 #30 dependency problem. --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4662db7f..ae2009b5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -30,15 +30,15 @@ RUN apk upgrade --update && \ pip install --upgrade pip && \ pip install \ PyDrive \ + chardet \ azure-storage \ boto \ lockfile \ - mediafire \ paramiko \ pycryptopp \ python-keystoneclient \ python-swiftclient \ - requests \ + requests==2.14.2 \ requests_oauthlib \ urllib3 \ dropbox==6.9.0 && \ From 5e3caa17cffb52c3a0a7d0e8a15347953d9474cf Mon Sep 17 00:00:00 2001 From: Fox Date: Thu, 14 Jun 2018 14:42:34 +0200 Subject: [PATCH 4/5] Fixed review in #31 --- README.md | 6 ++++- imagescripts/create_jobber.sh | 6 ++--- imagescripts/create_scripts.sh | 48 +++++++++++++++++++--------------- 3 files changed, 34 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index a386a4ab..7da24e81 100644 --- a/README.md +++ b/README.md @@ -438,7 +438,11 @@ This image creates at container startup some convenience scripts. | stopContainers | Stops the specified Docker containers | | remove-older-than | Delete older backups | | cleanCacheLocks | Cleanup of old Cache locks. | -| prepoststrategy `$path` | Execute all `.sh` files in `$path` folder in alphabetical order. | +| prepoststrategy `$execution_phase` `$duplicity_action` | Execute all `.sh` files for the specified exeuction phase and duplicity action in alphabetical order. | + +`$execution_phase` must be `preAction` or `postAction`. + +`$duplicity_action` must be `backup`, `verify` or `restpore`. Example triggering script inside running container: diff --git a/imagescripts/create_jobber.sh b/imagescripts/create_jobber.sh index 65967654..9553d6e2 100755 --- a/imagescripts/create_jobber.sh +++ b/imagescripts/create_jobber.sh @@ -13,13 +13,11 @@ cat > ${JOBBER_SCRIPT_DIR}/periodicBackup <<_EOF_ set -o errexit -PREPOSTSTRATEGY=/preexecute/backup -sourc ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy +${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy preAction backup source ${VOLUMERIZE_SCRIPT_DIR}/stopContainers ${DUPLICITY_COMMAND} ${JOBBER_PARAMETER_PROXY} ${DUPLICITY_MODE} ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_SOURCE} ${VOLUMERIZE_TARGET} source ${VOLUMERIZE_SCRIPT_DIR}/startContainers -PREPOSTSTRATEGY=/postexecute/backup -source ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy +${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy postAction backup _EOF_ JOBBER_CRON_SCHEDULE='0 0 4 * * *' diff --git a/imagescripts/create_scripts.sh b/imagescripts/create_scripts.sh index 82a9108a..e7f51a6f 100755 --- a/imagescripts/create_scripts.sh +++ b/imagescripts/create_scripts.sh @@ -13,8 +13,23 @@ cat > ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy << '_EOF_' set -o errexit -if [ -d "$PREPOSTSTRATEGY" ]; then - for f in $PREPOSTSTRATEGY/*; do +strategy_path="" + +case $1 in + preAction ) strategy_path=/preexecute ;; + postAction ) strategy_path=/postexecute ;; + *) echo "Error: prepoststrategy first parameter 'execution phase' must be preAction or postAction"; exit 1 ;; +esac + +case $2 in + backup | verify | restore ) ;; + *) echo "Error: porepoststrategy second parameter 'duplicity action' must be backup, verify or restore"; exit 1 ;; +esac + +strategy_path=$strategy_path/$2 + +if [ -d "$strategy_path" ]; then + for f in $strategy_path/*; do case "$f" in *.sh) echo "running $f"; . "$f" ;; *) echo "ignoring $f" ;; @@ -29,13 +44,12 @@ cat > ${VOLUMERIZE_SCRIPT_DIR}/backup <<_EOF_ set -o errexit -PREPOSTSTRATEGY=/preexecute/backup -source ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy +${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy preAction backup source ${VOLUMERIZE_SCRIPT_DIR}/stopContainers ${DUPLICITY_COMMAND} ${PARAMETER_PROXY} ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_SOURCE} ${VOLUMERIZE_TARGET} source ${VOLUMERIZE_SCRIPT_DIR}/startContainers PREPOSTSTRATEGY=/postexecute/backup -source ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy +${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy postAction backup _EOF_ cat > ${VOLUMERIZE_SCRIPT_DIR}/backupIncremental <<_EOF_ @@ -43,13 +57,11 @@ cat > ${VOLUMERIZE_SCRIPT_DIR}/backupIncremental <<_EOF_ set -o errexit -PREPOSTSTRATEGY=/preexecute/backup -source ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy +${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy preAction backup source ${VOLUMERIZE_SCRIPT_DIR}/stopContainers ${DUPLICITY_COMMAND} incremental ${PARAMETER_PROXY} ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_SOURCE} ${VOLUMERIZE_TARGET} source ${VOLUMERIZE_SCRIPT_DIR}/startContainers -PREPOSTSTRATEGY=/postexecute/backup -source ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy +${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy postAction backup _EOF_ cat > ${VOLUMERIZE_SCRIPT_DIR}/backupFull <<_EOF_ @@ -57,13 +69,11 @@ cat > ${VOLUMERIZE_SCRIPT_DIR}/backupFull <<_EOF_ set -o errexit -PREPOSTSTRATEGY=/preexecute/backup -source ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy +${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy preAction backup source ${VOLUMERIZE_SCRIPT_DIR}/stopContainers ${DUPLICITY_COMMAND} full ${PARAMETER_PROXY} ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_SOURCE} ${VOLUMERIZE_TARGET} source ${VOLUMERIZE_SCRIPT_DIR}/startContainers -PREPOSTSTRATEGY=/postexecute/backup -source ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy +${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy postAction backup _EOF_ cat > ${VOLUMERIZE_SCRIPT_DIR}/restore <<_EOF_ @@ -71,13 +81,11 @@ cat > ${VOLUMERIZE_SCRIPT_DIR}/restore <<_EOF_ set -o errexit -PREPOSTSTRATEGY=/preexecute/restore -source ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy +${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy preAction restore source ${VOLUMERIZE_SCRIPT_DIR}/stopContainers ${DUPLICITY_COMMAND} restore --force ${PARAMETER_PROXY} ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_TARGET} ${VOLUMERIZE_SOURCE} source ${VOLUMERIZE_SCRIPT_DIR}/startContainers -PREPOSTSTRATEGY=/postexecute/restore -source ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy +${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy postAction restore _EOF_ cat > ${VOLUMERIZE_SCRIPT_DIR}/verify <<_EOF_ @@ -85,11 +93,9 @@ cat > ${VOLUMERIZE_SCRIPT_DIR}/verify <<_EOF_ set -o errexit -PREPOSTSTRATEGY=/preexecute/verify -source ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy +${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy preAction verify ${DUPLICITY_COMMAND} verify --compare-data ${PARAMETER_PROXY} ${DUPLICITY_OPTIONS} ${VOLUMERIZE_INCUDES} ${VOLUMERIZE_TARGET} ${VOLUMERIZE_SOURCE} -PREPOSTSTRATEGY=/postexecute/verify -source ${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy +${VOLUMERIZE_SCRIPT_DIR}/prepoststrategy postAction verify _EOF_ cat > ${VOLUMERIZE_SCRIPT_DIR}/cleanup <<_EOF_ From e8cfe65faf7b865948e0fa2a60656a24f92264cc Mon Sep 17 00:00:00 2001 From: Fox Date: Thu, 14 Jun 2018 15:30:27 +0200 Subject: [PATCH 5/5] Prepost strategy for MongoDB --- prepost_strategies/mongodb/Dockerfile | 7 +++ prepost_strategies/mongodb/README.md | 55 +++++++++++++++++++ .../postexecute/restore/mongorestore.sh | 7 +++ .../mongodb/preexecute/backup/mongodump.sh | 9 +++ .../mongodb/preexecute/utils/check-env.sh | 43 +++++++++++++++ 5 files changed, 121 insertions(+) create mode 100644 prepost_strategies/mongodb/Dockerfile create mode 100644 prepost_strategies/mongodb/README.md create mode 100644 prepost_strategies/mongodb/postexecute/restore/mongorestore.sh create mode 100644 prepost_strategies/mongodb/preexecute/backup/mongodump.sh create mode 100644 prepost_strategies/mongodb/preexecute/utils/check-env.sh diff --git a/prepost_strategies/mongodb/Dockerfile b/prepost_strategies/mongodb/Dockerfile new file mode 100644 index 00000000..65c1cd25 --- /dev/null +++ b/prepost_strategies/mongodb/Dockerfile @@ -0,0 +1,7 @@ +FROM blacklabelops/volumerize + +RUN apk add --no-cache \ + mongodb-tools + +COPY postexecute /postexecute +COPY preexecute /preexecute \ No newline at end of file diff --git a/prepost_strategies/mongodb/README.md b/prepost_strategies/mongodb/README.md new file mode 100644 index 00000000..a0da0e38 --- /dev/null +++ b/prepost_strategies/mongodb/README.md @@ -0,0 +1,55 @@ +# Using a prepost strategy to create MongoDB backups + +Volumerize can execute scripts before and after the backup process. + +With this prepost strategy you can create dump of your MongoDB containers and save it with Volumerize. + +## Environment Variables + +Aside of the required environment variables by Volumerize, this prepost strategy will require a couple of extra variables. +MONGO_USERNAME MONGO_PASSWORD MONGO_HOST MONGO_PORT +| Name | Description | +| -------------- | ---------------------------------------------------------- | +| MONGO_USERNAME | Username of the user who will perform the restore or dump. | +| MONGO_PASSWORD | Password of the user who will perform the restore or dump. | +| MONGO_HOST | MongoDB IP or domain. | +| MONGO_PORT | MongoDB port. | + +## Example with Docker Compose + +```YAML +version: "3" + +services: + mongodb: + image: mongo + ports: + - 27017:27017 + environment: + - MONGO_INITDB_ROOT_USERNAME=root + - MONGO_INITDB_ROOT_PASSWORD=1234 + volumes: + - mongodb:/data/db + + volumerize: + build: ./prepost_strategies/mongodb/ + environment: + - VOLUMERIZE_SOURCE=/source + - VOLUMERIZE_TARGET=file:///backup + - MONGO_USERNAME=root + - MONGO_PASSWORD=1234 + - MONGO_PORT=27017 + - MONGO_HOST=mongodb + volumes: + - volumerize-cache:/volumerize-cache + - backup:/backup + depends_on: + - mongodb + +volumes: + volumerize-cache: + mongodb: + backup: +``` + +Then execute `docker-compose exec volumerize backup` to create a backup of your database and `docker-compose exec volumerize restore` to restore it from your backup. \ No newline at end of file diff --git a/prepost_strategies/mongodb/postexecute/restore/mongorestore.sh b/prepost_strategies/mongodb/postexecute/restore/mongorestore.sh new file mode 100644 index 00000000..55f654c8 --- /dev/null +++ b/prepost_strategies/mongodb/postexecute/restore/mongorestore.sh @@ -0,0 +1,7 @@ +source /preexecute/utils/check-env.sh + +check_env "mongorestore" "MONGO_USERNAME" "MONGO_PASSWORD" "MONGO_HOST" "MONGO_PORT" + +echo "mongorestore starts" +mongorestore --host ${MONGO_HOST} --port ${MONGO_PORT} --username ${MONGO_USERNAME} --password "${MONGO_PASSWORD}" ${VOLUMERIZE_SOURCE} +echo "Import done" \ No newline at end of file diff --git a/prepost_strategies/mongodb/preexecute/backup/mongodump.sh b/prepost_strategies/mongodb/preexecute/backup/mongodump.sh new file mode 100644 index 00000000..d4b1a3bd --- /dev/null +++ b/prepost_strategies/mongodb/preexecute/backup/mongodump.sh @@ -0,0 +1,9 @@ +source /preexecute/utils/check-env.sh + +check_env "mongodump" "MONGO_USERNAME" "MONGO_PASSWORD" "MONGO_HOST" "MONGO_PORT" + +echo "Creating $VOLUMERIZE_SOURCE folder if not exists" +mkdir -p $VOLUMERIZE_SOURCE + +echo "mongodump starts" +mongodump --host ${MONGO_HOST} --port ${MONGO_PORT} --username ${MONGO_USERNAME} --password "${MONGO_PASSWORD}" --out ${VOLUMERIZE_SOURCE} \ No newline at end of file diff --git a/prepost_strategies/mongodb/preexecute/utils/check-env.sh b/prepost_strategies/mongodb/preexecute/utils/check-env.sh new file mode 100644 index 00000000..d6643dcf --- /dev/null +++ b/prepost_strategies/mongodb/preexecute/utils/check-env.sh @@ -0,0 +1,43 @@ +#!/bin/sh + +## + # @private + # Executed if a variable is not set or is empty string + # @param1 - Name of the failed environment variable + ## +function _check_env_failed { + echo "Environment variable $1 is not set." + echo "Environment variables failed, exit 1" + exit 1 +} + +## + # @private + # Executed if a variable is setted + # @param1 - Name of the environment variable + ## +function _check_env_ok { + echo "Env var $1 ok." +} + +## + # Use it to check if environment variables are set + # @param1 - Name of the context + # @param2 to ∞ - Environment variables to check + ## +function check_env { + echo "Checking environment variables for $1." + + for e_var in "$@"; do + if [ $e_var = $1 ]; then continue; fi # Jump first arg + + # Check if env var is setted, if not raise error + if [ "${!e_var}" = "" ]; then + _check_env_failed $e_var; + else + _check_env_ok $e_var; + fi + + done + echo "Environment variables ok." +} \ No newline at end of file