From 08e33f670955f73742cdc2e6d71c10402e877c57 Mon Sep 17 00:00:00 2001 From: Samuel Baumgartner Date: Tue, 5 Dec 2017 17:50:08 +0100 Subject: [PATCH 01/18] Implement repo adding, add include function --- defaults/main.yml | 2 ++ tasks/install.deb.yml | 11 +++++++++++ templates/exclude.j2 | 3 +++ 3 files changed, 16 insertions(+) diff --git a/defaults/main.yml b/defaults/main.yml index 25a1c05..07b2fe8 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -92,3 +92,5 @@ backup_volsize: 50 backup_verbosity: 3 backup_exclude: [] # List of filemasks to exlude + +backup_include: [] # List of filemasks to exlude diff --git a/tasks/install.deb.yml b/tasks/install.deb.yml index 8ca6a2a..dd7b47e 100644 --- a/tasks/install.deb.yml +++ b/tasks/install.deb.yml @@ -2,6 +2,17 @@ - include_vars: "{{ansible_distribution}}.yml" +#- name: Adding key EBFF6B99D9B78493 (sonarr) +# apt_key: +# keyserver: keyserver.ubuntu.com +# id: EBFF6B99D9B78493 + +- name: Adding repository + apt_repository: + repo: "{{ backup_duplicity_ppa }}" + state: present + when: backup_duplicity_ppa is defined + - name: Install dependencies apt: name={{item}} with_items: diff --git a/templates/exclude.j2 b/templates/exclude.j2 index a15f330..9deee17 100644 --- a/templates/exclude.j2 +++ b/templates/exclude.j2 @@ -1,3 +1,6 @@ +{% for include in item.include|default(backup_include) %} ++ {{include}} +{% endfor %} {% for exclude in item.exclude|default(backup_exclude) %} - {{exclude}} {% endfor %} From 1b41903a1f101356e10a9298c0d75151304bf570 Mon Sep 17 00:00:00 2001 From: Samuel Baumgartner Date: Sun, 10 Dec 2017 21:36:08 +0100 Subject: [PATCH 02/18] python dependency for b2 storage --- defaults/main.yml | 4 ++++ meta/main.yml | 3 ++- tasks/install.deb.yml | 5 ----- vars/Ubuntu.yml | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 07b2fe8..ef2a9dd 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -1,9 +1,13 @@ --- +#python_install: b2 + backup_enabled: yes # Enable the role backup_remove: no # Set yes for uninstall the role from target system backup_cron: yes # Setup cron tasks for backup +backup_duplicity_ppa: ppa:duplicity-team/ppa # Install newest version from repo + backup_user: root # Run backups as user backup_group: "{{backup_user}}" diff --git a/meta/main.yml b/meta/main.yml index 87cc4c0..96bf5be 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -1,6 +1,7 @@ --- -dependencies: [] +dependencies: + - { role: Stouts.python, python_install: [b2] } galaxy_info: author: klen diff --git a/tasks/install.deb.yml b/tasks/install.deb.yml index dd7b47e..15a907a 100644 --- a/tasks/install.deb.yml +++ b/tasks/install.deb.yml @@ -2,11 +2,6 @@ - include_vars: "{{ansible_distribution}}.yml" -#- name: Adding key EBFF6B99D9B78493 (sonarr) -# apt_key: -# keyserver: keyserver.ubuntu.com -# id: EBFF6B99D9B78493 - - name: Adding repository apt_repository: repo: "{{ backup_duplicity_ppa }}" diff --git a/vars/Ubuntu.yml b/vars/Ubuntu.yml index ed97d53..73b314f 100644 --- a/vars/Ubuntu.yml +++ b/vars/Ubuntu.yml @@ -1 +1 @@ ---- +--- \ No newline at end of file From 09e40a24a5c8f673da233dac347f237a9c02205d Mon Sep 17 00:00:00 2001 From: Samuel Baumgartner Date: Tue, 19 Dec 2017 23:51:37 +0100 Subject: [PATCH 03/18] change python dependency to simpler role --- meta/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/main.yml b/meta/main.yml index 96bf5be..e8dc0bd 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -1,7 +1,7 @@ --- dependencies: - - { role: Stouts.python, python_install: [b2] } + - { role: ansible-role-python, python_install: [b2] } galaxy_info: author: klen From 96c9bcea31cbca13803a17cf63214570b58ccfd2 Mon Sep 17 00:00:00 2001 From: Samuel Baumgartner Date: Mon, 9 Apr 2018 10:09:51 +0200 Subject: [PATCH 04/18] add hubic --- defaults/main.yml | 8 ++++++++ meta/main.yml | 2 +- tasks/configure.yml | 9 +++++++++ templates/hubic_credentials.j2 | 6 ++++++ 4 files changed, 24 insertions(+), 1 deletion(-) create mode 100644 templates/hubic_credentials.j2 diff --git a/defaults/main.yml b/defaults/main.yml index ef2a9dd..2045d3c 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -98,3 +98,11 @@ backup_verbosity: 3 backup_exclude: [] # List of filemasks to exlude backup_include: [] # List of filemasks to exlude + +backup_credential_files: [] # List of credential file + +backup_credential_hubic_email: +backup_credential_hubic_password: +backup_credential_hubic_client_id: +backup_credential_hubic_client_secret: +backup_credential_hubic_redirect_uri: \ No newline at end of file diff --git a/meta/main.yml b/meta/main.yml index e8dc0bd..1338e07 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -1,7 +1,7 @@ --- dependencies: - - { role: ansible-role-python, python_install: [b2] } + - { role: ansible-role-python, python_install: [b2,pyrax] } galaxy_info: author: klen diff --git a/tasks/configure.yml b/tasks/configure.yml index c28918a..01f8c82 100644 --- a/tasks/configure.yml +++ b/tasks/configure.yml @@ -52,3 +52,12 @@ - name: backup-configure | Configure logrotate template: src=logrotate.j2 dest=/etc/logrotate.d/backup owner=root group=root mode=0644 + +- name: deploy credential files to home directory + become: true + become_user: "{{backup_user}}" + template: + src: "{{ item }}.j2" + dest: "{{ ansible_env.HOME }}/.{{ item }}" + with_items: + - "{{ backup_credential_files }}" \ No newline at end of file diff --git a/templates/hubic_credentials.j2 b/templates/hubic_credentials.j2 new file mode 100644 index 0000000..c6b72d5 --- /dev/null +++ b/templates/hubic_credentials.j2 @@ -0,0 +1,6 @@ +[hubic] +email = {{ backup_credential_hubic_email }} +password = {{ backup_credential_hubic_password }} +client_id = {{ backup_credential_hubic_client_id }} +client_secret = {{ backup_credential_hubic_client_secret }} +redirect_uri = {{ backup_credential_hubic_redirect_uri }} \ No newline at end of file From fb069ab4f233ad6c96098bb4349e1c7f19c1ef8d Mon Sep 17 00:00:00 2001 From: Samuel Baumgartner Date: Mon, 23 Apr 2018 10:50:15 +0200 Subject: [PATCH 05/18] pyrax is not needed anymore becuase hubic is a useless storage provider --- meta/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/main.yml b/meta/main.yml index 1338e07..e8dc0bd 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -1,7 +1,7 @@ --- dependencies: - - { role: ansible-role-python, python_install: [b2,pyrax] } + - { role: ansible-role-python, python_install: [b2] } galaxy_info: author: klen From ed2875431d59360e08f6986d26da3fdfb31eb5b0 Mon Sep 17 00:00:00 2001 From: Samuel Baumgartner Date: Mon, 23 Apr 2018 15:32:17 +0200 Subject: [PATCH 06/18] trying to fix install on CentOS --- tasks/install.deb.yml | 7 +------ tasks/install.red.yml | 11 +++++------ vars/Debian.yml | 2 +- vars/RedHat.yml | 7 +++++++ vars/Ubuntu.yml | 8 +++++++- 5 files changed, 21 insertions(+), 14 deletions(-) create mode 100644 vars/RedHat.yml diff --git a/tasks/install.deb.yml b/tasks/install.deb.yml index 15a907a..e8f99a7 100644 --- a/tasks/install.deb.yml +++ b/tasks/install.deb.yml @@ -9,12 +9,7 @@ when: backup_duplicity_ppa is defined - name: Install dependencies - apt: name={{item}} - with_items: - - cron - - gzip - - python-boto - - s3cmd + apt: name={{ dependencies }} - set_fact: backup_duplicity_pkg="{{backup_duplicity_pkg}}={{backup_duplicity_version}}" when: backup_duplicity_version diff --git a/tasks/install.red.yml b/tasks/install.red.yml index bbf1553..5d23184 100644 --- a/tasks/install.red.yml +++ b/tasks/install.red.yml @@ -1,5 +1,7 @@ --- +- include_vars: "{{ ansible_os_family }}.yml" + - name: Ensure libselinux-python is installed yum: name=libselinux-python @@ -14,12 +16,9 @@ state: present - name: Install dependencies - yum: name={{item}} - with_items: - - cronie - - gzip - - python-boto - - s3cmd + yum: name={{ dependencies }} + update_cache: yes + - set_fact: backup_duplicity_pkg="{{backup_duplicity_pkg}}-{{backup_duplicity_version}}" when: backup_duplicity_version diff --git a/vars/Debian.yml b/vars/Debian.yml index 2dc230e..4d72286 100644 --- a/vars/Debian.yml +++ b/vars/Debian.yml @@ -1,3 +1,3 @@ --- -backup_duplicity_ppa: false +backup_duplicity_ppa: false \ No newline at end of file diff --git a/vars/RedHat.yml b/vars/RedHat.yml new file mode 100644 index 0000000..faffce7 --- /dev/null +++ b/vars/RedHat.yml @@ -0,0 +1,7 @@ +--- + +dependencies: + - cronie + - gzip + - python-boto + - s3cmd \ No newline at end of file diff --git a/vars/Ubuntu.yml b/vars/Ubuntu.yml index 73b314f..330fe8a 100644 --- a/vars/Ubuntu.yml +++ b/vars/Ubuntu.yml @@ -1 +1,7 @@ ---- \ No newline at end of file +--- + +dependencies: + - cron + - gzip + - python-boto + - s3cmd \ No newline at end of file From dabdbecc679fb6a075a5170f199f42b8a7afc8ea Mon Sep 17 00:00:00 2001 From: Samuel Baumgartner Date: Sat, 5 May 2018 23:15:54 +0200 Subject: [PATCH 07/18] add the option to use a second cron job/schedule0 with a different action --- templates/cron.j2 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/templates/cron.j2 b/templates/cron.j2 index 7933c69..58848e0 100644 --- a/templates/cron.j2 +++ b/templates/cron.j2 @@ -6,5 +6,7 @@ {% if profile.schedule|default(None) %} {{profile.schedule}} {{profile.user|default(backup_user)}} /usr/bin/duply {{backup_home}}/{{profile.name}} {{profile.action|default('backup')}} >> {{backup_logdir}}/{{profile.name}}.log 2>&1 {% endif %} +{% if profile.schedule2|default(None) %} +{{profile.schedule2}} {{profile.user|default(backup_user)}} /usr/bin/duply {{backup_home}}/{{profile.name}} {{profile.action2|default('backup_verify_purge --force')}} >> {{backup_logdir}}/{{profile.name}}.log 2>&1 +{% endif %} {% endfor %} - From 4bbf1c204f9902063234f5da96487f6f194e3213 Mon Sep 17 00:00:00 2001 From: Samuel Baumgartner Date: Wed, 13 Jun 2018 17:24:44 +0200 Subject: [PATCH 08/18] change file globbing to a more generic way --- defaults/main.yml | 6 +++++- templates/exclude.j2 | 7 ++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 2045d3c..136b6f5 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -39,7 +39,11 @@ backup_profiles: [] # Setup backup profiles # max_age: 10D # target: s3://my.bucket/www # exclude: - # - *.pyc + # - "+ /var/www/folder1" + # - "+ /var/www/folder2" + # - "- **/folder3/*.log" + # - "+ **/folder3" + # - "- **" # - name: postgresql # schedule: 0 4 * * * # action: restore # Choose action: backup/restore (default is backup) diff --git a/templates/exclude.j2 b/templates/exclude.j2 index 9deee17..e3ffe0e 100644 --- a/templates/exclude.j2 +++ b/templates/exclude.j2 @@ -1,6 +1,3 @@ -{% for include in item.include|default(backup_include) %} -+ {{include}} -{% endfor %} -{% for exclude in item.exclude|default(backup_exclude) %} -- {{exclude}} +{% for item in item.exclude | default(backup_exclude) %} +{{ item }} {% endfor %} From f79b3928019435cc01d2ecc11ff82ff0263b650c Mon Sep 17 00:00:00 2001 From: Samuel Baumgartner Date: Mon, 18 Jun 2018 23:06:23 +0200 Subject: [PATCH 09/18] upgrade duply --- templates/duply.sh.j2 | 1179 +++++++++++++++++++++++------------------ 1 file changed, 671 insertions(+), 508 deletions(-) diff --git a/templates/duply.sh.j2 b/templates/duply.sh.j2 index b38e2c3..351334b 100755 --- a/templates/duply.sh.j2 +++ b/templates/duply.sh.j2 @@ -1,29 +1,29 @@ -#!/usr/bin/env bash +{% raw %}#!/usr/bin/env bash # -############################################################################### -# duply (grown out of ftplicity), is a shell front end to duplicity that # -# simplifies the usage by managing settings for backup jobs in profiles. # -# It supports executing multiple commands in a batch mode to enable single # -# line cron entries and executes pre/post backup scripts. # -# Since version 1.5.0 all duplicity backends are supported. Hence the name # -# changed from ftplicity to duply. # -# See http://duply.net or http://ftplicity.sourceforge.net/ for more info. # -# (c) 2006 Christiane Ruetten, Heise Zeitschriften Verlag, Germany # -# (c) 2008-2014 Edgar Soldin (changes since version 1.3) # -############################################################################### -# LICENSE: # -# This program is licensed under GPLv2. # -# Please read the accompanying license information in gpl.txt. # -############################################################################### +################################################################################ +# duply (grown out of ftplicity), is a shell front end to duplicity that # +# simplifies the usage by managing settings for backup jobs in profiles. # +# It supports executing multiple commands in a batch mode to enable single # +# line cron entries and executes pre/post backup scripts. # +# Since version 1.5.0 all duplicity backends are supported. Hence the name # +# changed from ftplicity to duply. # +# See http://duply.net or http://ftplicity.sourceforge.net/ for more info. # +# (c) 2006 Christiane Ruetten, Heise Zeitschriften Verlag, Germany # +# (c) 2008-2017 Edgar Soldin (changes since version 1.3) # +################################################################################ +# LICENSE: # +# This program is licensed under GPLv2. # +# Please read the accompanying license information in gpl.txt. # +################################################################################ # TODO/IDEAS/KNOWN PROBLEMS: # - possibility to restore time frames (incl. deleted files) -# realizable by listing each backup and restore from +# realizable by listing each backup and restore from # oldest to the newest, problem: not performant # - search file in all backups function and show available # versions with backups date (list old avail since 0.6.06) -# - edit profile opens conf file in vi +# - edit profile opens conf file in vi # - implement log-fd interpretation -# - add a duplicity option check against the options pending +# - add a duplicity option check against the options pending # deprecation since 0.5.10 namely --time-separator # --short-filenames # --old-filenames @@ -32,15 +32,100 @@ # - hint on install software if a piece is missing # - import/export profile from/to .tgz function !!! # -# # CHANGELOG: +# 2.0.4 (20.02.2018) +# - bugfix 114: "duply usage is not current" wrt. purgeFull/Incr +# - bugfix 115: typo in error message - "Not GPG_KEY entries" should be "No" +# - bugfix 117: no duply_ prefix when ARCH_DIR is set in conf +# - bugfix debian 882159: duply: occasionally shows negative runtimes +# +# 2.0.3 (29.08.2017) +# - bugfix: "line 2231: CMDS: bad array subscript" +# - bugfix 112: "env: illegal option -- u" on MacOSX +# +# 2.0.2 (23.05.2017) +# - bugfix: never insert creds into file:// targets +# - bugfix: avail profiles hint sometimes shortend the names by one char +# - bugfix 108: CMD_NEXT variable should ignore conditional commands (and, or) +# - export condition before/after next/prev command as CND_PREV,CND_NEXT now +# - bugfix 97: Unknown command should be ERROR, not WARNING +# +# 2.0.1 (16.11.2016) +# - bugfix 104: Duply 2.0 sets wrong archive dir, --name was always 'duply_' +# +# 2.0 (27.10.2016) +# made this a major version change, as we broke backward compatibility anyway +# (see last change in v1.10). got complaints that rightfully pointed out +# that should only come w/ a major version change. so, here we go ;) +# if your backend stops working w/ this version create a new profile and +# export the env vars needed as described in the comments of the conf file +# directly above the SOURCE setting. +# Changes: +# - making sure multi spaces in TARGET survive awk processing +# - new env var PROFILE exported to scripts +# - fix 102: expose a unique timestamp variable for pre/post scripts +# actually a featreq. exporting RUN_START nanosec unix timestamp +# - fix 101: GPG_AGENT_INFO is 'bogus' (thx Thomas Harning Jr.) +# - fix 96: duply cannot handle two consecutive spaces in paths +# +# 1.11.3 (29.5.2016) +# - fix wrong "WARNING: No running gpg-agent ..." when sign key was not set +# +# 1.11.2 (11.2.2016) +# - fix "gpg: unsafe" version print out +# - bugfix 91: v1.11 [r47] broke asymmetric encryption when using GPG_KEYS_ENC +# - bugfix 90: S3: TARGET_USER/PASS have no effect, added additional +# documentation about needed env vars to template conf file +# +# 1.11.1 (18.12.2015) +# - bugfix 89: "Duply has trouble with PYTHON-interpreter" on OSX homebrew +# - reverted duply's default PYTHON to 'python' +# +# 1.11 (24.11.2015) +# - remove obsolete --ssh-askpass routine +# - add PYTHON conf var to allow global override of used python interpreter +# - enforced usage of "python2" in PATH as default interpreter for internal +# use _and_ to run duplicity (setup.py changed the shebang to the fixed +# path /usr/bin/python until 0.7.05, which we circumvent this way) +# - featreq 36: support gpg-connect-agent as a means to detect if an agent is +# running (thx Thomas Harning Jr.), used gpg-agent for detection though +# - quotewrapped run_cmd parameters to protect it from spaces eg. in TMP path +# - key export routine respects gpg-agent usage now +# +# 1.10.1 (19.8.2015) +# - bugfix 86: Duply+Swift outputs warning +# - bugfix 87: Swift fails without BACKEND_URL +# +# 1.10 (31.7.2015) +# - featreq 37: busybox issues - fix awk, grep version detection, +# fix grep failure because --color=never switch is unsupported +# (thx Thomas Harning Jr. for reporting and helping to debug/fix it) +# - bugfix 81: --exclude-globbing-filelist is deprecated since 0.7.03 +# (thx Joachim Wiedorn, also for maintaining the debian package) +# - implemented base-/dirname as bash functions +# - featreq 31 " Support for duplicity Azure backend " - ignored a +# contributed patch by Scott McKenzie and instead opted for removing almost +# all code that deals with special env vars required by backends. +# adding and modifying these results in too much overhead so i dropped this +# feature. the future alternative for users is to consult the duplicity +# manpage and add the needed export definitions to the conf file. +# appended a commented example to the template conf below the auth section. +# +# 1.9.2 (21.6.2015) +# - bugfix: exporting keys with gpg2.1 works now (thx Philip Jocks) +# - documented GPG_OPTS needed for gpg2.1 to conf template (thx Troy Engel) +# - bugfix 82: GREP_OPTIONS=--color=always disrupted time calculation +# - added GPG conf var (see conf template for details) +# - added grep version output as it is an integral needed binary +# - added PYTHONPATH printout in version output +# # 1.9.1 (13.10.2014) # - export CMD_ERR now for scripts to detect if CMD_PREV failed/succeeded # - bugfix: CMD_PREV contained command even if it was skipped # # 1.9.0 (24.8.2014) # - bugfix: env vars were not exported when external script was executable -# - rework GPG_KEY handling, allow virtually anything now (uid, keyid etc.) +# - rework GPG_KEY handling, allow virtually anything now (uid, keyid etc.) # see gpg manpage, section "How to specify a user ID" # let gpg complain when the delivered values are invalid for whatever reason # - started to rework tmp space checking, exposed folder & writable check @@ -49,7 +134,7 @@ # 1.8.0 (13.7.2014) # - add command verifyPath to expose 'verify --file-to-restore' action # - add time parameter support to verify command -# - add section time formats to usage output +# - add section time formats to usage output # # 1.7.4 (24.6.2014) # - remove ubuntu one support, service is discontinued @@ -59,23 +144,23 @@ # - bugfix: test routines, gpg2 asked for passphrase although GPG_PW was set # # 1.7.2 (1.4.2014 "April,April") -# - bugfix: debian Bug#743190 "duply no longer allows restoration without +# - bugfix: debian Bug#743190 "duply no longer allows restoration without # gpg passphrase in conf file" # GPG_AGENT_INFO env var is now needed to trigger --use-agent # - bugfix: gpg keyenc test routines didn't work if GPG_PW was not set # # 1.7.1 (30.3.2014) -# - bugfix: purge-* commands renamed to purgeFull, purgeIncr due to -# incompatibility with new minus batch separator +# - bugfix: purge-* commands renamed to purgeFull, purgeIncr due to +# incompatibility with new minus batch separator # # 1.7.0 (20.3.2014) # - disabled gpg key id plausibility check, too many valid possibilities # - featreq 7 "Halt if precondition fails": # added and(+), or(-) batch command(separator) support -# - featreq 26 "pre/post script with shebang line": -# if a script is flagged executable it's executed in a subshell +# - featreq 26 "pre/post script with shebang line": +# if a script is flagged executable it's executed in a subshell # now as opposed to sourced to bash, which is the default -# - bugfix: do not check if dpbx, swift credentials are set anymore +# - bugfix: do not check if dpbx, swift credentials are set anymore # - bugfix: properly escape profile name, archdir if used as arguments # - add DUPL_PRECMD conf setting for use with e.g. trickle # @@ -88,7 +173,7 @@ # homedir can thus be configured to be located anywhere # - always import both secret and public key if avail from config profile # - new explanatory comments in initial exclude file -# - bugfix 7: Duply only imports one key at a time +# - bugfix 7: Duply only imports one key at a time # # 1.5.11 (19.07.2013) # - purge-incr command for remove-all-inc-of-but-n-full feature added @@ -103,7 +188,7 @@ # # 1.5.9 (22.11.2012) # - bugfix 3588926: filter --exclude* params for restore/fetch ate too much -# - restore/fetch now also ignores --include* or --exclude='foobar' +# - restore/fetch now also ignores --include* or --exclude='foobar' # # 1.5.8 (26.10.2012) # - bugfix 3575487: implement proper cloud files support @@ -112,7 +197,7 @@ # - bugfix 3531450: Cannot use space in target URL (file:///) anymore # # 1.5.6 (24.5.2012) -# - commands purge, purge-full have no default value anymore for security +# - commands purge, purge-full have no default value anymore for security # reasons; instead max value can be given via cmd line or must be set # in profile; else an error is shown. # - minor man page modifications @@ -143,33 +228,33 @@ # - bugfix 3312208: signing detection broke symmetric gpg test routine # # 1.5.5 (2.5.2011) -# - bugfix: fetch problem with space char in path, escape all params +# - bugfix: fetch problem with space char in path, escape all params # containing non word chars # - list available profiles, if given profile cannot be found # - added --use-agent configuration hint -# - bugfix 3174133: --exclude* params in conf DUPL_PARAMS broke +# - bugfix 3174133: --exclude* params in conf DUPL_PARAMS broke # fetch/restore # - version command now prints out 'using installed' info -# - featreq 3166169: autotrust imported keys, based on code submitted by -# Martin Ellis - imported keys are now automagically trusted ultimately +# - featreq 3166169: autotrust imported keys, based on code submitted by +# Martin Ellis - imported keys are now automagically trusted ultimately # - new txt2man feature to create manpages for package maintainers # # 1.5.4.2 (6.1.2011) # - new command changelog # - bugfix 3109884: freebsd awk segfaulted on printf '%*', use print again -# - bugfix: freebsd awk hangs on 'awk -W version' +# - bugfix: freebsd awk hangs on 'awk -W version' # - bugfix 3150244: mawk does not know '--version' # - minor help text improvements # - new env vars CMD_PREV,CMD_NEXT replacing CMD env var for scripts # # 1.5.4.1 (4.12.2010) # - output awk, python, bash version now in prolog -# - shebang uses /usr/bin/env now for freebsd compatibility, -# bash not in /bin/bash -# - new --disable-encryption parameter, +# - shebang uses /usr/bin/env now for freebsd compatibility, +# bash not in /bin/bash +# - new --disable-encryption parameter, # to override profile encr settings for one run # - added exclude-if-present setting to conf template -# - bug 3126972: GPG_PW only needed for signing/symmetric encryption +# - bug 3126972: GPG_PW only needed for signing/symmetric encryption # (even though duplicity still needs it) # # 1.5.4 (15.11.2010) @@ -186,8 +271,8 @@ # - bugfix 2996459: Duply erroneously escapes '-' symbol in username # - url_encode function is now pythonized # - rsync uses FTP_PASSWORD now if duplicity 0.6.10+ , else issue warning -# - feature 3059262: Make pre and post aware of parameters, -# internal parameters + CMD of pre or post +# - feature 3059262: Make pre and post aware of parameters, +# internal parameters + CMD of pre or post # # 1.5.2.3 (16.4.2010) # - bugfix: date again, should now work virtually anywhere @@ -234,15 +319,15 @@ # aid=2864410&group_id=217745&atid=1041147 # # 1.5.1 (21.09.2009) - duply (fka. ftplicity) -# - first things first: ftplicity (being able to support all backends since +# - first things first: ftplicity (being able to support all backends since # some time) will be called duply (fka. ftplicity) from now on. The addendum # is for the time being to circumvent confusion. -# - bugfix: exit code is 1 (error) not 0 (success), if at least on duplicity +# - bugfix: exit code is 1 (error) not 0 (success), if at least on duplicity # command failed # - s3[+http] now supported natively by translating user/pass to access_key/ -# secret_key environment variables needed by duplicity s3 boto backend +# secret_key environment variables needed by duplicity s3 boto backend # - bugfix: additional output lines do not confuse version check anymore -# - list command supports now age parameter (patch by stefan on feature +# - list command supports now age parameter (patch by stefan on feature # request tracker) # - bugfix: option/param pairs are now correctly passed on to duplicity # - bugfix: s3[+http] needs no TARGET_PASS if command is read only @@ -259,7 +344,7 @@ # # 1.5.0 (01.07.2009) # - removed ftp limitation, all duplicity backends should work now -# - bugfix: date for separator failed on openwrt busybox date, added a +# - bugfix: date for separator failed on openwrt busybox date, added a # detecting workaround, milliseconds are not available w/ busybox date # # 1.4.2.1 (14.05.2009) @@ -268,29 +353,29 @@ # 1.4.2 (22.04.2009) # - gpg keys are now exported as gpgkey.[id].asc , the suffix reflects the # armored ascii nature, the id helps if the key is switched for some reason -# im/export routines are updated accordingly (import is backward compatible -# to the old profile/gpgkey files) -# - profile argument is treated as path if it contains slashes +# im/export routines are updated accordingly (import is backward compatible +# to the old profile/gpgkey files) +# - profile argument is treated as path if it contains slashes # (for details see usage) -# - non-ftplicity options (all but --preview currently) are now passed -# on to duplicity +# - non-ftplicity options (all but --preview currently) are now passed +# on to duplicity # - removed need for stat in secure_conf, it is ls based now # - added profile folder readable check # - added gpg version & home info output # - awk utility availability is now checked, because it was mandatory already # - tmp space is now checked on writability and space requirement -# test fails on less than 25MB or configured $VOLSIZE, -# test warns if there is less than two times $VOLSIZE because -# that's required for --asynchronous-upload option -# - gpg functionality is tested now before executing duplicity +# test fails on less than 25MB or configured $VOLSIZE, +# test warns if there is less than two times $VOLSIZE because +# that's required for --asynchronous-upload option +# - gpg functionality is tested now before executing duplicity # test drive contains encryption, decryption, comparison, cleanup # this is meant to detect non trusted or other gpg errors early # - added possibility of doing symmetric encryption with duplicity # set GPG_KEY="" or simply comment it out -# - added hints in config template on the depreciation of +# - added hints in config template on the depreciation of # --short-filenames, --time-separator duplicity options # -# new versioning scheme 1.4.2b => 1.4.2, +# new versioning scheme 1.4.2b => 1.4.2, # beta b's are replaced by a patch count number e.g. 1.4.2.1 will be assigned # to the first bug fixing version and 1.4.2.2 to the second and so on # also the releases will now have a release date formatted (Day.Month.Year) @@ -311,26 +396,26 @@ # # 1.4.0b1 - bugfix: incr forces incremental backups on duplicity, # therefore backup translates to pre_bkp_post now -# - bugfix: new command bkp, which represents duplicity's +# - bugfix: new command bkp, which represents duplicity's # default action (incr or full if full_if_older matches # or no earlier backup chain is found) # # new versioning scheme 1.4 => 1.4.0, added new minor revision number -# this is meant to slow down the rapid version growing but still keep +# this is meant to slow down the rapid version growing but still keep # versions cleanly separated. -# only additional features will raise the new minor revision number. -# all releases start as beta, each bugfix release will raise the beta +# only additional features will raise the new minor revision number. +# all releases start as beta, each bugfix release will raise the beta # count, usually new features arrive before a version 'ripes' to stable -# +# # 1.4.0b # 1.4b - added startup info on version, time, selected profile # - added time output to separation lines -# - introduced: command purge-full implements duplicity's +# - introduced: command purge-full implements duplicity's # remove-all-but-n-full functionality (patch by unknown), # uses config variable $MAX_FULL_BACKUPS (default = 1) -# - purge config var $MAX_AGE defaults to 1M (month) now +# - purge config var $MAX_AGE defaults to 1M (month) now # - command full does not execute pre/post anymore -# use batch command pre_full_post if needed +# use batch command pre_full_post if needed # - introduced batch mode cmd1_cmd2_etc # (in turn removed the bvp command) # - unknown/undefined command issues a warning/error now @@ -338,13 +423,13 @@ # 1.3b3 - introduced pre/post commands to execute/debug scripts # - introduced bvp (backup, verify, purge) # - bugfix: removed need for awk gensub, now mawk compatible -# 1.3b2 - removed pre/post need executable bit set +# 1.3b2 - removed pre/post need executable bit set # - profiles now under ~/.ftplicity as folders # - root can keep profiles in /etc/ftplicity, folder must be # created by hand, existing profiles must be moved there # - removed ftplicity in path requirement # - bugfix: bash < v.3 did not know '=~' -# - bugfix: purge works again +# - bugfix: purge works again # 1.3 - introduces multiple profiles support # - modified some script errors/docs # - reordered gpg key check import routine @@ -352,21 +437,51 @@ # - added error_gpg (adds how to setup gpg key howto) # - bugfix: duplicity 0.4.4RC4+ parameter syntax changed # - duplicity_version_check routine introduced -# - added time separator, shortnames, volsize, full_if_older -# duplicity options to config file (inspired by stevie -# from http://weareroot.de) +# - added time separator, shortnames, volsize, full_if_older +# duplicity options to config file (inspired by stevie +# from http://weareroot.de) # 1.1.1 - bugfix: encryption reactivated # 1.1 - introduced config directory # 1.0 - first release -############################################################################### +################################################################################ + +# utility functions overriding binaries +# wrap grep to override possible env set GREP_OPTIONS=--color=always +function grep { + command env "GREP_OPTIONS=" grep "$@" +} + +# implement basename in plain bash +function basename { + local stripped="${1%/}" + echo "${stripped##*/}" +} + +# implement dirname in plain bash +function dirname { + echo ${1%/*} +} + +# implement basic which in plain bash +function which { + type -p "$@" +} + +# check availability of executables via file name or file paths +function lookup { + local bin="$1" + # look for file names in path via bash hash OR + # look for executables at given relative/absolute location + ( [ "${bin##*/}" == "$bin" ] && hash "$bin" 2>/dev/null ) || [ -x "$bin" ] +} # important definitions ####################################################### ME_LONG="$0" ME="$(basename $0)" ME_NAME="${ME%%.*}" -ME_VERSION="1.9.1" +ME_VERSION="2.0.4" ME_WEBSITE="http://duply.net" # default config values @@ -374,22 +489,25 @@ DEFAULT_SOURCE='/path/of/source' DEFAULT_TARGET='scheme://user[:password]@host[:port]/[/]path' DEFAULT_TARGET_USER='_backend_username_' DEFAULT_TARGET_PASS='_backend_password_' +DEFAULT_GPG='gpg' DEFAULT_GPG_KEY='_KEY_ID_' DEFAULT_GPG_PW='_GPG_PASSWORD_' +DEFAULT_PYTHON='python' # function definitions ########################## -function set_config { # sets config vars +{% endraw %} +function set_config { # sets global config vars local CONFHOME_COMPAT="$HOME/.ftplicity" + local CONFHOME="{{backup_home}}" local CONFHOME_ETC_COMPAT="/etc/ftplicity" local CONFHOME_ETC="{{backup_home}}" - local CONFHOME="{{backup_home}}" - +{% raw %} # confdir can be delivered as path (must contain /) - if [ `echo $FTPLCFG | grep /` ] ; then + if [ `echo $FTPLCFG | grep /` ] ; then CONFDIR=$(readlink -f $FTPLCFG 2>/dev/null || \ ( echo $FTPLCFG|grep -v '^/' 1>/dev/null 2>&1 \ && echo $(pwd)/${FTPLCFG} ) || \ - echo ${FTPLCFG}) + echo ${FTPLCFG}) # or DEFAULT in home/.duply folder (NEW) elif [ -d "${CONFHOME}" ]; then CONFDIR="${CONFHOME}/${FTPLCFG}" @@ -411,25 +529,22 @@ function set_config { # sets config vars # remove trailing slash, get profile name etc. CONFDIR="${CONFDIR%/}" - NAME="${CONFDIR##*/}" + PROFILE="${CONFDIR##*/}" CONF="$CONFDIR/conf" PRE="$CONFDIR/pre" POST="$CONFDIR/post" EXCLUDE="$CONFDIR/exclude" KEYFILE="$CONFDIR/gpgkey.asc" - } -{% raw %} - function version_info { # print version information cat </dev/null || awk -W version '' 2>/dev/null) | awk '/.+/{sub(/^[Aa][Ww][Kk][ \t]*/,"",$0);print $0;exit}') - PYTHON_VERSION=$(python -V 2>&1| awk '{print tolower($0);exit}') - GPG_INFO=`gpg --version 2>/dev/null| awk '/^gpg/{v=$1" "$3};/^Home/{print v" ("$0")"}'` - BASH_VERSION=$(bash --version | awk '/^GNU bash, version/{sub(/GNU bash, version[ ]+/,"",$0);print $0}') - echo -e "Using installed duplicity version ${DUPL_VERSION:-(not found)}${PYTHON_VERSION+, $PYTHON_VERSION}\ -${GPG_INFO:+, $GPG_INFO}${AWK_VERSION:+, awk '${AWK_VERSION}'}${BASH_VERSION:+, bash '${BASH_VERSION}'}." + local AWK_VERSION=$( lookup awk && (awk --version 2>/dev/null || awk -W version 2>&1) | awk 'NR<=2&&tolower($0)~/(busybox|awk)/{success=1;print;exit} END{if(success<1) print "unknown"}' || echo "$NOTFOUND" ) + local GREP_VERSION=$( lookup grep && grep --version 2>&1 | awk 'NR<=2&&tolower($0)~/(busybox|grep.*[0-9]+\.[0-9]+)/{success=1;print;exit} END{if(success<1) print "unknown"}' || echo "$NOTFOUND" ) + local PYTHON_RUNNER=$(python_binary) + local PYTHON_VERSION=$(lookup "$PYTHON_RUNNER" && "$PYTHON_RUNNER" -V 2>&1| awk '{print tolower($0);exit}' || echo "'$PYTHON_RUNNER' $NOTFOUND" ) + local GPG_INFO=$(gpg_avail && gpg --version 2>&1| awk '/^gpg.*[0-9\.]+$/&&length(v)<1{v=$1" "$3}/^Home:/{h=" ("$0")"}END{print v""h}' || echo "gpg $NOTFOUND") + local BASH_VERSION=$(bash --version | awk 'NR==1{IGNORECASE=1;sub(/GNU bash, version[ ]+/,"",$0);print $0}') + echo -e "Using installed duplicity version ${DUPL_VERSION:-$NOTFOUND}\ +${PYTHON_VERSION+, $PYTHON_VERSION${PYTHONPATH:+ 'PYTHONPATH=$PYTHONPATH'}}\ +${GPG_INFO:+, $GPG_INFO}${AWK_VERSION:+, awk '${AWK_VERSION}'}${GREP_VERSION:+, grep '${GREP_VERSION}'}\ +${BASH_VERSION:+, bash '${BASH_VERSION}'}." } function usage_info { # print usage information @@ -453,28 +577,29 @@ function usage_info { # print usage information cat <' (where ~ is the current users home directory). - Hint: + Hint: If the folder '/etc/${ME_NAME}' exists, the profiles for the super user root will be searched & created there. USAGE: - first time usage (profile creation): + first time usage (profile creation): $ME create - general usage in single or batch mode (see EXAMPLES): + general usage in single or batch mode (see EXAMPLES): $ME [[_|+|-][_|+|-]...] [ ...] For batches the conditional separators can also be written as pseudo commands @@ -484,36 +609,36 @@ USAGE: All conf parameters can also be defined in the environment instead. PROFILE: - Indicated by a path or a profile name (), which is resolved + Indicated by a path or a profile name (), which is resolved to '~/.${ME_NAME}/' (~ expands to environment variable \$HOME). Superuser root can place profiles under '/etc/${ME_NAME}'. Simply create - the folder manually before running $ME as superuser. - Note: - Already existing profiles in root's profile folder will cease to work - unless there are moved to the new location manually. + the folder manually before running $ME_NAME as superuser. + Note: + Already existing profiles in root's home folder will cease to work + unless they are moved to the new location manually. example 1: $ME humbug backup - Alternatively a _path_ might be used e.g. useful for quick testing, + Alternatively a _path_ might be used e.g. useful for quick testing, restoring or exotic locations. Shell expansion should work as usual. - Hint: - The path must contain at least one path separator '/', + Hint: + The path must contain at least one path separator '/', e.g. './test' instead of only 'test'. example 2: $ME ~/.${ME_NAME}/humbug backup SEPARATORS: - _ (underscore) + _ (underscore) neutral separator - + (plus sign), _and_ + + (plus sign), _and_ conditional AND the next command will only be executed if the previous succeeded - - (minus sign), _or_ + - (minus sign), _or_ conditional OR the next command will only be executed if the previous failed - example: + example: 'pre+bkp-verify_post' translates to 'pre_and_bkp_or_verify_post' COMMANDS: @@ -528,45 +653,46 @@ COMMANDS: bkp as above but without executing pre/post scripts full force full backup incr force incremental backup - list [] + list [] list all files in backup (as it was at , default: now) status prints backup sets and chains currently in repository - verify [] [--compare-data] + verify [] [--compare-data] list files changed, since age if given - verifyPath [] [--compare-data] + verifyPath [] [--compare-data] list changes of a file or folder path in backup compared to a local path, since age if given - restore [] + restore [] restore the complete backup to [as it was at ] - fetch [] + fetch [] fetch single file/folder from backup [as it was at ] - purge [] [--force] + purge [] [--force] list outdated backup files (older than \$MAX_AGE) [use --force to actually delete these files] - purgeFull [] [--force] + purgeFull [] [--force] list outdated backup files (\$MAX_FULL_BACKUPS being the number of - full backups and associated incrementals to keep, counting in + full backups and associated incrementals to keep, counting in reverse chronological order) [use --force to actually delete these files] - purgeIncr [] [--force] - list outdated incremental backups (\$MAX_FULLS_WITH_INCRS being + purgeIncr [] [--force] + list outdated incremental backups (\$MAX_FULLS_WITH_INCRS being the number of full backups which associated incrementals will be - kept, counting in reverse chronological order) + kept, counting in reverse chronological order) [use --force to actually delete these files] - cleanup [--force] + cleanup [--force] list broken backup chain files archives (e.g. after unfinished run) [use --force to actually delete these files] changelog print changelog / todo list - txt2man feature for package maintainers - create a manpage based on the - usage output. download txt2man from http://mvertes.free.fr/, put + txt2man feature for package maintainers - create a manpage based on the + usage output. download txt2man from http://mvertes.free.fr/, put it in the PATH and run '$ME txt2man' to create a man page. - version show version information of $ME and needed programs + version show version information of $ME_NAME and needed programs OPTIONS: - --force passed to duplicity (see commands: purge, purge-full, cleanup) + --force passed to duplicity (see commands: + purge, purgeFull, purgeIncr, cleanup) --preview do nothing but print out generated duplicity command lines - --disable-encryption + --disable-encryption disable encryption, overrides profile settings TIME FORMATS: @@ -578,34 +704,36 @@ TIME FORMATS: 1h78m (interval, 1 hour 78 minutes ago) PRE/POST SCRIPTS: - Useful internal duply variables will be readable in the scripts. - Some of interest may be + Some useful internal duply variables are exported to the scripts. - CONFDIR, SOURCE, TARGET_URL_, - GPG_, CMD_, CMD_ERR + PROFILE, CONFDIR, SOURCE, TARGET_URL_, + GPG_, CMD_, CMD_ERR, RUN_START, + CND_ (condition before/after next/prev command) - The CMD_* variables were introduced to allow different actions according to - the command the scripts were attached to e.g. 'pre_bkp_post_pre_verify_post' - will call the pre script two times, with CMD_NEXT variable set to 'bkp' + The CMD_* variables were introduced to allow different actions according to + the command the scripts were attached to e.g. 'pre_bkp_post_pre_verify_post' + will call the pre script two times, with CMD_NEXT variable set to 'bkp' on the first and to 'verify' on the second run. CMD_ERR holds the exit code of the CMD_PREV . EXAMPLES: - create profile 'humbug': - $ME humbug create (now edit the resulting conf file) - backup 'humbug' now: + create profile 'humbug': + $ME humbug create (don't forget to edit this new conf file) + backup 'humbug' now: $ME humbug backup - list available backup sets of profile 'humbug': + list available backup sets of profile 'humbug': $ME humbug status - list and delete obsolete backup archives of 'humbug': + list and delete outdated backups of 'humbug': $ME humbug purge --force - restore latest backup of 'humbug' to /mnt/restore: + restore latest backup of 'humbug' to /mnt/restore: $ME humbug restore /mnt/restore - restore /etc/passwd of 'humbug' from 4 days ago to /root/pw: + restore /etc/passwd of 'humbug' from 4 days ago to /root/pw: $ME humbug fetch etc/passwd /root/pw 4D (see "duplicity manpage", section TIME FORMATS) - a one line batch job on 'humbug' for cron execution: + a one line batch job on 'humbug' for cron execution: $ME humbug backup_verify_purge --force + batch job to run a full backup with pre/post scripts: + $ME humbug pre_full_post FILES: in profile folder '~/.${ME_NAME}/' or '/etc/${ME_NAME}' @@ -655,10 +783,10 @@ GPG_PW='${DEFAULT_GPG_PW}' # GPG_KEYS_ENC='[,,...]' - list of pubkeys to encrypt to # GPG_KEY_SIGN='|disabled' - a secret key for signing # GPG_PW='' - needed for signing, decryption and symmetric -# encryption. If you want to deliver different passphrases for e.g. +# encryption. If you want to deliver different passphrases for e.g. # several keys or symmetric encryption plus key signing you can use # gpg-agent. Simply make sure that GPG_AGENT_INFO is set in environment. -# also see "A NOTE ON SYMMETRIC ENCRYPTION AND SIGNING" in duplicity manpage +# also see "A NOTE ON SYMMETRIC ENCRYPTION AND SIGNING" in duplicity manpage # notes on en/decryption # private key and passphrase will only be needed for decryption or signing. # decryption happens on restore and incrementals (compare archdir contents). @@ -670,98 +798,102 @@ GPG_PW='${DEFAULT_GPG_PW}' # NOTE: available since duplicity 0.6.14, translates to SIGN_PASSPHRASE #GPG_PW_SIGN='' +# uncomment and set a file path or name force duply to use this gpg executable +# available in duplicity 0.7.04 and above (currently unreleased 06/2015) +#GPG='/usr/local/gpg-2.1/bin/gpg' + # gpg options passed from duplicity to gpg process (default='') -# e.g. "--trust-model pgp|classic|direct|always" +# e.g. "--trust-model pgp|classic|direct|always" # or "--compress-algo=bzip2 --bzip2-compress-level=9" # or "--personal-cipher-preferences AES256,AES192,AES..." # or "--homedir ~/.duply" - keep keyring and gpg settings duply specific +# or "--pinentry-mode loopback" - needed for GPG 2.1+ _and_ +# also enable allow-loopback-pinentry in your .gnupg/gpg-agent.conf #GPG_OPTS='' # disable preliminary tests with the following setting #GPG_TEST='disabled' -# credentials & server address of the backup target (URL-Format) -# syntax is -# scheme://[user:password@]host[:port]/[/]path -# for details see duplicity manpage, section URL Format -# http://duplicity.nongnu.org/duplicity.1.html#sect8 -# probably one out of -# # for cloudfiles backend user id is CLOUDFILES_USERNAME, password is -# # CLOUDFILES_APIKEY, you might need to set CLOUDFILES_AUTHURL manually -# cf+http://[user:password@]container_name -# dpbx:///some_dir -# file://[relative|/absolute]/local/path -# ftp[s]://user[:password]@other.host[:port]/some_dir -# gdocs://user[:password]@other.host/some_dir -# # for the google cloud storage (since duplicity 0.6.22) -# # user/password are GS_ACCESS_KEY_ID/GS_SECRET_ACCESS_KEY -# gs://bucket[/prefix] -# hsi://user[:password]@other.host/some_dir -# imap[s]://user[:password]@host.com[/from_address_prefix] -# mega://user[:password]@mega.co.nz/some_dir -# rsync://user[:password]@host.com[:port]::[/]module/some_dir -# # rsync over ssh (only keyauth) -# rsync://user@host.com[:port]/[relative|/absolute]_path -# # for the s3 user/password are AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY -# s3://[user:password@]host/bucket_name[/prefix] -# s3+http://[user:password@]bucket_name[/prefix] -# # scp and sftp are aliases for the ssh backend -# ssh://user[:password]@other.host[:port]/[/]some_dir -# # for authenticated swift define TARGET_USER or SWIFT_USERNAME, -# # TARGET_PASS or SWIFT_PASSWORD, SWIFT_AUTHURL (mandatory, the path to -# # your identity service, omitting leads to an error with swift), -# # optionally SWIFT_AUTHVERSION (which defaults to "1") -# swift://container_name -# tahoe://alias/directory -# webdav[s]://user[:password]@other.host/some_dir -# ATTENTION: characters other than A-Za-z0-9.-_.~ in the URL have -# to be replaced by their url encoded pendants, see -# http://en.wikipedia.org/wiki/Url_encoding -# if you define the credentials as TARGET_USER, TARGET_PASS below -# duply will try to url_encode them for you if the need arises +# backend, credentials & location of the backup target (URL-Format) +# generic syntax is +# scheme://[user[:password]@]host[:port]/[/]path +# eg. +# sftp://bob:secret@backupserver.com//home/bob/dupbkp +# for details and available backends see duplicity manpage, section URL Format +# http://duplicity.nongnu.org/duplicity.1.html#sect7 +# BE AWARE: +# some backends (cloudfiles, S3 etc.) need additional env vars to be set to +# work properly, read after the TARGET definition for more details. +# ATTENTION: +# characters other than A-Za-z0-9.-_.~ in the URL have to be +# replaced by their url encoded pendants, see +# http://en.wikipedia.org/wiki/Url_encoding +# if you define the credentials as TARGET_USER, TARGET_PASS below $ME +# will try to url_encode them for you if the need arises. TARGET='${DEFAULT_TARGET}' # optionally the username/password can be defined as extra variables # setting them here _and_ in TARGET results in an error +# ATTENTION: +# there are backends that do not support the user/pass auth scheme. +# prominent examples are S3, Azure, Cloudfiles. when in doubt consult the +# duplicity manpage. usually there is a NOTE section explaining if and which +# env vars should be set. #TARGET_USER='${DEFAULT_TARGET_USER}' #TARGET_PASS='${DEFAULT_TARGET_PASS}' +# eg. for cloud files backend it might look like this (uncomment for use!) +#export CLOUDFILES_USERNAME='someuser' +#export CLOUDFILES_APIKEY='somekey' +#export CLOUDFILES_AUTHURL ='someurl' +# the following is an incomplete list (: comma separated env vars list) +# Azure: AZURE_ACCOUNT_NAME, AZURE_ACCOUNT_KEY +# Cloudfiles: CLOUDFILES_USERNAME, CLOUDFILES_APIKEY, CLOUDFILES_AUTHURL +# Google Cloud Storage: GS_ACCESS_KEY_ID, GS_SECRET_ACCESS_KEY +# Pydrive: GOOGLE_DRIVE_ACCOUNT_KEY, GOOGLE_DRIVE_SETTINGS +# S3: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY +# Swift: SWIFT_USERNAME, SWIFT_PASSWORD, SWIFT_AUTHURL, +# SWIFT_TENANTNAME OR SWIFT_PREAUTHURL, SWIFT_PREAUTHTOKEN # base directory to backup SOURCE='${DEFAULT_SOURCE}' -# a command that runs duplicity e.g. +# a command that runs duplicity e.g. # shape bandwidth use via trickle # "trickle -s -u 640 -d 5120" # 5Mb up, 40Mb down" #DUPL_PRECMD="" +# override the used python interpreter, defaults to "python" +# e.g. "python2" or "/usr/bin/python2.7" +#PYTHON="python" + # exclude folders containing exclusion file (since duplicity 0.5.14) # Uncomment the following two lines to enable this setting. #FILENAME='.duplicity-ignore' #DUPL_PARAMS="\$DUPL_PARAMS --exclude-if-present '\$FILENAME'" -# Time frame for old backups to keep, Used for the "purge" command. +# Time frame for old backups to keep, Used for the "purge" command. # see duplicity man page, chapter TIME_FORMATS) #MAX_AGE=1M -# Number of full backups to keep. Used for the "purge-full" command. +# Number of full backups to keep. Used for the "purgeFull" command. # See duplicity man page, action "remove-all-but-n-full". #MAX_FULL_BACKUPS=1 # Number of full backups for which incrementals will be kept for. -# Used for the "purge-incr" command. +# Used for the "purgeIncr" command. # See duplicity man page, action "remove-all-inc-of-but-n-full". #MAX_FULLS_WITH_INCRS=1 -# activates duplicity --full-if-older-than option (since duplicity v0.4.4.RC3) -# forces a full backup if last full backup reaches a specified age, for the +# activates duplicity --full-if-older-than option (since duplicity v0.4.4.RC3) +# forces a full backup if last full backup reaches a specified age, for the # format of MAX_FULLBKP_AGE see duplicity man page, chapter TIME_FORMATS # Uncomment the following two lines to enable this setting. #MAX_FULLBKP_AGE=1M -#DUPL_PARAMS="\$DUPL_PARAMS --full-if-older-than \$MAX_FULLBKP_AGE " +#DUPL_PARAMS="\$DUPL_PARAMS --full-if-older-than \$MAX_FULLBKP_AGE " # sets duplicity --volsize option (available since v0.4.3.RC7) # set the size of backup chunks to VOLSIZE MB instead of the default 25MB. # VOLSIZE must be number of MB's to set the volume size to. -# Uncomment the following two lines to enable this setting. +# Uncomment the following two lines to enable this setting. #VOLSIZE=50 #DUPL_PARAMS="\$DUPL_PARAMS --volsize \$VOLSIZE " @@ -773,36 +905,36 @@ SOURCE='${DEFAULT_SOURCE}' # for a successful restoration process. (default is '/tmp', if not set) #TEMP_DIR=/tmp -# Modifies archive-dir option (since 0.6.0) Defines a folder that holds -# unencrypted meta data of the backup, enabling new incrementals without the -# need to decrypt backend metadata first. If empty or deleted somehow, the +# Modifies archive-dir option (since 0.6.0) Defines a folder that holds +# unencrypted meta data of the backup, enabling new incrementals without the +# need to decrypt backend metadata first. If empty or deleted somehow, the # private key and it's password are needed. -# NOTE: This is confidential data. Put it somewhere safe. It can grow quite +# NOTE: This is confidential data. Put it somewhere safe. It can grow quite # big over time so you might want to put it not in the home dir. # default '~/.cache/duplicity/duply_/' # if set '\${ARCH_DIR}/' #ARCH_DIR=/some/space/safe/.duply-cache # DEPRECATED setting -# sets duplicity --time-separator option (since v0.4.4.RC2) to allow users -# to change the time separator from ':' to another character that will work +# sets duplicity --time-separator option (since v0.4.4.RC2) to allow users +# to change the time separator from ':' to another character that will work # on their system. HINT: For Windows SMB shares, use --time-separator='_'. # NOTE: '-' is not valid as it conflicts with date separator. -# ATTENTION: only use this with duplicity < 0.5.10, since then default file -# naming is compatible and this option is pending depreciation +# ATTENTION: only use this with duplicity < 0.5.10, since then default file +# naming is compatible and this option is pending depreciation #DUPL_PARAMS="\$DUPL_PARAMS --time-separator _ " # DEPRECATED setting # activates duplicity --short-filenames option, when uploading to a file # system that can't have filenames longer than 30 characters (e.g. Mac OS 8) # or have problems with ':' as part of the filename (e.g. Microsoft Windows) -# ATTENTION: only use this with duplicity < 0.5.10, later versions default file +# ATTENTION: only use this with duplicity < 0.5.10, later versions default file # naming is compatible and this option is pending depreciation #DUPL_PARAMS="\$DUPL_PARAMS --short-filenames " # more duplicity command line options can be added in the following way # don't forget to leave a separating space char at the end -#DUPL_PARAMS="\$DUPL_PARAMS --put_your_options_here " +#DUPL_PARAMS="\$DUPL_PARAMS --put_your_options_here " EOF @@ -823,7 +955,7 @@ EOF cat <&1 | awk '/^duplicity /{print $2; exit;}'` - #DUPL_VERSION='0.6.08b' #,0.4.4.RC4,0.6.08b + #DUPL_VERSION='0.7.03' #'0.6.08b' #,0.4.4.RC4,0.6.08b DUPL_VERSION_VALUE=0 DUPL_VERSION_AWK=$(awk -v v="$DUPL_VERSION" 'BEGIN{ if (match(v,/[^\.0-9]+[0-9]*$/)){ @@ -957,9 +1089,9 @@ function duplicity_version_get { function duplicity_version_check { if [ $DUPL_VERSION_VALUE -eq 0 ]; then - inform "duplicity version check failed (please report, this is a bug)" + inform "duplicity version check failed (please report, this is a bug)" elif [ $DUPL_VERSION_VALUE -le 404 ] && [ ${DUPL_VERSION_RC:-4} -lt 4 ]; then - error "The installed version $DUPL_VERSION is incompatible with $ME v$ME_VERSION. + error "The installed version $DUPL_VERSION is incompatible with $ME_NAME v$ME_VERSION. You should upgrade your version of duplicity to at least v0.4.4RC4 or use the older ftplicity version 1.1.1 from $ME_WEBSITE." fi @@ -976,9 +1108,9 @@ function duplicity_version_lt { function run_script { # run pre/post scripts local ERR=0 local SCRIPT="$1" - if [ ! -z "$PREVIEW" ] ; then + if [ ! -z "$PREVIEW" ] ; then echo "$([ ! -x "$SCRIPT" ] && echo ". ")$SCRIPT" - elif [ -r "$SCRIPT" ] ; then + elif [ -r "$SCRIPT" ] ; then echo -n "Running '$SCRIPT' " if [ -x "$SCRIPT" ]; then OUT=$("$SCRIPT" 2>&1) @@ -1004,12 +1136,13 @@ function run_cmd { elif [ -n "$CMD_DISABLED" ]; then CMD_MSG="$CMD_MSG (DISABLED) - $CMD_DISABLED" else + echo -n -e "$CMD_MSG" CMD_OUT=` eval "$@" 2>&1 ` CMD_ERR=$? if [ "$CMD_ERR" = "0" ]; then - CMD_MSG="$CMD_MSG (OK)" + CMD_MSG=" (OK)" else - CMD_MSG="$CMD_MSG (FAILED)" + CMD_MSG=" (FAILED)" fi fi echo -e "$CMD_MSG" @@ -1056,8 +1189,15 @@ function duplicity_params_global { local DUPL_ARCHDIR='' if var_isset 'ARCH_DIR'; then DUPL_ARCHDIR="--archive-dir $(qw "${ARCH_DIR}")" + # reuse erronously duply_ prefixed folders from bug #117 + if [ -d "$ARCH_DIR/duply_${PROFILE}" ]; then + DUPL_ARCHDIR="${DUPL_ARCHDIR} --name $(qw "duply_${PROFILE}")" + else + DUPL_ARCHDIR="${DUPL_ARCHDIR} --name $(qw "${PROFILE}")" + fi + else + DUPL_ARCHDIR="--name $(qw "duply_${PROFILE}")" fi - DUPL_ARCHDIR="${DUPL_ARCHDIR} --name $(qw "duply_${NAME}")" fi DUPL_PARAMS_GLOBAL="${DUPL_ARCHDIR} ${DUPL_PARAM_ENC} \ @@ -1068,17 +1208,18 @@ DUPL_VARS_GLOBAL="TMPDIR='$TEMP_DIR' \ ${DUPL_ARG_ENC}" } -# filter the DUPL_PARAMS var from conf -function duplicity_params_conf { - # reuse cmd var from main loop - ## in/exclude parameters are currently not supported on restores - if [ "$cmd" = "fetch" ] || [ "$cmd" = "restore" ]; then - # filter exclude params from fetch/restore - echo "$DUPL_PARAMS" | awk '{gsub(/--(ex|in)clude[a-z-]*(([ \t]+|=)[^-][^ \t]+)?/,"");print}' - return - fi - echo "$DUPL_PARAMS" +# function to filter the DUPL_PARAMS var from user conf +function duplicity_params_conf { + # reuse cmd var from main loop + ## in/exclude parameters are currently not supported on restores + if [ "$cmd" = "fetch" ] || [ "$cmd" = "restore" ] || [ "$cmd" = "status" ]; then + # filter exclude params from fetch/restore + echo "$DUPL_PARAMS" | awk '{gsub(/--(ex|in)clude[a-z-]*(([ \t]+|=)[^-][^ \t]+)?/,"");print}' + return + fi + + echo "$DUPL_PARAMS" } function duplify { # the actual wrapper function @@ -1092,19 +1233,28 @@ function duplify { # the actual wrapper function else # wrap in quotes to protect from spaces [ ! $PARAMSNOW ] && \ - DUPL_CMD="$DUPL_CMD $(qw $param)" \ + DUPL_CMD="$DUPL_CMD $(qw "$param")" \ || \ - DUPL_CMD_PARAMS="$DUPL_CMD_PARAMS $(qw $param)" + DUPL_CMD_PARAMS="$DUPL_CMD_PARAMS $(qw "$param")" fi done # init global duplicity parameters same for all tasks duplicity_params_global - var_isset 'PREVIEW' && local RUN=echo || local RUN=eval -$RUN ${DUPL_VARS_GLOBAL} ${BACKEND_PARAMS} \ - ${DUPL_PRECMD} duplicity $DUPL_CMD $DUPL_PARAMS_GLOBAL $(duplicity_params_conf)\ - $GPG_USEAGENT $DUPL_CMD_PARAMS ${PREVIEW:+} + local RUN=eval BIN=duplicity DUPL_BIN + # run in cmd line preview mode if requested + var_isset 'PREVIEW' && RUN=echo + # try to resolve duplicity path for usage with python interpreter + DUPL_BIN=$(which "$BIN") || DUPL_BIN="$BIN" + # only run with a user specific python if configured (running by default + # breaks homebrew as they place a shell wrapper for duplicity in path) + [ -n "$PYTHON" ] && [ "$PYTHON" != "$DEFAULT_PYTHON" ] &&\ + BIN="$(qw "$(python_binary)") $(qw "$DUPL_BIN")" + +$RUN "${DUPL_VARS_GLOBAL} ${BACKEND_PARAMS} \ +${DUPL_PRECMD} $BIN $DUPL_CMD $DUPL_PARAMS_GLOBAL $(duplicity_params_conf)\ + $GPG_USEAGENT $(gpg_custom_binary) $DUPL_CMD_PARAMS" local ERR=$? return $ERR @@ -1115,7 +1265,7 @@ function secureconf { # secure the configuration dir local PERMS="$(ls -la "$CONFDIR/." | awk 'NR==2{print $1}')" if [ "${PERMS/#drwx------*/OK}" != 'OK' ] ; then chmod u+rwX,go= "$CONFDIR"; local ERR=$? - warning "The profile's folder + warning "The profile's folder '$CONFDIR' permissions are not safe ($PERMS). Secure them now. - ($(error_to_string $ERR))" fi @@ -1124,18 +1274,20 @@ permissions are not safe ($PERMS). Secure them now. - ($(error_to_string $ERR))" # params are $1=timeformatstring (default like date output), $2=epoch seconds since 1.1.1970 (default now) function date_fix { local DEFAULTFORMAT='%a %b %d %H:%M:%S %Z %Y' + local date + #[ "$1" == "%N" ] && return #test the no nsec test below # gnu date with -d @epoch date=$(date ${2:+-d @$2} ${1:++"$1"} 2> /dev/null) && \ echo $date && return # date bsd,osx with -r epoch date=$(date ${2:+-r $2} ${1:++"$1"} 2> /dev/null) && \ - echo $date && return + echo $date && return # date busybox with -d epoch -D %s date=$(date ${2:+-d $2 -D %s} ${1:++"$1"} 2> /dev/null) && \ echo $date && return ## some date commands do not support giving a time w/o setting it systemwide (irix,solaris,others?) # python fallback - date=$(python -c "import time;print time.strftime('${1:-$DEFAULTFORMAT}',time.localtime(${2}))" 2> /dev/null) && \ + date=$("$(python_binary)" -c "import time;print time.strftime('${1:-$DEFAULTFORMAT}',time.localtime(${2}))" 2> /dev/null) && \ echo $date && return # awk fallback date=$(awk "BEGIN{print strftime(\"${1:-$DEFAULTFORMAT}\"${2:+,$2})}" 2> /dev/null) && \ @@ -1149,9 +1301,19 @@ function date_fix { } function nsecs { - # only 9 digit returns, e.g. not all date(s) deliver nsecs - local NSECS=$(date +%N 2> /dev/null | head -1 |grep -e "^[[:digit:]]\{9\}$") - echo ${NSECS:-000000000} + local NSECS + # test if date supports nanosecond output + if ! var_isset NSECS_DISABLED; then + NSECS=$(date_fix %N 2> /dev/null | head -1 |grep -e "^[[:digit:]]\{9\}$") + [ -n "$NSECS" ] && NSECS_DISABLED=0 || NSECS_DISABLED=1 + fi + + # add 9 digits, not all date(s) deliver nsecs eg. busybox date + if [ "$NSECS_DISABLED" == "1" ]; then + date_fix %s000000000 + else + date_fix %s%N + fi } function nsecs_to_sec { @@ -1179,9 +1341,14 @@ function var_isset { return 1 } +function is_condition { + local CMD=$(tolower "$@") + [ "$CMD" == 'and' ] || [ "$CMD" == 'or' ] +} + function url_encode { # utilize python, silently do nothing on error - because no python no duplicity - OUT=$(python -c " + OUT=$("$(python_binary)" -c " try: import urllib.request as urllib except ImportError: import urllib print(urllib.${2}quote('$1')); @@ -1210,19 +1377,19 @@ function isnumber { } #function tmp_space { -# +# # if ! isnumber $VOLSIZE; then # inform "failed to determine free space (please report, this is a bug)" # return # fi -# +# # get free temp space # TEMP_FREE="$(df -P -k "$TEMP_DIR" 2>/dev/null | awk 'END{pos=(NF-2);if(pos>0) print $pos;}')" # # check for free space or FAIL # if [ $((${TEMP_FREE:-0}-${VOLSIZE:-0}*1024)) -lt 0-lt 0 ]; then # error "Temporary file space '$TEMP_DIR' free space is smaller ($((TEMP_FREE/1024))MB) #than one duplicity volume (${VOLSIZE}MB). -# +# # Hint: Free space or change TEMP_DIR setting." #fi # @@ -1261,17 +1428,17 @@ function gpg_import { local KEYFILES=( "$CONFDIR/gpgkey" $(gpg_keyfile "$KEY_ID") \ $(gpg_keyfile "$KEY_ID" PUB) $(gpg_keyfile "$KEY_ID" SEC)) - # Try autoimport from existing old gpgkey files + # Try autoimport from existing old gpgkey files # and new gpgkey.XXX.asc files (since v1.4.2) # and even newer gpgkey.XXX.[pub|sec].asc for (( i = 0 ; i < ${#KEYFILES[@]} ; i++ )); do FILE=${KEYFILES[$i]} if [ -f "$FILE" ]; then FOUND=1 - + CMD_MSG="Import keyfile '$FILE' to keyring" - run_cmd "$GPG" $GPG_OPTS --batch --import "$FILE" - if [ "$?" != "0" ]; then + run_cmd gpg $GPG_OPTS --batch --import $(qw "$FILE") + if [ "$?" != "0" ]; then warning "Import failed.${CMD_OUT:+\n$CMD_OUT}" ERR=1 # continue with next @@ -1286,25 +1453,25 @@ function gpg_import { # try to set trust automagically CMD_MSG="Autoset trust of key '$KEY_ID' to ultimate" - run_cmd echo $(gpg_fingerprint "$KEY_ID"):6: \| "$GPG" $GPG_OPTS --import-ownertrust --batch --logger-fd 1 - if [ "$?" = "0" ] && [ -z "$PREVIEW" ]; then + run_cmd echo $(gpg_fingerprint "$KEY_ID"):6: \| gpg $GPG_OPTS --import-ownertrust --batch --logger-fd 1 + if [ "$?" = "0" ] && [ -z "$PREVIEW" ]; then # success on all levels, we're done return $ERR fi # failover: user has to set trust manually - echo -e "For $ME to work you have to set the trust level + echo -e "For $ME_NAME to work you have to set the trust level with the command \"trust\" to \"ultimate\" (5) now. Exit the edit mode of gpg with \"quit\"." CMD_MSG="Running gpg to manually edit key '$KEY_ID'" - run_cmd sleep 5\; "$GPG" $GPG_OPTS --edit-key "$KEY_ID" + run_cmd sleep 5\; gpg $GPG_OPTS --edit-key $(qw "$KEY_ID") return $ERR } # see 'How to specify a user ID' on gpg manpage function gpg_fingerprint { - local PRINT=$("$GPG" $GPG_OPTS --fingerprint "$1" 2>&1|awk -F= 'NR==2{gsub(/ /,"",$2);$2=toupper($2); if ( $2 ~ /^[A-F0-9]+$/ && length($2) == 40 ) print $2; else exit 1}') \ + local PRINT=$(gpg $GPG_OPTS --fingerprint "$1" 2>&1|awk -F= 'NR==2{gsub(/ /,"",$2);$2=toupper($2); if ( $2 ~ /^[A-F0-9]+$/ && length($2) == 40 ) print $2; else exit 1}') \ && [ -n "$PRINT" ] && echo $PRINT && return 0 return 1 } @@ -1318,13 +1485,15 @@ function gpg_export_if_needed { FILE="$(gpg_keyfile "$KEY_ID" $KEY_TYPE)" if [ ! -f "$FILE" ] && eval gpg_$(tolower $KEY_TYPE)_avail \"$KEY_ID\"; then # exporting - CMD_MSG="Export $KEY_TYPE key '$KEY_ID'" - run_cmd $GPG $GPG_OPTS --armor --export"$(test "SEC" = "$KEY_TYPE" && echo -secret-keys)"" $(qw $KEY_ID) >> \"$TMPFILE\"" + CMD_MSG="Backup $KEY_TYPE key '$KEY_ID' to profile." + # gpg2.1 insists on passphrase here, gpg2.0- happily exports w/o it + # we pipe an empty string when GPG_PW is not set to avoid gpg silently waiting for input + run_cmd $(gpg_pass_pipein GPG_PW_SIGN GPG_PW) gpg $GPG_OPTS $GPG_USEAGENT $(gpg_param_passwd GPG_PW_SIGN GPG_PW) --armor --export"$(test "SEC" = "$KEY_TYPE" && echo -secret-keys)" $(qw "$KEY_ID") '>>' $(qw "$TMPFILE") CMD_ERR=$? if [ "$CMD_ERR" = "0" ]; then CMD_MSG="Write file '"$(basename "$FILE")"'" - run_cmd " mv \"$TMPFILE\" \"$FILE\"" + run_cmd mv $(qw "$TMPFILE") $(qw "$FILE") fi if [ "$CMD_ERR" != "0" ]; then @@ -1334,12 +1503,12 @@ function gpg_export_if_needed { fi # cleanup - rm "$TMPFILE" 1>/dev/null 2>&1 + rm $(qw "$TMPFILE") 1>/dev/null 2>&1 fi done done - - [ -n "$SUCCESS" ] && inform "$ME exported new keys to your profile. + + [ -n "$SUCCESS" ] && inform "$ME_NAME exported new keys to your profile. You should backup your changed profile folder now and store it in a safe place." } @@ -1361,9 +1530,9 @@ function gpg_key_cache { return 255 elif ! var_isset "$CACHE"; then if [ "$MODE" = "PUB" ]; then - RES=$("$GPG" $GPG_OPTS --list-key "$KEYID" > /dev/null 2>&1; echo -n $?) + RES=$(gpg $GPG_OPTS --list-key "$KEYID" > /dev/null 2>&1; echo -n $?) elif [ "$MODE" = "SEC" ]; then - RES=$("$GPG" $GPG_OPTS --list-secret-key "$KEYID" > /dev/null 2>&1; echo -n $?) + RES=$(gpg $GPG_OPTS --list-secret-key "$KEYID" > /dev/null 2>&1; echo -n $?) else return 255 fi @@ -1384,13 +1553,8 @@ function gpg_key_format { echo $1 | grep -q '^[0-9a-fA-F]\{8\}$' } -#function gpg_split_keyset { -# return -# awk "BEGIN{ keys=toupper(\"$@\"); gsub(/[^A-Z0-9]/,\" \",keys); print keys }" -#} - # splits a comma separated line into lines, respects escaped commas -function gpg_split_keyset2 { +function gpg_split_keyset { local LIST LIST=$(echo "$@" | awk '{ gsub(/,/,"\n",$0); gsub(/\\\n/,",",$0); print $0 }') echo -e "$LIST" @@ -1430,7 +1594,7 @@ function gpg_symmetric { # checks for max two params if they are set, typically GPG_PW & GPG_PW_SIGN function gpg_param_passwd { var_isset GPG_USEAGENT && exit 1 - + if ( [ -n "$1" ] && var_isset "$1" ) || ( [ -n "$2" ] && var_isset "$2" ); then echo "--passphrase-fd 0 --batch" fi @@ -1439,7 +1603,7 @@ function gpg_param_passwd { # select the earlist defined and create an "echo |" string function gpg_pass_pipein { var_isset GPG_USEAGENT && exit 1 - + for var in "$@" do if var_isset "$var"; then @@ -1447,24 +1611,54 @@ function gpg_pass_pipein { return 0 fi done - + return 1 } # checks if gpg-agent is available, returns error code # 0 on success -# 1 if GPG_AGENT_INFO is not set +# 1 if GPG_AGENT_INFO is not set (unused, should probably be merged w/ 3) # 2 if GPG_AGENT_INFO is stale +# 3 cannot connect to gpg-agent function gpg_agent_avail { - local ERR=1 + # GPG_AGENT_INFO is deprecated in gpg2.1, + # first try to connect to a possibly running agent here + local ERR=3 + gpg-agent > /dev/null 2>&1 && return 0 + + # detect stale pid in legacy GPG_AGENT_INFO env var if var_isset GPG_AGENT_INFO; then - ps -p $(echo $GPG_AGENT_INFO|awk -F: '{print $2}') > /dev/null 2>&1 &&\ - ERR=0 || ERR=2 + # check if a pid matching process is running at all + local GPG_AGENT_PID=$(echo $GPG_AGENT_INFO|awk -F: '{print $2}') + if isnumber "$GPG_AGENT_PID"; then + ps -p "$GPG_AGENT_PID" > /dev/null 2>&1 || ERR=2 + fi fi return $ERR } +function gpg_custom_binary { + var_isset GPG && [ "$GPG" != "$DEFAULT_GPG" ] &&\ + echo "--gpg-binary $(qw "$GPG")" +} + +function gpg_binary { + local BIN + var_isset GPG && BIN="$GPG" || BIN="$DEFAULT_GPG" + echo "$BIN" +} + +function gpg_avail { + lookup "$(gpg_binary)" +} + +# enforce the use our selected gpg binary +function gpg { + command "$(gpg_binary)" "$@" +} +export -f gpg + # start of script ####################################################################### # confidentiality first, all we create is only readable by us @@ -1474,9 +1668,9 @@ umask 077 [ -n "$ME_LONG" ] && [ -x "$ME_LONG" ] || error "$ME missing. Executable & available in path? ($ME_LONG)" if [ ${#@} -eq 1 ]; then - cmd="${1}" + cmd="${1}" else - FTPLCFG="${1}" ; cmd="${2}" + FTPLCFG="${1}" ; cmd="${2}" fi # deal with command before profile validation calls @@ -1497,7 +1691,7 @@ case "$cmd" in '$CONFDIR'. Hint: - If you _really_ want to create a new profile by this name you will + If you _really_ want to create a new profile by this name you will have to manually delete the existing profile folder first." exit 1 else @@ -1516,6 +1710,11 @@ Hint: exit 0 ;; version|-version|--version|-v|-V) + # profile can override GPG, so import it if it was given + var_isset FTPLCFG && { + set_config + [ -r "$CONF" ] && . "$CONF" || warning "Cannot import config '$CONF'." + } version_info_using exit 0 ;; @@ -1524,8 +1723,8 @@ Hint: # if we reach here, user either forgot profile or chose wrong profileless command if [ ${#@} -le 1 ]; then error "\ - Missing or wrong parameters. - Only the commands + Missing or wrong parameters. + Only the commands changelog, create, usage, txt2man, version can be called without selecting an existing profile first. Your command was '$cmd'. @@ -1539,30 +1738,35 @@ esac echo "Start $ME v$ME_VERSION, time is $(date_fix '%F %T')." # check system environment -DUPLICITY="$(which duplicity 2>/dev/null)" -[ -z "$DUPLICITY" ] && error_path "duplicity missing. installed und available in path?" + +# is duplicity avail +lookup duplicity || error_path "duplicity missing. installed und available in path?" # init, exec duplicity version check info duplicity_version_get duplicity_version_check -[ -z "$(which awk 2>/dev/null)" ] && error_path "awk missing. installed und available in path?" +# check for certain important helper programs +for f in awk grep "$(python_binary)"; do + lookup "$f" || \ + error_path "$f missing. installed und available in path?" +done ### read configuration set_config # check validity -if [ ! -d "$CONFDIR" ]; then +if [ ! -d "$CONFDIR" ]; then error "Selected profile '$FTPLCFG' does not resolve to a profile folder in '$CONFDIR'. Hints: - Select one of the available profiles: $(ls -1p $(dirname "$CONFDIR")| awk 'BEGIN{ORS="";OFS=""}/\/$/&&!/^\.+\/$/{print sep"\047"substr($0,0,length($0)-1)"\047";sep=","}'). +Select one of the available profiles: $(for d in "$(dirname "$CONFDIR")"/*/; do [ -e "$d" ] || [ -L "$d" ] || continue; printf "$sep'$(basename "$d")'"; sep=",";done) Use '$ME create' to create a new profile. Use '$ME usage' to get usage help." elif [ ! -x "$CONFDIR" ]; then error "\ Profile folder in '$CONFDIR' cannot be accessed. -Hint: +Hint: Check the filesystem permissions and set directory accessible e.g. 'chmod 700'." elif [ ! -f "$CONF" ] ; then error "'$CONF' not found." @@ -1583,7 +1787,7 @@ echo "Using profile '$CONFDIR'." secureconf # split TARGET in handy variables -TARGET_SPLIT_URL=$(echo $TARGET | awk '{ \ +TARGET_SPLIT_URL=$(echo "$TARGET" | awk '{ \ target=$0; match(target,/^([^\/:]+):\/\//); \ prot=substr(target,RSTART,RLENGTH);\ rest=substr(target,RSTART+RLENGTH); \ @@ -1610,25 +1814,19 @@ TARGET_SPLIT_URL=$(echo $TARGET | awk '{ \ gsub(/[\047]/,"\047\\\047\047",pass);\ print "TARGET_URL_PASS=$(url_decode \047"pass"\047)\n"}\ }') -eval ${TARGET_SPLIT_URL} - -# check if backend specific software is in path -[ -n "$(echo ${TARGET_URL_PROT} | grep -i -e '^ftp://$')" ] && \ - [ -z "$(which ncftp 2>/dev/null)" ] && error_path "Protocol 'ftp' needs ncftp. Installed und available in path?" -[ -n "$(echo ${TARGET_URL_PROT} | grep -i -e '^ftps://$')" ] && \ - [ -z "$(which lftp 2>/dev/null)" ] && error_path "Protocol 'ftps' needs lftp. Installed und available in path?" +eval "${TARGET_SPLIT_URL}" # fetch commmand from parameters ######################################################## -# Hint: cmds is also used to check if authentification info sufficient in the next step +# Hint: cmds is also used to check if authentification info sufficient in the next step cmds="$2"; shift 2 -# translate backup to batch command +# translate backup to batch command cmds=${cmds//backup/pre_bkp_post} # complain if command(s) missing [ -z $cmds ] && error " No command given. - Hint: + Hint: Use '$ME usage' to get usage help." # process params @@ -1659,100 +1857,96 @@ done # plausibility check config - VARS & KEY ################################################ # check if src, trg, trg pw -# auth info sufficient +# auth info sufficient # gpg key, gpg pwd (might be empty) set in config # OR key in local gpg db -# OR key can be imported from keyfile +# OR key can be imported from keyfile # OR fail if [ -z "$SOURCE" ] || [ "$SOURCE" == "${DEFAULT_SOURCE}" ]; then - error " Source Path (setting SOURCE) not set or still default value in conf file + error " Source Path (setting SOURCE) not set or still default value in conf file '$CONF'." elif [ -z "$TARGET" ] || [ "$TARGET" == "${DEFAULT_TARGET}" ]; then - error " Backup Target (setting TARGET) not set or still default value in conf file + error " Backup Target (setting TARGET) not set or still default value in conf file '$CONF'." elif var_isset 'TARGET_USER' && var_isset 'TARGET_URL_USER' && \ [ "${TARGET_USER}" != "${TARGET_URL_USER}" ]; then - error " TARGET_USER ('${TARGET_USER}') _and_ user in TARGET url ('${TARGET_URL_USER}') + error " TARGET_USER ('${TARGET_USER}') _and_ user in TARGET url ('${TARGET_URL_USER}') are configured with different values. There can be only one. - + Hint: Remove conflicting setting." elif var_isset 'TARGET_PASS' && var_isset 'TARGET_URL_PASS' && \ [ "${TARGET_PASS}" != "${TARGET_URL_PASS}" ]; then - error " TARGET_PASS ('${TARGET_PASS}') _and_ password in TARGET url ('${TARGET_URL_PASS}') + error " TARGET_PASS ('${TARGET_PASS}') _and_ password in TARGET url ('${TARGET_URL_PASS}') are configured with different values. There can be only one. - + Hint: Remove conflicting setting." fi -# check if authentication information sufficient -if ( ( ! var_isset 'TARGET_USER' && ! var_isset 'TARGET_URL_USER' ) && \ - ( ! var_isset 'TARGET_PASS' && ! var_isset 'TARGET_URL_PASS' ) ); then - # ok here some exceptions: - # protocols that do not need passwords - # s3[+http] only needs password for write operations - if [ -n "$( tolower "${TARGET_URL_PROT}" | grep -e '^\(dpbx\|file\|tahoe\|ssh\|scp\|sftp\|swift\)://$' )" ]; then - : # all is well file/tahoe do not need passwords, ssh might use key auth - elif [ -n "$(tolower "${TARGET_URL_PROT}" | grep -e '^s3\(\+http\)\?://$')" ] && \ - [ -z "$(echo ${cmds} | grep -e '\(bkp\|incr\|full\|purge\|cleanup\)')" ]; then - : # still fine, it's possible to read only access configured buckets anonymously - else - error " Backup target credentials needed but not set in conf file - '$CONF'. - Setting TARGET_USER or TARGET_PASS or the corresponding values in TARGET url - are missing. Some protocols only might need it for write access to the backup - repository (commands: bkp,backup,full,incr,purge) but not for read only access - (e.g. verify,list,restore,fetch). - - Hints: - Add the credentials (user,password) to the conf file. - To force an empty password set TARGET_PASS='' or TARGET='prot://user:@host..'. -" - fi -fi - # GPG config plausibility check1 (disabled check) ############################# if gpg_disabled; then : # encryption disabled, all is well elif [ -z "${GPG_KEY}${GPG_KEYS_ENC}${GPG_KEY_SIGN}" ] && ! var_isset 'GPG_PW'; then - warning "GPG_KEY, GPG_KEYS_ENC, GPG_KEY_SIGN and GPG_PW are empty/not set in conf file + warning "GPG_KEY, GPG_KEYS_ENC, GPG_KEY_SIGN and GPG_PW are empty/not set in conf file '$CONF'. Will disable encryption for duplicity now. -Hint: - If you really want to use _no_ encryption you can disable this warning by +Hint: + If you really want to use _no_ encryption you can disable this warning by setting GPG_KEY='disabled' in conf file." GPG_KEY='disabled' fi # GPG availability check (now we know if gpg is really needed)################# -if ! gpg_disabled; then - GPG="$(which gpg 2>/dev/null)" - [ -z "$GPG" ] && error_path "gpg missing. installed und available in path?" +if ! gpg_disabled; then + gpg_avail || error_path "gpg '$(gpg_binary)' missing. installed und available in path?" fi - # Output versions info ######################################################## using_info # GPG create key settings, config check2 (needs gpg) ########################## if gpg_disabled; then - : # the following tests are not necessary + : # the following tests are not necessary else -# key set? -if [ "$GPG_KEY" == "${DEFAULT_GPG_KEY}" ]; then - error_gpg "Encryption Key GPG_KEY still default in conf file +# we test this early as any invocation gpg2.1+ starts gpg-agent automatically +GPG_AGENT_ERR=$(gpg_agent_avail ; echo $?) + +# enc key still default? +if [ "$GPG_KEY" == "${DEFAULT_GPG_KEY}" ]; then + error_gpg "Encryption Key GPG_KEY still default in conf file '$CONF'." fi # create array of gpg encr keys, for further processing OIFS="$IFS" IFS=$'\n' -GPG_KEYS_ENC_ARRAY=( $( gpg_split_keyset2 ${GPG_KEY},${GPG_KEYS_ENC} ) ) +GPG_KEYS_ENC_ARRAY=( $( gpg_split_keyset ${GPG_KEY},${GPG_KEYS_ENC} ) ) IFS="$OIFS" +# pw set? +# symmetric needs one, always +if gpg_symmetric && ( [ -z "$GPG_PW" ] || [ "$GPG_PW" == "${DEFAULT_GPG_PW}" ] ) \ + ; then + error_gpg "Encryption passphrase GPG_PW (needed for symmetric encryption) +is empty/not set or still default value in conf file +'$CONF'." +fi +# this is a technicality, we can only pump one pass via pipe into gpg +# but symmetric already always needs one for encryption +if gpg_symmetric && var_isset GPG_PW && var_isset GPG_PW_SIGN &&\ + [ -n "$GPG_PW_SIGN" ] && [ "$GPG_PW" != "$GPG_PW_SIGN" ]; then + error_gpg "GPG_PW _and_ GPG_PW_SIGN are defined but not identical in config +'$CONF'. +This is unfortunately impossible. For details see duplicity manpage, +section 'A Note On Symmetric Encryption And Signing'. + +Tip: Separate signing keys may have empty passwords e.g. GPG_PW_SIGN=''. +Tip2: Use gpg-agent." +fi + # check gpg encr public keys availability for (( i = 0 ; i < ${#GPG_KEYS_ENC_ARRAY[@]} ; i++ )); do KEY_ID="${GPG_KEYS_ENC_ARRAY[$i]}" @@ -1773,9 +1967,9 @@ if ! gpg_signing; then elif ! var_isset 'GPG_KEY_SIGN'; then KEY_ID="${GPG_KEYS_ENC_ARRAY[0]}" if [ -z "${KEY_ID}" ]; then - echo "Signing disabled. Not GPG_KEY entries in config." + echo "Signing disabled. No GPG_KEY entries in config." GPG_KEY_SIGN='disabled' - else + else # use avail OR try import OR fail if gpg_sec_avail "${KEY_ID}"; then GPG_KEY_SIGN="${KEY_ID}" @@ -1805,26 +1999,7 @@ else fi fi -# pw set? -# symmetric needs one, always -if gpg_symmetric && ( [ -z "$GPG_PW" ] || [ "$GPG_PW" == "${DEFAULT_GPG_PW}" ] ) \ - ; then - error_gpg "Encryption passphrase GPG_PW (needed for symmetric encryption) -is empty/not set or still default value in conf file -'$CONF'." -fi -# this is a technicality, we can only pump one pass via pipe into gpg -# but symmetric already always needs one for encryption -if gpg_symmetric && var_isset GPG_PW && var_isset GPG_PW_SIGN &&\ - [ -n "$GPG_PW_SIGN" ] && [ "$GPG_PW" != "$GPG_PW_SIGN" ]; then - error_gpg "GPG_PW _and_ GPG_PW_SIGN are defined but not identical in config -'$CONF'. -This is unfortunately impossible. For details see duplicity manpage, -section 'A Note On Symmetric Encryption And Signing'. - -Tip: Separate signing keys may have empty passwords e.g. GPG_PW_SIGN=''. -Tip2: Use gpg-agent." -fi +# using GPG_AGENT_ERR set early above, try to autoenable gpg-agent or issue some warnings # key enc can deal without, but might profit from gpg-agent # if GPG_PW is not set alltogether # if signing key is different from first (main) enc key (we can only pipe one pass into gpg) @@ -1832,33 +2007,28 @@ if ! gpg_symmetric && \ ( ! var_isset GPG_PW || \ ( gpg_signing && ! var_isset GPG_PW_SIGN && [ "$GPG_KEY_SIGN" != "${GPG_KEYS_ENC_ARRAY[0]}" ] ) ); then - GPG_AGENT_ERR=$(gpg_agent_avail ; echo $?) if [ "$GPG_AGENT_ERR" -eq 1 ]; then - echo "Cannot use gpg-agent. GPG_AGENT_INFO not set." + warning "Cannot use gpg-agent. GPG_AGENT_INFO not set." elif [ "$GPG_AGENT_ERR" -eq 2 ]; then - echo "Cannot use gpg-agent! GPG_AGENT_INFO contains stale pid." + warning "Cannot use gpg-agent! GPG_AGENT_INFO contains stale pid." + elif [ "$GPG_AGENT_ERR" -eq 3 ]; then + warning "No running gpg-agent found although GPG_PW or GPG_PW_SIGN (enc != sign key) not set." else - echo "Autoenable use of gpg-agent. GPG_PW or GPG_PW_SIGN (enc != sign key) not set." + echo "Enable gpg-agent usage. Running gpg-agent instance found and GPG_PW or GPG_PW_SIGN (enc != sign key) not set." GPG_USEAGENT="--use-agent" fi fi -# end GPG config plausibility check2 +# end GPG config plausibility check2 fi # config plausibility check - SPACE ########################################### -# is tmp is a folder -CMD_MSG="Checking TEMP_DIR '${TEMP_DIR}' is a folder" -run_cmd test -d "$TEMP_DIR" +# is tmp is a folder and writable +CMD_MSG="Checking TEMP_DIR '${TEMP_DIR}' is a folder and writable" +run_cmd test -d $(qw "$TEMP_DIR") '&&' test -w $(qw "$TEMP_DIR") if [ "$?" != "0" ]; then - error "Temporary file space '$TEMP_DIR' is not a directory." -fi -# is tmp writeable -CMD_MSG="Checking TEMP_DIR '${TEMP_DIR}' is writable" -run_cmd test -w "$TEMP_DIR" -if [ "$?" != "0" ]; then - error "Temporary file space '$TEMP_DIR' not writable." + error "Temporary file space '$TEMP_DIR' is not a directory or writable." fi @@ -1868,22 +2038,22 @@ VOLSIZE=${VOLSIZE:-25} echo $@ $DUPL_PARAMS | grep -q -e '--asynchronous-upload' && FACTOR=2 || FACTOR=1 # TODO: check for enough (async= upload space and WARN only -# use function tmp_space -echo TODO: reimplent tmp space check +# use function tmp_space +#echo TODO: reimplent tmp space check # test - GPG SANITY ##################################################################### # if encryption is disabled, skip this whole section if gpg_disabled; then echo -e "Test - En/Decryption skipped. (GPG disabled)" -elif [ "$GPG_TEST" = "disabled" ]; then +elif [ "$GPG_TEST" = "disabled" ]; then echo -e "Test - En/Decryption skipped. (Testing disabled)" else GPG_TEST="$TEMP_DIR/${ME_NAME}.$$.$(date_fix %s)" -function cleanup_gpgtest { +function cleanup_gpgtest { echo -en "Cleanup - Delete '${GPG_TEST}_*'" - rm ${GPG_TEST}_* 2>/dev/null && echo "(OK)" || echo "(FAILED)" + rm "${GPG_TEST}"_* 2>/dev/null && echo "(OK)" || echo "(FAILED)" } # signing enabled? @@ -1900,10 +2070,10 @@ if [ ${#GPG_KEYS_ENC_ARRAY[@]} -gt 0 ]; then done # check encrypting CMD_MSG="Test - Encrypt to '$(join "','" "${GPG_KEYS_ENC_ARRAY[@]}")'${CMD_MSG_SIGN:+ & $CMD_MSG_SIGN}" - run_cmd $(gpg_pass_pipein GPG_PW_SIGN GPG_PW) $GPG $CMD_PARAM_SIGN $(gpg_param_passwd GPG_PW_SIGN GPG_PW) $CMD_PARAMS $GPG_USEAGENT --status-fd 1 $GPG_OPTS -o "${GPG_TEST}_ENC" -e "$ME_LONG" + run_cmd $(gpg_pass_pipein GPG_PW_SIGN GPG_PW) gpg $CMD_PARAM_SIGN $(gpg_param_passwd GPG_PW_SIGN GPG_PW) $CMD_PARAMS $GPG_USEAGENT --status-fd 1 $GPG_OPTS -o $(qw "${GPG_TEST}_ENC") -e $(qw "$ME_LONG") CMD_ERR=$? - if [ "$CMD_ERR" != "0" ]; then + if [ "$CMD_ERR" != "0" ]; then KEY_NOTRUST=$(echo "$CMD_OUT"|awk '/^\[GNUPG:\] INV_RECP 10/ { print $4 }') [ -n "$KEY_NOTRUST" ] && HINT="Key '${KEY_NOTRUST}' seems to be untrusted. If you really trust this key try to 'gpg --edit-key "$KEY_NOTRUST"' and raise the trust level to ultimate. If you @@ -1914,10 +2084,10 @@ if [ ${#GPG_KEYS_ENC_ARRAY[@]} -gt 0 ]; then # check decrypting CMD_MSG="Test - Decrypt" gpg_key_decryptable || CMD_DISABLED="No matching secret key available." - run_cmd $(gpg_pass_pipein GPG_PW) "$GPG" $(gpg_param_passwd GPG_PW) $GPG_OPTS -o "${GPG_TEST}_DEC" $GPG_USEAGENT -d "${GPG_TEST}_ENC" + run_cmd $(gpg_pass_pipein GPG_PW) gpg $(gpg_param_passwd GPG_PW) $GPG_OPTS -o $(qw "${GPG_TEST}_DEC") $GPG_USEAGENT -d $(qw "${GPG_TEST}_ENC") CMD_ERR=$? - if [ "$CMD_ERR" != "0" ]; then + if [ "$CMD_ERR" != "0" ]; then error_gpg_test "Decryption failed.${CMD_OUT:+\n$CMD_OUT}" fi @@ -1925,17 +2095,17 @@ if [ ${#GPG_KEYS_ENC_ARRAY[@]} -gt 0 ]; then else # check encrypting CMD_MSG="Test - Encryption with passphrase${CMD_MSG_SIGN:+ & $CMD_MSG_SIGN}" - run_cmd $(gpg_pass_pipein GPG_PW) "$GPG" $GPG_OPTS $CMD_PARAM_SIGN --passphrase-fd 0 -o "${GPG_TEST}_ENC" --batch -c "$ME_LONG" + run_cmd $(gpg_pass_pipein GPG_PW) gpg $GPG_OPTS $CMD_PARAM_SIGN --passphrase-fd 0 -o $(qw "${GPG_TEST}_ENC") --batch -c $(qw "$ME_LONG") CMD_ERR=$? - if [ "$CMD_ERR" != "0" ]; then + if [ "$CMD_ERR" != "0" ]; then error_gpg_test "Encryption failed.${CMD_OUT:+\n$CMD_OUT}" fi # check decrypting CMD_MSG="Test - Decryption with passphrase" - run_cmd $(gpg_pass_pipein GPG_PW) "$GPG" $GPG_OPTS --passphrase-fd 0 -o "${GPG_TEST}_DEC" --batch -d "${GPG_TEST}_ENC" + run_cmd $(gpg_pass_pipein GPG_PW) gpg $GPG_OPTS --passphrase-fd 0 -o $(qw "${GPG_TEST}_DEC") --batch -d $(qw "${GPG_TEST}_ENC") CMD_ERR=$? - if [ "$CMD_ERR" != "0" ]; then + if [ "$CMD_ERR" != "0" ]; then error_gpg_test "Decryption failed.${CMD_OUT:+\n$CMD_OUT}" fi fi @@ -1945,7 +2115,7 @@ CMD_MSG="Test - Compare" [ -r "${GPG_TEST}_DEC" ] || CMD_DISABLED="File not found. Nothing to compare." run_cmd "test \"\$(cat '$ME_LONG')\" = \"\$(cat '${GPG_TEST}_DEC')\"" CMD_ERR=$? -if [ "$CMD_ERR" = "0" ]; then +if [ "$CMD_ERR" = "0" ]; then cleanup_gpgtest else error_gpg_test "Comparision failed.${CMD_OUT:+\n$CMD_OUT}" @@ -1969,115 +2139,77 @@ gpg_export_if_needed "${GPG_KEYS_ENC_ARRAY[@]}" "$(gpg_signing && echo $GPG_KEY_ var_isset 'TARGET_URL_USER' && TARGET_URL_USER="$(url_decode "$TARGET_URL_USER")" var_isset 'TARGET_URL_PASS' && TARGET_URL_PASS="$(url_decode "$TARGET_URL_PASS")" -# defined TARGET_USER&PASS vars replace their URL pendants +# defined TARGET_USER&PASS vars replace their URL pendants # (double defs already dealt with) var_isset 'TARGET_USER' && TARGET_URL_USER="$TARGET_USER" var_isset 'TARGET_PASS' && TARGET_URL_PASS="$TARGET_PASS" -# build target backend data depending on protocol -case "$(tolower "${TARGET_URL_PROT%%:*}")" in - 's3'|'s3+http') - BACKEND_PARAMS="AWS_ACCESS_KEY_ID='${TARGET_URL_USER}' AWS_SECRET_ACCESS_KEY='${TARGET_URL_PASS}'" - BACKEND_URL="${TARGET_URL_PROT}${TARGET_URL_HOSTPATH}" - ;; - 'gs') - BACKEND_PARAMS="GS_ACCESS_KEY_ID='${TARGET_URL_USER}' GS_SECRET_ACCESS_KEY='${TARGET_URL_PASS}'" - BACKEND_URL="${TARGET_URL_PROT}${TARGET_URL_HOSTPATH}" - ;; +TARGET_URL_PROT_lowercase="$(tolower "${TARGET_URL_PROT%%:*}")" + +# issue some warnings +case "$TARGET_URL_PROT_lowercase" in 'cf+http') - # respect potentially set cloudfile env vars - var_isset 'CLOUDFILES_USERNAME' && TARGET_URL_USER="$CLOUDFILES_USERNAME" - var_isset 'CLOUDFILES_APIKEY' && TARGET_URL_PASS="$CLOUDFILES_APIKEY" - # add them to duplicity params - var_isset 'TARGET_URL_USER' && \ - BACKEND_PARAMS="CLOUDFILES_USERNAME=$(qw "${TARGET_URL_USER}")" - var_isset 'TARGET_URL_PASS' && \ - BACKEND_PARAMS="$BACKEND_PARAMS CLOUDFILES_APIKEY=$(qw "${TARGET_URL_PASS}")" - BACKEND_URL="${TARGET_URL_PROT}${TARGET_URL_HOSTPATH}" # info on missing AUTH_URL if ! var_isset 'CLOUDFILES_AUTHURL'; then - echo -e "INFO: No CLOUDFILES_AUTHURL defined (in conf).\n Will use default from python-cloudfiles (probably rackspace)." - else - BACKEND_PARAMS="$BACKEND_PARAMS CLOUDFILES_AUTHURL=$(qw "${CLOUDFILES_AUTHURL}")" + inform "No CLOUDFILES_AUTHURL exported (in conf). +Will use default which is probably rackspace." fi ;; - 'file'|'tahoe'|'dpbx') - BACKEND_URL="${TARGET_URL_PROT}${TARGET_URL_HOSTPATH}" - ;; 'swift') - BACKEND_URL="${TARGET_URL_PROT}${TARGET_URL_HOSTPATH}" - # respect possibly set swift env vars - var_isset 'SWIFT_USERNAME' && TARGET_URL_USER="$SWIFT_USERNAME" - var_isset 'SWIFT_PASSWORD' && TARGET_URL_PASS="$SWIFT_PASSWORD" - # add them to duplicity params like with cloudfile to make it look standardized - var_isset 'TARGET_URL_USER' && \ - BACKEND_PARAMS="$BACKEND_PARAMS SWIFT_USERNAME=$(qw "${TARGET_URL_USER}")" - var_isset 'SWIFT_AUTHURL' && \ - BACKEND_PARAMS="$BACKEND_PARAMS SWIFT_AUTHURL=$(qw "${SWIFT_AUTHURL}")" - ( var_isset 'TARGET_URL_USER' && ! var_isset 'SWIFT_AUTHURL' ) &&\ + # info on possibly missing AUTH_URL + var_isset 'SWIFT_AUTHURL' ||\ warning "\ -Swift will probably fail because the conf var SWIFT_AUTHURL was not defined!" - var_isset 'SWIFT_TENANTNAME' && \ - BACKEND_PARAMS="$BACKEND_PARAMS SWIFT_TENANTNAME=$(qw "${SWIFT_TENANTNAME}")" - var_isset 'SWIFT_AUTHVERSION' && \ - BACKEND_PARAMS="$BACKEND_PARAMS SWIFT_AUTHVERSION=$(qw "${SWIFT_AUTHVERSION}")" - var_isset 'TARGET_URL_PASS' && \ - BACKEND_PARAMS="$BACKEND_PARAMS SWIFT_PASSWORD=$(qw "${TARGET_URL_PASS}")" - ;; +Swift will probably fail because the conf var SWIFT_AUTHURL was not exported!" + ;; 'rsync') # everything in url (this backend does not support pass in env var) # this is obsolete from version 0.6.10 (buggy), hopefully fixed in 0.6.11 # print warning older version is detected - var_isset 'TARGET_URL_USER' && BACKEND_CREDS="$(url_encode "${TARGET_URL_USER}")" - if duplicity_version_lt 610; then + duplicity_version_lt 610 && warning "\ -Duplicity version '$DUPL_VERSION' does not support providing the password as -env var for rsync backend. For security reasons you should consider to +Duplicity version '$DUPL_VERSION' does not support providing the password as +env var for rsync backend. For security reasons you should consider to update to a version greater than '0.6.10' of duplicity." - var_isset 'TARGET_URL_PASS' && BACKEND_CREDS="${BACKEND_CREDS}:$(url_encode "${TARGET_URL_PASS}")" - else - var_isset 'TARGET_URL_PASS' && BACKEND_PARAMS="FTP_PASSWORD=$(qw "${TARGET_URL_PASS}")" - fi - var_isset 'BACKEND_CREDS' && BACKEND_CREDS="${BACKEND_CREDS}@" - BACKEND_URL="${TARGET_URL_PROT}${BACKEND_CREDS}${TARGET_URL_HOSTPATH}" + ;; +esac + + +# for all protocols we put username in url and pass into env var +# for sec�rity reasons, we url_encode username to protect special chars +# first sortout backends with special ways to handle password +case "$TARGET_URL_PROT_lowercase" in + 'imap'|'imaps') + var_isset 'TARGET_URL_PASS' && BACKEND_PARAMS="IMAP_PASSWORD=$(qw "${TARGET_URL_PASS}")" ;; *) - # for all other protocols we put username in url and pass into env var - # for sec˙rity reasons, we url_encode username to protect special chars - var_isset 'TARGET_URL_USER' && - BACKEND_CREDS="$(url_encode "${TARGET_URL_USER}")@" - # sortout backends with special ways to handle password - case "$(tolower "${TARGET_URL_PROT%%:*}")" in - 'imap'|'imaps') - var_isset 'TARGET_URL_PASS' && BACKEND_PARAMS="IMAP_PASSWORD=$(qw "${TARGET_URL_PASS}")" - ;; - 'ssh'|'sftp'|'scp') - # ssh backend wants to be told that theres a pass to use - var_isset 'TARGET_URL_PASS' && \ - DUPL_PARAMS="$DUPL_PARAMS --ssh-askpass" && \ - BACKEND_PARAMS="FTP_PASSWORD=$(qw "${TARGET_URL_PASS}")" - ;; - *) - # rest uses FTP_PASS var - var_isset 'TARGET_URL_PASS' && \ - BACKEND_PARAMS="FTP_PASSWORD=$(qw "${TARGET_URL_PASS}")" - ;; - esac - BACKEND_URL="${TARGET_URL_PROT}${BACKEND_CREDS}${TARGET_URL_HOSTPATH}" + # rest uses FTP_PASS var + var_isset 'TARGET_URL_PASS' && \ + BACKEND_PARAMS="FTP_PASSWORD=$(qw "${TARGET_URL_PASS}")" ;; esac +# insert url encoded username into target url if needed +if var_isset 'TARGET_URL_USER' && [ "$TARGET_URL_PROT_lowercase" != "file" ]; then + BACKEND_URL="${TARGET_URL_PROT}$(url_encode "${TARGET_URL_USER}")@${TARGET_URL_HOSTPATH}" +else + BACKEND_URL="$TARGET" +fi + -# protect eval from special chars in url (e.g. open ')' in password, +# protect eval from special chars in url (e.g. open ')' in password, # spaces in path, quotes) happens above in duplify() via quotewrap() SOURCE="$SOURCE" BACKEND_URL="$BACKEND_URL" EXCLUDE="$EXCLUDE" +# since 0.7.03 --exclude-globbing-filelist is deprecated +EXCLUDE_PARAM="--exclude$(duplicity_version_lt 703 && echo -globbing)-filelist" # replace magic separators to condition command equivalents (+=and,-=or) cmds=$(awk -v cmds="$cmds" "BEGIN{ gsub(/\+/,\"_and_\",cmds); gsub(/\-/,\"_or_\",cmds); print cmds}") # convert cmds to array, lowercase for safety CMDS=( $(awk "BEGIN{ cmds=tolower(\"$cmds\"); gsub(/_/,\" \",cmds); print cmds }") ) +unset FTPL_ERR + # run cmds for cmd in ${CMDS[*]}; do @@ -2109,21 +2241,48 @@ if [ -n "$SKIP_NOW" ]; then continue fi -# get prev/nextcmd vars -nextno=$(($CMD_NO+1)) -[ "$nextno" -lt "${#CMDS[@]}" ] && CMD_NEXT=${CMDS[$nextno]} || CMD_NEXT='END' -# get previous command minus skipped commands +unset CMD_VALUE CMD_NEXT CMD_PREV CND_NEXT CND_PREV + +# get next cmd,cnd vars +nextno=$(( $CMD_NO + 1 )) +while ! var_isset 'CMD_NEXT' +do + if [ "$nextno" -lt "${#CMDS[@]}" ]; then + CMD_VALUE=${CMDS[$nextno]} + is_condition "$CMD_VALUE" && CND_NEXT="$CMD_VALUE" || CMD_NEXT="$CMD_VALUE" + else + CMD_NEXT='END' + fi + nextno=$(($nextno+1)) +done + +# get prev cnd, cnd are skipped pseudocmds +prevno=$(( $CMD_NO - 1 )) +[ "$prevno" -ge 0 ] && is_condition "${CMDS[$prevno]}" && CND_PREV=${CMDS[$prevno]} + +# get prev cmd command minus skipped commands, only executed prevno=$(( $CMD_NO - ${CMD_SKIPPED-0} - 1 )); unset CMD_SKIPPED -[ "$prevno" -ge 0 ] && CMD_PREV=${CMDS[$prevno]} || CMD_PREV='START' +while ! var_isset 'CMD_PREV' +do + if [ "$prevno" -ge 0 ]; then + CMD_VALUE=${CMDS[$prevno]} + is_condition "$CMD_VALUE" || CMD_PREV="$CMD_VALUE" + else + CMD_PREV='START' + fi + prevno=$(($prevno-1)) +done + +# save start time +RUN_START=$(nsecs) # export some useful env vars for external scripts/programs to use -export CONFDIR SOURCE TARGET_URL_PROT TARGET_URL_HOSTPATH \ +export PROFILE CONFDIR SOURCE TARGET_URL_PROT TARGET_URL_HOSTPATH \ TARGET_URL_USER TARGET_URL_PASS \ GPG_KEYS_ENC=$(join "\n" "${GPG_KEYS_ENC_ARRAY[@]}") GPG_KEY_SIGN \ - GPG_PW CMD_PREV CMD_NEXT CMD_ERR + GPG_PW CMD_PREV CMD_NEXT CMD_ERR CND_PREV CND_NEXT\ + RUN_START -# save start time -RUN_START=$(date_fix %s)$(nsecs) # user info echo; separator "Start running command $(toupper $cmd) at $(date_from_nsecs $RUN_START)" @@ -2138,31 +2297,31 @@ case "$(tolower $cmd)" in ( run_script "$script" ) ;; 'bkp') - duplify -- "${dupl_opts[@]}" --exclude-globbing-filelist "$EXCLUDE" \ + duplify -- "${dupl_opts[@]}" $EXCLUDE_PARAM "$EXCLUDE" \ "$SOURCE" "$BACKEND_URL" ;; 'incr') - duplify incr -- "${dupl_opts[@]}" --exclude-globbing-filelist "$EXCLUDE" \ + duplify incr -- "${dupl_opts[@]}" $EXCLUDE_PARAM "$EXCLUDE" \ "$SOURCE" "$BACKEND_URL" ;; 'full') - duplify full -- "${dupl_opts[@]}" --exclude-globbing-filelist "$EXCLUDE" \ + duplify full -- "${dupl_opts[@]}" $EXCLUDE_PARAM "$EXCLUDE" \ "$SOURCE" "$BACKEND_URL" ;; 'verify') TIME="${ftpl_pars[0]:+"-t ${ftpl_pars[0]}"}" - duplify verify -- $TIME "${dupl_opts[@]}" --exclude-globbing-filelist "$EXCLUDE" \ + duplify verify -- $TIME "${dupl_opts[@]}" $EXCLUDE_PARAM "$EXCLUDE" \ "$BACKEND_URL" "$SOURCE" ;; 'verifypath') TIME="${ftpl_pars[2]:+"-t ${ftpl_pars[2]}"}" - IN_PATH="${ftpl_pars[0]}"; OUT_PATH="${ftpl_pars[1]}"; + IN_PATH="${ftpl_pars[0]}"; OUT_PATH="${ftpl_pars[1]}"; ( [ -z "$IN_PATH" ] || [ -z "$OUT_PATH" ] ) && error " Missing parameter or for verifyPath. - - Hint: + + Hint: Syntax is -> $ME verifyPath []" - duplify verify -- $TIME "${dupl_opts[@]}" --exclude-globbing-filelist "$EXCLUDE" \ + duplify verify -- $TIME "${dupl_opts[@]}" $EXCLUDE_PARAM "$EXCLUDE" \ --file-to-restore "$IN_PATH" "$BACKEND_URL" "$OUT_PATH" ;; 'list') @@ -2176,55 +2335,59 @@ case "$(tolower $cmd)" in 'purge') MAX_AGE=${ftpl_pars[0]:-$MAX_AGE} [ -z "$MAX_AGE" ] && error " Missing parameter . Can be set in profile or as command line parameter." - + duplify remove-older-than "${MAX_AGE}" \ -- "${dupl_opts[@]}" "$BACKEND_URL" ;; 'purgefull') MAX_FULL_BACKUPS=${ftpl_pars[0]:-$MAX_FULL_BACKUPS} [ -z "$MAX_FULL_BACKUPS" ] && error " Missing parameter . Can be set in profile or as command line parameter." - + duplify remove-all-but-n-full "${MAX_FULL_BACKUPS}" \ -- "${dupl_opts[@]}" "$BACKEND_URL" ;; 'purgeincr') MAX_FULLS_WITH_INCRS=${ftpl_pars[0]:-$MAX_FULLS_WITH_INCRS} [ -z "$MAX_FULLS_WITH_INCRS" ] && error " Missing parameter . Can be set in profile or as command line parameter." - + duplify remove-all-inc-of-but-n-full "${MAX_FULLS_WITH_INCRS}" \ -- "${dupl_opts[@]}" "$BACKEND_URL" ;; 'restore') - OUT_PATH="${ftpl_pars[0]:-$SOURCE}"; TIME="${ftpl_pars[1]:-now}"; + OUT_PATH="${ftpl_pars[0]}"; TIME="${ftpl_pars[1]:-now}"; [ -z "$OUT_PATH" ] && error " Missing parameter target_path for restore. - - Hint: + + Hint: Syntax is -> $ME restore []" - - duplify -- -t "$TIME" "${dupl_opts[@]}" "$BACKEND_URL" "$OUT_PATH" && run_script $CONFDIR/restore; + + duplify -- -t "$TIME" "${dupl_opts[@]}" "$BACKEND_URL" "$OUT_PATH" ;; 'fetch') - IN_PATH="${ftpl_pars[0]}"; OUT_PATH="${ftpl_pars[1]}"; + IN_PATH="${ftpl_pars[0]}"; OUT_PATH="${ftpl_pars[1]}"; TIME="${ftpl_pars[2]:-now}"; ( [ -z "$IN_PATH" ] || [ -z "$OUT_PATH" ] ) && error " Missing parameter or for fetch. - - Hint: + + Hint: Syntax is -> $ME fetch []" - + # duplicity 0.4.7 doesnt like cmd restore in combination with --file-to-restore duplify -- --restore-time "$TIME" "${dupl_opts[@]}" \ --file-to-restore "$IN_PATH" "$BACKEND_URL" "$OUT_PATH" ;; 'status') duplify collection-status -- "${dupl_opts[@]}" "$BACKEND_URL" - ;; + ;; *) - warning "Unknown command '$cmd'." + error " Unknown command '$cmd'. + + Hint: + Use '$ME usage' to get usage help." ;; esac CMD_ERR=$? -RUN_END=$(date_fix %s)$(nsecs) ; RUNTIME=$(( $RUN_END - $RUN_START )) +RUN_END=$(nsecs) +RUNTIME=$(( $RUN_END - $RUN_START )) # print message on error; set error code if [ "$CMD_ERR" -ne 0 ]; then @@ -2238,4 +2401,4 @@ Runtime $(printf "%02d:%02d:%02d.%03d" $((RUNTIME/1000000000/60/60)) $((RUNTIME/ done exit ${FTPL_ERR} -{% endraw %} +{% endraw %} \ No newline at end of file From d168196c714bc998fc057ffbcea9bf1bb99a9921 Mon Sep 17 00:00:00 2001 From: Samuel Baumgartner Date: Fri, 20 Jul 2018 22:32:21 +0200 Subject: [PATCH 10/18] Fix RedHat install --- meta/main.yml | 2 +- tasks/install.red.yml | 8 ++++++-- vars/RedHat.yml | 4 ++-- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/meta/main.yml b/meta/main.yml index e8dc0bd..485b090 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -1,7 +1,7 @@ --- dependencies: - - { role: ansible-role-python, python_install: [b2] } + - { role: ansible-role-python, python_install: ["setuptools>=20.2", b2] } galaxy_info: author: klen diff --git a/tasks/install.red.yml b/tasks/install.red.yml index 5d23184..a6208d9 100644 --- a/tasks/install.red.yml +++ b/tasks/install.red.yml @@ -15,10 +15,14 @@ key: "/etc/pki/rpm-gpg/RPM-GPG-KEY-EPEL-{{ ansible_distribution_major_version }}" state: present +- name: Remove urllib3 because it will be installed as a dependency later on with rpm + pip: + name: urllib3 + state: absent + - name: Install dependencies yum: name={{ dependencies }} - update_cache: yes - + update_cache: yes - set_fact: backup_duplicity_pkg="{{backup_duplicity_pkg}}-{{backup_duplicity_version}}" when: backup_duplicity_version diff --git a/vars/RedHat.yml b/vars/RedHat.yml index faffce7..029ec2e 100644 --- a/vars/RedHat.yml +++ b/vars/RedHat.yml @@ -3,5 +3,5 @@ dependencies: - cronie - gzip - - python-boto - - s3cmd \ No newline at end of file + - python2-boto #amazon + - s3cmd #amazon From 1381dd08847a10140855a6f4a0ab4fe1ddcc6823 Mon Sep 17 00:00:00 2001 From: Samuel Baumgartner Date: Thu, 1 Nov 2018 15:34:28 +0100 Subject: [PATCH 11/18] fix update_cache in yum task --- tasks/install.red.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tasks/install.red.yml b/tasks/install.red.yml index a6208d9..1d769ec 100644 --- a/tasks/install.red.yml +++ b/tasks/install.red.yml @@ -21,8 +21,9 @@ state: absent - name: Install dependencies - yum: name={{ dependencies }} - update_cache: yes + yum: + name: "{{ dependencies }}" + update_cache: yes - set_fact: backup_duplicity_pkg="{{backup_duplicity_pkg}}-{{backup_duplicity_version}}" when: backup_duplicity_version From f9470007a04525e0e162aa71099be6d0d9937deb Mon Sep 17 00:00:00 2001 From: Samuel Baumgartner Date: Wed, 17 Apr 2019 14:51:09 +0200 Subject: [PATCH 12/18] fix problem with setuptools dependency --- meta/main.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/meta/main.yml b/meta/main.yml index 485b090..8b72e48 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -1,7 +1,8 @@ --- dependencies: - - { role: ansible-role-python, python_install: ["setuptools>=20.2", b2] } + - { role: ansible-role-python, python_install: ["setuptools>=20.2"] } + - { role: ansible-role-python, python_install: [b2] } galaxy_info: author: klen From e5be51665dce77eab27219d990f850520638cf67 Mon Sep 17 00:00:00 2001 From: Samuel Baumgartner Date: Mon, 3 Jun 2019 11:25:18 +0200 Subject: [PATCH 13/18] postgres-backup: dont aks for pw when localhost --- templates/pre.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/templates/pre.j2 b/templates/pre.j2 index 10ae897..21d0e5d 100644 --- a/templates/pre.j2 +++ b/templates/pre.j2 @@ -8,7 +8,11 @@ WORKDIR={{backup_work}}/{{item.name}} {% if item.source.startswith('postgresql://') %} DBNAME={{item.source.split('postgresql://')[-1]}} +{% if backup_postgres_host == "localhost" %} +pg_dump -U {{backup_postgres_user}} {{ ('-p %s' % backup_postgres_port) if backup_postgres_port else ''}} -c $DBNAME -f ${WORKDIR}/dump +{% else %} pg_dump -U {{backup_postgres_user}} {{ '-h ' + backup_postgres_host if backup_postgres_host else ''}} {{ ('-p %s' % backup_postgres_port) if backup_postgres_port else ''}} -c $DBNAME -f ${WORKDIR}/dump +{% endif %} {% elif item.source.startswith('mysql://') %} From 000a2d4cd825bd037fe44d24e31c0b2202188b57 Mon Sep 17 00:00:00 2001 From: Samuel Baumgartner Date: Fri, 21 Jun 2019 19:21:25 +0200 Subject: [PATCH 14/18] allow option for disabling encryption --- templates/conf.j2 | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/templates/conf.j2 b/templates/conf.j2 index 9a1acb9..8416f01 100644 --- a/templates/conf.j2 +++ b/templates/conf.j2 @@ -8,7 +8,12 @@ {% else %} GPG_KEY='{{ item.gpg_key|default(backup_gpg_key) }}' {% endif %} +{% if item.gpg_pw|default(backup_gpg_pw) == 'disabled' %} +GPG_KEY='disabled' +#GPG_PW='{{ item.gpg_pw|default(backup_gpg_pw) }}' +{% else %} GPG_PW='{{ item.gpg_pw|default(backup_gpg_pw) }}' +{% endif %} {% if item.gpg_keys_enc|default(None) %} GPG_KEYS_ENC='{{item.gpg_keys_enc}}' {% endif %} From 3be11d985f8d70d097c6cc41f56ce3d62ce0f8f2 Mon Sep 17 00:00:00 2001 From: Samuel Baumgartner Date: Sun, 7 Jun 2020 21:36:23 +0200 Subject: [PATCH 15/18] 20.04 changes --- defaults/main.yml | 2 +- meta/main.yml | 5 +++-- tasks/install.deb.yml | 12 ++++++++++++ vars/Ubuntu.yml | 2 +- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 136b6f5..c32e7e7 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -6,7 +6,7 @@ backup_enabled: yes # Enable the role backup_remove: no # Set yes for uninstall the role from target system backup_cron: yes # Setup cron tasks for backup -backup_duplicity_ppa: ppa:duplicity-team/ppa # Install newest version from repo +backup_duplicity_ppa: ppa:duplicity-team/duplicity-release-git # Install newest version from repo backup_user: root # Run backups as user backup_group: "{{backup_user}}" diff --git a/meta/main.yml b/meta/main.yml index 8b72e48..b22bf62 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -1,8 +1,9 @@ --- dependencies: - - { role: ansible-role-python, python_install: ["setuptools>=20.2"] } - - { role: ansible-role-python, python_install: [b2] } + - { role: geerlingguy.pip, pip_package: python3-pip, pip_install_packages: [b2]} +# - { role: ansible-role-python, python_install: ["setuptools>=20.2"] } +# - { role: ansible-role-python, python_install: [b2] } galaxy_info: author: klen diff --git a/tasks/install.deb.yml b/tasks/install.deb.yml index e8f99a7..1d14307 100644 --- a/tasks/install.deb.yml +++ b/tasks/install.deb.yml @@ -2,6 +2,18 @@ - include_vars: "{{ansible_distribution}}.yml" +# This is required until duplicity works with python3.8 (ubuntu 20.04) +- name: Install python 3.7 - add repo + apt_repository: + repo: ppa:deadsnakes/ppa + state: present + update_cache: yes + +- name: Install python 3.7 - install package + package: + name: python3.7 + state: present + - name: Adding repository apt_repository: repo: "{{ backup_duplicity_ppa }}" diff --git a/vars/Ubuntu.yml b/vars/Ubuntu.yml index 330fe8a..8d8fdbe 100644 --- a/vars/Ubuntu.yml +++ b/vars/Ubuntu.yml @@ -3,5 +3,5 @@ dependencies: - cron - gzip - - python-boto + - python3-boto - s3cmd \ No newline at end of file From 4c1441521316a6192660c2ca1d040f77a970796e Mon Sep 17 00:00:00 2001 From: Samuel Baumgartner Date: Mon, 8 Jun 2020 16:19:25 +0200 Subject: [PATCH 16/18] upgrade duply --- templates/duply.sh.j2 | 397 ++++++++++++++++++++++++++++++------------ 1 file changed, 283 insertions(+), 114 deletions(-) diff --git a/templates/duply.sh.j2 b/templates/duply.sh.j2 index 351334b..df68fe0 100755 --- a/templates/duply.sh.j2 +++ b/templates/duply.sh.j2 @@ -9,11 +9,11 @@ # changed from ftplicity to duply. # # See http://duply.net or http://ftplicity.sourceforge.net/ for more info. # # (c) 2006 Christiane Ruetten, Heise Zeitschriften Verlag, Germany # -# (c) 2008-2017 Edgar Soldin (changes since version 1.3) # +# (c) 2008-2019 Edgar Soldin (changes since version 1.3) # ################################################################################ # LICENSE: # # This program is licensed under GPLv2. # -# Please read the accompanying license information in gpl.txt. # +# Please read the accompanying license information in gpl-2.0.txt. # ################################################################################ # TODO/IDEAS/KNOWN PROBLEMS: # - possibility to restore time frames (incl. deleted files) @@ -27,12 +27,37 @@ # deprecation since 0.5.10 namely --time-separator # --short-filenames # --old-filenames -# - add 'exclude_' list usage eg. exclude_verify +# - add 'exclude_' list usage e.g. exclude_verify # - featreq 25: a download/install duplicity option # - hint on install software if a piece is missing # - import/export profile from/to .tgz function !!! # # CHANGELOG: +# 2.2.2 (24.02.2020) +# - bugfix 120: Failures in "Autoset trust of key" during restore +# because of gpg2.2 fingerprint output change +# +# 2.2.1 (22.01.2020) +# - featreq 46: Example systemd units & Howto, courtesy of Jozef Riha +# - featreq 47: Clarify message about keeping the profile, also by Jozef Riha +# - fix abbreviation spelling of 'e.g.' +# +# 2.2 (30.12.2018) +# - featreq 44: implement grouping for batch commands +# new separators are [] (square brackets) or groupIn/groupOut +# command 'backup' translates now to [pre_bkp_post] to be skipped as +# one block in case a condition was set in the batch instruction +# +# 2.1 (23.07.2018) +# - be more verbose when duplicity version detection fails +# - using info shows python binary's path for easier identification now +# - reworked python interpreter handling, it's either +# configured per PYTHON var +# unconfigured, parsed from duplicity shebang +# or set to current duplicity default 'python2' (was 'python' until now) +# - do not quotewrap strings because of slashes (e.g. paths) anymore +# - bugfix: improved in/exclude stripping from conf DUPL_PARAMS +# # 2.0.4 (20.02.2018) # - bugfix 114: "duply usage is not current" wrt. purgeFull/Incr # - bugfix 115: typo in error message - "Not GPG_KEY entries" should be "No" @@ -89,7 +114,7 @@ # path /usr/bin/python until 0.7.05, which we circumvent this way) # - featreq 36: support gpg-connect-agent as a means to detect if an agent is # running (thx Thomas Harning Jr.), used gpg-agent for detection though -# - quotewrapped run_cmd parameters to protect it from spaces eg. in TMP path +# - quotewrapped run_cmd parameters to protect it from spaces e.g. in TMP path # - key export routine respects gpg-agent usage now # # 1.10.1 (19.8.2015) @@ -476,12 +501,26 @@ function lookup { ( [ "${bin##*/}" == "$bin" ] && hash "$bin" 2>/dev/null ) || [ -x "$bin" ] } +# the python binary to use, exit code 0 when configured, else 1 +function python_binary { + # if unset, parse from duplicity shebang + if ! var_isset 'PYTHON'; then + duplicity_python_binary_parse; + echo $DUPL_PYTHON_BIN; + return 1; + else + # tell if PYTHON was configured manually + echo $PYTHON; + return 0 + fi +} + # important definitions ####################################################### ME_LONG="$0" ME="$(basename $0)" ME_NAME="${ME%%.*}" -ME_VERSION="2.0.4" +ME_VERSION="2.2.2" ME_WEBSITE="http://duply.net" # default config values @@ -492,16 +531,16 @@ DEFAULT_TARGET_PASS='_backend_password_' DEFAULT_GPG='gpg' DEFAULT_GPG_KEY='_KEY_ID_' DEFAULT_GPG_PW='_GPG_PASSWORD_' -DEFAULT_PYTHON='python' +DEFAULT_PYTHON='python2' # function definitions ########################## -{% endraw %} + function set_config { # sets global config vars local CONFHOME_COMPAT="$HOME/.ftplicity" - local CONFHOME="{{backup_home}}" + local CONFHOME="$HOME/.duply" local CONFHOME_ETC_COMPAT="/etc/ftplicity" - local CONFHOME_ETC="{{backup_home}}" -{% raw %} + local CONFHOME_ETC="/etc/duply" + # confdir can be delivered as path (must contain /) if [ `echo $FTPLCFG | grep /` ] ; then CONFDIR=$(readlink -f $FTPLCFG 2>/dev/null || \ @@ -552,22 +591,21 @@ $(version_info) END } -function python_binary { - echo "${PYTHON-$DEFAULT_PYTHON}" -} - function using_info { - lookup duplicity && duplicity_version_get - local NOTFOUND="MISSING" + # init needed vars into global name space + lookup duplicity && { duplicity_python_binary_parse; duplicity_version_get; } + local NOTFOUND="INVALID" + local AWK_VERSION GREP_VERSION PYTHON_RUNNER # freebsd awk (--version only), debian mawk (-W version only), deliver '' so awk does not wait for input - local AWK_VERSION=$( lookup awk && (awk --version 2>/dev/null || awk -W version 2>&1) | awk 'NR<=2&&tolower($0)~/(busybox|awk)/{success=1;print;exit} END{if(success<1) print "unknown"}' || echo "$NOTFOUND" ) - local GREP_VERSION=$( lookup grep && grep --version 2>&1 | awk 'NR<=2&&tolower($0)~/(busybox|grep.*[0-9]+\.[0-9]+)/{success=1;print;exit} END{if(success<1) print "unknown"}' || echo "$NOTFOUND" ) - local PYTHON_RUNNER=$(python_binary) + AWK_VERSION=$( lookup awk && (awk --version 2>/dev/null || awk -W version 2>&1) | awk 'NR<=2&&tolower($0)~/(busybox|awk)/{success=1;print;exit} END{if(success<1) print "unknown"}' || echo "$NOTFOUND" ) + GREP_VERSION=$( lookup grep && grep --version 2>&1 | awk 'NR<=2&&tolower($0)~/(busybox|grep.*[0-9]+\.[0-9]+)/{success=1;print;exit} END{if(success<1) print "unknown"}' || echo "$NOTFOUND" ) + PYTHON_RUNNER=$(python_binary) local PYTHON_VERSION=$(lookup "$PYTHON_RUNNER" && "$PYTHON_RUNNER" -V 2>&1| awk '{print tolower($0);exit}' || echo "'$PYTHON_RUNNER' $NOTFOUND" ) local GPG_INFO=$(gpg_avail && gpg --version 2>&1| awk '/^gpg.*[0-9\.]+$/&&length(v)<1{v=$1" "$3}/^Home:/{h=" ("$0")"}END{print v""h}' || echo "gpg $NOTFOUND") local BASH_VERSION=$(bash --version | awk 'NR==1{IGNORECASE=1;sub(/GNU bash, version[ ]+/,"",$0);print $0}') + # print out echo -e "Using installed duplicity version ${DUPL_VERSION:-$NOTFOUND}\ -${PYTHON_VERSION+, $PYTHON_VERSION${PYTHONPATH:+ 'PYTHONPATH=$PYTHONPATH'}}\ +${PYTHON_VERSION+, $PYTHON_VERSION ${PYTHON_RUNNER:+($(which "$PYTHON_RUNNER"))}${PYTHONPATH:+ 'PYTHONPATH=$PYTHONPATH'}}\ ${GPG_INFO:+, $GPG_INFO}${AWK_VERSION:+, awk '${AWK_VERSION}'}${GREP_VERSION:+, grep '${GREP_VERSION}'}\ ${BASH_VERSION:+, bash '${BASH_VERSION}'}." } @@ -583,7 +621,7 @@ DESCRIPTION: It simplifies running duplicity with cron or on command line by: - keeping recurring settings in profiles per backup job - - enabling batch operations eg. backup_verify+purge + - enabling batch operations e.g. backup_verify+purge - executing pre/post scripts (different actions possible depending on previous or next command or it's exit status) - precondition checking for flawless duplicity operation @@ -637,16 +675,21 @@ SEPARATORS: - (minus sign), _or_ conditional OR the next command will only be executed if the previous failed + [] (square brackets), _groupIn_/_groupOut_ + enables grouping of commands example: - 'pre+bkp-verify_post' translates to 'pre_and_bkp_or_verify_post' + 'pre+[bkp-verify]_post' translates to + 'pre_and_groupIn_bkp_or_verify_groupOut_post' COMMANDS: usage get usage help text - and/or pseudo commands for better batch cmd readability (see SEPARATORS) + and/or/groupIn/groupOut + pseudo commands used in batches (see SEPARATORS above) + create creates a configuration profile - backup backup with pre/post script execution (batch: pre_bkp_post), + backup backup with pre/post script execution (batch: [pre_bkp_post]), full (if full_if_older matches or no earlier backup is found) incremental (in all other cases) pre/post execute '/$(basename "$PRE")', '/$(basename "$POST")' scripts @@ -707,8 +750,9 @@ PRE/POST SCRIPTS: Some useful internal duply variables are exported to the scripts. PROFILE, CONFDIR, SOURCE, TARGET_URL_, - GPG_, CMD_, CMD_ERR, RUN_START, - CND_ (condition before/after next/prev command) + GPG_, CMD_ERR, RUN_START, + CMD_ (previous/next command), + CND_ (condition before/after) The CMD_* variables were introduced to allow different actions according to the command the scripts were attached to e.g. 'pre_bkp_post_pre_verify_post' @@ -817,7 +861,7 @@ GPG_PW='${DEFAULT_GPG_PW}' # backend, credentials & location of the backup target (URL-Format) # generic syntax is # scheme://[user[:password]@]host[:port]/[/]path -# eg. +# e.g. # sftp://bob:secret@backupserver.com//home/bob/dupbkp # for details and available backends see duplicity manpage, section URL Format # http://duplicity.nongnu.org/duplicity.1.html#sect7 @@ -840,7 +884,7 @@ TARGET='${DEFAULT_TARGET}' # env vars should be set. #TARGET_USER='${DEFAULT_TARGET_USER}' #TARGET_PASS='${DEFAULT_TARGET_PASS}' -# eg. for cloud files backend it might look like this (uncomment for use!) +# e.g. for cloud files backend it might look like this (uncomment for use!) #export CLOUDFILES_USERNAME='someuser' #export CLOUDFILES_APIKEY='somekey' #export CLOUDFILES_AUTHURL ='someurl' @@ -861,7 +905,8 @@ SOURCE='${DEFAULT_SOURCE}' # "trickle -s -u 640 -d 5120" # 5Mb up, 40Mb down" #DUPL_PRECMD="" -# override the used python interpreter, defaults to "python" +# override the used python interpreter, defaults to +# - parsed result of duplicity's shebang or 'python2' # e.g. "python2" or "/usr/bin/python2.7" #PYTHON="python" @@ -971,12 +1016,23 @@ function hint_profile { cat <&1 | awk '/^duplicity /{print $2; exit;}'` - #DUPL_VERSION='0.7.03' #'0.6.08b' #,0.4.4.RC4,0.6.08b - DUPL_VERSION_VALUE=0 - DUPL_VERSION_AWK=$(awk -v v="$DUPL_VERSION" 'BEGIN{ - if (match(v,/[^\.0-9]+[0-9]*$/)){ - rest=substr(v,RSTART,RLENGTH);v=substr(v,0,RSTART-1);} - if (pos=match(rest,/RC([0-9]+)$/)) rc=substr(rest,pos+2) - split(v,f,"[. ]"); if(f[1]f[2]f[3]~/^[0-9]+$/) vvalue=f[1]*10000+f[2]*100+f[3]; else vvalue=0 - print "#"v"_"rest"("rc"):"f[1]"-"f[2]"-"f[3] - print "DUPL_VERSION_VALUE=\047"vvalue"\047" - print "DUPL_VERSION_RC=\047"rc"\047" - print "DUPL_VERSION_SUFFIX=\047"rest"\047" - }') - eval "$DUPL_VERSION_AWK" - #echo -e ",$DUPL_VERSION,$DUPL_VERSION_VALUE,$DUPL_VERSION_RC,$DUPL_VERSION_SUFFIX," -} - -function duplicity_version_check { - if [ $DUPL_VERSION_VALUE -eq 0 ]; then - inform "duplicity version check failed (please report, this is a bug)" - elif [ $DUPL_VERSION_VALUE -le 404 ] && [ ${DUPL_VERSION_RC:-4} -lt 4 ]; then - error "The installed version $DUPL_VERSION is incompatible with $ME_NAME v$ME_VERSION. + # nothing to do, just print + var_isset DUPL_VERSION && return + + local DUPL_VERSION_OUT DUPL_VERSION_AWK PYTHON_BIN CMD='duplicity' + # only run with a user specific python if configured (running by default + # breaks homebrew as they place a shell wrapper for duplicity in path) + PYTHON_BIN="$(python_binary)" &&\ + CMD="$(qw "$PYTHON_BIN") $(which $CMD)" + CMD="$CMD --version 2>&1" + DUPL_VERSION_OUT=$(eval "$CMD") + DUPL_VERSION=`echo $DUPL_VERSION_OUT | awk '/^duplicity /{print $2; exit;}'` + #DUPL_VERSION='0.7.03' #'0.6.08b' #,0.4.4.RC4,0.6.08b + DUPL_VERSION_VALUE=0 + DUPL_VERSION_AWK=$(awk -v v="$DUPL_VERSION" 'BEGIN{ + if (match(v,/[^\.0-9]+[0-9]*$/)){ + rest=substr(v,RSTART,RLENGTH);v=substr(v,0,RSTART-1);} + if (pos=match(rest,/RC([0-9]+)$/)) rc=substr(rest,pos+2) + split(v,f,"[. ]"); if(f[1]f[2]f[3]~/^[0-9]+$/) vvalue=f[1]*10000+f[2]*100+f[3]; else vvalue=0 + print "#"v"_"rest"("rc"):"f[1]"-"f[2]"-"f[3] + print "DUPL_VERSION_VALUE=\047"vvalue"\047" + print "DUPL_VERSION_RC=\047"rc"\047" + print "DUPL_VERSION_SUFFIX=\047"rest"\047" + }') + eval "$DUPL_VERSION_AWK" + #echo -e ",$DUPL_VERSION,$DUPL_VERSION_VALUE,$DUPL_VERSION_RC,$DUPL_VERSION_SUFFIX," + + # doublecheck findings and report error + if [ $DUPL_VERSION_VALUE -eq 0 ]; then + inform "duplicity version check failed (please report, this is a bug) +the command + $CMD +resulted in + $DUPL_VERSION_OUT +" + elif [ $DUPL_VERSION_VALUE -le 404 ] && [ ${DUPL_VERSION_RC:-4} -lt 4 ]; then + error "The installed version $DUPL_VERSION is incompatible with $ME_NAME v$ME_VERSION. You should upgrade your version of duplicity to at least v0.4.4RC4 or use the older ftplicity version 1.1.1 from $ME_WEBSITE." - fi + fi } function duplicity_version_ge { @@ -1105,6 +1174,21 @@ function duplicity_version_lt { ! duplicity_version_ge "$1" } +# parse interpreter from duplicity shebang +function duplicity_python_binary_parse { + # cached result + ( var_isset 'PYTHON' || var_isset 'DUPL_PYTHON_BIN' ) && return + + # parse it or warn + local DUPL_BIN=$(which duplicity) + DUPL_PYTHON_BIN=$(awk 'NR==1&&/^#!/{sub(/^#!( *\/usr\/bin\/env *)?/,""); print}' < "$DUPL_BIN") + if ! echo "$DUPL_PYTHON_BIN" | grep -q -i 'python'; then + warning "Could not parse the python interpreter used from duplicity ($DUPL_BIN). Result was '$DUPL_PYTHON_BIN'. +Will assume it is '$DEFAULT_PYTHON'." + DUPL_PYTHON_BIN="$DEFAULT_PYTHON" + fi +} + function run_script { # run pre/post scripts local ERR=0 local SCRIPT="$1" @@ -1156,7 +1240,7 @@ function qw { quotewrap "$@"; } function quotewrap { local param="$@" # quote strings having non word chars (e.g. spaces) - if echo "$param" | awk '/[^A-Za-z0-9_\.\-]/{exit 0}{exit 1}'; then + if echo "$param" | awk '/[^A-Za-z0-9_\.\-\/]/{exit 0}{exit 1}'; then echo "$param" | awk '{\ gsub(/[\047]/,"\047\\\047\047",$0);\ gsub(/[\042]/,"\047\\\042\047",$0);\ @@ -1208,20 +1292,42 @@ DUPL_VARS_GLOBAL="TMPDIR='$TEMP_DIR' \ ${DUPL_ARG_ENC}" } - # function to filter the DUPL_PARAMS var from user conf function duplicity_params_conf { # reuse cmd var from main loop ## in/exclude parameters are currently not supported on restores if [ "$cmd" = "fetch" ] || [ "$cmd" = "restore" ] || [ "$cmd" = "status" ]; then - # filter exclude params from fetch/restore - echo "$DUPL_PARAMS" | awk '{gsub(/--(ex|in)clude[a-z-]*(([ \t]+|=)[^-][^ \t]+)?/,"");print}' + # filter exclude params from fetch/restore/status + eval "stripXcludes $DUPL_PARAMS" return fi - + + # nothing done, print unchanged echo "$DUPL_PARAMS" } +# strip in/exclude parameters from param string +function stripXcludes { + local STRIPNEXT OUT; + for p in "$@"; do + if [ -n "$STRIPNEXT" ]; then + unset STRIPNEXT + # strip the value of previous parameter + continue + elif echo "$p" | awk '/^\-\-(in|ex)clude(\-[a-zA-Z]+)?$/{exit 0;}{exit 1;}'; then + # strips e.g. --include /foo/bar + STRIPNEXT="yes" + continue + elif echo "$p" | awk '/^\-\-(in|ex)clude(\-[a-zA-Z]+)?=/{exit 0;}{exit 1;}'; then + # strips e.g. --include=/foo/bar + continue + fi + + OUT="$OUT $(qw "$p")" + done + echo "$OUT" +} + function duplify { # the actual wrapper function local PARAMSNOW DUPL_CMD DUPL_CMD_PARAMS @@ -1242,18 +1348,18 @@ function duplify { # the actual wrapper function # init global duplicity parameters same for all tasks duplicity_params_global - local RUN=eval BIN=duplicity DUPL_BIN + local RUN=eval BIN=duplicity DUPL_BIN PYTHON_BIN # run in cmd line preview mode if requested var_isset 'PREVIEW' && RUN=echo # try to resolve duplicity path for usage with python interpreter DUPL_BIN=$(which "$BIN") || DUPL_BIN="$BIN" # only run with a user specific python if configured (running by default # breaks homebrew as they place a shell wrapper for duplicity in path) - [ -n "$PYTHON" ] && [ "$PYTHON" != "$DEFAULT_PYTHON" ] &&\ - BIN="$(qw "$(python_binary)") $(qw "$DUPL_BIN")" + PYTHON_BIN="$(python_binary)" &&\ + BIN="$(qw "$PYTHON_BIN") $(qw "$DUPL_BIN")" -$RUN "${DUPL_VARS_GLOBAL} ${BACKEND_PARAMS} \ -${DUPL_PRECMD} $BIN $DUPL_CMD $DUPL_PARAMS_GLOBAL $(duplicity_params_conf)\ +$RUN "${DUPL_VARS_GLOBAL} ${BACKEND_PARAMS}\ + ${DUPL_PRECMD} $BIN $DUPL_CMD $DUPL_PARAMS_GLOBAL $(duplicity_params_conf)\ $GPG_USEAGENT $(gpg_custom_binary) $DUPL_CMD_PARAMS" local ERR=$? @@ -1308,7 +1414,7 @@ function nsecs { [ -n "$NSECS" ] && NSECS_DISABLED=0 || NSECS_DISABLED=1 fi - # add 9 digits, not all date(s) deliver nsecs eg. busybox date + # add 9 digits, not all date(s) deliver nsecs e.g. busybox date if [ "$NSECS_DISABLED" == "1" ]; then date_fix %s000000000 else @@ -1342,10 +1448,20 @@ function var_isset { } function is_condition { - local CMD=$(tolower "$@") + local CMD=$(tolower "$1") [ "$CMD" == 'and' ] || [ "$CMD" == 'or' ] } +function is_groupMarker { + local CMD=$(tolower "$1") + [ "$CMD" == 'groupin' ] || [ "$CMD" == 'groupout' ] +} + +function is_command { + local CMD=$(tolower "$1") + ! is_condition "$CMD" && ! is_groupMarker "$CMD" +} + function url_encode { # utilize python, silently do nothing on error - because no python no duplicity OUT=$("$(python_binary)" -c " @@ -1471,9 +1587,8 @@ Exit the edit mode of gpg with \"quit\"." # see 'How to specify a user ID' on gpg manpage function gpg_fingerprint { - local PRINT=$(gpg $GPG_OPTS --fingerprint "$1" 2>&1|awk -F= 'NR==2{gsub(/ /,"",$2);$2=toupper($2); if ( $2 ~ /^[A-F0-9]+$/ && length($2) == 40 ) print $2; else exit 1}') \ - && [ -n "$PRINT" ] && echo $PRINT && return 0 - return 1 + gpg $GPG_OPTS --fingerprint "$1" 2>&1 | \ + awk 'NR==2{sub(/^.*=/,"");gsub(/[ \t]/,""); if ( $0 !~ /^[A-F0-9]+$/ || length($0) != 40 ) exit 1; print}' } function gpg_export_if_needed { @@ -1653,7 +1768,7 @@ function gpg_avail { lookup "$(gpg_binary)" } -# enforce the use our selected gpg binary +# enforce the use of our selected gpg binary function gpg { command "$(gpg_binary)" "$@" } @@ -1710,7 +1825,7 @@ Hint: exit 0 ;; version|-version|--version|-v|-V) - # profile can override GPG, so import it if it was given + # profile can override GPG/PYTHON, so import it if it was given var_isset FTPLCFG && { set_config [ -r "$CONF" ] && . "$CONF" || warning "Cannot import config '$CONF'." @@ -1742,8 +1857,8 @@ echo "Start $ME v$ME_VERSION, time is $(date_fix '%F %T')." # is duplicity avail lookup duplicity || error_path "duplicity missing. installed und available in path?" # init, exec duplicity version check info +duplicity_python_binary_parse duplicity_version_get -duplicity_version_check # check for certain important helper programs for f in awk grep "$(python_binary)"; do @@ -1820,9 +1935,6 @@ eval "${TARGET_SPLIT_URL}" # Hint: cmds is also used to check if authentification info sufficient in the next step cmds="$2"; shift 2 -# translate backup to batch command -cmds=${cmds//backup/pre_bkp_post} - # complain if command(s) missing [ -z $cmds ] && error " No command given. @@ -1847,7 +1959,7 @@ for param in "$@"; do # forward parameter[/option pairs] to duplicity dupl_opts["${#dupl_opts[@]}"]=${param} else - # anything else must be a parameter (eg. for fetch, ...) + # anything else must be a parameter (e.g. for fetch, ...) ftpl_pars["${#ftpl_pars[@]}"]=${param} fi last_param=${param} @@ -2203,9 +2315,18 @@ EXCLUDE="$EXCLUDE" # since 0.7.03 --exclude-globbing-filelist is deprecated EXCLUDE_PARAM="--exclude$(duplicity_version_lt 703 && echo -globbing)-filelist" -# replace magic separators to condition command equivalents (+=and,-=or) -cmds=$(awk -v cmds="$cmds" "BEGIN{ gsub(/\+/,\"_and_\",cmds); gsub(/\-/,\"_or_\",cmds); print cmds}") +# translate backup to batch command +cmds=${cmds//backup/groupIn_pre_bkp_post_groupOut} + +# replace magic separators to command equivalents (+=and,-=or,[=groupIn,]=groupOut) +cmds=$(awk -v cmds="$cmds" "BEGIN{ \ + gsub(/\+/,\"_and_\",cmds);\ + gsub(/\-/,\"_or_\",cmds);\ + gsub(/\[/,\"_groupIn_\",cmds);\ + gsub(/\]/,\"_groupOut_\",cmds);\ + print cmds}") # convert cmds to array, lowercase for safety +declare -a CMDS CMDS=( $(awk "BEGIN{ cmds=tolower(\"$cmds\"); gsub(/_/,\" \",cmds); print cmds }") ) unset FTPL_ERR @@ -2218,61 +2339,109 @@ do # raise index in cmd array for pre/post param var_isset 'CMD_NO' && CMD_NO=$((++CMD_NO)) || CMD_NO=0 -# deal with condition "commands" -unset SKIP_NOW -if var_isset 'CMD_SKIP' && [ $CMD_SKIP -gt 0 ]; then - echo -e "\n--- Skipping command $(toupper $cmd) ! ---" - CMD_SKIP=$(($CMD_SKIP - 1)) - SKIP_NOW="yes" -elif [ "$cmd" == 'and' ] && [ "$CMD_ERR" -ne "0" ]; then - CMD_SKIP=1 - SKIP_NOW="yes" -elif [ "$cmd" == 'or' ] && [ "$CMD_ERR" -eq "0" ]; then - CMD_SKIP=1 - SKIP_NOW="yes" -elif [ "$cmd" == 'and' ] || [ "$cmd" == 'or' ]; then - unset 'CMD_SKIP'; - SKIP_NOW="yes" -fi - -# sum up how many commands we skip and actually skip -if [ -n "$SKIP_NOW" ]; then - CMD_SKIPPED=$((${CMD_SKIPPED-0} + 1)) - continue -fi - unset CMD_VALUE CMD_NEXT CMD_PREV CND_NEXT CND_PREV # get next cmd,cnd vars -nextno=$(( $CMD_NO + 1 )) +nextno=$(( $CMD_NO )) while ! var_isset 'CMD_NEXT' do + nextno=$(($nextno+1)) if [ "$nextno" -lt "${#CMDS[@]}" ]; then CMD_VALUE=${CMDS[$nextno]} - is_condition "$CMD_VALUE" && CND_NEXT="$CMD_VALUE" || CMD_NEXT="$CMD_VALUE" + is_condition "$CMD_VALUE" && CND_NEXT="$CMD_VALUE" && continue + is_groupMarker "$CMD_VALUE" && continue + CMD_NEXT="$CMD_VALUE" else CMD_NEXT='END' fi - nextno=$(($nextno+1)) done -# get prev cnd, cnd are skipped pseudocmds -prevno=$(( $CMD_NO - 1 )) -[ "$prevno" -ge 0 ] && is_condition "${CMDS[$prevno]}" && CND_PREV=${CMDS[$prevno]} +# get prev cnd, cnds are skipped pseudocmds +prevno=$(( $CMD_NO )); +while ! var_isset 'CND_PREV' +do + prevno=$(($prevno-1)) + if [ "$prevno" -ge 0 ]; then + CMD_VALUE=${CMDS[$prevno]} + is_condition "$CMD_VALUE" && CND_PREV="$CMD_VALUE" && break + is_command "$CMD_VALUE" && break + else + break + fi +done # get prev cmd command minus skipped commands, only executed -prevno=$(( $CMD_NO - ${CMD_SKIPPED-0} - 1 )); unset CMD_SKIPPED +prevno=$(( $CMD_NO - ${CMD_SKIPPED-0} )); while ! var_isset 'CMD_PREV' do + prevno=$(($prevno-1)) if [ "$prevno" -ge 0 ]; then CMD_VALUE=${CMDS[$prevno]} - is_condition "$CMD_VALUE" || CMD_PREV="$CMD_VALUE" + is_condition "$CMD_VALUE" && CND_PREV="$CMD_VALUE" && continue + is_groupMarker "$CMD_VALUE" && continue + CMD_PREV="$CMD_VALUE" else CMD_PREV='START' fi - prevno=$(($prevno-1)) done +function get_cmd_skip_count { + # find closing bracket, get group skip count + local nextno=$CMD_NO + local GRP_OPEN=0 + local GRP_SKIP=0 + local CMD_VALUE + while [ "$nextno" -lt "${#CMDS[@]}" ] + do + nextno=$(($nextno+1)) + CMD_VALUE=${CMDS[$nextno]} + GRP_SKIP=$(( ${GRP_SKIP} + 1 )); + if is_command "$CMD_VALUE" && [ "$GRP_OPEN" -lt 1 ]; then + break; + elif [ "$CMD_VALUE" == 'groupin' ]; then + GRP_OPEN=$(( ${GRP_OPEN} + 1 )) + elif [ "$CMD_VALUE" == 'groupout' ]; then + GRP_OPEN=$(( ${GRP_OPEN} - 1 )) + if [ "$GRP_OPEN" -lt 1 ]; then + break; + fi + fi + done + + echo $GRP_SKIP; +} + +# decision time: are we skipping already or dealing with condition "commands" or other non-cmds? +unset SKIP_NOW +if var_isset 'CMD_SKIP' && [ $CMD_SKIP -gt 0 ]; then + # skip cnd/grp cmds silently + is_command "$cmd" && echo -e "\n--- Skipping command $(toupper $cmd) ! ---" + CMD_SKIP=$(($CMD_SKIP - 1)) + SKIP_NOW="yes" +elif ! var_isset 'PREVIEW' && [ "$cmd" == 'and' ] && [ "$CMD_ERR" -ne "0" ]; then + CMD_SKIP=$(get_cmd_skip_count) + # incl. this "cmd" + CMD_SKIP=$(( $CMD_SKIP + 1 )) + unset CMD_SKIPPED + SKIP_NOW="yes" +elif ! var_isset 'PREVIEW' && [ "$cmd" == 'or' ] && [ "$CMD_ERR" -eq "0" ]; then + CMD_SKIP=$(get_cmd_skip_count) + # incl. this "cmd" + CMD_SKIP=$(( $CMD_SKIP + 1 )) + unset CMD_SKIPPED + SKIP_NOW="yes" +elif is_condition "$cmd" || is_groupMarker "$cmd"; then + unset 'CMD_SKIP'; + SKIP_NOW="yes" +fi + +# let's do the skip now +if [ -n "$SKIP_NOW" ]; then + # sum up how many commands we actually skipped for the prev var routines + CMD_SKIPPED=$((${CMD_SKIPPED-0} + 1)) + continue +fi + # save start time RUN_START=$(nsecs) From 9087f0e89d847b1b88b7a77ba6b6e7aea0edf6c1 Mon Sep 17 00:00:00 2001 From: Samuel Baumgartner Date: Sun, 28 Jun 2020 12:56:11 +0200 Subject: [PATCH 17/18] clean-up --- meta/main.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/meta/main.yml b/meta/main.yml index b22bf62..2c340c8 100644 --- a/meta/main.yml +++ b/meta/main.yml @@ -2,8 +2,6 @@ dependencies: - { role: geerlingguy.pip, pip_package: python3-pip, pip_install_packages: [b2]} -# - { role: ansible-role-python, python_install: ["setuptools>=20.2"] } -# - { role: ansible-role-python, python_install: [b2] } galaxy_info: author: klen From ffaa2b705613dac9e75a5f022ab771166df6aa6c Mon Sep 17 00:00:00 2001 From: Samuel Baumgartner Date: Sun, 5 Jul 2020 00:45:39 +0200 Subject: [PATCH 18/18] multi var backup file list --- defaults/main.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index c32e7e7..5b6220c 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -9,7 +9,7 @@ backup_cron: yes # Setup cron tasks for backup backup_duplicity_ppa: ppa:duplicity-team/duplicity-release-git # Install newest version from repo backup_user: root # Run backups as user -backup_group: "{{backup_user}}" +backup_group: "{{ backup_user }}" backup_home: /etc/duply # Backup configuration directory backup_work: /var/duply # Working directory @@ -109,4 +109,8 @@ backup_credential_hubic_email: backup_credential_hubic_password: backup_credential_hubic_client_id: backup_credential_hubic_client_secret: -backup_credential_hubic_redirect_uri: \ No newline at end of file +backup_credential_hubic_redirect_uri: + +backup_profiles_exclude_list_host: [] +backup_profiles_exclude_list_group: [] +backup_profiles_exclude_list_all: []