From 424b753df5f91fa3434ff3d623b2fac01a7529ed Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 27 Apr 2022 18:23:05 -0700 Subject: [PATCH 01/31] full pre-commit config --- .pre-commit-config.yaml | 48 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 68648e39..40070bf4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -3,3 +3,51 @@ repos: rev: "v4.1.0" hooks: - id: check-merge-conflict + - id: end-of-file-fixer + exclude: >- + ^docs/[^/]*\.svg$ + - id: requirements-txt-fixer + - id: trailing-whitespace + types: [python] + - id: check-case-conflict + - id: check-json + - id: check-xml + - id: check-toml + - id: check-xml + - id: check-yaml + - id: debug-statements + - id: check-added-large-files + - id: check-symlinks + - id: debug-statements + - id: detect-aws-credentials + args: ["--allow-missing-credentials"] + - id: detect-private-key + exclude: ^examples|(?:tests/ssl)/ + - repo: https://github.com/hadialqattan/pycln + rev: v1.2.5 + hooks: + - id: pycln + args: ["--all"] + - repo: https://github.com/asottile/yesqa + rev: v1.3.0 + hooks: + - id: yesqa + - repo: https://github.com/pycqa/isort + rev: "5.10.1" + hooks: + - id: isort + args: ["--profile", "black"] + - repo: https://github.com/asottile/pyupgrade + rev: "v2.31.1" + hooks: + - id: pyupgrade + args: ["--py36-plus", "--keep-mock"] + - repo: https://github.com/pre-commit/mirrors-autopep8 + rev: "v1.6.0" + hooks: + - id: autopep8 + - repo: https://github.com/PyCQA/flake8 + rev: "4.0.1" + hooks: + - id: flake8 + exclude: "^docs/" From 5869c424717a691b9789e265acc178ba5939d842 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 27 Apr 2022 18:44:14 -0700 Subject: [PATCH 02/31] skip class extraction --- .sourcery.yaml | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 .sourcery.yaml diff --git a/.sourcery.yaml b/.sourcery.yaml new file mode 100644 index 00000000..738078bd --- /dev/null +++ b/.sourcery.yaml @@ -0,0 +1,2 @@ +refactor: + skip: ['class-extract-method'] \ No newline at end of file From 6de46b2e54afc87f9967875dbed7cdaeaa53a94e Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Wed, 27 Apr 2022 18:44:33 -0700 Subject: [PATCH 03/31] sourcery refactoring --- dbbackup/checks.py | 10 +++---- dbbackup/db/base.py | 6 ++--- dbbackup/db/mongodb.py | 26 +++++++++--------- dbbackup/db/mysql.py | 28 +++++++++++--------- dbbackup/db/sqlite.py | 6 ++--- dbbackup/management/commands/_base.py | 6 ++--- dbbackup/management/commands/dbbackup.py | 4 +-- dbbackup/management/commands/dbrestore.py | 6 ++--- dbbackup/management/commands/listbackups.py | 3 +-- dbbackup/management/commands/mediabackup.py | 4 +-- dbbackup/management/commands/mediarestore.py | 7 +++-- dbbackup/storage.py | 2 +- dbbackup/tests/commands/test_mediabackup.py | 6 ++--- dbbackup/tests/test_utils.py | 6 ++--- dbbackup/tests/utils.py | 15 +++++------ dbbackup/utils.py | 19 +++++++------ 16 files changed, 73 insertions(+), 81 deletions(-) diff --git a/dbbackup/checks.py b/dbbackup/checks.py index 62eda358..a4d1626f 100644 --- a/dbbackup/checks.py +++ b/dbbackup/checks.py @@ -34,13 +34,11 @@ def check_settings(app_configs, **kwargs): if not settings.STORAGE or not isinstance(settings.STORAGE, str): errors.append(W002) - if not callable(settings.FILENAME_TEMPLATE): - if '{datetime}' not in settings.FILENAME_TEMPLATE: - errors.append(W003) + if not callable(settings.FILENAME_TEMPLATE) and '{datetime}' not in settings.FILENAME_TEMPLATE: + errors.append(W003) - if not callable(settings.MEDIA_FILENAME_TEMPLATE): - if '{datetime}' not in settings.MEDIA_FILENAME_TEMPLATE: - errors.append(W004) + if not callable(settings.MEDIA_FILENAME_TEMPLATE) and '{datetime}' not in settings.MEDIA_FILENAME_TEMPLATE: + errors.append(W004) if re.search(r'[^A-Za-z0-9%_-]', settings.DATE_FORMAT): errors.append(W005) diff --git a/dbbackup/db/base.py b/dbbackup/db/base.py index cc3119bd..6223bf54 100644 --- a/dbbackup/db/base.py +++ b/dbbackup/db/base.py @@ -81,8 +81,7 @@ def generate_filename(self, server_name=None): server_name) def create_dump(self): - dump = self._create_dump() - return dump + return self._create_dump() def _create_dump(self): """ @@ -95,8 +94,7 @@ def restore_dump(self, dump): :param dump: Dump file :type dump: file """ - result = self._restore_dump(dump) - return result + return self._restore_dump(dump) def _restore_dump(self, dump): """ diff --git a/dbbackup/db/mongodb.py b/dbbackup/db/mongodb.py index 5b318964..626428c1 100644 --- a/dbbackup/db/mongodb.py +++ b/dbbackup/db/mongodb.py @@ -13,20 +13,21 @@ class MongoDumpConnector(BaseCommandDBConnector): drop = True def _create_dump(self): - cmd = '{} --db {}'.format(self.dump_cmd, self.settings['NAME']) + cmd = f"{self.dump_cmd} --db {self.settings['NAME']}" host = self.settings.get('HOST') or 'localhost' port = self.settings.get('PORT') or 27017 - cmd += ' --host {}:{}'.format(host, port) + cmd += f' --host {host}:{port}' if self.settings.get('USER'): - cmd += ' --username {}'.format(self.settings['USER']) + cmd += f" --username {self.settings['USER']}" if self.settings.get('PASSWORD'): - cmd += ' --password {}'.format(utils.get_escaped_command_arg(self.settings['PASSWORD'])) + cmd += f" --password {utils.get_escaped_command_arg(self.settings['PASSWORD'])}" + if self.settings.get('AUTH_SOURCE'): - cmd += ' --authenticationDatabase {}'.format(self.settings['AUTH_SOURCE']) + cmd += f" --authenticationDatabase {self.settings['AUTH_SOURCE']}" for collection in self.exclude: - cmd += ' --excludeCollection {}'.format(collection) + cmd += f' --excludeCollection {collection}' cmd += ' --archive' - cmd = '{} {} {}'.format(self.dump_prefix, cmd, self.dump_suffix) + cmd = f'{self.dump_prefix} {cmd} {self.dump_suffix}' stdout, stderr = self.run_command(cmd, env=self.dump_env) return stdout @@ -34,17 +35,18 @@ def _restore_dump(self, dump): cmd = self.restore_cmd host = self.settings.get('HOST') or 'localhost' port = self.settings.get('PORT') or 27017 - cmd += ' --host {}:{}'.format(host, port) + cmd += f' --host {host}:{port}' if self.settings.get('USER'): - cmd += ' --username {}'.format(self.settings['USER']) + cmd += f" --username {self.settings['USER']}" if self.settings.get('PASSWORD'): - cmd += ' --password {}'.format(utils.get_escaped_command_arg(self.settings['PASSWORD'])) + cmd += f" --password {utils.get_escaped_command_arg(self.settings['PASSWORD'])}" + if self.settings.get('AUTH_SOURCE'): - cmd += ' --authenticationDatabase {}'.format(self.settings['AUTH_SOURCE']) + cmd += f" --authenticationDatabase {self.settings['AUTH_SOURCE']}" if self.object_check: cmd += ' --objcheck' if self.drop: cmd += ' --drop' cmd += ' --archive' - cmd = '{} {} {}'.format(self.restore_prefix, cmd, self.restore_suffix) + cmd = f'{self.restore_prefix} {cmd} {self.restore_suffix}' return self.run_command(cmd, stdin=dump, env=self.restore_env) diff --git a/dbbackup/db/mysql.py b/dbbackup/db/mysql.py index 16d91745..86a8545a 100644 --- a/dbbackup/db/mysql.py +++ b/dbbackup/db/mysql.py @@ -11,31 +11,33 @@ class MysqlDumpConnector(BaseCommandDBConnector): restore_cmd = 'mysql' def _create_dump(self): - cmd = '{} {} --quick'.format(self.dump_cmd, self.settings['NAME']) + cmd = f"{self.dump_cmd} {self.settings['NAME']} --quick" if self.settings.get('HOST'): - cmd += ' --host={}'.format(self.settings['HOST']) + cmd += f" --host={self.settings['HOST']}" if self.settings.get('PORT'): - cmd += ' --port={}'.format(self.settings['PORT']) + cmd += f" --port={self.settings['PORT']}" if self.settings.get('USER'): - cmd += ' --user={}'.format(self.settings['USER']) + cmd += f" --user={self.settings['USER']}" if self.settings.get('PASSWORD'): - cmd += ' --password={}'.format(utils.get_escaped_command_arg(self.settings['PASSWORD'])) + cmd += f" --password={utils.get_escaped_command_arg(self.settings['PASSWORD'])}" + for table in self.exclude: - cmd += ' --ignore-table={}.{}'.format(self.settings['NAME'], table) - cmd = '{} {} {}'.format(self.dump_prefix, cmd, self.dump_suffix) + cmd += f" --ignore-table={self.settings['NAME']}.{table}" + cmd = f'{self.dump_prefix} {cmd} {self.dump_suffix}' stdout, stderr = self.run_command(cmd, env=self.dump_env) return stdout def _restore_dump(self, dump): - cmd = '{} {}'.format(self.restore_cmd, self.settings['NAME']) + cmd = f"{self.restore_cmd} {self.settings['NAME']}" if self.settings.get('HOST'): - cmd += ' --host={}'.format(self.settings['HOST']) + cmd += f" --host={self.settings['HOST']}" if self.settings.get('PORT'): - cmd += ' --port={}'.format(self.settings['PORT']) + cmd += f" --port={self.settings['PORT']}" if self.settings.get('USER'): - cmd += ' --user={}'.format(self.settings['USER']) + cmd += f" --user={self.settings['USER']}" if self.settings.get('PASSWORD'): - cmd += ' --password={}'.format(utils.get_escaped_command_arg(self.settings['PASSWORD'])) - cmd = '{} {} {}'.format(self.restore_prefix, cmd, self.restore_suffix) + cmd += f" --password={utils.get_escaped_command_arg(self.settings['PASSWORD'])}" + + cmd = f'{self.restore_prefix} {cmd} {self.restore_suffix}' stdout, stderr = self.run_command(cmd, stdin=dump, env=self.restore_env) return stdout, stderr diff --git a/dbbackup/db/sqlite.py b/dbbackup/db/sqlite.py index c65ddc99..605186be 100644 --- a/dbbackup/db/sqlite.py +++ b/dbbackup/db/sqlite.py @@ -73,10 +73,8 @@ def restore_dump(self, dump): for line in dump.readlines(): try: cursor.execute(line.decode('UTF-8')) - except OperationalError as err: - warnings.warn("Error in db restore: {}".format(err)) - except IntegrityError as err: - warnings.warn("Error in db restore: {}".format(err)) + except (OperationalError, IntegrityError) as err: + warnings.warn(f"Error in db restore: {err}") class SqliteCPConnector(BaseDBConnector): diff --git a/dbbackup/management/commands/_base.py b/dbbackup/management/commands/_base.py index 978239f8..7571e1a0 100644 --- a/dbbackup/management/commands/_base.py +++ b/dbbackup/management/commands/_base.py @@ -51,8 +51,8 @@ class BaseDbBackupCommand(BaseCommand): def __init__(self, *args, **kwargs): self.option_list = self.base_option_list + self.option_list if django.VERSION < (1, 10): - options = tuple([optparse_make_option(*_args, **_kwargs) - for _args, _kwargs in self.option_list]) + options = tuple(optparse_make_option(*_args, **_kwargs) for _args, _kwargs in self.option_list) + self.option_list = options + BaseCommand.option_list super(BaseDbBackupCommand, self).__init__(*args, **kwargs) @@ -110,7 +110,7 @@ def _get_backup_file(self, database=None, servername=None): database=database, servername=servername) except StorageError as err: - raise CommandError(err.args[0]) + raise CommandError(err.args[0]) from err input_file = self.read_from_storage(input_filename) return input_filename, input_file diff --git a/dbbackup/management/commands/dbbackup.py b/dbbackup/management/commands/dbbackup.py index 541e8d89..59fecbd7 100644 --- a/dbbackup/management/commands/dbbackup.py +++ b/dbbackup/management/commands/dbbackup.py @@ -67,7 +67,7 @@ def handle(self, **options): if self.clean: self._cleanup_old_backups(database=database_key) except StorageError as err: - raise CommandError(err) + raise CommandError(err) from err def _save_new_backup(self, database): """ @@ -85,7 +85,7 @@ def _save_new_backup(self, database): encrypted_file, filename = utils.encrypt_file(outputfile, filename) outputfile = encrypted_file # Set file name - filename = self.filename if self.filename else filename + filename = self.filename or filename self.logger.debug("Backup size: %s", utils.handle_size(outputfile)) # Store backup outputfile.seek(0) diff --git a/dbbackup/management/commands/dbrestore.py b/dbbackup/management/commands/dbrestore.py index 4cacfab3..3839f5e0 100644 --- a/dbbackup/management/commands/dbrestore.py +++ b/dbbackup/management/commands/dbrestore.py @@ -52,7 +52,7 @@ def handle(self, *args, **options): self.storage = get_storage() self._restore_backup() except StorageError as err: - raise CommandError(err) + raise CommandError(err) from err def _get_database(self, options): """Get the database to restore.""" @@ -64,7 +64,7 @@ def _get_database(self, options): raise CommandError(errmsg) database_name = list(settings.DATABASES.keys())[0] if database_name not in settings.DATABASES: - raise CommandError("Database %s does not exist." % database_name) + raise CommandError(f"Database {database_name} does not exist.") return database_name, settings.DATABASES[database_name] def _restore_backup(self): @@ -73,7 +73,7 @@ def _restore_backup(self): servername=self.servername) self.logger.info("Restoring backup for database '%s' and server '%s'", self.database_name, self.servername) - self.logger.info("Restoring: %s" % input_filename) + self.logger.info(f"Restoring: {input_filename}") if self.decrypt: unencrypted_file, input_filename = utils.unencrypt_file(input_file, input_filename, diff --git a/dbbackup/management/commands/listbackups.py b/dbbackup/management/commands/listbackups.py index 6e8ba39f..3c3856cf 100644 --- a/dbbackup/management/commands/listbackups.py +++ b/dbbackup/management/commands/listbackups.py @@ -40,8 +40,7 @@ def get_backup_attrs(self, options): filters = dict([(k, v) for k, v in options.items() if k in FILTER_KEYS]) filenames = self.storage.list_backups(**filters) - files_attr = [ + return [ {'datetime': utils.filename_to_date(filename).strftime('%x %X'), 'name': filename} for filename in filenames] - return files_attr diff --git a/dbbackup/management/commands/mediabackup.py b/dbbackup/management/commands/mediabackup.py index 542f3ff0..87527a14 100644 --- a/dbbackup/management/commands/mediabackup.py +++ b/dbbackup/management/commands/mediabackup.py @@ -54,7 +54,7 @@ def handle(self, **options): self._cleanup_old_backups(servername=self.servername) except StorageError as err: - raise CommandError(err) + raise CommandError(err) from err def _explore_storage(self): """Generator of all files contained in media storage.""" @@ -89,7 +89,7 @@ def backup_mediafiles(self): if self.filename: filename = self.filename else: - extension = "tar%s" % ('.gz' if self.compress else '') + extension = f"tar{'.gz' if self.compress else ''}" filename = utils.filename_generate(extension, servername=self.servername, content_type=self.content_type) diff --git a/dbbackup/management/commands/mediarestore.py b/dbbackup/management/commands/mediarestore.py index def825fa..431ce46f 100644 --- a/dbbackup/management/commands/mediarestore.py +++ b/dbbackup/management/commands/mediarestore.py @@ -54,11 +54,10 @@ def handle(self, *args, **options): def _upload_file(self, name, media_file): if self.media_storage.exists(name): - if self.replace: - self.media_storage.delete(name) - self.logger.info("%s deleted", name) - else: + if not self.replace: return + self.media_storage.delete(name) + self.logger.info("%s deleted", name) self.media_storage.save(name, media_file) self.logger.info("%s uploaded", name) diff --git a/dbbackup/storage.py b/dbbackup/storage.py index 3a0719cf..c4f4c10b 100644 --- a/dbbackup/storage.py +++ b/dbbackup/storage.py @@ -68,7 +68,7 @@ def __init__(self, storage_path=None, **options): self.name = self.storageCls.__name__ def __str__(self): - return 'dbbackup-%s' % self.storage.__str__() + return f'dbbackup-{self.storage.__str__()}' def delete_file(self, filepath): self.logger.debug('Deleting file %s', filepath) diff --git a/dbbackup/tests/commands/test_mediabackup.py b/dbbackup/tests/commands/test_mediabackup.py index a2c7a64f..c7ecc6ae 100644 --- a/dbbackup/tests/commands/test_mediabackup.py +++ b/dbbackup/tests/commands/test_mediabackup.py @@ -1,6 +1,8 @@ """ Tests for mediabackup command. """ + +import contextlib import os import tempfile from django.test import TestCase @@ -25,10 +27,8 @@ def setUp(self): def tearDown(self): if self.command.path is not None: - try: + with contextlib.suppress(OSError): os.remove(self.command.path) - except OSError: - pass def test_func(self): self.command.backup_mediafiles() diff --git a/dbbackup/tests/test_utils.py b/dbbackup/tests/test_utils.py index ce549833..3f33edb2 100644 --- a/dbbackup/tests/test_utils.py +++ b/dbbackup/tests/test_utils.py @@ -51,7 +51,7 @@ def test_func(self): self.assertEqual(len(mail.outbox), 1) sent_mail = mail.outbox[0] - expected_subject = '%s%s' % (settings.EMAIL_SUBJECT_PREFIX, subject) + expected_subject = f'{settings.EMAIL_SUBJECT_PREFIX}{subject}' expected_to = settings.ADMINS[0][1] expected_from = settings.SERVER_EMAIL @@ -208,7 +208,7 @@ class Filename_To_DatestringTest(TestCase): def test_func(self): now = datetime.now() datefmt = settings.DATE_FORMAT - filename = '%s-foo.gz.gpg' % datetime.strftime(now, datefmt) + filename = f'{datetime.strftime(now, datefmt)}-foo.gz.gpg' datestring = utils.filename_to_datestring(filename, datefmt) self.assertIn(datestring, filename) @@ -222,7 +222,7 @@ class Filename_To_DateTest(TestCase): def test_func(self): now = datetime.now() datefmt = settings.DATE_FORMAT - filename = '%s-foo.gz.gpg' % datetime.strftime(now, datefmt) + filename = f'{datetime.strftime(now, datefmt)}-foo.gz.gpg' date = utils.filename_to_date(filename, datefmt) self.assertEqual(date.timetuple()[:5], now.timetuple()[:5]) diff --git a/dbbackup/tests/utils.py b/dbbackup/tests/utils.py index 0285a288..1126937c 100644 --- a/dbbackup/tests/utils.py +++ b/dbbackup/tests/utils.py @@ -1,3 +1,4 @@ +import contextlib import os import subprocess @@ -73,25 +74,21 @@ def delete(self, name): def clean_gpg_keys(): - try: + with contextlib.suppress(Exception): cmd = ("gpg --batch --yes --delete-key '%s'" % GPG_FINGERPRINT) subprocess.call(cmd, stdout=DEV_NULL, stderr=DEV_NULL) - except: - pass - try: + with contextlib.suppress(Exception): cmd = ("gpg --batch --yes --delete-secrect-key '%s'" % GPG_FINGERPRINT) subprocess.call(cmd, stdout=DEV_NULL, stderr=DEV_NULL) - except: - pass def add_private_gpg(): - cmd = ('gpg --import %s' % GPG_PRIVATE_PATH).split() + cmd = f'gpg --import {GPG_PRIVATE_PATH}'.split() subprocess.call(cmd, stdout=DEV_NULL, stderr=DEV_NULL) def add_public_gpg(): - cmd = ('gpg --import %s' % GPG_PUBLIC_PATH).split() + cmd = f'gpg --import {GPG_PUBLIC_PATH}'.split() subprocess.call(cmd, stdout=DEV_NULL, stderr=DEV_NULL) @@ -99,7 +96,7 @@ def add_public_gpg(): def callable_for_filename_template(datetime, **kwargs): - return '%s_foo' % datetime + return f'{datetime}_foo' def get_dump(database=TEST_DATABASE): diff --git a/dbbackup/utils.py b/dbbackup/utils.py index 3c61cc6a..fd4c164f 100644 --- a/dbbackup/utils.py +++ b/dbbackup/utils.py @@ -69,9 +69,9 @@ def bytes_to_str(byteVal, decimals=1): for unit, byte in BYTES: if (byteVal >= byte): if decimals == 0: - return '%s %s' % (int(round(byteVal / byte, 0)), unit) - return '%s %s' % (round(byteVal / byte, decimals), unit) - return '%s B' % byteVal + return f'{int(round(byteVal / byte, 0))} {unit}' + return f'{round(byteVal / byte, decimals)} {unit}' + return f'{byteVal} B' def handle_size(filehandle): @@ -93,9 +93,8 @@ def mail_admins(subject, message, fail_silently=False, connection=None, """Sends a message to the admins, as defined by the DBBACKUP_ADMINS setting.""" if not settings.ADMINS: return - mail = EmailMultiAlternatives('%s%s' % (settings.EMAIL_SUBJECT_PREFIX, subject), - message, settings.SERVER_EMAIL, [a[1] for a in settings.ADMINS], - connection=connection) + mail = EmailMultiAlternatives(f'{settings.EMAIL_SUBJECT_PREFIX}{subject}', message, settings.SERVER_EMAIL, [a[1] for a in settings.ADMINS], connection=connection) + if html_message: mail.attach_alternative(html_message, 'text/html') mail.send(fail_silently=fail_silently) @@ -165,7 +164,7 @@ def encrypt_file(inputfile, filename): import gnupg tempdir = tempfile.mkdtemp(dir=settings.TMP_DIR) try: - filename = '%s.gpg' % filename + filename = f'{filename}.gpg' filepath = os.path.join(tempdir, filename) try: inputfile.seek(0) @@ -176,7 +175,7 @@ def encrypt_file(inputfile, filename): always_trust=always_trust) inputfile.close() if not result: - msg = 'Encryption failed; status: %s' % result.status + msg = f'Encryption failed; status: {result.status}' raise EncryptionError(msg) return create_spooled_temporary_file(filepath), filename finally: @@ -243,7 +242,7 @@ def compress_file(inputfile, filename): :rtype: :class:`tempfile.SpooledTemporaryFile`, ``str`` """ outputfile = create_spooled_temporary_file() - new_filename = filename + '.gz' + new_filename = f'{filename}.gz' zipfile = gzip.GzipFile(filename=filename, fileobj=outputfile, mode="wb") try: inputfile.seek(0) @@ -335,7 +334,7 @@ def datefmt_to_regex(datefmt): new_string = datefmt for pat, reg in PATTERN_MATCHNG: new_string = new_string.replace(pat, reg) - return re.compile(r'(%s)' % new_string) + return re.compile(f'({new_string})') def filename_to_datestring(filename, datefmt=None): From 1b23dfdc1985927b92b677a8a39436ccbfb01816 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 29 Apr 2022 01:09:48 -0700 Subject: [PATCH 04/31] add more pre-commit hooks --- .pre-commit-config.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 40070bf4..a37392ed 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -51,3 +51,13 @@ repos: hooks: - id: flake8 exclude: "^docs/" + - repo: https://github.com/Lucas-C/pre-commit-hooks-markup + rev: "v1.0.1" + hooks: + - id: rst-linter + files: >- + ^[^/]+[.]rst$ + - repo: https://github.com/adrienverge/yamllint + rev: "v1.26.3" + hooks: + - id: yamllint \ No newline at end of file From 61b8d9185b0ac6dcdba8139c2fd858f7256a4e58 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 29 Apr 2022 01:21:21 -0700 Subject: [PATCH 05/31] add conf files --- .flake8 | 5 +++++ .isort.cfg | 3 +++ .pylintrc | 14 +------------- .vscode/settings.json | 31 +++++++++++++++++++++++++++++++ .yamllint.yaml | 6 ++++++ pyproject.toml | 3 +++ requirements/dev.txt | 5 +++++ 7 files changed, 54 insertions(+), 13 deletions(-) create mode 100644 .flake8 create mode 100644 .isort.cfg create mode 100644 .vscode/settings.json create mode 100644 .yamllint.yaml create mode 100644 pyproject.toml create mode 100644 requirements/dev.txt diff --git a/.flake8 b/.flake8 new file mode 100644 index 00000000..896a803c --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +ignore = + E501, + E203, + W503 \ No newline at end of file diff --git a/.isort.cfg b/.isort.cfg new file mode 100644 index 00000000..437feb47 --- /dev/null +++ b/.isort.cfg @@ -0,0 +1,3 @@ +[isort] +profile=black +skip=migrations diff --git a/.pylintrc b/.pylintrc index 56d02c2c..0385136f 100644 --- a/.pylintrc +++ b/.pylintrc @@ -3,18 +3,11 @@ # path. You may set this option multiple times. ignore=test - # Pickle collected data for later comparisons. persistent=yes -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - - [MESSAGES CONTROL] -disable=redefined-builtin,too-many-arguments,too-few-public-methods,missing-docstring,invalid-name,abstract-method,no-self-use - +disable=broad-except, fixme, missing-module-docstring, missing-class-docstring, missing-function-docstring, too-many-arguments, too-few-public-methods, abstract-method [TYPECHECK] # List of members which are set dynamically and missed by pylint inference @@ -23,13 +16,8 @@ disable=redefined-builtin,too-many-arguments,too-few-public-methods,missing-docs generated-members=async_request,objects [VARIABLES] - # Tells wether we should check for unused import in __init__ files. init-import=no # A regular expression matching names used for dummy variables (i.e. not used). dummy-variables-rgx=_|dummy - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 00000000..25bb2227 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,31 @@ +{ + "editor.detectIndentation": false, + "editor.formatOnSave": true, + "python.linting.flake8Enabled": true, + "python.linting.pylintEnabled": true, + "python.languageServer": "Pylance", + "python.analysis.typeCheckingMode": "off", + "python.formatting.provider": "black", + "python.sortImports.args": [ + "--src=${workspaceFolder}" + ], + "terminal.integrated.scrollback": 10000, + "git.autofetch": true, + "prettier.tabWidth": 4, + "prettier.useTabs": true, + "prettier.endOfLine": "auto", + "files.associations": { + "**/requirements/*.txt": "pip-requirements", + }, + "[jsonc]": { + "editor.defaultFormatter": "vscode.json-language-features" + }, + "[json]": { + "editor.defaultFormatter": "vscode.json-language-features" + }, + "[python]": { + "editor.defaultFormatter": "ms-python.python" + }, + "html.format.endWithNewline": true, + "files.insertFinalNewline": true +} diff --git a/.yamllint.yaml b/.yamllint.yaml new file mode 100644 index 00000000..64bcf90c --- /dev/null +++ b/.yamllint.yaml @@ -0,0 +1,6 @@ +--- +extends: default + +rules: + line-length: disable + comments: disable diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..7507676c --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,3 @@ +[tool.black] +target-version = ['py39'] +extend-exclude = 'migrations' diff --git a/requirements/dev.txt b/requirements/dev.txt new file mode 100644 index 00000000..10d8bfcb --- /dev/null +++ b/requirements/dev.txt @@ -0,0 +1,5 @@ +black +flake8 +pylint +rope +twine From 5ba0795cf79d9a85b2fc21c778bff7b533293a26 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 29 Apr 2022 08:23:00 +0000 Subject: [PATCH 06/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .flake8 | 2 +- .pre-commit-config.yaml | 2 +- .sourcery.yaml | 2 +- README.rst | 2 +- contributing.md | 2 +- dbbackup/db/base.py | 19 +++++----- dbbackup/db/mongodb.py | 1 + dbbackup/db/mysql.py | 1 + dbbackup/db/postgresql.py | 26 +++++++------- dbbackup/db/sqlite.py | 16 ++++----- dbbackup/log.py | 3 +- dbbackup/management/commands/_base.py | 10 +++--- dbbackup/management/commands/dbbackup.py | 8 ++--- dbbackup/management/commands/dbrestore.py | 6 ++-- dbbackup/management/commands/listbackups.py | 9 +++-- dbbackup/management/commands/mediabackup.py | 9 +++-- dbbackup/management/commands/mediarestore.py | 4 +-- dbbackup/settings.py | 7 ++-- dbbackup/storage.py | 6 ++-- dbbackup/tests/commands/test_dbbackup.py | 7 ++-- dbbackup/tests/commands/test_dbrestore.py | 26 +++++++++----- dbbackup/tests/commands/test_mediabackup.py | 6 ++-- dbbackup/tests/functional/test_commands.py | 18 +++++----- dbbackup/tests/settings.py | 3 +- dbbackup/tests/test_checks.py | 3 +- dbbackup/tests/test_connectors/test_base.py | 4 +-- .../tests/test_connectors/test_mongodb.py | 2 -- dbbackup/tests/test_connectors/test_mysql.py | 2 -- .../tests/test_connectors/test_postgresql.py | 2 -- dbbackup/tests/test_connectors/test_sqlite.py | 2 -- dbbackup/tests/test_log.py | 8 +++-- dbbackup/tests/test_storage.py | 11 +++--- dbbackup/tests/test_utils.py | 1 - .../testapp/management/commands/count.py | 1 + .../tests/testapp/management/commands/feed.py | 2 +- .../tests/testapp/migrations/0001_initial.py | 5 +-- dbbackup/tests/testapp/models.py | 1 - dbbackup/tests/testapp/urls.py | 2 -- dbbackup/tests/testapp/views.py | 2 -- dbbackup/tests/utils.py | 8 ++--- dbbackup/utils.py | 3 +- docs/conf.py | 36 +++++++++---------- docs/configuration.rst | 2 -- requirements.txt | 2 +- requirements/build.txt | 6 ++-- requirements/docs.txt | 4 +-- requirements/tests.txt | 2 +- setup.py | 3 +- 48 files changed, 153 insertions(+), 156 deletions(-) diff --git a/.flake8 b/.flake8 index 896a803c..7268a896 100644 --- a/.flake8 +++ b/.flake8 @@ -2,4 +2,4 @@ ignore = E501, E203, - W503 \ No newline at end of file + W503 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a37392ed..4bfc5e6c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -60,4 +60,4 @@ repos: - repo: https://github.com/adrienverge/yamllint rev: "v1.26.3" hooks: - - id: yamllint \ No newline at end of file + - id: yamllint diff --git a/.sourcery.yaml b/.sourcery.yaml index 738078bd..be7bb7cb 100644 --- a/.sourcery.yaml +++ b/.sourcery.yaml @@ -1,2 +1,2 @@ refactor: - skip: ['class-extract-method'] \ No newline at end of file + skip: ['class-extract-method'] diff --git a/README.rst b/README.rst index dbab659f..249b5103 100644 --- a/README.rst +++ b/README.rst @@ -212,4 +212,4 @@ We use GitHub Actions as continuous integration tools. .. _`Read The Docs`: https://django-dbbackup.readthedocs.org/ .. _`GitHub issues`: https://github.com/jazzband/django-dbbackup/issues .. _`pull requests`: https://github.com/jazzband/django-dbbackup/pulls -.. _Coveralls: https://coveralls.io/github/jazzband/django-dbbackup \ No newline at end of file +.. _Coveralls: https://coveralls.io/github/jazzband/django-dbbackup diff --git a/contributing.md b/contributing.md index 10d79191..ad78220b 100644 --- a/contributing.md +++ b/contributing.md @@ -1,3 +1,3 @@ [![Jazzband](https://jazzband.co/static/img/jazzband.svg)](https://jazzband.co/) -This is a [Jazzband](https://jazzband.co/) project. By contributing you agree to abide by the [Contributor Code of Conduct](https://jazzband.co/about/conduct) and follow the [guidelines](https://jazzband.co/about/guidelines). \ No newline at end of file +This is a [Jazzband](https://jazzband.co/) project. By contributing you agree to abide by the [Contributor Code of Conduct](https://jazzband.co/about/conduct) and follow the [guidelines](https://jazzband.co/about/guidelines). diff --git a/dbbackup/db/base.py b/dbbackup/db/base.py index 6223bf54..3e0ee06c 100644 --- a/dbbackup/db/base.py +++ b/dbbackup/db/base.py @@ -1,15 +1,17 @@ """ Base database connectors """ +import logging import os import shlex -from django.core.files.base import File -from tempfile import SpooledTemporaryFile -from subprocess import Popen from importlib import import_module -import logging +from subprocess import Popen +from tempfile import SpooledTemporaryFile + +from django.core.files.base import File from dbbackup import settings, utils + from . import exceptions logger = logging.getLogger('dbbackup.command') @@ -38,7 +40,8 @@ def get_connector(database_name=None): """ Get a connector from its database key in setttings. """ - from django.db import connections, DEFAULT_DB_ALIAS + from django.db import DEFAULT_DB_ALIAS, connections + # Get DB database_name = database_name or DEFAULT_DB_ALIAS connection = connections[database_name] @@ -52,7 +55,7 @@ def get_connector(database_name=None): return connector(database_name, **connector_settings) -class BaseDBConnector(object): +class BaseDBConnector: """ Base class for create database connector. This kind of object creates interaction with database and allow backup and restore operations. @@ -61,7 +64,7 @@ class BaseDBConnector(object): exclude = [] def __init__(self, database_name=None, **kwargs): - from django.db import connections, DEFAULT_DB_ALIAS + from django.db import DEFAULT_DB_ALIAS, connections self.database_name = database_name or DEFAULT_DB_ALIAS self.connection = connections[self.database_name] for attr, value in kwargs.items(): @@ -159,4 +162,4 @@ def run_command(self, command, stdin=None, env=None): return stdout, stderr except OSError as err: raise exceptions.CommandConnectorError( - "Error running: {}\n{}".format(command, str(err))) + f"Error running: {command}\n{str(err)}") diff --git a/dbbackup/db/mongodb.py b/dbbackup/db/mongodb.py index 626428c1..fb662d07 100644 --- a/dbbackup/db/mongodb.py +++ b/dbbackup/db/mongodb.py @@ -1,4 +1,5 @@ from dbbackup import utils + from .base import BaseCommandDBConnector diff --git a/dbbackup/db/mysql.py b/dbbackup/db/mysql.py index 86a8545a..84fd1378 100644 --- a/dbbackup/db/mysql.py +++ b/dbbackup/db/mysql.py @@ -1,4 +1,5 @@ from dbbackup import utils + from .base import BaseCommandDBConnector diff --git a/dbbackup/db/postgresql.py b/dbbackup/db/postgresql.py index 89707ca0..82432c56 100644 --- a/dbbackup/db/postgresql.py +++ b/dbbackup/db/postgresql.py @@ -1,5 +1,5 @@ -from urllib.parse import quote import logging +from urllib.parse import quote from .base import BaseCommandDBConnector from .exceptions import DumpError @@ -15,7 +15,7 @@ def create_postgres_uri(self): dbname = self.settings.get('NAME') or '' user = quote(self.settings.get('USER') or '') password = self.settings.get('PASSWORD') or '' - password = ':{}'.format(quote(password)) if password else '' + password = f':{quote(password)}' if password else '' if not user: password = '' else: @@ -38,20 +38,20 @@ class PgDumpConnector(BaseCommandDBConnector): drop = True def _create_dump(self): - cmd = '{} '.format(self.dump_cmd) + cmd = f'{self.dump_cmd} ' cmd = cmd + create_postgres_uri(self) for table in self.exclude: - cmd += ' --exclude-table-data={}'.format(table) + cmd += f' --exclude-table-data={table}' if self.drop: cmd += ' --clean' - cmd = '{} {} {}'.format(self.dump_prefix, cmd, self.dump_suffix) + cmd = f'{self.dump_prefix} {cmd} {self.dump_suffix}' stdout, stderr = self.run_command(cmd, env=self.dump_env) return stdout def _restore_dump(self, dump): - cmd = '{} '.format(self.restore_cmd) + cmd = f'{self.restore_cmd} ' cmd = cmd + create_postgres_uri(self) # without this, psql terminates with an exit value of 0 regardless of errors @@ -59,7 +59,7 @@ def _restore_dump(self, dump): if self.single_transaction: cmd += ' --single-transaction' cmd += ' {}'.format(self.settings['NAME']) - cmd = '{} {} {}'.format(self.restore_prefix, cmd, self.restore_suffix) + cmd = f'{self.restore_prefix} {cmd} {self.restore_suffix}' stdout, stderr = self.run_command(cmd, stdin=dump, env=self.restore_env) return stdout, stderr @@ -85,7 +85,7 @@ def _enable_postgis(self): def _restore_dump(self, dump): if self.settings.get('ADMIN_USER'): self._enable_postgis() - return super(PgDumpGisConnector, self)._restore_dump(dump) + return super()._restore_dump(dump) class PgDumpBinaryConnector(PgDumpConnector): @@ -100,24 +100,24 @@ class PgDumpBinaryConnector(PgDumpConnector): drop = True def _create_dump(self): - cmd = '{} '.format(self.dump_cmd) + cmd = f'{self.dump_cmd} ' cmd = cmd + create_postgres_uri(self) cmd += ' --format=custom' for table in self.exclude: - cmd += ' --exclude-table-data={}'.format(table) - cmd = '{} {} {}'.format(self.dump_prefix, cmd, self.dump_suffix) + cmd += f' --exclude-table-data={table}' + cmd = f'{self.dump_prefix} {cmd} {self.dump_suffix}' stdout, stderr = self.run_command(cmd, env=self.dump_env) return stdout def _restore_dump(self, dump): dbname = create_postgres_uri(self) - cmd = '{} {}'.format(self.restore_cmd, dbname) + cmd = f'{self.restore_cmd} {dbname}' if self.single_transaction: cmd += ' --single-transaction' if self.drop: cmd += ' --clean' - cmd = '{} {} {}'.format(self.restore_prefix, cmd, self.restore_suffix) + cmd = f'{self.restore_prefix} {cmd} {self.restore_suffix}' stdout, stderr = self.run_command(cmd, stdin=dump, env=self.restore_env) return stdout, stderr diff --git a/dbbackup/db/sqlite.py b/dbbackup/db/sqlite.py index 605186be..fccaa949 100644 --- a/dbbackup/db/sqlite.py +++ b/dbbackup/db/sqlite.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - import warnings from io import BytesIO from shutil import copyfileobj @@ -27,6 +25,7 @@ class SqliteConnector(BaseDBConnector): Create a dump at SQL layer like could make ``.dumps`` in sqlite3. Restore by evaluate the created SQL. """ + def _write_dump(self, fileobj): cursor = self.connection.cursor() cursor.execute(DUMP_TABLES) @@ -38,24 +37,24 @@ def _write_dump(self, fileobj): # Make SQL commands in 1 line sql = sql.replace('\n ', '') sql = sql.replace('\n)', ')') - fileobj.write("{};\n".format(sql).encode('UTF-8')) + fileobj.write(f"{sql};\n".encode('UTF-8')) else: - fileobj.write("{};\n".format(sql)) + fileobj.write(f"{sql};\n") table_name_ident = table_name.replace('"', '""') - res = cursor.execute('PRAGMA table_info("{0}")'.format(table_name_ident)) + res = cursor.execute(f'PRAGMA table_info("{table_name_ident}")') column_names = [str(table_info[1]) for table_info in res.fetchall()] q = """SELECT 'INSERT INTO "{0}" VALUES({1})' FROM "{0}";\n""".format( table_name_ident, - ",".join("""'||quote("{0}")||'""".format(col.replace('"', '""')) + ",".join("""'||quote("{}")||'""".format(col.replace('"', '""')) for col in column_names)) query_res = cursor.execute(q) for row in query_res: - fileobj.write("{};\n".format(row[0]).encode('UTF-8')) + fileobj.write(f"{row[0]};\n".encode('UTF-8')) schema_res = cursor.execute(DUMP_ETC) for name, type, sql in schema_res.fetchall(): if sql.startswith("CREATE INDEX"): sql = sql.replace('CREATE INDEX', 'CREATE INDEX IF NOT EXISTS') - fileobj.write('{};\n'.format(sql).encode('UTF-8')) + fileobj.write(f'{sql};\n'.encode('UTF-8')) cursor.close() def create_dump(self): @@ -82,6 +81,7 @@ class SqliteCPConnector(BaseDBConnector): Create a dump by copy the binary data file. Restore by simply copy to the good location. """ + def create_dump(self): path = self.connection.settings_dict['NAME'] dump = BytesIO() diff --git a/dbbackup/log.py b/dbbackup/log.py index 9bb94936..452ad151 100644 --- a/dbbackup/log.py +++ b/dbbackup/log.py @@ -1,4 +1,5 @@ import logging + import django from django.utils.log import AdminEmailHandler @@ -9,7 +10,7 @@ def emit(self, record): if django.VERSION < (1, 8): from . import utils django.core.mail.mail_admins = utils.mail_admins - super(DbbackupAdminEmailHandler, self).emit(record) + super().emit(record) def send_mail(self, subject, message, *args, **kwargs): from . import utils diff --git a/dbbackup/management/commands/_base.py b/dbbackup/management/commands/_base.py index 7571e1a0..0407bd38 100644 --- a/dbbackup/management/commands/_base.py +++ b/dbbackup/management/commands/_base.py @@ -54,14 +54,14 @@ def __init__(self, *args, **kwargs): options = tuple(optparse_make_option(*_args, **_kwargs) for _args, _kwargs in self.option_list) self.option_list = options + BaseCommand.option_list - super(BaseDbBackupCommand, self).__init__(*args, **kwargs) + super().__init__(*args, **kwargs) def add_arguments(self, parser): for args, kwargs in self.option_list: - kwargs = dict([ - (k, v) for k, v in kwargs.items() - if not k.startswith('_') and - k not in USELESS_ARGS]) + kwargs = { + k: v for k, v in kwargs.items() + if not k.startswith('_') + and k not in USELESS_ARGS} parser.add_argument(*args, **kwargs) def _set_logger_level(self): diff --git a/dbbackup/management/commands/dbbackup.py b/dbbackup/management/commands/dbbackup.py index 59fecbd7..5efa8e6e 100644 --- a/dbbackup/management/commands/dbbackup.py +++ b/dbbackup/management/commands/dbbackup.py @@ -1,15 +1,13 @@ """ Command for backup database. """ -from __future__ import (absolute_import, division, - print_function, unicode_literals) from django.core.management.base import CommandError -from ._base import BaseDbBackupCommand, make_option +from ... import settings, utils from ...db.base import get_connector -from ...storage import get_storage, StorageError -from ... import utils, settings +from ...storage import StorageError, get_storage +from ._base import BaseDbBackupCommand, make_option class Command(BaseDbBackupCommand): diff --git a/dbbackup/management/commands/dbrestore.py b/dbbackup/management/commands/dbrestore.py index 3839f5e0..ef8ef586 100644 --- a/dbbackup/management/commands/dbrestore.py +++ b/dbbackup/management/commands/dbrestore.py @@ -1,17 +1,15 @@ """ Restore database. """ -from __future__ import (absolute_import, division, - print_function, unicode_literals) from django.conf import settings from django.core.management.base import CommandError from django.db import connection -from ._base import BaseDbBackupCommand, make_option from ... import utils from ...db.base import get_connector -from ...storage import get_storage, StorageError +from ...storage import StorageError, get_storage +from ._base import BaseDbBackupCommand, make_option class Command(BaseDbBackupCommand): diff --git a/dbbackup/management/commands/listbackups.py b/dbbackup/management/commands/listbackups.py index 3c3856cf..6cc8d7e9 100644 --- a/dbbackup/management/commands/listbackups.py +++ b/dbbackup/management/commands/listbackups.py @@ -1,11 +1,10 @@ """ List backups. """ -from __future__ import (absolute_import, division, - print_function, unicode_literals) + from ... import utils -from ._base import BaseDbBackupCommand, make_option from ...storage import get_storage +from ._base import BaseDbBackupCommand, make_option ROW_TEMPLATE = '{name:40} {datetime:20}' FILTER_KEYS = ('encrypted', 'compressed', 'content_type', 'database') @@ -37,8 +36,8 @@ def handle(self, **options): self.stdout.write(row) def get_backup_attrs(self, options): - filters = dict([(k, v) for k, v in options.items() - if k in FILTER_KEYS]) + filters = {k: v for k, v in options.items() + if k in FILTER_KEYS} filenames = self.storage.list_backups(**filters) return [ {'datetime': utils.filename_to_date(filename).strftime('%x %X'), diff --git a/dbbackup/management/commands/mediabackup.py b/dbbackup/management/commands/mediabackup.py index 87527a14..5c4195ad 100644 --- a/dbbackup/management/commands/mediabackup.py +++ b/dbbackup/management/commands/mediabackup.py @@ -1,17 +1,16 @@ """ Save media files. """ -from __future__ import (absolute_import, division, - print_function, unicode_literals) + import os import tarfile -from django.core.management.base import CommandError from django.core.files.storage import get_storage_class +from django.core.management.base import CommandError -from ._base import BaseDbBackupCommand, make_option from ... import utils -from ...storage import get_storage, StorageError +from ...storage import StorageError, get_storage +from ._base import BaseDbBackupCommand, make_option class Command(BaseDbBackupCommand): diff --git a/dbbackup/management/commands/mediarestore.py b/dbbackup/management/commands/mediarestore.py index 431ce46f..5403bede 100644 --- a/dbbackup/management/commands/mediarestore.py +++ b/dbbackup/management/commands/mediarestore.py @@ -5,9 +5,9 @@ from django.core.files.storage import get_storage_class -from ._base import BaseDbBackupCommand, make_option -from ...storage import get_storage from ... import utils +from ...storage import get_storage +from ._base import BaseDbBackupCommand, make_option class Command(BaseDbBackupCommand): diff --git a/dbbackup/settings.py b/dbbackup/settings.py index e62a423a..d0e3b888 100644 --- a/dbbackup/settings.py +++ b/dbbackup/settings.py @@ -1,7 +1,8 @@ # DO NOT IMPORT THIS BEFORE django.configure() has been run! -import tempfile import socket +import tempfile + from django.conf import settings DATABASES = getattr(settings, 'DBBACKUP_DATABASES', list(settings.DATABASES.keys())) @@ -22,8 +23,8 @@ MEDIA_PATH = getattr(settings, 'DBBACKUP_MEDIA_PATH', settings.MEDIA_ROOT) DATE_FORMAT = getattr(settings, 'DBBACKUP_DATE_FORMAT', '%Y-%m-%d-%H%M%S') -FILENAME_TEMPLATE = getattr(settings, 'DBBACKUP_FILENAME_TEMPLATE', '{databasename}-{servername}-{datetime}.{extension}') # noqa -MEDIA_FILENAME_TEMPLATE = getattr(settings, 'DBBACKUP_MEDIA_FILENAME_TEMPLATE', '{servername}-{datetime}.{extension}') # noqa +FILENAME_TEMPLATE = getattr(settings, 'DBBACKUP_FILENAME_TEMPLATE', '{databasename}-{servername}-{datetime}.{extension}') # noqa +MEDIA_FILENAME_TEMPLATE = getattr(settings, 'DBBACKUP_MEDIA_FILENAME_TEMPLATE', '{servername}-{datetime}.{extension}') # noqa GPG_ALWAYS_TRUST = getattr(settings, 'DBBACKUP_GPG_ALWAYS_TRUST', False) GPG_RECIPIENT = GPG_ALWAYS_TRUST = getattr(settings, 'DBBACKUP_GPG_RECIPIENT', None) diff --git a/dbbackup/storage.py b/dbbackup/storage.py index c4f4c10b..081049ac 100644 --- a/dbbackup/storage.py +++ b/dbbackup/storage.py @@ -2,8 +2,10 @@ Utils for handle files. """ import logging + from django.core.exceptions import ImproperlyConfigured from django.core.files.storage import get_storage_class + from . import settings, utils @@ -38,7 +40,7 @@ class FileNotFound(StorageError): pass -class Storage(object): +class Storage: """ This object make high-level storage operations for upload/download or list and filter files. It uses a Django storage object for low-level @@ -62,7 +64,7 @@ def __init__(self, storage_path=None, **options): self._storage_path = storage_path or settings.STORAGE options = options.copy() options.update(settings.STORAGE_OPTIONS) - options = dict([(key.lower(), value) for key, value in options.items()]) + options = {key.lower(): value for key, value in options.items()} self.storageCls = get_storage_class(self._storage_path) self.storage = self.storageCls(**options) self.name = self.storageCls.__name__ diff --git a/dbbackup/tests/commands/test_dbbackup.py b/dbbackup/tests/commands/test_dbbackup.py index 19df725b..b57c1b78 100644 --- a/dbbackup/tests/commands/test_dbbackup.py +++ b/dbbackup/tests/commands/test_dbbackup.py @@ -2,15 +2,14 @@ Tests for dbbackup command. """ import os -from mock import patch from django.test import TestCase +from mock import patch -from dbbackup.management.commands.dbbackup import Command as DbbackupCommand from dbbackup.db.base import get_connector +from dbbackup.management.commands.dbbackup import Command as DbbackupCommand from dbbackup.storage import get_storage -from dbbackup.tests.utils import (TEST_DATABASE, add_public_gpg, clean_gpg_keys, - DEV_NULL) +from dbbackup.tests.utils import DEV_NULL, TEST_DATABASE, add_public_gpg, clean_gpg_keys @patch('dbbackup.settings.GPG_RECIPIENT', 'test@test') diff --git a/dbbackup/tests/commands/test_dbrestore.py b/dbbackup/tests/commands/test_dbrestore.py index 72f8cffa..6397db18 100644 --- a/dbbackup/tests/commands/test_dbrestore.py +++ b/dbbackup/tests/commands/test_dbrestore.py @@ -1,24 +1,32 @@ """ Tests for dbrestore command. """ -from mock import patch -from tempfile import mktemp from shutil import copyfileobj +from tempfile import mktemp -from django.test import TestCase -from django.core.management.base import CommandError -from django.core.files import File from django.conf import settings +from django.core.files import File +from django.core.management.base import CommandError +from django.test import TestCase +from mock import patch from dbbackup import utils from dbbackup.db.base import get_connector from dbbackup.db.mongodb import MongoDumpConnector from dbbackup.management.commands.dbrestore import Command as DbrestoreCommand -from dbbackup.storage import get_storage from dbbackup.settings import HOSTNAME -from dbbackup.tests.utils import (TEST_DATABASE, add_private_gpg, DEV_NULL, - clean_gpg_keys, HANDLED_FILES, TEST_MONGODB, TARED_FILE, - get_dump, get_dump_name) +from dbbackup.storage import get_storage +from dbbackup.tests.utils import ( + DEV_NULL, + HANDLED_FILES, + TARED_FILE, + TEST_DATABASE, + TEST_MONGODB, + add_private_gpg, + clean_gpg_keys, + get_dump, + get_dump_name, +) @patch('dbbackup.management.commands._base.input', return_value='y') diff --git a/dbbackup/tests/commands/test_mediabackup.py b/dbbackup/tests/commands/test_mediabackup.py index c7ecc6ae..ceabe283 100644 --- a/dbbackup/tests/commands/test_mediabackup.py +++ b/dbbackup/tests/commands/test_mediabackup.py @@ -5,8 +5,10 @@ import contextlib import os import tempfile -from django.test import TestCase + from django.core.files.storage import get_storage_class +from django.test import TestCase + from dbbackup.management.commands.mediabackup import Command as DbbackupCommand from dbbackup.storage import get_storage from dbbackup.tests.utils import DEV_NULL, HANDLED_FILES, add_public_gpg @@ -64,7 +66,7 @@ def test_write_local_file(self): self.command.backup_mediafiles() self.assertTrue(os.path.exists(self.command.path)) self.assertEqual(0, len(HANDLED_FILES['written_files'])) - + def test_output_filename(self): self.command.filename = "my_new_name.tar" self.command.backup_mediafiles() diff --git a/dbbackup/tests/functional/test_commands.py b/dbbackup/tests/functional/test_commands.py index 0123417c..e3043d09 100644 --- a/dbbackup/tests/functional/test_commands.py +++ b/dbbackup/tests/functional/test_commands.py @@ -1,17 +1,19 @@ import os import tempfile -from mock import patch -from django.test import TransactionTestCase as TestCase -from django.core.management import execute_from_command_line from django.conf import settings - -from dbbackup.tests.utils import (TEST_DATABASE, HANDLED_FILES, - clean_gpg_keys, add_public_gpg, - add_private_gpg, get_dump, - get_dump_name) +from django.core.management import execute_from_command_line +from django.test import TransactionTestCase as TestCase +from mock import patch from dbbackup.tests.testapp import models +from dbbackup.tests.utils import ( + HANDLED_FILES, + TEST_DATABASE, + add_private_gpg, + add_public_gpg, + clean_gpg_keys, +) class DbBackupCommandTest(TestCase): diff --git a/dbbackup/tests/settings.py b/dbbackup/tests/settings.py index 0aff0fd5..7243a132 100644 --- a/dbbackup/tests/settings.py +++ b/dbbackup/tests/settings.py @@ -2,8 +2,9 @@ Configuration and launcher for dbbackup tests. """ import os -import tempfile import sys +import tempfile + from dotenv import load_dotenv test = len(sys.argv) <= 1 or sys.argv[1] == 'test' diff --git a/dbbackup/tests/test_checks.py b/dbbackup/tests/test_checks.py index 32257472..adc48500 100644 --- a/dbbackup/tests/test_checks.py +++ b/dbbackup/tests/test_checks.py @@ -1,5 +1,6 @@ -from mock import patch from django.test import TestCase +from mock import patch + try: from dbbackup import checks from dbbackup.apps import DbbackupConfig diff --git a/dbbackup/tests/test_connectors/test_base.py b/dbbackup/tests/test_connectors/test_base.py index 8482dc3c..fd95e4da 100644 --- a/dbbackup/tests/test_connectors/test_base.py +++ b/dbbackup/tests/test_connectors/test_base.py @@ -1,12 +1,10 @@ -from __future__ import unicode_literals - import os from tempfile import SpooledTemporaryFile from django.test import TestCase -from dbbackup.db.base import get_connector, BaseDBConnector, BaseCommandDBConnector from dbbackup.db import exceptions +from dbbackup.db.base import BaseCommandDBConnector, BaseDBConnector, get_connector class GetConnectorTest(TestCase): diff --git a/dbbackup/tests/test_connectors/test_mongodb.py b/dbbackup/tests/test_connectors/test_mongodb.py index 12708d55..1ebf7bb0 100644 --- a/dbbackup/tests/test_connectors/test_mongodb.py +++ b/dbbackup/tests/test_connectors/test_mongodb.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - from io import BytesIO from django.test import TestCase diff --git a/dbbackup/tests/test_connectors/test_mysql.py b/dbbackup/tests/test_connectors/test_mysql.py index aefcb8ed..ce1c265e 100644 --- a/dbbackup/tests/test_connectors/test_mysql.py +++ b/dbbackup/tests/test_connectors/test_mysql.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - from io import BytesIO from django.test import TestCase diff --git a/dbbackup/tests/test_connectors/test_postgresql.py b/dbbackup/tests/test_connectors/test_postgresql.py index 887184a8..55c44c58 100644 --- a/dbbackup/tests/test_connectors/test_postgresql.py +++ b/dbbackup/tests/test_connectors/test_postgresql.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - from io import BytesIO from django.test import TestCase diff --git a/dbbackup/tests/test_connectors/test_sqlite.py b/dbbackup/tests/test_connectors/test_sqlite.py index bfe7ed0c..d54ca095 100644 --- a/dbbackup/tests/test_connectors/test_sqlite.py +++ b/dbbackup/tests/test_connectors/test_sqlite.py @@ -1,5 +1,3 @@ -from __future__ import unicode_literals - from io import BytesIO from django.test import TestCase diff --git a/dbbackup/tests/test_log.py b/dbbackup/tests/test_log.py index be6551f8..2a618485 100644 --- a/dbbackup/tests/test_log.py +++ b/dbbackup/tests/test_log.py @@ -1,11 +1,13 @@ import logging -from mock import patch + import django -from django.test import TestCase from django.core import mail -from dbbackup import log +from django.test import TestCase +from mock import patch from testfixtures import log_capture +from dbbackup import log + class LoggerDefaultTestCase(TestCase): @log_capture() diff --git a/dbbackup/tests/test_storage.py b/dbbackup/tests/test_storage.py index b46eb824..f3f22ac7 100644 --- a/dbbackup/tests/test_storage.py +++ b/dbbackup/tests/test_storage.py @@ -1,8 +1,9 @@ -from mock import patch from django.test import TestCase -from dbbackup.storage import get_storage, Storage -from dbbackup.tests.utils import HANDLED_FILES, FakeStorage +from mock import patch + from dbbackup import utils +from dbbackup.storage import Storage, get_storage +from dbbackup.tests.utils import HANDLED_FILES, FakeStorage DEFAULT_STORAGE_PATH = 'django.core.files.storage.FileSystemStorage' STORAGE_OPTIONS = {'location': '/tmp'} @@ -61,7 +62,7 @@ def setUp(self): def test_nofilter(self): files = self.storage.list_backups() - self.assertEqual(len(HANDLED_FILES['written_files'])-1, len(files)) + self.assertEqual(len(HANDLED_FILES['written_files']) - 1, len(files)) for file in files: self.assertNotEqual('file_without_date', file) @@ -174,4 +175,4 @@ def test_func(self): @patch('dbbackup.settings.CLEANUP_KEEP_FILTER', keep_only_even_files) def test_keep_filter(self): self.storage.clean_old_backups(keep_number=1) - self.assertListEqual(['2015-02-07-042810.bak'], HANDLED_FILES['deleted_files']) \ No newline at end of file + self.assertListEqual(['2015-02-07-042810.bak'], HANDLED_FILES['deleted_files']) diff --git a/dbbackup/tests/test_utils.py b/dbbackup/tests/test_utils.py index 3f33edb2..06af128f 100644 --- a/dbbackup/tests/test_utils.py +++ b/dbbackup/tests/test_utils.py @@ -12,7 +12,6 @@ from dbbackup import settings, utils from dbbackup.tests.utils import ( COMPRESSED_FILE, - DEV_NULL, ENCRYPTED_FILE, add_private_gpg, add_public_gpg, diff --git a/dbbackup/tests/testapp/management/commands/count.py b/dbbackup/tests/testapp/management/commands/count.py index 32117d41..7bfb0798 100644 --- a/dbbackup/tests/testapp/management/commands/count.py +++ b/dbbackup/tests/testapp/management/commands/count.py @@ -1,4 +1,5 @@ from django.core.management.base import BaseCommand + from dbbackup.tests.testapp.models import CharModel diff --git a/dbbackup/tests/testapp/management/commands/feed.py b/dbbackup/tests/testapp/management/commands/feed.py index 37f15a85..bd3d4b4f 100644 --- a/dbbackup/tests/testapp/management/commands/feed.py +++ b/dbbackup/tests/testapp/management/commands/feed.py @@ -1,6 +1,6 @@ from django.core.management.base import BaseCommand + from dbbackup.tests.testapp.models import CharModel -from dbbackup.tests.utils import FakeStorage class Command(BaseCommand): diff --git a/dbbackup/tests/testapp/migrations/0001_initial.py b/dbbackup/tests/testapp/migrations/0001_initial.py index d225561e..0189338c 100644 --- a/dbbackup/tests/testapp/migrations/0001_initial.py +++ b/dbbackup/tests/testapp/migrations/0001_initial.py @@ -1,7 +1,4 @@ -# -*- coding: utf-8 -*- -from __future__ import unicode_literals - -from django.db import models, migrations +from django.db import migrations, models class Migration(migrations.Migration): diff --git a/dbbackup/tests/testapp/models.py b/dbbackup/tests/testapp/models.py index 6012d250..11e676d4 100644 --- a/dbbackup/tests/testapp/models.py +++ b/dbbackup/tests/testapp/models.py @@ -1,4 +1,3 @@ -from __future__ import unicode_literals from django.db import models ___all__ = ('CharModel', 'IntegerModel', 'TextModel', 'BooleanModel' diff --git a/dbbackup/tests/testapp/urls.py b/dbbackup/tests/testapp/urls.py index df25a40a..5eab1517 100644 --- a/dbbackup/tests/testapp/urls.py +++ b/dbbackup/tests/testapp/urls.py @@ -1,5 +1,3 @@ -from django.urls import include, re_path - urlpatterns = ( # url(r'^admin/', include(admin.site.urls)), ) diff --git a/dbbackup/tests/testapp/views.py b/dbbackup/tests/testapp/views.py index 91ea44a2..60f00ef0 100644 --- a/dbbackup/tests/testapp/views.py +++ b/dbbackup/tests/testapp/views.py @@ -1,3 +1 @@ -from django.shortcuts import render - # Create your views here. diff --git a/dbbackup/tests/utils.py b/dbbackup/tests/utils.py index 1126937c..082a1da4 100644 --- a/dbbackup/tests/utils.py +++ b/dbbackup/tests/utils.py @@ -30,13 +30,16 @@ class handled_files(dict): You should use the constant instance ``HANDLED_FILES`` and clean it before tests. """ + def __init__(self): - super(handled_files, self).__init__() + super().__init__() self.clean() def clean(self): self['written_files'] = [] self['deleted_files'] = [] + + HANDLED_FILES = handled_files() @@ -92,9 +95,6 @@ def add_public_gpg(): subprocess.call(cmd, stdout=DEV_NULL, stderr=DEV_NULL) - - - def callable_for_filename_template(datetime, **kwargs): return f'{datetime}_foo' diff --git a/dbbackup/utils.py b/dbbackup/utils.py index fd4c164f..ca7ec3d1 100644 --- a/dbbackup/utils.py +++ b/dbbackup/utils.py @@ -1,7 +1,6 @@ """ Utility functions for dbbackup. """ -from __future__ import absolute_import, division, print_function, unicode_literals import gzip import logging @@ -115,7 +114,7 @@ def wrapper(*args, **kwargs): logger = logging.getLogger('dbbackup') exc_type, exc_value, tb = sys.exc_info() tb_str = ''.join(traceback.format_tb(tb)) - msg = '%s: %s\n%s' % (exc_type.__name__, exc_value, tb_str) + msg = f'{exc_type.__name__}: {exc_value}\n{tb_str}' logger.error(msg) raise finally: diff --git a/docs/conf.py b/docs/conf.py index b6b55763..f70ba171 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- # # django-dbbackup documentation build configuration file, created by # sphinx-quickstart on Sun May 18 13:35:53 2014. @@ -11,7 +10,9 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +import dbbackup +import os +import sys # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -42,8 +43,8 @@ master_doc = 'index' # General information about the project. -project = u'django-dbbackup' -copyright = u'2016, Michael Shepanski' +project = 'django-dbbackup' +copyright = '2016, Michael Shepanski' # basepath path = os.path.dirname( @@ -57,7 +58,6 @@ os.environ['DJANGO_SETTINGS_MODULE'] = 'dbbackup.tests.settings' -import dbbackup # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -188,21 +188,21 @@ # -- Options for LaTeX output -------------------------------------------------- latex_elements = { -# The paper size ('letterpaper' or 'a4paper'). -#'papersize': 'letterpaper', + # The paper size ('letterpaper' or 'a4paper'). + # 'papersize': 'letterpaper', -# The font size ('10pt', '11pt' or '12pt'). -#'pointsize': '10pt', + # The font size ('10pt', '11pt' or '12pt'). + # 'pointsize': '10pt', -# Additional stuff for the LaTeX preamble. -#'preamble': '', + # Additional stuff for the LaTeX preamble. + # 'preamble': '', } # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'django-dbbackup.tex', u'django-dbbackup Documentation', - u'Michael Shepanski', 'manual'), + ('index', 'django-dbbackup.tex', 'django-dbbackup Documentation', + 'Michael Shepanski', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -231,8 +231,8 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'django-dbbackup', u'django-dbbackup Documentation', - [u'Michael Shepanski'], 1) + ('index', 'django-dbbackup', 'django-dbbackup Documentation', + ['Michael Shepanski'], 1) ] # If true, show URL addresses after external links. @@ -245,9 +245,9 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'django-dbbackup', u'django-dbbackup Documentation', - u'Michael Shepanski', 'django-dbbackup', 'One line description of project.', - 'Miscellaneous'), + ('index', 'django-dbbackup', 'django-dbbackup Documentation', + 'Michael Shepanski', 'django-dbbackup', 'One line description of project.', + 'Miscellaneous'), ] # Documents to append as an appendix to all manuals. diff --git a/docs/configuration.rst b/docs/configuration.rst index 5ededb6a..5ebf4fd1 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -188,5 +188,3 @@ You have to use a storage for your backups, see `Storage settings`_ for more. .. _`Database settings`: databases.html .. _`Storage settings`: storage.html - - diff --git a/requirements.txt b/requirements.txt index 076335e5..83d6b43d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -pytz Django>=2.2 +pytz diff --git a/requirements/build.txt b/requirements/build.txt index 24b84397..82e745cf 100644 --- a/requirements/build.txt +++ b/requirements/build.txt @@ -1,5 +1,5 @@ -setuptools -wheel build +setuptools +tox twine -tox \ No newline at end of file +wheel diff --git a/requirements/docs.txt b/requirements/docs.txt index 9d06a8c1..d1fd12c9 100644 --- a/requirements/docs.txt +++ b/requirements/docs.txt @@ -1,4 +1,4 @@ -sphinx docutils -sphinx-django-command python-dotenv +sphinx +sphinx-django-command diff --git a/requirements/tests.txt b/requirements/tests.txt index 6582b0ef..e294db57 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -9,5 +9,5 @@ python-dotenv python-gnupg pytz testfixtures -tox-gh-actions tox +tox-gh-actions diff --git a/setup.py b/setup.py index f6563708..3d4e6cb1 100644 --- a/setup.py +++ b/setup.py @@ -2,8 +2,7 @@ from pathlib import Path -from setuptools import setup, find_packages - +from setuptools import find_packages, setup project_dir = Path(__file__).parent From 099614dbf8dff521d3eaaac80384cbbe912b1d9d Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 29 Apr 2022 01:29:24 -0700 Subject: [PATCH 07/31] remove broken comma --- .vscode/settings.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 25bb2227..c6d47d83 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -15,7 +15,7 @@ "prettier.useTabs": true, "prettier.endOfLine": "auto", "files.associations": { - "**/requirements/*.txt": "pip-requirements", + "**/requirements/*.txt": "pip-requirements" }, "[jsonc]": { "editor.defaultFormatter": "vscode.json-language-features" From 1c9a2f3edd7f859615787aaf8246accbe0d17402 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 29 Apr 2022 08:29:39 +0000 Subject: [PATCH 08/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- dbbackup/db/sqlite.py | 6 +++--- dbbackup/settings.py | 4 ++-- docs/conf.py | 3 ++- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/dbbackup/db/sqlite.py b/dbbackup/db/sqlite.py index fccaa949..cb5df1b9 100644 --- a/dbbackup/db/sqlite.py +++ b/dbbackup/db/sqlite.py @@ -37,7 +37,7 @@ def _write_dump(self, fileobj): # Make SQL commands in 1 line sql = sql.replace('\n ', '') sql = sql.replace('\n)', ')') - fileobj.write(f"{sql};\n".encode('UTF-8')) + fileobj.write(f"{sql};\n".encode()) else: fileobj.write(f"{sql};\n") table_name_ident = table_name.replace('"', '""') @@ -49,12 +49,12 @@ def _write_dump(self, fileobj): for col in column_names)) query_res = cursor.execute(q) for row in query_res: - fileobj.write(f"{row[0]};\n".encode('UTF-8')) + fileobj.write(f"{row[0]};\n".encode()) schema_res = cursor.execute(DUMP_ETC) for name, type, sql in schema_res.fetchall(): if sql.startswith("CREATE INDEX"): sql = sql.replace('CREATE INDEX', 'CREATE INDEX IF NOT EXISTS') - fileobj.write(f'{sql};\n'.encode('UTF-8')) + fileobj.write(f'{sql};\n'.encode()) cursor.close() def create_dump(self): diff --git a/dbbackup/settings.py b/dbbackup/settings.py index d0e3b888..fa8b59ec 100644 --- a/dbbackup/settings.py +++ b/dbbackup/settings.py @@ -23,8 +23,8 @@ MEDIA_PATH = getattr(settings, 'DBBACKUP_MEDIA_PATH', settings.MEDIA_ROOT) DATE_FORMAT = getattr(settings, 'DBBACKUP_DATE_FORMAT', '%Y-%m-%d-%H%M%S') -FILENAME_TEMPLATE = getattr(settings, 'DBBACKUP_FILENAME_TEMPLATE', '{databasename}-{servername}-{datetime}.{extension}') # noqa -MEDIA_FILENAME_TEMPLATE = getattr(settings, 'DBBACKUP_MEDIA_FILENAME_TEMPLATE', '{servername}-{datetime}.{extension}') # noqa +FILENAME_TEMPLATE = getattr(settings, 'DBBACKUP_FILENAME_TEMPLATE', '{databasename}-{servername}-{datetime}.{extension}') +MEDIA_FILENAME_TEMPLATE = getattr(settings, 'DBBACKUP_MEDIA_FILENAME_TEMPLATE', '{servername}-{datetime}.{extension}') GPG_ALWAYS_TRUST = getattr(settings, 'DBBACKUP_GPG_ALWAYS_TRUST', False) GPG_RECIPIENT = GPG_ALWAYS_TRUST = getattr(settings, 'DBBACKUP_GPG_RECIPIENT', None) diff --git a/docs/conf.py b/docs/conf.py index f70ba171..7e400a82 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -10,10 +10,11 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import dbbackup import os import sys +import dbbackup + # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. From 4b590216616b8b7e530f6d7fa0bb0483f28bf4cd Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 29 Apr 2022 01:37:32 -0700 Subject: [PATCH 09/31] misc fixes --- dbbackup/db/mongodb.py | 45 +++++----- dbbackup/db/mysql.py | 27 +++--- dbbackup/db/postgresql.py | 94 +++++++++++---------- dbbackup/tests/test_connectors/test_base.py | 66 +++++++-------- 4 files changed, 118 insertions(+), 114 deletions(-) diff --git a/dbbackup/db/mongodb.py b/dbbackup/db/mongodb.py index fb662d07..28c76bd0 100644 --- a/dbbackup/db/mongodb.py +++ b/dbbackup/db/mongodb.py @@ -8,46 +8,47 @@ class MongoDumpConnector(BaseCommandDBConnector): MongoDB connector, creates dump with ``mongodump`` and restore with ``mongorestore``. """ - dump_cmd = 'mongodump' - restore_cmd = 'mongorestore' + + dump_cmd = "mongodump" + restore_cmd = "mongorestore" object_check = True drop = True def _create_dump(self): cmd = f"{self.dump_cmd} --db {self.settings['NAME']}" - host = self.settings.get('HOST') or 'localhost' - port = self.settings.get('PORT') or 27017 - cmd += f' --host {host}:{port}' - if self.settings.get('USER'): + host = self.settings.get("HOST") or "localhost" + port = self.settings.get("PORT") or 27017 + cmd += f" --host {host}:{port}" + if self.settings.get("USER"): cmd += f" --username {self.settings['USER']}" - if self.settings.get('PASSWORD'): + if self.settings.get("PASSWORD"): cmd += f" --password {utils.get_escaped_command_arg(self.settings['PASSWORD'])}" - if self.settings.get('AUTH_SOURCE'): + if self.settings.get("AUTH_SOURCE"): cmd += f" --authenticationDatabase {self.settings['AUTH_SOURCE']}" for collection in self.exclude: - cmd += f' --excludeCollection {collection}' - cmd += ' --archive' - cmd = f'{self.dump_prefix} {cmd} {self.dump_suffix}' - stdout, stderr = self.run_command(cmd, env=self.dump_env) + cmd += f" --excludeCollection {collection}" + cmd += " --archive" + cmd = f"{self.dump_prefix} {cmd} {self.dump_suffix}" + stdout, _stderr = self.run_command(cmd, env=self.dump_env) return stdout def _restore_dump(self, dump): cmd = self.restore_cmd - host = self.settings.get('HOST') or 'localhost' - port = self.settings.get('PORT') or 27017 - cmd += f' --host {host}:{port}' - if self.settings.get('USER'): + host = self.settings.get("HOST") or "localhost" + port = self.settings.get("PORT") or 27017 + cmd += f" --host {host}:{port}" + if self.settings.get("USER"): cmd += f" --username {self.settings['USER']}" - if self.settings.get('PASSWORD'): + if self.settings.get("PASSWORD"): cmd += f" --password {utils.get_escaped_command_arg(self.settings['PASSWORD'])}" - if self.settings.get('AUTH_SOURCE'): + if self.settings.get("AUTH_SOURCE"): cmd += f" --authenticationDatabase {self.settings['AUTH_SOURCE']}" if self.object_check: - cmd += ' --objcheck' + cmd += " --objcheck" if self.drop: - cmd += ' --drop' - cmd += ' --archive' - cmd = f'{self.restore_prefix} {cmd} {self.restore_suffix}' + cmd += " --drop" + cmd += " --archive" + cmd = f"{self.restore_prefix} {cmd} {self.restore_suffix}" return self.run_command(cmd, stdin=dump, env=self.restore_env) diff --git a/dbbackup/db/mysql.py b/dbbackup/db/mysql.py index 84fd1378..7e35516f 100644 --- a/dbbackup/db/mysql.py +++ b/dbbackup/db/mysql.py @@ -8,37 +8,38 @@ class MysqlDumpConnector(BaseCommandDBConnector): MySQL connector, creates dump with ``mysqldump`` and restore with ``mysql``. """ - dump_cmd = 'mysqldump' - restore_cmd = 'mysql' + + dump_cmd = "mysqldump" + restore_cmd = "mysql" def _create_dump(self): cmd = f"{self.dump_cmd} {self.settings['NAME']} --quick" - if self.settings.get('HOST'): + if self.settings.get("HOST"): cmd += f" --host={self.settings['HOST']}" - if self.settings.get('PORT'): + if self.settings.get("PORT"): cmd += f" --port={self.settings['PORT']}" - if self.settings.get('USER'): + if self.settings.get("USER"): cmd += f" --user={self.settings['USER']}" - if self.settings.get('PASSWORD'): + if self.settings.get("PASSWORD"): cmd += f" --password={utils.get_escaped_command_arg(self.settings['PASSWORD'])}" for table in self.exclude: cmd += f" --ignore-table={self.settings['NAME']}.{table}" - cmd = f'{self.dump_prefix} {cmd} {self.dump_suffix}' - stdout, stderr = self.run_command(cmd, env=self.dump_env) + cmd = f"{self.dump_prefix} {cmd} {self.dump_suffix}" + stdout, _stderr = self.run_command(cmd, env=self.dump_env) return stdout def _restore_dump(self, dump): cmd = f"{self.restore_cmd} {self.settings['NAME']}" - if self.settings.get('HOST'): + if self.settings.get("HOST"): cmd += f" --host={self.settings['HOST']}" - if self.settings.get('PORT'): + if self.settings.get("PORT"): cmd += f" --port={self.settings['PORT']}" - if self.settings.get('USER'): + if self.settings.get("USER"): cmd += f" --user={self.settings['USER']}" - if self.settings.get('PASSWORD'): + if self.settings.get("PASSWORD"): cmd += f" --password={utils.get_escaped_command_arg(self.settings['PASSWORD'])}" - cmd = f'{self.restore_prefix} {cmd} {self.restore_suffix}' + cmd = f"{self.restore_prefix} {cmd} {self.restore_suffix}" stdout, stderr = self.run_command(cmd, stdin=dump, env=self.restore_env) return stdout, stderr diff --git a/dbbackup/db/postgresql.py b/dbbackup/db/postgresql.py index 82432c56..d4e2ad89 100644 --- a/dbbackup/db/postgresql.py +++ b/dbbackup/db/postgresql.py @@ -4,25 +4,25 @@ from .base import BaseCommandDBConnector from .exceptions import DumpError -logger = logging.getLogger('dbbackup.command') +logger = logging.getLogger("dbbackup.command") def create_postgres_uri(self): - host = self.settings.get('HOST') + host = self.settings.get("HOST") if not host: - raise DumpError('A host name is required') + raise DumpError("A host name is required") - dbname = self.settings.get('NAME') or '' - user = quote(self.settings.get('USER') or '') - password = self.settings.get('PASSWORD') or '' - password = f':{quote(password)}' if password else '' + dbname = self.settings.get("NAME") or "" + user = quote(self.settings.get("USER") or "") + password = self.settings.get("PASSWORD") or "" + password = f":{quote(password)}" if password else "" if not user: - password = '' + password = "" else: - host = '@' + host + host = f"@{host}" - port = ':{}'.format(self.settings.get('PORT')) if self.settings.get('PORT') else '' - dbname = f'--dbname=postgresql://{user}{password}{host}{port}/{dbname}' + port = ":{}".format(self.settings.get("PORT")) if self.settings.get("PORT") else "" + dbname = f"--dbname=postgresql://{user}{password}{host}{port}/{dbname}" return dbname @@ -31,35 +31,36 @@ class PgDumpConnector(BaseCommandDBConnector): PostgreSQL connector, it uses pg_dump`` to create an SQL text file and ``psql`` for restore it. """ - extension = 'psql' - dump_cmd = 'pg_dump' - restore_cmd = 'psql' + + extension = "psql" + dump_cmd = "pg_dump" + restore_cmd = "psql" single_transaction = True drop = True def _create_dump(self): - cmd = f'{self.dump_cmd} ' + cmd = f"{self.dump_cmd} " cmd = cmd + create_postgres_uri(self) for table in self.exclude: - cmd += f' --exclude-table-data={table}' + cmd += f" --exclude-table-data={table}" if self.drop: - cmd += ' --clean' + cmd += " --clean" - cmd = f'{self.dump_prefix} {cmd} {self.dump_suffix}' - stdout, stderr = self.run_command(cmd, env=self.dump_env) + cmd = f"{self.dump_prefix} {cmd} {self.dump_suffix}" + stdout, _stderr = self.run_command(cmd, env=self.dump_env) return stdout def _restore_dump(self, dump): - cmd = f'{self.restore_cmd} ' + cmd = f"{self.restore_cmd} " cmd = cmd + create_postgres_uri(self) # without this, psql terminates with an exit value of 0 regardless of errors - cmd += ' --set ON_ERROR_STOP=on' + cmd += " --set ON_ERROR_STOP=on" if self.single_transaction: - cmd += ' --single-transaction' - cmd += ' {}'.format(self.settings['NAME']) - cmd = f'{self.restore_prefix} {cmd} {self.restore_suffix}' + cmd += " --single-transaction" + cmd += " {}".format(self.settings["NAME"]) + cmd = f"{self.restore_prefix} {cmd} {self.restore_suffix}" stdout, stderr = self.run_command(cmd, stdin=dump, env=self.restore_env) return stdout, stderr @@ -69,21 +70,21 @@ class PgDumpGisConnector(PgDumpConnector): PostgreGIS connector, same than :class:`PgDumpGisConnector` but enable postgis if not made. """ - psql_cmd = 'psql' + + psql_cmd = "psql" def _enable_postgis(self): - cmd = '{} -c "CREATE EXTENSION IF NOT EXISTS postgis;"'.format( - self.psql_cmd) - cmd += ' --username={}'.format(self.settings['ADMIN_USER']) - cmd += ' --no-password' - if self.settings.get('HOST'): - cmd += ' --host={}'.format(self.settings['HOST']) - if self.settings.get('PORT'): - cmd += ' --port={}'.format(self.settings['PORT']) + cmd = '{} -c "CREATE EXTENSION IF NOT EXISTS postgis;"'.format(self.psql_cmd) + cmd += " --username={}".format(self.settings["ADMIN_USER"]) + cmd += " --no-password" + if self.settings.get("HOST"): + cmd += " --host={}".format(self.settings["HOST"]) + if self.settings.get("PORT"): + cmd += " --port={}".format(self.settings["PORT"]) return self.run_command(cmd) def _restore_dump(self, dump): - if self.settings.get('ADMIN_USER'): + if self.settings.get("ADMIN_USER"): self._enable_postgis() return super()._restore_dump(dump) @@ -93,31 +94,32 @@ class PgDumpBinaryConnector(PgDumpConnector): PostgreSQL connector, it uses pg_dump`` to create an SQL text file and ``pg_restore`` for restore it. """ - extension = 'psql.bin' - dump_cmd = 'pg_dump' - restore_cmd = 'pg_restore' + + extension = "psql.bin" + dump_cmd = "pg_dump" + restore_cmd = "pg_restore" single_transaction = True drop = True def _create_dump(self): - cmd = f'{self.dump_cmd} ' + cmd = f"{self.dump_cmd} " cmd = cmd + create_postgres_uri(self) - cmd += ' --format=custom' + cmd += " --format=custom" for table in self.exclude: - cmd += f' --exclude-table-data={table}' - cmd = f'{self.dump_prefix} {cmd} {self.dump_suffix}' - stdout, stderr = self.run_command(cmd, env=self.dump_env) + cmd += f" --exclude-table-data={table}" + cmd = f"{self.dump_prefix} {cmd} {self.dump_suffix}" + stdout, _stderr = self.run_command(cmd, env=self.dump_env) return stdout def _restore_dump(self, dump): dbname = create_postgres_uri(self) - cmd = f'{self.restore_cmd} {dbname}' + cmd = f"{self.restore_cmd} {dbname}" if self.single_transaction: - cmd += ' --single-transaction' + cmd += " --single-transaction" if self.drop: - cmd += ' --clean' - cmd = f'{self.restore_prefix} {cmd} {self.restore_suffix}' + cmd += " --clean" + cmd = f"{self.restore_prefix} {cmd} {self.restore_suffix}" stdout, stderr = self.run_command(cmd, stdin=dump, env=self.restore_env) return stdout, stderr diff --git a/dbbackup/tests/test_connectors/test_base.py b/dbbackup/tests/test_connectors/test_base.py index fd95e4da..7a7de602 100644 --- a/dbbackup/tests/test_connectors/test_base.py +++ b/dbbackup/tests/test_connectors/test_base.py @@ -15,76 +15,76 @@ def test_get_connector(self): class BaseDBConnectorTest(TestCase): def test_init(self): - connector = BaseDBConnector() + BaseDBConnector() def test_settings(self): connector = BaseDBConnector() - connector.settings + connector.settings # pylint: disable=pointless-statement def test_generate_filename(self): connector = BaseDBConnector() - filename = connector.generate_filename() + connector.generate_filename() class BaseCommandDBConnectorTest(TestCase): def test_run_command(self): connector = BaseCommandDBConnector() - stdout, stderr = connector.run_command('echo 123') - self.assertEqual(stdout.read(), b'123\n') - self.assertEqual(stderr.read(), b'') + stdout, _stderr = connector.run_command("echo 123") + self.assertEqual(stdout.read(), b"123\n") + self.assertEqual(stderr.read(), b"") def test_run_command_error(self): connector = BaseCommandDBConnector() with self.assertRaises(exceptions.CommandConnectorError): - connector.run_command('echa 123') + connector.run_command("echa 123") def test_run_command_stdin(self): connector = BaseCommandDBConnector() stdin = SpooledTemporaryFile() - stdin.write(b'foo') + stdin.write(b"foo") stdin.seek(0) # Run - stdout, stderr = connector.run_command('cat', stdin=stdin) - self.assertEqual(stdout.read(), b'foo') + stdout, _stderr = connector.run_command("cat", stdin=stdin) + self.assertEqual(stdout.read(), b"foo") self.assertFalse(stderr.read()) def test_run_command_with_env(self): connector = BaseCommandDBConnector() # Empty env - stdout, stderr = connector.run_command('env') + stdout, _stderr = connector.run_command("env") self.assertTrue(stdout.read()) # env from self.env - connector.env = {'foo': 'bar'} - stdout, stderr = connector.run_command('env') - self.assertIn(b'foo=bar\n', stdout.read()) + connector.env = {"foo": "bar"} + stdout, _stderr = connector.run_command("env") + self.assertIn(b"foo=bar\n", stdout.read()) # method overide gloabal env - stdout, stderr = connector.run_command('env', env={'foo': 'ham'}) - self.assertIn(b'foo=ham\n', stdout.read()) + stdout, _stderr = connector.run_command("env", env={"foo": "ham"}) + self.assertIn(b"foo=ham\n", stdout.read()) # get a var from parent env - os.environ['bar'] = 'foo' - stdout, stderr = connector.run_command('env') - self.assertIn(b'bar=foo\n', stdout.read()) + os.environ["bar"] = "foo" + stdout, _stderr = connector.run_command("env") + self.assertIn(b"bar=foo\n", stdout.read()) # Conf overides parendt env - connector.env = {'bar': 'bar'} - stdout, stderr = connector.run_command('env') - self.assertIn(b'bar=bar\n', stdout.read()) + connector.env = {"bar": "bar"} + stdout, _stderr = connector.run_command("env") + self.assertIn(b"bar=bar\n", stdout.read()) # method overides all - stdout, stderr = connector.run_command('env', env={'bar': 'ham'}) - self.assertIn(b'bar=ham\n', stdout.read()) + stdout, _stderr = connector.run_command("env", env={"bar": "ham"}) + self.assertIn(b"bar=ham\n", stdout.read()) def test_run_command_with_parent_env(self): connector = BaseCommandDBConnector(use_parent_env=False) # Empty env - stdout, stderr = connector.run_command('env') + stdout, _stderr = connector.run_command("env") self.assertFalse(stdout.read()) # env from self.env - connector.env = {'foo': 'bar'} - stdout, stderr = connector.run_command('env') - self.assertEqual(stdout.read(), b'foo=bar\n') + connector.env = {"foo": "bar"} + stdout, _stderr = connector.run_command("env") + self.assertEqual(stdout.read(), b"foo=bar\n") # method overide gloabal env - stdout, stderr = connector.run_command('env', env={'foo': 'ham'}) - self.assertEqual(stdout.read(), b'foo=ham\n') + stdout, _stderr = connector.run_command("env", env={"foo": "ham"}) + self.assertEqual(stdout.read(), b"foo=ham\n") # no var from parent env - os.environ['bar'] = 'foo' - stdout, stderr = connector.run_command('env') - self.assertNotIn(b'bar=foo\n', stdout.read()) + os.environ["bar"] = "foo" + stdout, _stderr = connector.run_command("env") + self.assertNotIn(b"bar=foo\n", stdout.read()) From 6a87dbb68f92ed3125d9fbe71798e03be2eb5172 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 29 Apr 2022 01:40:39 -0700 Subject: [PATCH 10/31] yaml stuff --- .github/workflows/build.yml | 60 +++++++++++++++++------------------ .github/workflows/release.yml | 51 ++++++++++++++--------------- .pre-commit-config.yaml | 1 + .sourcery.yaml | 3 +- .yamllint.yaml | 1 + 5 files changed, 60 insertions(+), 56 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 38bc7443..055132e8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,35 +1,35 @@ +--- name: Test on: - push: - branches: [ master ] - pull_request: - branches: [ master ] + push: + branches: [master] + pull_request: + branches: [master] jobs: - build: - - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install -r requirements/tests.txt - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Linting - run: flake8 - # Environments are selected using tox-gh-actions configuration in tox.ini. - - name: Test with tox - run: tox - - name: Upload coverage - uses: codecov/codecov-action@v1 - with: - name: Python ${{ matrix.python-version }} Codecov + build: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.6", "3.7", "3.8", "3.9", "3.10"] + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install -r requirements/tests.txt + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Linting + run: flake8 + # Environments are selected using tox-gh-actions configuration in tox.ini. + - name: Test with tox + run: tox + - name: Upload coverage + uses: codecov/codecov-action@v1 + with: + name: Python ${{ matrix.python-version }} Codecov diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 243eea93..29869fa2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,35 +1,36 @@ +--- name: Publish PyPI Release on: - release: - types: [published] + release: + types: [published] jobs: - release-package: - runs-on: ubuntu-latest + release-package: + runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 + steps: + - uses: actions/checkout@v2 - - name: Set up Python - uses: actions/setup-python@v2 - with: - python-version: "3.x" + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.x" - - name: Install dependencies - run: | - python -m pip install -U pip - python -m pip install -r requirements/build.txt - python -m pip install -r requirements.txt + - name: Install dependencies + run: | + python -m pip install -U pip + python -m pip install -r requirements/build.txt + python -m pip install -r requirements.txt - - name: Build package - run: | - python -m build --sdist --wheel --outdir dist . - twine check dist/* + - name: Build package + run: | + python -m build --sdist --wheel --outdir dist . + twine check dist/* - - name: Upload packages to Jazzband - uses: pypa/gh-action-pypi-publish@master - with: - user: jazzband - password: ${{ secrets.JAZZBAND_RELEASE_KEY }} - repository_url: https://jazzband.co/projects/django-dbbackup/upload + - name: Upload packages to Jazzband + uses: pypa/gh-action-pypi-publish@master + with: + user: jazzband + password: ${{ secrets.JAZZBAND_RELEASE_KEY }} + repository_url: https://jazzband.co/projects/django-dbbackup/upload diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4bfc5e6c..c687a2e2 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,4 @@ +--- repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: "v4.1.0" diff --git a/.sourcery.yaml b/.sourcery.yaml index be7bb7cb..d82adfa5 100644 --- a/.sourcery.yaml +++ b/.sourcery.yaml @@ -1,2 +1,3 @@ +--- refactor: - skip: ['class-extract-method'] + skip: ["class-extract-method"] diff --git a/.yamllint.yaml b/.yamllint.yaml index 64bcf90c..59161a46 100644 --- a/.yamllint.yaml +++ b/.yamllint.yaml @@ -4,3 +4,4 @@ extends: default rules: line-length: disable comments: disable + truthy: disable From 9f6232d30d9888aface556c7f41fd65f6fd08ba0 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 29 Apr 2022 01:41:31 -0700 Subject: [PATCH 11/31] datestring fix --- dbbackup/tests/test_utils.py | 127 +++++++++++++++++++---------------- 1 file changed, 68 insertions(+), 59 deletions(-) diff --git a/dbbackup/tests/test_utils.py b/dbbackup/tests/test_utils.py index 06af128f..4cc915f9 100644 --- a/dbbackup/tests/test_utils.py +++ b/dbbackup/tests/test_utils.py @@ -36,21 +36,21 @@ def test_2_decimal(self): class Handle_SizeTest(TestCase): def test_func(self): - filehandle = StringIO('Test string') + filehandle = StringIO("Test string") value = utils.handle_size(filehandle=filehandle) - self.assertEqual(value, '11.0 B') + self.assertEqual(value, "11.0 B") class MailAdminsTest(TestCase): def test_func(self): - subject = 'foo subject' - msg = 'bar message' + subject = "foo subject" + msg = "bar message" utils.mail_admins(subject, msg) self.assertEqual(len(mail.outbox), 1) sent_mail = mail.outbox[0] - expected_subject = f'{settings.EMAIL_SUBJECT_PREFIX}{subject}' + expected_subject = f"{settings.EMAIL_SUBJECT_PREFIX}{subject}" expected_to = settings.ADMINS[0][1] expected_from = settings.SERVER_EMAIL @@ -59,10 +59,10 @@ def test_func(self): self.assertEqual(sent_mail.to[0], expected_to) self.assertEqual(sent_mail.from_email, expected_from) - @patch('dbbackup.settings.ADMINS', None) + @patch("dbbackup.settings.ADMINS", None) def test_no_admin(self): - subject = 'foo subject' - msg = 'bar message' + subject = "foo subject" + msg = "bar message" self.assertIsNone(utils.mail_admins(subject, msg)) self.assertEqual(len(mail.outbox), 0) @@ -71,27 +71,30 @@ class Email_Uncaught_ExceptionTest(TestCase): def test_success(self): def func(): pass + utils.email_uncaught_exception(func) self.assertEqual(len(mail.outbox), 0) - @patch('dbbackup.settings.SEND_EMAIL', False) + @patch("dbbackup.settings.SEND_EMAIL", False) def test_raise_error_without_mail(self): def func(): - raise Exception('Foo') + raise Exception("Foo") + with self.assertRaises(Exception): utils.email_uncaught_exception(func)() self.assertEqual(len(mail.outbox), 0) - @patch('dbbackup.settings.SEND_EMAIL', True) - @patch('dbbackup.settings.FAILURE_RECIPIENTS', ['foo@bar']) + @patch("dbbackup.settings.SEND_EMAIL", True) + @patch("dbbackup.settings.FAILURE_RECIPIENTS", ["foo@bar"]) def test_raise_with_mail(self): def func(): - raise Exception('Foo') + raise Exception("Foo") + with self.assertRaises(Exception): utils.email_uncaught_exception(func)() self.assertEqual(len(mail.outbox), 1) error_mail = mail.outbox[0] - self.assertEqual(['foo@bar'], error_mail.to) + self.assertEqual(["foo@bar"], error_mail.to) self.assertIn("Exception('Foo')", error_mail.subject) if django.VERSION >= (1, 7): self.assertIn("Exception('Foo')", error_mail.body) @@ -100,8 +103,8 @@ def func(): class Encrypt_FileTest(TestCase): def setUp(self): self.path = tempfile.mktemp() - with open(self.path, 'a') as fd: - fd.write('foo') + with open(self.path, "a") as fd: + fd.write("foo") add_public_gpg() def tearDown(self): @@ -110,8 +113,9 @@ def tearDown(self): def test_func(self, *args): with open(self.path) as fd: - encrypted_file, filename = utils.encrypt_file(inputfile=fd, - filename='foo.txt') + encrypted_file, filename = utils.encrypt_file( + inputfile=fd, filename="foo.txt" + ) encrypted_file.seek(0) self.assertTrue(encrypted_file.read()) @@ -123,43 +127,44 @@ def setUp(self): def tearDown(self): clean_gpg_keys() - @patch('dbbackup.utils.input', return_value=None) - @patch('dbbackup.utils.getpass', return_value=None) + @patch("dbbackup.utils.input", return_value=None) + @patch("dbbackup.utils.getpass", return_value=None) def test_unencrypt(self, *args): - inputfile = open(ENCRYPTED_FILE, 'r+b') - uncryptfile, filename = utils.unencrypt_file(inputfile, 'foofile.gpg') + inputfile = open(ENCRYPTED_FILE, "r+b") + uncryptfile, filename = utils.unencrypt_file(inputfile, "foofile.gpg") uncryptfile.seek(0) - self.assertEqual(b'foo\n', uncryptfile.read()) + self.assertEqual(b"foo\n", uncryptfile.read()) class Compress_FileTest(TestCase): def setUp(self): self.path = tempfile.mktemp() - with open(self.path, 'a+b') as fd: - fd.write(b'foo') + with open(self.path, "a+b") as fd: + fd.write(b"foo") def tearDown(self): os.remove(self.path) def test_func(self, *args): with open(self.path) as fd: - compressed_file, filename = utils.encrypt_file(inputfile=fd, - filename='foo.txt') + compressed_file, filename = utils.encrypt_file( + inputfile=fd, filename="foo.txt" + ) class Uncompress_FileTest(TestCase): def test_func(self): - inputfile = open(COMPRESSED_FILE, 'rb') - fd, filename = utils.uncompress_file(inputfile, 'foo.gz') + inputfile = open(COMPRESSED_FILE, "rb") + fd, filename = utils.uncompress_file(inputfile, "foo.gz") fd.seek(0) - self.assertEqual(fd.read(), b'foo\n') + self.assertEqual(fd.read(), b"foo\n") class Create_Spooled_Temporary_FileTest(TestCase): def setUp(self): self.path = tempfile.mktemp() - with open(self.path, 'a') as fd: - fd.write('foo') + with open(self.path, "a") as fd: + fd.write("foo") def tearDown(self): os.remove(self.path) @@ -169,16 +174,17 @@ def test_func(self, *args): class TimestampTest(TestCase): - def test_naive_value(self): with self.settings(USE_TZ=False): timestamp = utils.timestamp(datetime(2015, 8, 15, 8, 15, 12, 0)) - self.assertEqual(timestamp, '2015-08-15-081512') + self.assertEqual(timestamp, "2015-08-15-081512") def test_aware_value(self): - with self.settings(USE_TZ=True) and self.settings(TIME_ZONE='Europe/Rome'): - timestamp = utils.timestamp(datetime(2015, 8, 15, 8, 15, 12, 0, tzinfo=pytz.utc)) - self.assertEqual(timestamp, '2015-08-15-101512') + with self.settings(USE_TZ=True) and self.settings(TIME_ZONE="Europe/Rome"): + timestamp = utils.timestamp( + datetime(2015, 8, 15, 8, 15, 12, 0, tzinfo=pytz.utc) + ) + self.assertEqual(timestamp, "2015-08-15-101512") class Datefmt_To_Regex(TestCase): @@ -193,11 +199,11 @@ def test_patterns(self): def test_complex_pattern(self): now = datetime.now() - datefmt = 'Foo%a_%A-%w-%d-%b-%B_%m_%y_%Y-%H-%I-%M_%S_%f_%j-%U-%W-Bar' + datefmt = "Foo%a_%A-%w-%d-%b-%B_%m_%y_%Y-%H-%I-%M_%S_%f_%j-%U-%W-Bar" date_string = datetime.strftime(now, datefmt) regex = utils.datefmt_to_regex(datefmt) - self.assertTrue(regex.pattern.startswith('(Foo')) - self.assertTrue(regex.pattern.endswith('Bar)')) + self.assertTrue(regex.pattern.startswith("(Foo")) + self.assertTrue(regex.pattern.endswith("Bar)")) match = regex.match(date_string) self.assertTrue(match) self.assertEqual(match.groups()[0], date_string) @@ -207,12 +213,12 @@ class Filename_To_DatestringTest(TestCase): def test_func(self): now = datetime.now() datefmt = settings.DATE_FORMAT - filename = f'{datetime.strftime(now, datefmt)}-foo.gz.gpg' + filename = f"{datetime.strftime(now, datefmt)}-foo.gz.gpg" datestring = utils.filename_to_datestring(filename, datefmt) self.assertIn(datestring, filename) def test_generated_filename(self): - filename = utils.filename_generate('bak', 'default') + filename = utils.filename_generate("bak", "default") datestring = utils.filename_to_datestring(filename) self.assertIn(datestring, filename) @@ -221,49 +227,52 @@ class Filename_To_DateTest(TestCase): def test_func(self): now = datetime.now() datefmt = settings.DATE_FORMAT - filename = f'{datetime.strftime(now, datefmt)}-foo.gz.gpg' + filename = f"{datetime.strftime(now, datefmt)}-foo.gz.gpg" date = utils.filename_to_date(filename, datefmt) self.assertEqual(date.timetuple()[:5], now.timetuple()[:5]) def test_generated_filename(self): - filename = utils.filename_generate('bak', 'default') - datestring = utils.filename_to_date(filename) + filename = utils.filename_generate("bak", "default") + utils.filename_to_date(filename) -@patch('dbbackup.settings.HOSTNAME', 'test') +@patch("dbbackup.settings.HOSTNAME", "test") class Filename_GenerateTest(TestCase): - @patch('dbbackup.settings.FILENAME_TEMPLATE', '---{databasename}--{servername}-{datetime}.{extension}') + @patch( + "dbbackup.settings.FILENAME_TEMPLATE", + "---{databasename}--{servername}-{datetime}.{extension}", + ) def test_func(self, *args): - extension = 'foo' + extension = "foo" generated_name = utils.filename_generate(extension) - self.assertTrue('--' not in generated_name) - self.assertFalse(generated_name.startswith('-')) + self.assertTrue("--" not in generated_name) + self.assertFalse(generated_name.startswith("-")) def test_db(self, *args): - extension = 'foo' + extension = "foo" generated_name = utils.filename_generate(extension) self.assertTrue(generated_name.startswith(settings.HOSTNAME)) self.assertTrue(generated_name.endswith(extension)) def test_media(self, *args): - extension = 'foo' - generated_name = utils.filename_generate(extension, content_type='media') + extension = "foo" + generated_name = utils.filename_generate(extension, content_type="media") self.assertTrue(generated_name.startswith(settings.HOSTNAME)) self.assertTrue(generated_name.endswith(extension)) - @patch('django.utils.timezone.settings.USE_TZ', True) + @patch("django.utils.timezone.settings.USE_TZ", True) def test_tz_true(self): - filename = utils.filename_generate('bak', 'default') + filename = utils.filename_generate("bak", "default") datestring = utils.filename_to_datestring(filename) self.assertIn(datestring, filename) - @patch('dbbackup.settings.FILENAME_TEMPLATE', callable_for_filename_template) + @patch("dbbackup.settings.FILENAME_TEMPLATE", callable_for_filename_template) def test_template_is_callable(self, *args): - extension = 'foo' + extension = "foo" generated_name = utils.filename_generate(extension) - self.assertTrue(generated_name.endswith('foo')) + self.assertTrue(generated_name.endswith("foo")) class QuoteCommandArg(TestCase): def test_arg_with_space(self): - assert utils.get_escaped_command_arg('foo bar') == '\'foo bar\'' + assert utils.get_escaped_command_arg("foo bar") == "'foo bar'" From da6dbe0488befe7fea0b917dc98f72249d86c8fe Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 29 Apr 2022 08:41:52 +0000 Subject: [PATCH 12/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- dbbackup/db/postgresql.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbbackup/db/postgresql.py b/dbbackup/db/postgresql.py index d4e2ad89..35b7269f 100644 --- a/dbbackup/db/postgresql.py +++ b/dbbackup/db/postgresql.py @@ -74,7 +74,7 @@ class PgDumpGisConnector(PgDumpConnector): psql_cmd = "psql" def _enable_postgis(self): - cmd = '{} -c "CREATE EXTENSION IF NOT EXISTS postgis;"'.format(self.psql_cmd) + cmd = f'{self.psql_cmd} -c "CREATE EXTENSION IF NOT EXISTS postgis;"' cmd += " --username={}".format(self.settings["ADMIN_USER"]) cmd += " --no-password" if self.settings.get("HOST"): From e55b3c770a962eaedd9293e27815fa0115e604e2 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 29 Apr 2022 01:45:39 -0700 Subject: [PATCH 13/31] fix flake8 errors --- dbbackup/tests/commands/test_base.py | 125 +++++++++++--------- dbbackup/tests/test_connectors/test_base.py | 4 +- 2 files changed, 68 insertions(+), 61 deletions(-) diff --git a/dbbackup/tests/commands/test_base.py b/dbbackup/tests/commands/test_base.py index a9b8abf3..f1fd1490 100644 --- a/dbbackup/tests/commands/test_base.py +++ b/dbbackup/tests/commands/test_base.py @@ -51,25 +51,25 @@ def setUp(self): self.command.storage = get_storage() def test_read_from_storage(self): - HANDLED_FILES['written_files'].append(['foo', File(BytesIO(b'bar'))]) - file_ = self.command.read_from_storage('foo') - self.assertEqual(file_.read(), b'bar') + HANDLED_FILES["written_files"].append(["foo", File(BytesIO(b"bar"))]) + file_ = self.command.read_from_storage("foo") + self.assertEqual(file_.read(), b"bar") def test_write_to_storage(self): - self.command.write_to_storage(BytesIO(b'foo'), 'bar') - self.assertEqual(HANDLED_FILES['written_files'][0][0], 'bar') + self.command.write_to_storage(BytesIO(b"foo"), "bar") + self.assertEqual(HANDLED_FILES["written_files"][0][0], "bar") def test_read_local_file(self): # setUp - self.command.path = '/tmp/foo.bak' - open(self.command.path, 'w').close() + self.command.path = "/tmp/foo.bak" + open(self.command.path, "w").close() # Test - output_file = self.command.read_local_file(self.command.path) + self.command.read_local_file(self.command.path) # tearDown os.remove(self.command.path) def test_write_local_file(self): - fd, path = File(BytesIO(b"foo")), '/tmp/foo.bak' + fd, path = File(BytesIO(b"foo")), "/tmp/foo.bak" self.command.write_local_file(fd, path) self.assertTrue(os.path.exists(path)) # tearDown @@ -77,22 +77,22 @@ def test_write_local_file(self): def test_ask_confirmation(self): # Yes - with patch('dbbackup.management.commands._base.input', return_value='y'): + with patch("dbbackup.management.commands._base.input", return_value="y"): self.command._ask_confirmation() - with patch('dbbackup.management.commands._base.input', return_value='Y'): + with patch("dbbackup.management.commands._base.input", return_value="Y"): self.command._ask_confirmation() - with patch('dbbackup.management.commands._base.input', return_value=''): + with patch("dbbackup.management.commands._base.input", return_value=""): self.command._ask_confirmation() - with patch('dbbackup.management.commands._base.input', return_value='foo'): + with patch("dbbackup.management.commands._base.input", return_value="foo"): self.command._ask_confirmation() # No - with patch('dbbackup.management.commands._base.input', return_value='n'): + with patch("dbbackup.management.commands._base.input", return_value="n"): with self.assertRaises(SystemExit): self.command._ask_confirmation() - with patch('dbbackup.management.commands._base.input', return_value='N'): + with patch("dbbackup.management.commands._base.input", return_value="N"): with self.assertRaises(SystemExit): self.command._ask_confirmation() - with patch('dbbackup.management.commands._base.input', return_value='No'): + with patch("dbbackup.management.commands._base.input", return_value="No"): with self.assertRaises(SystemExit): self.command._ask_confirmation() @@ -104,52 +104,59 @@ def setUp(self): self.command.stdout = DEV_NULL self.command.encrypt = False self.command.compress = False - self.command.servername = 'foo-server' + self.command.servername = "foo-server" self.command.storage = get_storage() - HANDLED_FILES['written_files'] = [(f, None) for f in [ - 'fooserver-2015-02-06-042810.tar', - 'fooserver-2015-02-07-042810.tar', - 'fooserver-2015-02-08-042810.tar', - 'foodb-fooserver-2015-02-06-042810.dump', - 'foodb-fooserver-2015-02-07-042810.dump', - 'foodb-fooserver-2015-02-08-042810.dump', - 'bardb-fooserver-2015-02-06-042810.dump', - 'bardb-fooserver-2015-02-07-042810.dump', - 'bardb-fooserver-2015-02-08-042810.dump', - 'hamdb-hamserver-2015-02-06-042810.dump', - 'hamdb-hamserver-2015-02-07-042810.dump', - 'hamdb-hamserver-2015-02-08-042810.dump', - ]] - - @patch('dbbackup.settings.CLEANUP_KEEP', 1) + HANDLED_FILES["written_files"] = [ + (f, None) + for f in [ + "fooserver-2015-02-06-042810.tar", + "fooserver-2015-02-07-042810.tar", + "fooserver-2015-02-08-042810.tar", + "foodb-fooserver-2015-02-06-042810.dump", + "foodb-fooserver-2015-02-07-042810.dump", + "foodb-fooserver-2015-02-08-042810.dump", + "bardb-fooserver-2015-02-06-042810.dump", + "bardb-fooserver-2015-02-07-042810.dump", + "bardb-fooserver-2015-02-08-042810.dump", + "hamdb-hamserver-2015-02-06-042810.dump", + "hamdb-hamserver-2015-02-07-042810.dump", + "hamdb-hamserver-2015-02-08-042810.dump", + ] + ] + + @patch("dbbackup.settings.CLEANUP_KEEP", 1) def test_clean_db(self): - self.command.content_type = 'db' - self.command.database = 'foodb' - self.command._cleanup_old_backups(database='foodb') - self.assertEqual(2, len(HANDLED_FILES['deleted_files'])) - self.assertNotIn('foodb-fooserver-2015-02-08-042810.dump', - HANDLED_FILES['deleted_files']) - - @patch('dbbackup.settings.CLEANUP_KEEP', 1) + self.command.content_type = "db" + self.command.database = "foodb" + self.command._cleanup_old_backups(database="foodb") + self.assertEqual(2, len(HANDLED_FILES["deleted_files"])) + self.assertNotIn( + "foodb-fooserver-2015-02-08-042810.dump", HANDLED_FILES["deleted_files"] + ) + + @patch("dbbackup.settings.CLEANUP_KEEP", 1) def test_clean_other_db(self): - self.command.content_type = 'db' - self.command._cleanup_old_backups(database='bardb') - self.assertEqual(2, len(HANDLED_FILES['deleted_files'])) - self.assertNotIn('bardb-fooserver-2015-02-08-042810.dump', - HANDLED_FILES['deleted_files']) - - @patch('dbbackup.settings.CLEANUP_KEEP', 1) + self.command.content_type = "db" + self.command._cleanup_old_backups(database="bardb") + self.assertEqual(2, len(HANDLED_FILES["deleted_files"])) + self.assertNotIn( + "bardb-fooserver-2015-02-08-042810.dump", HANDLED_FILES["deleted_files"] + ) + + @patch("dbbackup.settings.CLEANUP_KEEP", 1) def test_clean_other_server_db(self): - self.command.content_type = 'db' - self.command._cleanup_old_backups(database='bardb') - self.assertEqual(2, len(HANDLED_FILES['deleted_files'])) - self.assertNotIn('bardb-fooserver-2015-02-08-042810.dump', - HANDLED_FILES['deleted_files']) - - @patch('dbbackup.settings.CLEANUP_KEEP_MEDIA', 1) + self.command.content_type = "db" + self.command._cleanup_old_backups(database="bardb") + self.assertEqual(2, len(HANDLED_FILES["deleted_files"])) + self.assertNotIn( + "bardb-fooserver-2015-02-08-042810.dump", HANDLED_FILES["deleted_files"] + ) + + @patch("dbbackup.settings.CLEANUP_KEEP_MEDIA", 1) def test_clean_media(self): - self.command.content_type = 'media' + self.command.content_type = "media" self.command._cleanup_old_backups() - self.assertEqual(2, len(HANDLED_FILES['deleted_files'])) - self.assertNotIn('foo-server-2015-02-08-042810.tar', - HANDLED_FILES['deleted_files']) + self.assertEqual(2, len(HANDLED_FILES["deleted_files"])) + self.assertNotIn( + "foo-server-2015-02-08-042810.tar", HANDLED_FILES["deleted_files"] + ) diff --git a/dbbackup/tests/test_connectors/test_base.py b/dbbackup/tests/test_connectors/test_base.py index 7a7de602..973ad8fa 100644 --- a/dbbackup/tests/test_connectors/test_base.py +++ b/dbbackup/tests/test_connectors/test_base.py @@ -29,7 +29,7 @@ def test_generate_filename(self): class BaseCommandDBConnectorTest(TestCase): def test_run_command(self): connector = BaseCommandDBConnector() - stdout, _stderr = connector.run_command("echo 123") + stdout, stderr = connector.run_command("echo 123") self.assertEqual(stdout.read(), b"123\n") self.assertEqual(stderr.read(), b"") @@ -44,7 +44,7 @@ def test_run_command_stdin(self): stdin.write(b"foo") stdin.seek(0) # Run - stdout, _stderr = connector.run_command("cat", stdin=stdin) + stdout, stderr = connector.run_command("cat", stdin=stdin) self.assertEqual(stdout.read(), b"foo") self.assertFalse(stderr.read()) From 388cab24aa4bd9ea9b16cf2d43d4f3c51b0e472c Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 29 Apr 2022 01:49:14 -0700 Subject: [PATCH 14/31] fix tests add suggested vscode extensions Revert "misc fixes" This reverts commit 4b590216616b8b7e530f6d7fa0bb0483f28bf4cd. [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci remove trailing comma fix flake8 warnings fix autofield warning Revert "datestring fix" This reverts commit 9f6232d30d9888aface556c7f41fd65f6fd08ba0. test datestring thing once more add black to precommit [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- .pre-commit-config.yaml | 4 + .vscode/extensions.json | 12 + dbbackup/__init__.py | 2 +- dbbackup/apps.py | 8 +- dbbackup/checks.py | 64 ++++-- dbbackup/db/base.py | 79 ++++--- dbbackup/db/mongodb.py | 2 +- dbbackup/db/mysql.py | 2 +- dbbackup/db/postgresql.py | 6 +- dbbackup/db/sqlite.py | 31 +-- dbbackup/log.py | 7 +- dbbackup/management/commands/_base.py | 68 +++--- dbbackup/management/commands/dbbackup.py | 93 +++++--- dbbackup/management/commands/dbrestore.py | 82 ++++--- dbbackup/management/commands/listbackups.py | 64 ++++-- dbbackup/management/commands/mediabackup.py | 73 +++--- dbbackup/management/commands/mediarestore.py | 86 ++++--- dbbackup/settings.py | 54 +++-- dbbackup/storage.py | 118 ++++++---- dbbackup/tests/commands/test_dbbackup.py | 22 +- dbbackup/tests/commands/test_dbrestore.py | 64 +++--- dbbackup/tests/commands/test_listbackups.py | 86 ++++--- dbbackup/tests/commands/test_mediabackup.py | 24 +- dbbackup/tests/functional/test_commands.py | 154 ++++++------- dbbackup/tests/settings.py | 97 ++++---- dbbackup/tests/test_checks.py | 22 +- dbbackup/tests/test_connectors/test_base.py | 22 +- .../tests/test_connectors/test_mongodb.py | 78 ++++--- dbbackup/tests/test_connectors/test_mysql.py | 116 +++++----- .../tests/test_connectors/test_postgresql.py | 210 ++++++++++-------- dbbackup/tests/test_connectors/test_sqlite.py | 8 +- dbbackup/tests/test_log.py | 128 +++++------ dbbackup/tests/test_storage.py | 134 +++++------ dbbackup/tests/test_utils.py | 4 +- .../tests/testapp/management/commands/feed.py | 2 +- .../tests/testapp/migrations/0001_initial.py | 62 ++++-- dbbackup/tests/testapp/models.py | 16 +- dbbackup/tests/utils.py | 70 +++--- dbbackup/utils.py | 152 +++++++------ docs/conf.py | 146 ++++++------ runtests.py | 5 +- setup.py | 48 ++-- 42 files changed, 1463 insertions(+), 1062 deletions(-) create mode 100644 .vscode/extensions.json diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c687a2e2..32e99cba 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -38,6 +38,10 @@ repos: hooks: - id: isort args: ["--profile", "black"] + - repo: https://github.com/psf/black + rev: "22.3.0" + hooks: + - id: black - repo: https://github.com/asottile/pyupgrade rev: "v2.31.1" hooks: diff --git a/.vscode/extensions.json b/.vscode/extensions.json new file mode 100644 index 00000000..32e34a2d --- /dev/null +++ b/.vscode/extensions.json @@ -0,0 +1,12 @@ +{ + "recommendations": [ + "eamodio.gitlens", + "github.vscode-pull-request-github", + "knisterpeter.vscode-github", + "esbenp.prettier-vscode", + "ms-python.vscode-pylance", + "ms-python.python", + "gruntfuggly.todo-tree", + "sourcery.sourcery" + ] +} diff --git a/dbbackup/__init__.py b/dbbackup/__init__.py index c04d41f5..e3b35fcf 100644 --- a/dbbackup/__init__.py +++ b/dbbackup/__init__.py @@ -3,4 +3,4 @@ import django if django.VERSION < (3, 2): - default_app_config = 'dbbackup.apps.DbbackupConfig' + default_app_config = "dbbackup.apps.DbbackupConfig" diff --git a/dbbackup/apps.py b/dbbackup/apps.py index 575ee3b5..889af87e 100644 --- a/dbbackup/apps.py +++ b/dbbackup/apps.py @@ -10,9 +10,11 @@ class DbbackupConfig(AppConfig): """ Config for DBBackup application. """ - name = 'dbbackup' - label = 'dbbackup' - verbose_name = gettext_lazy('Backup and restore') + + name = "dbbackup" + label = "dbbackup" + verbose_name = gettext_lazy("Backup and restore") + default_auto_field = "django.db.models.AutoField" def ready(self): log.load() diff --git a/dbbackup/checks.py b/dbbackup/checks.py index a4d1626f..9fbf76c4 100644 --- a/dbbackup/checks.py +++ b/dbbackup/checks.py @@ -4,25 +4,37 @@ from dbbackup import settings -W001 = Warning('Invalid HOSTNAME parameter', - hint='Set a non empty string to this settings.DBBACKUP_HOSTNAME', - id='dbbackup.W001') -W002 = Warning('Invalid STORAGE parameter', - hint='Set a valid path to a storage in settings.DBBACKUP_STORAGE', - id='dbbackup.W002') -W003 = Warning('Invalid FILENAME_TEMPLATE parameter', - hint='Include {datetime} to settings.DBBACKUP_FILENAME_TEMPLATE', - id='dbbackup.W003') -W004 = Warning('Invalid MEDIA_FILENAME_TEMPLATE parameter', - hint='Include {datetime} to settings.DBBACKUP_MEDIA_FILENAME_TEMPLATE', - id='dbbackup.W004') -W005 = Warning('Invalid DATE_FORMAT parameter', - hint='settings.DBBACKUP_DATE_FORMAT can contain only [A-Za-z0-9%_-]', - id='dbbackup.W005') -W006 = Warning('FAILURE_RECIPIENTS has been deprecated', - hint='settings.DBBACKUP_FAILURE_RECIPIENTS is replaced by ' - 'settings.DBBACKUP_ADMINS', - id='dbbackup.W006') +W001 = Warning( + "Invalid HOSTNAME parameter", + hint="Set a non empty string to this settings.DBBACKUP_HOSTNAME", + id="dbbackup.W001", +) +W002 = Warning( + "Invalid STORAGE parameter", + hint="Set a valid path to a storage in settings.DBBACKUP_STORAGE", + id="dbbackup.W002", +) +W003 = Warning( + "Invalid FILENAME_TEMPLATE parameter", + hint="Include {datetime} to settings.DBBACKUP_FILENAME_TEMPLATE", + id="dbbackup.W003", +) +W004 = Warning( + "Invalid MEDIA_FILENAME_TEMPLATE parameter", + hint="Include {datetime} to settings.DBBACKUP_MEDIA_FILENAME_TEMPLATE", + id="dbbackup.W004", +) +W005 = Warning( + "Invalid DATE_FORMAT parameter", + hint="settings.DBBACKUP_DATE_FORMAT can contain only [A-Za-z0-9%_-]", + id="dbbackup.W005", +) +W006 = Warning( + "FAILURE_RECIPIENTS has been deprecated", + hint="settings.DBBACKUP_FAILURE_RECIPIENTS is replaced by " + "settings.DBBACKUP_ADMINS", + id="dbbackup.W006", +) @register(Tags.compatibility) @@ -34,16 +46,22 @@ def check_settings(app_configs, **kwargs): if not settings.STORAGE or not isinstance(settings.STORAGE, str): errors.append(W002) - if not callable(settings.FILENAME_TEMPLATE) and '{datetime}' not in settings.FILENAME_TEMPLATE: + if ( + not callable(settings.FILENAME_TEMPLATE) + and "{datetime}" not in settings.FILENAME_TEMPLATE + ): errors.append(W003) - if not callable(settings.MEDIA_FILENAME_TEMPLATE) and '{datetime}' not in settings.MEDIA_FILENAME_TEMPLATE: + if ( + not callable(settings.MEDIA_FILENAME_TEMPLATE) + and "{datetime}" not in settings.MEDIA_FILENAME_TEMPLATE + ): errors.append(W004) - if re.search(r'[^A-Za-z0-9%_-]', settings.DATE_FORMAT): + if re.search(r"[^A-Za-z0-9%_-]", settings.DATE_FORMAT): errors.append(W005) - if getattr(settings, 'FAILURE_RECIPIENTS', None) is not None: + if getattr(settings, "FAILURE_RECIPIENTS", None) is not None: errors.append(W006) return errors diff --git a/dbbackup/db/base.py b/dbbackup/db/base.py index 3e0ee06c..96d6fae6 100644 --- a/dbbackup/db/base.py +++ b/dbbackup/db/base.py @@ -14,22 +14,22 @@ from . import exceptions -logger = logging.getLogger('dbbackup.command') +logger = logging.getLogger("dbbackup.command") logger.setLevel(logging.DEBUG) CONNECTOR_MAPPING = { - 'django.db.backends.sqlite3': 'dbbackup.db.sqlite.SqliteConnector', - 'django.db.backends.mysql': 'dbbackup.db.mysql.MysqlDumpConnector', - 'django.db.backends.postgresql': 'dbbackup.db.postgresql.PgDumpBinaryConnector', - 'django.db.backends.postgresql_psycopg2': 'dbbackup.db.postgresql.PgDumpBinaryConnector', - 'django.db.backends.oracle': None, - 'django_mongodb_engine': 'dbbackup.db.mongodb.MongoDumpConnector', - 'djongo': 'dbbackup.db.mongodb.MongoDumpConnector', - 'django.contrib.gis.db.backends.postgis': 'dbbackup.db.postgresql.PgDumpGisConnector', - 'django.contrib.gis.db.backends.mysql': 'dbbackup.db.mysql.MysqlDumpConnector', - 'django.contrib.gis.db.backends.oracle': None, - 'django.contrib.gis.db.backends.spatialite': 'dbbackup.db.sqlite.SqliteConnector', + "django.db.backends.sqlite3": "dbbackup.db.sqlite.SqliteConnector", + "django.db.backends.mysql": "dbbackup.db.mysql.MysqlDumpConnector", + "django.db.backends.postgresql": "dbbackup.db.postgresql.PgDumpBinaryConnector", + "django.db.backends.postgresql_psycopg2": "dbbackup.db.postgresql.PgDumpBinaryConnector", + "django.db.backends.oracle": None, + "django_mongodb_engine": "dbbackup.db.mongodb.MongoDumpConnector", + "djongo": "dbbackup.db.mongodb.MongoDumpConnector", + "django.contrib.gis.db.backends.postgis": "dbbackup.db.postgresql.PgDumpGisConnector", + "django.contrib.gis.db.backends.mysql": "dbbackup.db.mysql.MysqlDumpConnector", + "django.contrib.gis.db.backends.oracle": None, + "django.contrib.gis.db.backends.spatialite": "dbbackup.db.sqlite.SqliteConnector", } if settings.CUSTOM_CONNECTOR_MAPPING: @@ -45,12 +45,12 @@ def get_connector(database_name=None): # Get DB database_name = database_name or DEFAULT_DB_ALIAS connection = connections[database_name] - engine = connection.settings_dict['ENGINE'] + engine = connection.settings_dict["ENGINE"] connector_settings = settings.CONNECTORS.get(database_name, {}) - connector_path = connector_settings.get('CONNECTOR', CONNECTOR_MAPPING[engine]) - connector_module_path = '.'.join(connector_path.split('.')[:-1]) + connector_path = connector_settings.get("CONNECTOR", CONNECTOR_MAPPING[engine]) + connector_module_path = ".".join(connector_path.split(".")[:-1]) module = import_module(connector_module_path) - connector_name = connector_path.split('.')[-1] + connector_name = connector_path.split(".")[-1] connector = getattr(module, connector_name) return connector(database_name, **connector_settings) @@ -60,11 +60,13 @@ class BaseDBConnector: Base class for create database connector. This kind of object creates interaction with database and allow backup and restore operations. """ - extension = 'dump' + + extension = "dump" exclude = [] def __init__(self, database_name=None, **kwargs): from django.db import DEFAULT_DB_ALIAS, connections + self.database_name = database_name or DEFAULT_DB_ALIAS self.connection = connections[self.database_name] for attr, value in kwargs.items(): @@ -73,15 +75,14 @@ def __init__(self, database_name=None, **kwargs): @property def settings(self): """Mix of database and connector settings.""" - if not hasattr(self, '_settings'): + if not hasattr(self, "_settings"): sett = self.connection.settings_dict.copy() sett.update(settings.CONNECTORS.get(self.database_name, {})) self._settings = sett return self._settings def generate_filename(self, server_name=None): - return utils.filename_generate(self.extension, self.database_name, - server_name) + return utils.filename_generate(self.extension, self.database_name, server_name) def create_dump(self): return self._create_dump() @@ -112,10 +113,11 @@ class BaseCommandDBConnector(BaseDBConnector): """ Base class for create database connector based on command line tools. """ - dump_prefix = '' - dump_suffix = '' - restore_prefix = '' - restore_suffix = '' + + dump_prefix = "" + dump_suffix = "" + restore_prefix = "" + restore_suffix = "" use_parent_env = True env = {} @@ -137,29 +139,40 @@ def run_command(self, command, stdin=None, env=None): """ logger.debug(command) cmd = shlex.split(command) - stdout = SpooledTemporaryFile(max_size=settings.TMP_FILE_MAX_SIZE, - dir=settings.TMP_DIR) - stderr = SpooledTemporaryFile(max_size=settings.TMP_FILE_MAX_SIZE, - dir=settings.TMP_DIR) + stdout = SpooledTemporaryFile( + max_size=settings.TMP_FILE_MAX_SIZE, dir=settings.TMP_DIR + ) + stderr = SpooledTemporaryFile( + max_size=settings.TMP_FILE_MAX_SIZE, dir=settings.TMP_DIR + ) full_env = os.environ.copy() if self.use_parent_env else {} full_env.update(self.env) full_env.update(env or {}) try: if isinstance(stdin, File): process = Popen( - cmd, stdin=stdin.open("rb"), stdout=stdout, stderr=stderr, - env=full_env + cmd, + stdin=stdin.open("rb"), + stdout=stdout, + stderr=stderr, + env=full_env, ) else: - process = Popen(cmd, stdin=stdin, stdout=stdout, stderr=stderr, env=full_env) + process = Popen( + cmd, stdin=stdin, stdout=stdout, stderr=stderr, env=full_env + ) process.wait() if process.poll(): stderr.seek(0) raise exceptions.CommandConnectorError( - "Error running: {}\n{}".format(command, stderr.read().decode('utf-8'))) + "Error running: {}\n{}".format( + command, stderr.read().decode("utf-8") + ) + ) stdout.seek(0) stderr.seek(0) return stdout, stderr except OSError as err: raise exceptions.CommandConnectorError( - f"Error running: {command}\n{str(err)}") + f"Error running: {command}\n{str(err)}" + ) diff --git a/dbbackup/db/mongodb.py b/dbbackup/db/mongodb.py index 28c76bd0..d05d9528 100644 --- a/dbbackup/db/mongodb.py +++ b/dbbackup/db/mongodb.py @@ -30,7 +30,7 @@ def _create_dump(self): cmd += f" --excludeCollection {collection}" cmd += " --archive" cmd = f"{self.dump_prefix} {cmd} {self.dump_suffix}" - stdout, _stderr = self.run_command(cmd, env=self.dump_env) + stdout, stderr = self.run_command(cmd, env=self.dump_env) return stdout def _restore_dump(self, dump): diff --git a/dbbackup/db/mysql.py b/dbbackup/db/mysql.py index 7e35516f..7a07fe3c 100644 --- a/dbbackup/db/mysql.py +++ b/dbbackup/db/mysql.py @@ -26,7 +26,7 @@ def _create_dump(self): for table in self.exclude: cmd += f" --ignore-table={self.settings['NAME']}.{table}" cmd = f"{self.dump_prefix} {cmd} {self.dump_suffix}" - stdout, _stderr = self.run_command(cmd, env=self.dump_env) + stdout, stderr = self.run_command(cmd, env=self.dump_env) return stdout def _restore_dump(self, dump): diff --git a/dbbackup/db/postgresql.py b/dbbackup/db/postgresql.py index 35b7269f..b7433d31 100644 --- a/dbbackup/db/postgresql.py +++ b/dbbackup/db/postgresql.py @@ -19,7 +19,7 @@ def create_postgres_uri(self): if not user: password = "" else: - host = f"@{host}" + host = "@" + host port = ":{}".format(self.settings.get("PORT")) if self.settings.get("PORT") else "" dbname = f"--dbname=postgresql://{user}{password}{host}{port}/{dbname}" @@ -48,7 +48,7 @@ def _create_dump(self): cmd += " --clean" cmd = f"{self.dump_prefix} {cmd} {self.dump_suffix}" - stdout, _stderr = self.run_command(cmd, env=self.dump_env) + stdout, stderr = self.run_command(cmd, env=self.dump_env) return stdout def _restore_dump(self, dump): @@ -109,7 +109,7 @@ def _create_dump(self): for table in self.exclude: cmd += f" --exclude-table-data={table}" cmd = f"{self.dump_prefix} {cmd} {self.dump_suffix}" - stdout, _stderr = self.run_command(cmd, env=self.dump_env) + stdout, stderr = self.run_command(cmd, env=self.dump_env) return stdout def _restore_dump(self, dump): diff --git a/dbbackup/db/sqlite.py b/dbbackup/db/sqlite.py index cb5df1b9..ff886f95 100644 --- a/dbbackup/db/sqlite.py +++ b/dbbackup/db/sqlite.py @@ -30,13 +30,13 @@ def _write_dump(self, fileobj): cursor = self.connection.cursor() cursor.execute(DUMP_TABLES) for table_name, type, sql in cursor.fetchall(): - if table_name.startswith('sqlite_') or table_name in self.exclude: + if table_name.startswith("sqlite_") or table_name in self.exclude: continue - elif sql.startswith('CREATE TABLE'): - sql = sql.replace('CREATE TABLE', 'CREATE TABLE IF NOT EXISTS') + elif sql.startswith("CREATE TABLE"): + sql = sql.replace("CREATE TABLE", "CREATE TABLE IF NOT EXISTS") # Make SQL commands in 1 line - sql = sql.replace('\n ', '') - sql = sql.replace('\n)', ')') + sql = sql.replace("\n ", "") + sql = sql.replace("\n)", ")") fileobj.write(f"{sql};\n".encode()) else: fileobj.write(f"{sql};\n") @@ -45,16 +45,19 @@ def _write_dump(self, fileobj): column_names = [str(table_info[1]) for table_info in res.fetchall()] q = """SELECT 'INSERT INTO "{0}" VALUES({1})' FROM "{0}";\n""".format( table_name_ident, - ",".join("""'||quote("{}")||'""".format(col.replace('"', '""')) - for col in column_names)) + ",".join( + """'||quote("{}")||'""".format(col.replace('"', '""')) + for col in column_names + ), + ) query_res = cursor.execute(q) for row in query_res: fileobj.write(f"{row[0]};\n".encode()) schema_res = cursor.execute(DUMP_ETC) for name, type, sql in schema_res.fetchall(): if sql.startswith("CREATE INDEX"): - sql = sql.replace('CREATE INDEX', 'CREATE INDEX IF NOT EXISTS') - fileobj.write(f'{sql};\n'.encode()) + sql = sql.replace("CREATE INDEX", "CREATE INDEX IF NOT EXISTS") + fileobj.write(f"{sql};\n".encode()) cursor.close() def create_dump(self): @@ -71,7 +74,7 @@ def restore_dump(self, dump): cursor = self.connection.cursor() for line in dump.readlines(): try: - cursor.execute(line.decode('UTF-8')) + cursor.execute(line.decode("UTF-8")) except (OperationalError, IntegrityError) as err: warnings.warn(f"Error in db restore: {err}") @@ -83,14 +86,14 @@ class SqliteCPConnector(BaseDBConnector): """ def create_dump(self): - path = self.connection.settings_dict['NAME'] + path = self.connection.settings_dict["NAME"] dump = BytesIO() - with open(path, 'rb') as db_file: + with open(path, "rb") as db_file: copyfileobj(db_file, dump) dump.seek(0) return dump def restore_dump(self, dump): - path = self.connection.settings_dict['NAME'] - with open(path, 'wb') as db_file: + path = self.connection.settings_dict["NAME"] + with open(path, "wb") as db_file: copyfileobj(dump, db_file) diff --git a/dbbackup/log.py b/dbbackup/log.py index 452ad151..a7566b05 100644 --- a/dbbackup/log.py +++ b/dbbackup/log.py @@ -9,17 +9,22 @@ def emit(self, record): # Monkey patch for old Django versions without send_mail method if django.VERSION < (1, 8): from . import utils + django.core.mail.mail_admins = utils.mail_admins super().emit(record) def send_mail(self, subject, message, *args, **kwargs): from . import utils - utils.mail_admins(subject, message, *args, connection=self.connection(), **kwargs) + + utils.mail_admins( + subject, message, *args, connection=self.connection(), **kwargs + ) class MailEnabledFilter(logging.Filter): def filter(self, record): from .settings import SEND_EMAIL + return SEND_EMAIL diff --git a/dbbackup/management/commands/_base.py b/dbbackup/management/commands/_base.py index 0407bd38..48b88578 100644 --- a/dbbackup/management/commands/_base.py +++ b/dbbackup/management/commands/_base.py @@ -11,14 +11,14 @@ from ...storage import StorageError -USELESS_ARGS = ('callback', 'callback_args', 'callback_kwargs', 'metavar') +USELESS_ARGS = ("callback", "callback_args", "callback_kwargs", "metavar") TYPES = { - 'string': str, - 'int': int, - 'long': int, - 'float': float, - 'complex': complex, - 'choice': list + "string": str, + "int": int, + "long": int, + "float": float, + "complex": complex, + "choice": list, } LOGGING_VERBOSITY = { 0: logging.WARN, @@ -36,22 +36,36 @@ class BaseDbBackupCommand(BaseCommand): """ Base command class used for create all dbbackup command. """ + base_option_list = ( - make_option("--noinput", action='store_false', dest='interactive', default=True, - help='Tells Django to NOT prompt the user for input of any kind.'), - make_option('-q', "--quiet", action='store_true', default=False, - help='Tells Django to NOT output other text than errors.') + make_option( + "--noinput", + action="store_false", + dest="interactive", + default=True, + help="Tells Django to NOT prompt the user for input of any kind.", + ), + make_option( + "-q", + "--quiet", + action="store_true", + default=False, + help="Tells Django to NOT output other text than errors.", + ), ) option_list = () verbosity = 1 quiet = False - logger = logging.getLogger('dbbackup.command') + logger = logging.getLogger("dbbackup.command") def __init__(self, *args, **kwargs): self.option_list = self.base_option_list + self.option_list if django.VERSION < (1, 10): - options = tuple(optparse_make_option(*_args, **_kwargs) for _args, _kwargs in self.option_list) + options = tuple( + optparse_make_option(*_args, **_kwargs) + for _args, _kwargs in self.option_list + ) self.option_list = options + BaseCommand.option_list super().__init__(*args, **kwargs) @@ -59,9 +73,10 @@ def __init__(self, *args, **kwargs): def add_arguments(self, parser): for args, kwargs in self.option_list: kwargs = { - k: v for k, v in kwargs.items() - if not k.startswith('_') - and k not in USELESS_ARGS} + k: v + for k, v in kwargs.items() + if not k.startswith("_") and k not in USELESS_ARGS + } parser.add_argument(*args, **kwargs) def _set_logger_level(self): @@ -70,7 +85,7 @@ def _set_logger_level(self): def _ask_confirmation(self): answer = input("Are you sure you want to continue? [Y/n] ") - if answer.lower().startswith('n'): + if answer.lower().startswith("n"): self.logger.info("Quitting") sys.exit(0) @@ -83,13 +98,13 @@ def write_to_storage(self, file, path): def read_local_file(self, path): """Open file in read mode on local filesystem.""" - return open(path, 'rb') + return open(path, "rb") def write_local_file(self, outputfile, path): """Write file to the desired path.""" self.logger.info("Writing file to %s", path) outputfile.seek(0) - with open(path, 'wb') as fd: + with open(path, "wb") as fd: copyfileobj(outputfile, fd) def _get_backup_file(self, database=None, servername=None): @@ -108,7 +123,8 @@ def _get_backup_file(self, database=None, servername=None): compressed=self.uncompress, content_type=self.content_type, database=database, - servername=servername) + servername=servername, + ) except StorageError as err: raise CommandError(err.args[0]) from err input_file = self.read_from_storage(input_filename) @@ -119,8 +135,10 @@ def _cleanup_old_backups(self, database=None, servername=None): Cleanup old backups, keeping the number of backups specified by DBBACKUP_CLEANUP_KEEP and any backups that occur on first of the month. """ - self.storage.clean_old_backups(encrypted=self.encrypt, - compressed=self.compress, - content_type=self.content_type, - database=database, - servername=servername) + self.storage.clean_old_backups( + encrypted=self.encrypt, + compressed=self.compress, + content_type=self.content_type, + database=database, + servername=servername, + ) diff --git a/dbbackup/management/commands/dbbackup.py b/dbbackup/management/commands/dbbackup.py index 5efa8e6e..7172e12f 100644 --- a/dbbackup/management/commands/dbbackup.py +++ b/dbbackup/management/commands/dbbackup.py @@ -11,54 +11,83 @@ class Command(BaseDbBackupCommand): - help = "Backup a database, encrypt and/or compress and write to " \ - "storage.""" - content_type = 'db' + help = "Backup a database, encrypt and/or compress and write to " "storage." "" + content_type = "db" option_list = BaseDbBackupCommand.option_list + ( - make_option("-c", "--clean", dest='clean', action="store_true", - default=False, help="Clean up old backup files"), - make_option("-d", "--database", - help="Database(s) to backup specified by key separated by" - " commas(default: all)"), - make_option("-s", "--servername", - help="Specify server name to include in backup filename"), - make_option("-z", "--compress", action="store_true", default=False, - help="Compress the backup files"), - make_option("-e", "--encrypt", action="store_true", default=False, - help="Encrypt the backup files"), - make_option("-o", "--output-filename", default=None, - help="Specify filename on storage"), - make_option("-O", "--output-path", default=None, - help="Specify where to store on local filesystem"), - make_option("-x", "--exclude-tables", default=None, - help="Exclude tables from backup") + make_option( + "-c", + "--clean", + dest="clean", + action="store_true", + default=False, + help="Clean up old backup files", + ), + make_option( + "-d", + "--database", + help="Database(s) to backup specified by key separated by" + " commas(default: all)", + ), + make_option( + "-s", + "--servername", + help="Specify server name to include in backup filename", + ), + make_option( + "-z", + "--compress", + action="store_true", + default=False, + help="Compress the backup files", + ), + make_option( + "-e", + "--encrypt", + action="store_true", + default=False, + help="Encrypt the backup files", + ), + make_option( + "-o", "--output-filename", default=None, help="Specify filename on storage" + ), + make_option( + "-O", + "--output-path", + default=None, + help="Specify where to store on local filesystem", + ), + make_option( + "-x", "--exclude-tables", default=None, help="Exclude tables from backup" + ), ) @utils.email_uncaught_exception def handle(self, **options): - self.verbosity = options.get('verbosity') - self.quiet = options.get('quiet') + self.verbosity = options.get("verbosity") + self.quiet = options.get("quiet") self._set_logger_level() - self.clean = options.get('clean') + self.clean = options.get("clean") - self.servername = options.get('servername') - self.compress = options.get('compress') - self.encrypt = options.get('encrypt') + self.servername = options.get("servername") + self.compress = options.get("compress") + self.encrypt = options.get("encrypt") - self.filename = options.get('output_filename') - self.path = options.get('output_path') + self.filename = options.get("output_filename") + self.path = options.get("output_path") self.exclude_tables = options.get("exclude_tables") self.storage = get_storage() - self.database = options.get('database') or '' - database_keys = self.database.split(',') or settings.DATABASES + self.database = options.get("database") or "" + database_keys = self.database.split(",") or settings.DATABASES for database_key in database_keys: self.connector = get_connector(database_key) if self.connector and self.exclude_tables: - self.connector.exclude.extend(list(self.exclude_tables.replace(" ", "").split(','))) + self.connector.exclude.extend( + list(self.exclude_tables.replace(" ", "").split(",")) + ) database = self.connector.settings try: self._save_new_backup(database) @@ -71,7 +100,7 @@ def _save_new_backup(self, database): """ Save a new backup file. """ - self.logger.info("Backing Up Database: %s", database['NAME']) + self.logger.info("Backing Up Database: %s", database["NAME"]) # Get backup and name filename = self.connector.generate_filename(self.servername) outputfile = self.connector.create_dump() diff --git a/dbbackup/management/commands/dbrestore.py b/dbbackup/management/commands/dbrestore.py index ef8ef586..e29999cd 100644 --- a/dbbackup/management/commands/dbrestore.py +++ b/dbbackup/management/commands/dbrestore.py @@ -15,37 +15,54 @@ class Command(BaseDbBackupCommand): help = """Restore a database backup from storage, encrypted and/or compressed.""" - content_type = 'db' + content_type = "db" option_list = BaseDbBackupCommand.option_list + ( make_option("-d", "--database", help="Database to restore"), make_option("-i", "--input-filename", help="Specify filename to backup from"), - make_option("-I", "--input-path", help="Specify path on local filesystem to backup from"), - make_option("-s", "--servername", - help="If backup file is not specified, filter the " - "existing ones with the given servername"), - make_option("-c", "--decrypt", default=False, action='store_true', - help="Decrypt data before restoring"), - make_option("-p", "--passphrase", help="Passphrase for decrypt file", default=None), - make_option("-z", "--uncompress", action='store_true', default=False, - help="Uncompress gzip data before restoring") + make_option( + "-I", "--input-path", help="Specify path on local filesystem to backup from" + ), + make_option( + "-s", + "--servername", + help="If backup file is not specified, filter the " + "existing ones with the given servername", + ), + make_option( + "-c", + "--decrypt", + default=False, + action="store_true", + help="Decrypt data before restoring", + ), + make_option( + "-p", "--passphrase", help="Passphrase for decrypt file", default=None + ), + make_option( + "-z", + "--uncompress", + action="store_true", + default=False, + help="Uncompress gzip data before restoring", + ), ) def handle(self, *args, **options): """Django command handler.""" - self.verbosity = int(options.get('verbosity')) - self.quiet = options.get('quiet') + self.verbosity = int(options.get("verbosity")) + self.quiet = options.get("quiet") self._set_logger_level() try: connection.close() - self.filename = options.get('input_filename') - self.path = options.get('input_path') - self.servername = options.get('servername') - self.decrypt = options.get('decrypt') - self.uncompress = options.get('uncompress') - self.passphrase = options.get('passphrase') - self.interactive = options.get('interactive') + self.filename = options.get("input_filename") + self.path = options.get("input_path") + self.servername = options.get("servername") + self.decrypt = options.get("decrypt") + self.uncompress = options.get("uncompress") + self.passphrase = options.get("passphrase") + self.interactive = options.get("interactive") self.database_name, self.database = self._get_database(options) self.storage = get_storage() self._restore_backup() @@ -54,11 +71,13 @@ def handle(self, *args, **options): def _get_database(self, options): """Get the database to restore.""" - database_name = options.get('database') + database_name = options.get("database") if not database_name: if len(settings.DATABASES) > 1: - errmsg = "Because this project contains more than one database, you"\ + errmsg = ( + "Because this project contains more than one database, you" " must specify the --database option." + ) raise CommandError(errmsg) database_name = list(settings.DATABASES.keys())[0] if database_name not in settings.DATABASES: @@ -67,19 +86,26 @@ def _get_database(self, options): def _restore_backup(self): """Restore the specified database.""" - input_filename, input_file = self._get_backup_file(database=self.database_name, - servername=self.servername) - self.logger.info("Restoring backup for database '%s' and server '%s'", - self.database_name, self.servername) + input_filename, input_file = self._get_backup_file( + database=self.database_name, servername=self.servername + ) + self.logger.info( + "Restoring backup for database '%s' and server '%s'", + self.database_name, + self.servername, + ) self.logger.info(f"Restoring: {input_filename}") if self.decrypt: - unencrypted_file, input_filename = utils.unencrypt_file(input_file, input_filename, - self.passphrase) + unencrypted_file, input_filename = utils.unencrypt_file( + input_file, input_filename, self.passphrase + ) input_file.close() input_file = unencrypted_file if self.uncompress: - uncompressed_file, input_filename = utils.uncompress_file(input_file, input_filename) + uncompressed_file, input_filename = utils.uncompress_file( + input_file, input_filename + ) input_file.close() input_file = uncompressed_file diff --git a/dbbackup/management/commands/listbackups.py b/dbbackup/management/commands/listbackups.py index 6cc8d7e9..a1dea78a 100644 --- a/dbbackup/management/commands/listbackups.py +++ b/dbbackup/management/commands/listbackups.py @@ -6,40 +6,68 @@ from ...storage import get_storage from ._base import BaseDbBackupCommand, make_option -ROW_TEMPLATE = '{name:40} {datetime:20}' -FILTER_KEYS = ('encrypted', 'compressed', 'content_type', 'database') +ROW_TEMPLATE = "{name:40} {datetime:20}" +FILTER_KEYS = ("encrypted", "compressed", "content_type", "database") class Command(BaseDbBackupCommand): option_list = ( make_option("-d", "--database", help="Filter by database name"), - make_option("-z", "--compressed", help="Exclude non-compressed", action="store_true", - default=None, dest="compressed"), - make_option("-Z", "--not-compressed", help="Exclude compressed", action="store_false", - default=None, dest="compressed"), - make_option("-e", "--encrypted", help="Exclude non-encrypted", action="store_true", - default=None, dest="encrypted"), - make_option("-E", "--not-encrypted", help="Exclude encrypted", action="store_false", - default=None, dest="encrypted"), - make_option("-c", "--content-type", help="Filter by content type 'db' or 'media'"), + make_option( + "-z", + "--compressed", + help="Exclude non-compressed", + action="store_true", + default=None, + dest="compressed", + ), + make_option( + "-Z", + "--not-compressed", + help="Exclude compressed", + action="store_false", + default=None, + dest="compressed", + ), + make_option( + "-e", + "--encrypted", + help="Exclude non-encrypted", + action="store_true", + default=None, + dest="encrypted", + ), + make_option( + "-E", + "--not-encrypted", + help="Exclude encrypted", + action="store_false", + default=None, + dest="encrypted", + ), + make_option( + "-c", "--content-type", help="Filter by content type 'db' or 'media'" + ), ) def handle(self, **options): - self.quiet = options.get('quiet') + self.quiet = options.get("quiet") self.storage = get_storage() files_attr = self.get_backup_attrs(options) if not self.quiet: - title = ROW_TEMPLATE.format(name='Name', datetime='Datetime') + title = ROW_TEMPLATE.format(name="Name", datetime="Datetime") self.stdout.write(title) for file_attr in files_attr: row = ROW_TEMPLATE.format(**file_attr) self.stdout.write(row) def get_backup_attrs(self, options): - filters = {k: v for k, v in options.items() - if k in FILTER_KEYS} + filters = {k: v for k, v in options.items() if k in FILTER_KEYS} filenames = self.storage.list_backups(**filters) return [ - {'datetime': utils.filename_to_date(filename).strftime('%x %X'), - 'name': filename} - for filename in filenames] + { + "datetime": utils.filename_to_date(filename).strftime("%x %X"), + "name": filename, + } + for filename in filenames + ] diff --git a/dbbackup/management/commands/mediabackup.py b/dbbackup/management/commands/mediabackup.py index 5c4195ad..aea73201 100644 --- a/dbbackup/management/commands/mediabackup.py +++ b/dbbackup/management/commands/mediabackup.py @@ -19,37 +19,60 @@ class Command(BaseDbBackupCommand): content_type = "media" option_list = BaseDbBackupCommand.option_list + ( - make_option("-c", "--clean", help="Clean up old backup files", action="store_true", - default=False), - make_option("-s", "--servername", - help="Specify server name to include in backup filename"), - make_option("-z", "--compress", help="Compress the archive", action="store_true", - default=False), - make_option("-e", "--encrypt", help="Encrypt the backup files", action="store_true", - default=False), - make_option("-o", "--output-filename", default=None, - help="Specify filename on storage"), - make_option("-O", "--output-path", default=None, - help="Specify where to store on local filesystem",) + make_option( + "-c", + "--clean", + help="Clean up old backup files", + action="store_true", + default=False, + ), + make_option( + "-s", + "--servername", + help="Specify server name to include in backup filename", + ), + make_option( + "-z", + "--compress", + help="Compress the archive", + action="store_true", + default=False, + ), + make_option( + "-e", + "--encrypt", + help="Encrypt the backup files", + action="store_true", + default=False, + ), + make_option( + "-o", "--output-filename", default=None, help="Specify filename on storage" + ), + make_option( + "-O", + "--output-path", + default=None, + help="Specify where to store on local filesystem", + ), ) @utils.email_uncaught_exception def handle(self, **options): - self.verbosity = options.get('verbosity') - self.quiet = options.get('quiet') + self.verbosity = options.get("verbosity") + self.quiet = options.get("quiet") self._set_logger_level() - self.encrypt = options.get('encrypt', False) - self.compress = options.get('compress', False) - self.servername = options.get('servername') + self.encrypt = options.get("encrypt", False) + self.compress = options.get("compress", False) + self.servername = options.get("servername") - self.filename = options.get('output_filename') - self.path = options.get('output_path') + self.filename = options.get("output_filename") + self.path = options.get("output_path") try: self.media_storage = get_storage_class()() self.storage = get_storage() self.backup_mediafiles() - if options.get('clean'): + if options.get("clean"): self._cleanup_old_backups(servername=self.servername) except StorageError as err: @@ -57,7 +80,7 @@ def handle(self, **options): def _explore_storage(self): """Generator of all files contained in media storage.""" - path = '' + path = "" dirs = [path] while dirs: path = dirs.pop() @@ -69,7 +92,7 @@ def _explore_storage(self): def _create_tar(self, name): """Create TAR file.""" fileobj = utils.create_spooled_temporary_file() - mode = 'w:gz' if self.compress else 'w' + mode = "w:gz" if self.compress else "w" tar_file = tarfile.open(name=name, fileobj=fileobj, mode=mode) for media_filename in self._explore_storage(): tarinfo = tarfile.TarInfo(media_filename) @@ -89,9 +112,9 @@ def backup_mediafiles(self): filename = self.filename else: extension = f"tar{'.gz' if self.compress else ''}" - filename = utils.filename_generate(extension, - servername=self.servername, - content_type=self.content_type) + filename = utils.filename_generate( + extension, servername=self.servername, content_type=self.content_type + ) tarball = self._create_tar(filename) # Apply trans diff --git a/dbbackup/management/commands/mediarestore.py b/dbbackup/management/commands/mediarestore.py index 5403bede..3e135ace 100644 --- a/dbbackup/management/commands/mediarestore.py +++ b/dbbackup/management/commands/mediarestore.py @@ -13,40 +13,61 @@ class Command(BaseDbBackupCommand): help = """Restore a media backup from storage, encrypted and/or compressed.""" - content_type = 'media' + content_type = "media" option_list = ( - make_option("-i", "--input-filename", action='store', - help="Specify filename to backup from"), - make_option("-I", "--input-path", - help="Specify path on local filesystem to backup from"), - make_option("-s", "--servername", - help="If backup file is not specified, filter the existing ones with the " - "given servername"), - make_option("-e", "--decrypt", default=False, action='store_true', - help="Decrypt data before restoring"), - make_option("-p", "--passphrase", default=None, help="Passphrase for decrypt file"), - make_option("-z", "--uncompress", action='store_true', - help="Uncompress gzip data before restoring"), - make_option("-r", "--replace", help="Replace existing files", action='store_true'), + make_option( + "-i", + "--input-filename", + action="store", + help="Specify filename to backup from", + ), + make_option( + "-I", "--input-path", help="Specify path on local filesystem to backup from" + ), + make_option( + "-s", + "--servername", + help="If backup file is not specified, filter the existing ones with the " + "given servername", + ), + make_option( + "-e", + "--decrypt", + default=False, + action="store_true", + help="Decrypt data before restoring", + ), + make_option( + "-p", "--passphrase", default=None, help="Passphrase for decrypt file" + ), + make_option( + "-z", + "--uncompress", + action="store_true", + help="Uncompress gzip data before restoring", + ), + make_option( + "-r", "--replace", help="Replace existing files", action="store_true" + ), ) def handle(self, *args, **options): """Django command handler.""" - self.verbosity = int(options.get('verbosity')) - self.quiet = options.get('quiet') + self.verbosity = int(options.get("verbosity")) + self.quiet = options.get("quiet") self._set_logger_level() - self.servername = options.get('servername') - self.decrypt = options.get('decrypt') - self.uncompress = options.get('uncompress') + self.servername = options.get("servername") + self.decrypt = options.get("decrypt") + self.uncompress = options.get("uncompress") - self.filename = options.get('input_filename') - self.path = options.get('input_path') + self.filename = options.get("input_filename") + self.path = options.get("input_path") - self.replace = options.get('replace') - self.passphrase = options.get('passphrase') - self.interactive = options.get('interactive') + self.replace = options.get("replace") + self.passphrase = options.get("passphrase") + self.interactive = options.get("interactive") self.storage = get_storage() self.media_storage = get_storage_class()() @@ -67,8 +88,9 @@ def _restore_backup(self): self.logger.info("Restoring: %s", input_filename) if self.decrypt: - unencrypted_file, input_filename = utils.unencrypt_file(input_file, input_filename, - self.passphrase) + unencrypted_file, input_filename = utils.unencrypt_file( + input_file, input_filename, self.passphrase + ) input_file.close() input_file = unencrypted_file @@ -77,15 +99,17 @@ def _restore_backup(self): self._ask_confirmation() input_file.seek(0) - tar_file = tarfile.open(fileobj=input_file, mode='r:gz') \ - if self.uncompress \ - else tarfile.open(fileobj=input_file, mode='r:') + tar_file = ( + tarfile.open(fileobj=input_file, mode="r:gz") + if self.uncompress + else tarfile.open(fileobj=input_file, mode="r:") + ) # Restore file 1 by 1 for media_file_info in tar_file: - if media_file_info.path == 'media': + if media_file_info.path == "media": continue # Don't copy root directory media_file = tar_file.extractfile(media_file_info) if media_file is None: continue # Skip directories - name = media_file_info.path.replace('media/', '') + name = media_file_info.path.replace("media/", "") self._upload_file(name, media_file) diff --git a/dbbackup/settings.py b/dbbackup/settings.py index fa8b59ec..d7a3689b 100644 --- a/dbbackup/settings.py +++ b/dbbackup/settings.py @@ -5,43 +5,51 @@ from django.conf import settings -DATABASES = getattr(settings, 'DBBACKUP_DATABASES', list(settings.DATABASES.keys())) +DATABASES = getattr(settings, "DBBACKUP_DATABASES", list(settings.DATABASES.keys())) # Fake host -HOSTNAME = getattr(settings, 'DBBACKUP_HOSTNAME', socket.gethostname()) +HOSTNAME = getattr(settings, "DBBACKUP_HOSTNAME", socket.gethostname()) # Directory to use for temporary files -TMP_DIR = getattr(settings, 'DBBACKUP_TMP_DIR', tempfile.gettempdir()) -TMP_FILE_MAX_SIZE = getattr(settings, 'DBBACKUP_TMP_FILE_MAX_SIZE', 10 * 1024 * 1024) -TMP_FILE_READ_SIZE = getattr(settings, 'DBBACKUP_TMP_FILE_READ_SIZE', 1024 * 1000) +TMP_DIR = getattr(settings, "DBBACKUP_TMP_DIR", tempfile.gettempdir()) +TMP_FILE_MAX_SIZE = getattr(settings, "DBBACKUP_TMP_FILE_MAX_SIZE", 10 * 1024 * 1024) +TMP_FILE_READ_SIZE = getattr(settings, "DBBACKUP_TMP_FILE_READ_SIZE", 1024 * 1000) # Number of old backup files to keep -CLEANUP_KEEP = getattr(settings, 'DBBACKUP_CLEANUP_KEEP', 10) -CLEANUP_KEEP_MEDIA = getattr(settings, 'DBBACKUP_CLEANUP_KEEP_MEDIA', CLEANUP_KEEP) -CLEANUP_KEEP_FILTER = getattr(settings, 'DBBACKUP_CLEANUP_KEEP_FILTER', lambda x: False) +CLEANUP_KEEP = getattr(settings, "DBBACKUP_CLEANUP_KEEP", 10) +CLEANUP_KEEP_MEDIA = getattr(settings, "DBBACKUP_CLEANUP_KEEP_MEDIA", CLEANUP_KEEP) +CLEANUP_KEEP_FILTER = getattr(settings, "DBBACKUP_CLEANUP_KEEP_FILTER", lambda x: False) -MEDIA_PATH = getattr(settings, 'DBBACKUP_MEDIA_PATH', settings.MEDIA_ROOT) +MEDIA_PATH = getattr(settings, "DBBACKUP_MEDIA_PATH", settings.MEDIA_ROOT) -DATE_FORMAT = getattr(settings, 'DBBACKUP_DATE_FORMAT', '%Y-%m-%d-%H%M%S') -FILENAME_TEMPLATE = getattr(settings, 'DBBACKUP_FILENAME_TEMPLATE', '{databasename}-{servername}-{datetime}.{extension}') -MEDIA_FILENAME_TEMPLATE = getattr(settings, 'DBBACKUP_MEDIA_FILENAME_TEMPLATE', '{servername}-{datetime}.{extension}') +DATE_FORMAT = getattr(settings, "DBBACKUP_DATE_FORMAT", "%Y-%m-%d-%H%M%S") +FILENAME_TEMPLATE = getattr( + settings, + "DBBACKUP_FILENAME_TEMPLATE", + "{databasename}-{servername}-{datetime}.{extension}", +) +MEDIA_FILENAME_TEMPLATE = getattr( + settings, "DBBACKUP_MEDIA_FILENAME_TEMPLATE", "{servername}-{datetime}.{extension}" +) -GPG_ALWAYS_TRUST = getattr(settings, 'DBBACKUP_GPG_ALWAYS_TRUST', False) -GPG_RECIPIENT = GPG_ALWAYS_TRUST = getattr(settings, 'DBBACKUP_GPG_RECIPIENT', None) +GPG_ALWAYS_TRUST = getattr(settings, "DBBACKUP_GPG_ALWAYS_TRUST", False) +GPG_RECIPIENT = GPG_ALWAYS_TRUST = getattr(settings, "DBBACKUP_GPG_RECIPIENT", None) -STORAGE = getattr(settings, 'DBBACKUP_STORAGE', 'django.core.files.storage.FileSystemStorage') -STORAGE_OPTIONS = getattr(settings, 'DBBACKUP_STORAGE_OPTIONS', {}) +STORAGE = getattr( + settings, "DBBACKUP_STORAGE", "django.core.files.storage.FileSystemStorage" +) +STORAGE_OPTIONS = getattr(settings, "DBBACKUP_STORAGE_OPTIONS", {}) -CONNECTORS = getattr(settings, 'DBBACKUP_CONNECTORS', {}) +CONNECTORS = getattr(settings, "DBBACKUP_CONNECTORS", {}) -CUSTOM_CONNECTOR_MAPPING = getattr(settings, 'DBBACKUP_CONNECTOR_MAPPING', {}) +CUSTOM_CONNECTOR_MAPPING = getattr(settings, "DBBACKUP_CONNECTOR_MAPPING", {}) # Mail -SEND_EMAIL = getattr(settings, 'DBBACKUP_SEND_EMAIL', True) -SERVER_EMAIL = getattr(settings, 'DBBACKUP_SERVER_EMAIL', settings.SERVER_EMAIL) -FAILURE_RECIPIENTS = getattr(settings, 'DBBACKUP_FAILURE_RECIPIENTS', None) +SEND_EMAIL = getattr(settings, "DBBACKUP_SEND_EMAIL", True) +SERVER_EMAIL = getattr(settings, "DBBACKUP_SERVER_EMAIL", settings.SERVER_EMAIL) +FAILURE_RECIPIENTS = getattr(settings, "DBBACKUP_FAILURE_RECIPIENTS", None) if FAILURE_RECIPIENTS is None: - ADMINS = getattr(settings, 'DBBACKUP_ADMIN', settings.ADMINS) + ADMINS = getattr(settings, "DBBACKUP_ADMIN", settings.ADMINS) else: ADMINS = FAILURE_RECIPIENTS -EMAIL_SUBJECT_PREFIX = getattr(settings, 'DBBACKUP_EMAIL_SUBJECT_PREFIX', '[dbbackup] ') +EMAIL_SUBJECT_PREFIX = getattr(settings, "DBBACKUP_EMAIL_SUBJECT_PREFIX", "[dbbackup] ") diff --git a/dbbackup/storage.py b/dbbackup/storage.py index 081049ac..24fa9ab8 100644 --- a/dbbackup/storage.py +++ b/dbbackup/storage.py @@ -27,8 +27,9 @@ def get_storage(path=None, options=None): path = path or settings.STORAGE options = options or settings.STORAGE_OPTIONS if not path: - raise ImproperlyConfigured('You must specify a storage class using ' - 'DBBACKUP_STORAGE settings.') + raise ImproperlyConfigured( + "You must specify a storage class using " "DBBACKUP_STORAGE settings." + ) return Storage(path, **options) @@ -46,10 +47,11 @@ class Storage: list and filter files. It uses a Django storage object for low-level operations. """ + @property def logger(self): - if not hasattr(self, '_logger'): - self._logger = logging.getLogger('dbbackup.storage') + if not hasattr(self, "_logger"): + self._logger = logging.getLogger("dbbackup.storage") return self._logger def __init__(self, storage_path=None, **options): @@ -70,28 +72,34 @@ def __init__(self, storage_path=None, **options): self.name = self.storageCls.__name__ def __str__(self): - return f'dbbackup-{self.storage.__str__()}' + return f"dbbackup-{self.storage.__str__()}" def delete_file(self, filepath): - self.logger.debug('Deleting file %s', filepath) + self.logger.debug("Deleting file %s", filepath) self.storage.delete(name=filepath) - def list_directory(self, path=''): + def list_directory(self, path=""): return self.storage.listdir(path)[1] def write_file(self, filehandle, filename): - self.logger.debug('Writing file %s', filename) + self.logger.debug("Writing file %s", filename) self.storage.save(name=filename, content=filehandle) def read_file(self, filepath): - self.logger.debug('Reading file %s', filepath) - file_ = self.storage.open(name=filepath, mode='rb') - if not getattr(file_, 'name', None): + self.logger.debug("Reading file %s", filepath) + file_ = self.storage.open(name=filepath, mode="rb") + if not getattr(file_, "name", None): file_.name = filepath return file_ - def list_backups(self, encrypted=None, compressed=None, content_type=None, - database=None, servername=None): + def list_backups( + self, + encrypted=None, + compressed=None, + content_type=None, + database=None, + servername=None, + ): """ List stored files except given filter. If filter is None, it won't be used. ``content_type`` must be ``'db'`` for database backups or @@ -117,28 +125,33 @@ def list_backups(self, encrypted=None, compressed=None, content_type=None, :returns: List of files :rtype: ``list`` of ``str`` """ - if content_type not in ('db', 'media', None): - msg = "Bad content_type %s, must be 'db', 'media', or None" % ( - content_type) + if content_type not in ("db", "media", None): + msg = "Bad content_type %s, must be 'db', 'media', or None" % (content_type) raise TypeError(msg) # TODO: Make better filter for include only backups files = [f for f in self.list_directory() if utils.filename_to_datestring(f)] if encrypted is not None: - files = [f for f in files if ('.gpg' in f) == encrypted] + files = [f for f in files if (".gpg" in f) == encrypted] if compressed is not None: - files = [f for f in files if ('.gz' in f) == compressed] - if content_type == 'media': - files = [f for f in files if '.tar' in f] - elif content_type == 'db': - files = [f for f in files if '.tar' not in f] + files = [f for f in files if (".gz" in f) == compressed] + if content_type == "media": + files = [f for f in files if ".tar" in f] + elif content_type == "db": + files = [f for f in files if ".tar" not in f] if database: files = [f for f in files if database in f] if servername: files = [f for f in files if servername in f] return files - def get_latest_backup(self, encrypted=None, compressed=None, - content_type=None, database=None, servername=None): + def get_latest_backup( + self, + encrypted=None, + compressed=None, + content_type=None, + database=None, + servername=None, + ): """ Return the latest backup file name. @@ -164,15 +177,25 @@ def get_latest_backup(self, encrypted=None, compressed=None, :raises: FileNotFound: If no backup file is found """ - files = self.list_backups(encrypted=encrypted, compressed=compressed, - content_type=content_type, database=database, - servername=servername) + files = self.list_backups( + encrypted=encrypted, + compressed=compressed, + content_type=content_type, + database=database, + servername=servername, + ) if not files: raise FileNotFound("There's no backup file available.") return max(files, key=utils.filename_to_date) - def get_older_backup(self, encrypted=None, compressed=None, - content_type=None, database=None, servername=None): + def get_older_backup( + self, + encrypted=None, + compressed=None, + content_type=None, + database=None, + servername=None, + ): """ Return the older backup's file name. @@ -198,16 +221,26 @@ def get_older_backup(self, encrypted=None, compressed=None, :raises: FileNotFound: If no backup file is found """ - files = self.list_backups(encrypted=encrypted, compressed=compressed, - content_type=content_type, database=database, - servername=servername) + files = self.list_backups( + encrypted=encrypted, + compressed=compressed, + content_type=content_type, + database=database, + servername=servername, + ) if not files: raise FileNotFound("There's no backup file available.") return min(files, key=utils.filename_to_date) - def clean_old_backups(self, encrypted=None, compressed=None, - content_type=None, database=None, servername=None, - keep_number=None): + def clean_old_backups( + self, + encrypted=None, + compressed=None, + content_type=None, + database=None, + servername=None, + keep_number=None, + ): """ Delete olders backups and hold the number defined. @@ -232,12 +265,19 @@ def clean_old_backups(self, encrypted=None, compressed=None, :type keep_number: ``int`` or ``None`` """ if keep_number is None: - keep_number = settings.CLEANUP_KEEP if content_type == 'db' \ + keep_number = ( + settings.CLEANUP_KEEP + if content_type == "db" else settings.CLEANUP_KEEP_MEDIA + ) keep_filter = settings.CLEANUP_KEEP_FILTER - files = self.list_backups(encrypted=encrypted, compressed=compressed, - content_type=content_type, database=database, - servername=servername) + files = self.list_backups( + encrypted=encrypted, + compressed=compressed, + content_type=content_type, + database=database, + servername=servername, + ) files = sorted(files, key=utils.filename_to_date, reverse=True) files_to_delete = [fi for i, fi in enumerate(files) if i >= keep_number] for filename in files_to_delete: diff --git a/dbbackup/tests/commands/test_dbbackup.py b/dbbackup/tests/commands/test_dbbackup.py index b57c1b78..c18fec6e 100644 --- a/dbbackup/tests/commands/test_dbbackup.py +++ b/dbbackup/tests/commands/test_dbbackup.py @@ -12,15 +12,15 @@ from dbbackup.tests.utils import DEV_NULL, TEST_DATABASE, add_public_gpg, clean_gpg_keys -@patch('dbbackup.settings.GPG_RECIPIENT', 'test@test') -@patch('sys.stdout', DEV_NULL) +@patch("dbbackup.settings.GPG_RECIPIENT", "test@test") +@patch("sys.stdout", DEV_NULL) class DbbackupCommandSaveNewBackupTest(TestCase): def setUp(self): self.command = DbbackupCommand() - self.command.servername = 'foo-server' + self.command.servername = "foo-server" self.command.encrypt = False self.command.compress = False - self.command.database = TEST_DATABASE['NAME'] + self.command.database = TEST_DATABASE["NAME"] self.command.storage = get_storage() self.command.connector = get_connector() self.command.stdout = DEV_NULL @@ -43,28 +43,28 @@ def test_encrypt(self): self.command._save_new_backup(TEST_DATABASE) def test_path(self): - self.command.path = '/tmp/foo.bak' + self.command.path = "/tmp/foo.bak" self.command._save_new_backup(TEST_DATABASE) self.assertTrue(os.path.exists(self.command.path)) # tearDown os.remove(self.command.path) -@patch('dbbackup.settings.GPG_RECIPIENT', 'test@test') -@patch('sys.stdout', DEV_NULL) -@patch('dbbackup.db.sqlite.SqliteConnector.create_dump') -@patch('dbbackup.utils.handle_size', returned_value=4.2) +@patch("dbbackup.settings.GPG_RECIPIENT", "test@test") +@patch("sys.stdout", DEV_NULL) +@patch("dbbackup.db.sqlite.SqliteConnector.create_dump") +@patch("dbbackup.utils.handle_size", returned_value=4.2) class DbbackupCommandSaveNewMongoBackupTest(TestCase): def setUp(self): self.command = DbbackupCommand() - self.command.servername = 'foo-server' + self.command.servername = "foo-server" self.command.encrypt = False self.command.compress = False self.command.storage = get_storage() self.command.stdout = DEV_NULL self.command.filename = None self.command.path = None - self.command.connector = get_connector('default') + self.command.connector = get_connector("default") def tearDown(self): clean_gpg_keys() diff --git a/dbbackup/tests/commands/test_dbrestore.py b/dbbackup/tests/commands/test_dbrestore.py index 6397db18..20512927 100644 --- a/dbbackup/tests/commands/test_dbrestore.py +++ b/dbbackup/tests/commands/test_dbrestore.py @@ -29,22 +29,22 @@ ) -@patch('dbbackup.management.commands._base.input', return_value='y') +@patch("dbbackup.management.commands._base.input", return_value="y") class DbrestoreCommandRestoreBackupTest(TestCase): def setUp(self): self.command = DbrestoreCommand() self.command.stdout = DEV_NULL self.command.uncompress = False self.command.decrypt = False - self.command.backup_extension = 'bak' - self.command.filename = 'foofile' + self.command.backup_extension = "bak" + self.command.filename = "foofile" self.command.database = TEST_DATABASE self.command.passphrase = None self.command.interactive = True self.command.storage = get_storage() self.command.servername = HOSTNAME - self.command.database_name = 'default' - self.command.connector = get_connector('default') + self.command.database_name = "default" + self.command.connector = get_connector("default") HANDLED_FILES.clean() def tearDown(self): @@ -52,8 +52,9 @@ def tearDown(self): def test_no_filename(self, *args): # Prepare backup - HANDLED_FILES['written_files'].append( - (utils.filename_generate('default'), File(get_dump()))) + HANDLED_FILES["written_files"].append( + (utils.filename_generate("default"), File(get_dump())) + ) # Check self.command.path = None self.command.filename = None @@ -67,19 +68,23 @@ def test_no_backup_found(self, *args): def test_uncompress(self, *args): self.command.path = None - compressed_file, self.command.filename = utils.compress_file(get_dump(), get_dump_name()) - HANDLED_FILES['written_files'].append( + compressed_file, self.command.filename = utils.compress_file( + get_dump(), get_dump_name() + ) + HANDLED_FILES["written_files"].append( (self.command.filename, File(compressed_file)) ) self.command.uncompress = True self.command._restore_backup() - @patch('dbbackup.utils.getpass', return_value=None) + @patch("dbbackup.utils.getpass", return_value=None) def test_decrypt(self, *args): self.command.path = None self.command.decrypt = True - encrypted_file, self.command.filename = utils.encrypt_file(get_dump(), get_dump_name()) - HANDLED_FILES['written_files'].append( + encrypted_file, self.command.filename = utils.encrypt_file( + get_dump(), get_dump_name() + ) + HANDLED_FILES["written_files"].append( (self.command.filename, File(encrypted_file)) ) self.command._restore_backup() @@ -87,15 +92,13 @@ def test_decrypt(self, *args): def test_path(self, *args): temp_dump = get_dump() dump_path = mktemp() - with open(dump_path, 'wb') as dump: + with open(dump_path, "wb") as dump: copyfileobj(temp_dump, dump) self.command.path = dump.name self.command._restore_backup() self.command.decrypt = False self.command.filepath = get_dump_name() - HANDLED_FILES['written_files'].append( - (self.command.filepath, get_dump()) - ) + HANDLED_FILES["written_files"].append((self.command.filepath, get_dump())) self.command._restore_backup() @@ -104,39 +107,42 @@ def setUp(self): self.command = DbrestoreCommand() def test_give_db_name(self): - name, db = self.command._get_database({'database': 'default'}) - self.assertEqual(name, 'default') - self.assertEqual(db, settings.DATABASES['default']) + name, db = self.command._get_database({"database": "default"}) + self.assertEqual(name, "default") + self.assertEqual(db, settings.DATABASES["default"]) def test_no_given_db(self): name, db = self.command._get_database({}) - self.assertEqual(name, 'default') - self.assertEqual(db, settings.DATABASES['default']) + self.assertEqual(name, "default") + self.assertEqual(db, settings.DATABASES["default"]) - @patch('django.conf.settings.DATABASES', {'db1': {}, 'db2': {}}) + @patch("django.conf.settings.DATABASES", {"db1": {}, "db2": {}}) def test_no_given_db_multidb(self): with self.assertRaises(CommandError): self.command._get_database({}) -@patch('dbbackup.management.commands._base.input', return_value='y') -@patch('dbbackup.management.commands.dbrestore.get_connector', return_value=MongoDumpConnector()) -@patch('dbbackup.db.mongodb.MongoDumpConnector.restore_dump') +@patch("dbbackup.management.commands._base.input", return_value="y") +@patch( + "dbbackup.management.commands.dbrestore.get_connector", + return_value=MongoDumpConnector(), +) +@patch("dbbackup.db.mongodb.MongoDumpConnector.restore_dump") class DbMongoRestoreCommandRestoreBackupTest(TestCase): def setUp(self): self.command = DbrestoreCommand() self.command.stdout = DEV_NULL self.command.uncompress = False self.command.decrypt = False - self.command.backup_extension = 'bak' + self.command.backup_extension = "bak" self.command.path = None - self.command.filename = 'foofile' + self.command.filename = "foofile" self.command.database = TEST_MONGODB self.command.passphrase = None self.command.interactive = True self.command.storage = get_storage() self.command.connector = MongoDumpConnector() - self.command.database_name = 'mongo' + self.command.database_name = "mongo" self.command.servername = HOSTNAME HANDLED_FILES.clean() add_private_gpg() @@ -144,6 +150,6 @@ def setUp(self): def test_mongo_settings_backup_command(self, mock_runcommands, *args): self.command.storage.file_read = TARED_FILE self.command.filename = TARED_FILE - HANDLED_FILES['written_files'].append((TARED_FILE, open(TARED_FILE, 'rb'))) + HANDLED_FILES["written_files"].append((TARED_FILE, open(TARED_FILE, "rb"))) self.command._restore_backup() self.assertTrue(mock_runcommands.called) diff --git a/dbbackup/tests/commands/test_listbackups.py b/dbbackup/tests/commands/test_listbackups.py index 348b9d69..f541e527 100644 --- a/dbbackup/tests/commands/test_listbackups.py +++ b/dbbackup/tests/commands/test_listbackups.py @@ -13,84 +13,100 @@ class ListbackupsCommandTest(TestCase): def setUp(self): self.command = ListbackupsCommand() self.command.storage = get_storage() - HANDLED_FILES['written_files'] = [(f, None) for f in [ - '2015-02-06-042810.bak', - '2015-02-07-042810.bak', - '2015-02-08-042810.bak', - ]] + HANDLED_FILES["written_files"] = [ + (f, None) + for f in [ + "2015-02-06-042810.bak", + "2015-02-07-042810.bak", + "2015-02-08-042810.bak", + ] + ] def test_get_backup_attrs(self): options = {} attrs = self.command.get_backup_attrs(options) - self.assertEqual(len(HANDLED_FILES['written_files']), len(attrs)) + self.assertEqual(len(HANDLED_FILES["written_files"]), len(attrs)) class ListbackupsCommandArgComputingTest(TestCase): def setUp(self): - HANDLED_FILES['written_files'] = [(f, None) for f in [ - '2015-02-06-042810_foo.db', '2015-02-06-042810_foo.db.gz', - '2015-02-06-042810_foo.db.gpg', '2015-02-06-042810_foo.db.gz.gpg', - '2015-02-06-042810_foo.tar', '2015-02-06-042810_foo.tar.gz', - '2015-02-06-042810_foo.tar.gpg', '2015-02-06-042810_foo.tar.gz.gpg', - '2015-02-06-042810_bar.db', '2015-02-06-042810_bar.db.gz', - '2015-02-06-042810_bar.db.gpg', '2015-02-06-042810_bar.db.gz.gpg', - '2015-02-06-042810_bar.tar', '2015-02-06-042810_bar.tar.gz', - '2015-02-06-042810_bar.tar.gpg', '2015-02-06-042810_bar.tar.gz.gpg', - ]] + HANDLED_FILES["written_files"] = [ + (f, None) + for f in [ + "2015-02-06-042810_foo.db", + "2015-02-06-042810_foo.db.gz", + "2015-02-06-042810_foo.db.gpg", + "2015-02-06-042810_foo.db.gz.gpg", + "2015-02-06-042810_foo.tar", + "2015-02-06-042810_foo.tar.gz", + "2015-02-06-042810_foo.tar.gpg", + "2015-02-06-042810_foo.tar.gz.gpg", + "2015-02-06-042810_bar.db", + "2015-02-06-042810_bar.db.gz", + "2015-02-06-042810_bar.db.gpg", + "2015-02-06-042810_bar.db.gz.gpg", + "2015-02-06-042810_bar.tar", + "2015-02-06-042810_bar.tar.gz", + "2015-02-06-042810_bar.tar.gpg", + "2015-02-06-042810_bar.tar.gz.gpg", + ] + ] def test_list(self): - execute_from_command_line(['', 'listbackups']) + execute_from_command_line(["", "listbackups"]) def test_filter_encrypted(self): stdout = StringIO() - with patch('sys.stdout', stdout): - execute_from_command_line(['', 'listbackups', '--encrypted', '-q']) + with patch("sys.stdout", stdout): + execute_from_command_line(["", "listbackups", "--encrypted", "-q"]) stdout.seek(0) stdout.readline() for line in stdout.readlines(): - self.assertIn('.gpg', line) + self.assertIn(".gpg", line) def test_filter_not_encrypted(self): stdout = StringIO() - with patch('sys.stdout', stdout): - execute_from_command_line(['', 'listbackups', '--not-encrypted', '-q']) + with patch("sys.stdout", stdout): + execute_from_command_line(["", "listbackups", "--not-encrypted", "-q"]) stdout.seek(0) stdout.readline() for line in stdout.readlines(): - self.assertNotIn('.gpg', line) + self.assertNotIn(".gpg", line) def test_filter_compressed(self): stdout = StringIO() - with patch('sys.stdout', stdout): - execute_from_command_line(['', 'listbackups', '--compressed', '-q']) + with patch("sys.stdout", stdout): + execute_from_command_line(["", "listbackups", "--compressed", "-q"]) stdout.seek(0) stdout.readline() for line in stdout.readlines(): - self.assertIn('.gz', line) + self.assertIn(".gz", line) def test_filter_not_compressed(self): stdout = StringIO() - with patch('sys.stdout', stdout): - execute_from_command_line(['', 'listbackups', '--not-compressed', '-q']) + with patch("sys.stdout", stdout): + execute_from_command_line(["", "listbackups", "--not-compressed", "-q"]) stdout.seek(0) stdout.readline() for line in stdout.readlines(): - self.assertNotIn('.gz', line) + self.assertNotIn(".gz", line) def test_filter_db(self): stdout = StringIO() - with patch('sys.stdout', stdout): - execute_from_command_line(['', 'listbackups', '--content-type', 'db', '-q']) + with patch("sys.stdout", stdout): + execute_from_command_line(["", "listbackups", "--content-type", "db", "-q"]) stdout.seek(0) stdout.readline() for line in stdout.readlines(): - self.assertIn('.db', line) + self.assertIn(".db", line) def test_filter_media(self): stdout = StringIO() - with patch('sys.stdout', stdout): - execute_from_command_line(['', 'listbackups', '--content-type', 'media', '-q']) + with patch("sys.stdout", stdout): + execute_from_command_line( + ["", "listbackups", "--content-type", "media", "-q"] + ) stdout.seek(0) stdout.readline() for line in stdout.readlines(): - self.assertIn('.tar', line) + self.assertIn(".tar", line) diff --git a/dbbackup/tests/commands/test_mediabackup.py b/dbbackup/tests/commands/test_mediabackup.py index ceabe283..c3e9e5ab 100644 --- a/dbbackup/tests/commands/test_mediabackup.py +++ b/dbbackup/tests/commands/test_mediabackup.py @@ -18,7 +18,7 @@ class MediabackupBackupMediafilesTest(TestCase): def setUp(self): HANDLED_FILES.clean() self.command = DbbackupCommand() - self.command.servername = 'foo-server' + self.command.servername = "foo-server" self.command.storage = get_storage() self.command.stdout = DEV_NULL self.command.compress = False @@ -34,40 +34,40 @@ def tearDown(self): def test_func(self): self.command.backup_mediafiles() - self.assertEqual(1, len(HANDLED_FILES['written_files'])) + self.assertEqual(1, len(HANDLED_FILES["written_files"])) def test_compress(self): self.command.compress = True self.command.backup_mediafiles() - self.assertEqual(1, len(HANDLED_FILES['written_files'])) - self.assertTrue(HANDLED_FILES['written_files'][0][0].endswith('.gz')) + self.assertEqual(1, len(HANDLED_FILES["written_files"])) + self.assertTrue(HANDLED_FILES["written_files"][0][0].endswith(".gz")) def test_encrypt(self): self.command.encrypt = True add_public_gpg() self.command.backup_mediafiles() - self.assertEqual(1, len(HANDLED_FILES['written_files'])) - outputfile = HANDLED_FILES['written_files'][0][1] + self.assertEqual(1, len(HANDLED_FILES["written_files"])) + outputfile = HANDLED_FILES["written_files"][0][1] outputfile.seek(0) - self.assertTrue(outputfile.read().startswith(b'-----BEGIN PGP MESSAGE-----')) + self.assertTrue(outputfile.read().startswith(b"-----BEGIN PGP MESSAGE-----")) def test_compress_and_encrypt(self): self.command.compress = True self.command.encrypt = True add_public_gpg() self.command.backup_mediafiles() - self.assertEqual(1, len(HANDLED_FILES['written_files'])) - outputfile = HANDLED_FILES['written_files'][0][1] + self.assertEqual(1, len(HANDLED_FILES["written_files"])) + outputfile = HANDLED_FILES["written_files"][0][1] outputfile.seek(0) - self.assertTrue(outputfile.read().startswith(b'-----BEGIN PGP MESSAGE-----')) + self.assertTrue(outputfile.read().startswith(b"-----BEGIN PGP MESSAGE-----")) def test_write_local_file(self): self.command.path = tempfile.mktemp() self.command.backup_mediafiles() self.assertTrue(os.path.exists(self.command.path)) - self.assertEqual(0, len(HANDLED_FILES['written_files'])) + self.assertEqual(0, len(HANDLED_FILES["written_files"])) def test_output_filename(self): self.command.filename = "my_new_name.tar" self.command.backup_mediafiles() - self.assertEqual(HANDLED_FILES['written_files'][0][0], self.command.filename) + self.assertEqual(HANDLED_FILES["written_files"][0][0], self.command.filename) diff --git a/dbbackup/tests/functional/test_commands.py b/dbbackup/tests/functional/test_commands.py index e3043d09..dd250649 100644 --- a/dbbackup/tests/functional/test_commands.py +++ b/dbbackup/tests/functional/test_commands.py @@ -20,116 +20,116 @@ class DbBackupCommandTest(TestCase): def setUp(self): HANDLED_FILES.clean() add_public_gpg() - open(TEST_DATABASE['NAME'], 'a').close() - self.instance = models.CharModel.objects.create(field='foo') + open(TEST_DATABASE["NAME"], "a").close() + self.instance = models.CharModel.objects.create(field="foo") def tearDown(self): clean_gpg_keys() def test_database(self): - argv = ['', 'dbbackup', '--database=default'] + argv = ["", "dbbackup", "--database=default"] execute_from_command_line(argv) - self.assertEqual(1, len(HANDLED_FILES['written_files'])) - filename, outputfile = HANDLED_FILES['written_files'][0] + self.assertEqual(1, len(HANDLED_FILES["written_files"])) + filename, outputfile = HANDLED_FILES["written_files"][0] # Test file content outputfile.seek(0) self.assertTrue(outputfile.read()) def test_encrypt(self): - argv = ['', 'dbbackup', '--encrypt'] + argv = ["", "dbbackup", "--encrypt"] execute_from_command_line(argv) - self.assertEqual(1, len(HANDLED_FILES['written_files'])) - filename, outputfile = HANDLED_FILES['written_files'][0] - self.assertTrue(filename.endswith('.gpg')) + self.assertEqual(1, len(HANDLED_FILES["written_files"])) + filename, outputfile = HANDLED_FILES["written_files"][0] + self.assertTrue(filename.endswith(".gpg")) # Test file content - outputfile = HANDLED_FILES['written_files'][0][1] + outputfile = HANDLED_FILES["written_files"][0][1] outputfile.seek(0) - self.assertTrue(outputfile.read().startswith(b'-----BEGIN PGP MESSAGE-----')) + self.assertTrue(outputfile.read().startswith(b"-----BEGIN PGP MESSAGE-----")) def test_compress(self): - argv = ['', 'dbbackup', '--compress'] + argv = ["", "dbbackup", "--compress"] execute_from_command_line(argv) - self.assertEqual(1, len(HANDLED_FILES['written_files'])) - filename, outputfile = HANDLED_FILES['written_files'][0] - self.assertTrue(filename.endswith('.gz')) + self.assertEqual(1, len(HANDLED_FILES["written_files"])) + filename, outputfile = HANDLED_FILES["written_files"][0] + self.assertTrue(filename.endswith(".gz")) def test_compress_and_encrypt(self): - argv = ['', 'dbbackup', '--compress', '--encrypt'] + argv = ["", "dbbackup", "--compress", "--encrypt"] execute_from_command_line(argv) - self.assertEqual(1, len(HANDLED_FILES['written_files'])) - filename, outputfile = HANDLED_FILES['written_files'][0] - self.assertTrue(filename.endswith('.gz.gpg')) + self.assertEqual(1, len(HANDLED_FILES["written_files"])) + filename, outputfile = HANDLED_FILES["written_files"][0] + self.assertTrue(filename.endswith(".gz.gpg")) # Test file content - outputfile = HANDLED_FILES['written_files'][0][1] + outputfile = HANDLED_FILES["written_files"][0][1] outputfile.seek(0) - self.assertTrue(outputfile.read().startswith(b'-----BEGIN PGP MESSAGE-----')) + self.assertTrue(outputfile.read().startswith(b"-----BEGIN PGP MESSAGE-----")) -@patch('dbbackup.management.commands._base.input', return_value='y') +@patch("dbbackup.management.commands._base.input", return_value="y") class DbRestoreCommandTest(TestCase): def setUp(self): HANDLED_FILES.clean() add_public_gpg() add_private_gpg() - open(TEST_DATABASE['NAME'], 'a').close() - self.instance = models.CharModel.objects.create(field='foo') + open(TEST_DATABASE["NAME"], "a").close() + self.instance = models.CharModel.objects.create(field="foo") def tearDown(self): clean_gpg_keys() def test_restore(self, *args): # Create backup - execute_from_command_line(['', 'dbbackup']) + execute_from_command_line(["", "dbbackup"]) self.instance.delete() # Restore - execute_from_command_line(['', 'dbrestore']) + execute_from_command_line(["", "dbrestore"]) restored = models.CharModel.objects.all().exists() self.assertTrue(restored) - @patch('dbbackup.utils.getpass', return_value=None) + @patch("dbbackup.utils.getpass", return_value=None) def test_encrypted(self, *args): # Create backup - execute_from_command_line(['', 'dbbackup', '--encrypt']) + execute_from_command_line(["", "dbbackup", "--encrypt"]) self.instance.delete() # Restore - execute_from_command_line(['', 'dbrestore', '--decrypt']) + execute_from_command_line(["", "dbrestore", "--decrypt"]) restored = models.CharModel.objects.all().exists() self.assertTrue(restored) def test_compressed(self, *args): # Create backup - execute_from_command_line(['', 'dbbackup', '--compress']) + execute_from_command_line(["", "dbbackup", "--compress"]) self.instance.delete() # Restore - execute_from_command_line(['', 'dbrestore', '--uncompress']) + execute_from_command_line(["", "dbrestore", "--uncompress"]) def test_no_backup_available(self, *args): with self.assertRaises(SystemExit): - execute_from_command_line(['', 'dbrestore']) + execute_from_command_line(["", "dbrestore"]) - @patch('dbbackup.utils.getpass', return_value=None) + @patch("dbbackup.utils.getpass", return_value=None) def test_available_but_not_encrypted(self, *args): # Create backup - execute_from_command_line(['', 'dbbackup']) + execute_from_command_line(["", "dbbackup"]) # Restore with self.assertRaises(SystemExit): - execute_from_command_line(['', 'dbrestore', '--decrypt']) + execute_from_command_line(["", "dbrestore", "--decrypt"]) def test_available_but_not_compressed(self, *args): # Create backup - execute_from_command_line(['', 'dbbackup']) + execute_from_command_line(["", "dbbackup"]) # Restore with self.assertRaises(SystemExit): - execute_from_command_line(['', 'dbrestore', '--uncompress']) + execute_from_command_line(["", "dbrestore", "--uncompress"]) def test_specify_db(self, *args): # Create backup - execute_from_command_line(['', 'dbbackup', '--database', 'default']) + execute_from_command_line(["", "dbbackup", "--database", "default"]) # Test wrong name with self.assertRaises(SystemExit): - execute_from_command_line(['', 'dbrestore', '--database', 'foo']) + execute_from_command_line(["", "dbrestore", "--database", "foo"]) # Restore - execute_from_command_line(['', 'dbrestore', '--database', 'default']) + execute_from_command_line(["", "dbrestore", "--database", "default"]) class MediaBackupCommandTest(TestCase): @@ -141,38 +141,38 @@ def tearDown(self): clean_gpg_keys() def test_encrypt(self): - argv = ['', 'mediabackup', '--encrypt'] + argv = ["", "mediabackup", "--encrypt"] execute_from_command_line(argv) - self.assertEqual(1, len(HANDLED_FILES['written_files'])) - filename, outputfile = HANDLED_FILES['written_files'][0] - self.assertTrue('.gpg' in filename) + self.assertEqual(1, len(HANDLED_FILES["written_files"])) + filename, outputfile = HANDLED_FILES["written_files"][0] + self.assertTrue(".gpg" in filename) # Test file content - outputfile = HANDLED_FILES['written_files'][0][1] + outputfile = HANDLED_FILES["written_files"][0][1] outputfile.seek(0) - self.assertTrue(outputfile.read().startswith(b'-----BEGIN PGP MESSAGE-----')) + self.assertTrue(outputfile.read().startswith(b"-----BEGIN PGP MESSAGE-----")) def test_compress(self): - argv = ['', 'mediabackup', '--compress'] + argv = ["", "mediabackup", "--compress"] execute_from_command_line(argv) - self.assertEqual(1, len(HANDLED_FILES['written_files'])) - filename, outputfile = HANDLED_FILES['written_files'][0] - self.assertTrue('.gz' in filename) + self.assertEqual(1, len(HANDLED_FILES["written_files"])) + filename, outputfile = HANDLED_FILES["written_files"][0] + self.assertTrue(".gz" in filename) - @patch('dbbackup.utils.getpass', return_value=None) + @patch("dbbackup.utils.getpass", return_value=None) def test_compress_and_encrypted(self, getpass_mock): - argv = ['', 'mediabackup', '--compress', '--encrypt'] + argv = ["", "mediabackup", "--compress", "--encrypt"] execute_from_command_line(argv) - self.assertEqual(1, len(HANDLED_FILES['written_files'])) - filename, outputfile = HANDLED_FILES['written_files'][0] - self.assertTrue('.gpg' in filename) - self.assertTrue('.gz' in filename) + self.assertEqual(1, len(HANDLED_FILES["written_files"])) + filename, outputfile = HANDLED_FILES["written_files"][0] + self.assertTrue(".gpg" in filename) + self.assertTrue(".gz" in filename) # Test file content - outputfile = HANDLED_FILES['written_files'][0][1] + outputfile = HANDLED_FILES["written_files"][0][1] outputfile.seek(0) - self.assertTrue(outputfile.read().startswith(b'-----BEGIN PGP MESSAGE-----')) + self.assertTrue(outputfile.read().startswith(b"-----BEGIN PGP MESSAGE-----")) -@patch('dbbackup.management.commands._base.input', return_value='y') +@patch("dbbackup.management.commands._base.input", return_value="y") class MediaRestoreCommandTest(TestCase): def setUp(self): HANDLED_FILES.clean() @@ -186,8 +186,8 @@ def tearDown(self): def _create_file(self, name=None): name = name or tempfile._RandomNameSequence().next() path = os.path.join(settings.MEDIA_ROOT, name) - with open(path, 'a+b') as fd: - fd.write(b'foo') + with open(path, "a+b") as fd: + fd.write(b"foo") def _emtpy_media(self): for fi in os.listdir(settings.MEDIA_ROOT): @@ -198,47 +198,47 @@ def _is_restored(self): def test_restore(self, *args): # Create backup - self._create_file('foo') - execute_from_command_line(['', 'mediabackup']) + self._create_file("foo") + execute_from_command_line(["", "mediabackup"]) self._emtpy_media() # Restore - execute_from_command_line(['', 'mediarestore']) + execute_from_command_line(["", "mediarestore"]) self.assertTrue(self._is_restored()) - @patch('dbbackup.utils.getpass', return_value=None) + @patch("dbbackup.utils.getpass", return_value=None) def test_encrypted(self, *args): # Create backup - self._create_file('foo') - execute_from_command_line(['', 'mediabackup', '--encrypt']) + self._create_file("foo") + execute_from_command_line(["", "mediabackup", "--encrypt"]) self._emtpy_media() # Restore - execute_from_command_line(['', 'mediarestore', '--decrypt']) + execute_from_command_line(["", "mediarestore", "--decrypt"]) self.assertTrue(self._is_restored()) def test_compressed(self, *args): # Create backup - self._create_file('foo') - execute_from_command_line(['', 'mediabackup', '--compress']) + self._create_file("foo") + execute_from_command_line(["", "mediabackup", "--compress"]) self._emtpy_media() # Restore - execute_from_command_line(['', 'mediarestore', '--uncompress']) + execute_from_command_line(["", "mediarestore", "--uncompress"]) self.assertTrue(self._is_restored()) def test_no_backup_available(self, *args): with self.assertRaises(SystemExit): - execute_from_command_line(['', 'mediarestore']) + execute_from_command_line(["", "mediarestore"]) - @patch('dbbackup.utils.getpass', return_value=None) + @patch("dbbackup.utils.getpass", return_value=None) def test_available_but_not_encrypted(self, *args): # Create backup - execute_from_command_line(['', 'mediabackup']) + execute_from_command_line(["", "mediabackup"]) # Restore with self.assertRaises(SystemExit): - execute_from_command_line(['', 'mediarestore', '--decrypt']) + execute_from_command_line(["", "mediarestore", "--decrypt"]) def test_available_but_not_compressed(self, *args): # Create backup - execute_from_command_line(['', 'mediabackup']) + execute_from_command_line(["", "mediabackup"]) # Restore with self.assertRaises(SystemExit): - execute_from_command_line(['', 'mediarestore', '--uncompress']) + execute_from_command_line(["", "mediarestore", "--uncompress"]) diff --git a/dbbackup/tests/settings.py b/dbbackup/tests/settings.py index 7243a132..fd596801 100644 --- a/dbbackup/tests/settings.py +++ b/dbbackup/tests/settings.py @@ -7,87 +7,84 @@ from dotenv import load_dotenv -test = len(sys.argv) <= 1 or sys.argv[1] == 'test' +test = len(sys.argv) <= 1 or sys.argv[1] == "test" if not test: load_dotenv() DEBUG = False BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TESTAPP_DIR = os.path.join(BASE_DIR, 'testapp/') -BLOB_DIR = os.path.join(TESTAPP_DIR, 'blobs/') +TESTAPP_DIR = os.path.join(BASE_DIR, "testapp/") +BLOB_DIR = os.path.join(TESTAPP_DIR, "blobs/") -ADMINS = ( - ('ham', 'foo@bar'), -) -ALLOWED_HOSTS = ['*'] +ADMINS = (("ham", "foo@bar"),) +ALLOWED_HOSTS = ["*"] MIDDLEWARE_CLASSES = () -ROOT_URLCONF = 'dbbackup.tests.testapp.urls' +ROOT_URLCONF = "dbbackup.tests.testapp.urls" SECRET_KEY = "it's a secret to everyone" SITE_ID = 1 -MEDIA_ROOT = os.environ.get('MEDIA_ROOT') or tempfile.mkdtemp() +MEDIA_ROOT = os.environ.get("MEDIA_ROOT") or tempfile.mkdtemp() INSTALLED_APPS = ( - 'dbbackup', - 'dbbackup.tests.testapp', + "dbbackup", + "dbbackup.tests.testapp", ) DATABASES = { - 'default': { - "ENGINE": os.environ.get('DB_ENGINE', "django.db.backends.sqlite3"), - "NAME": os.environ.get('DB_NAME', ":memory:"), - "USER": os.environ.get('DB_USER'), - "PASSWORD": os.environ.get('DB_PASSWORD'), - "HOST": os.environ.get('DB_HOST'), + "default": { + "ENGINE": os.environ.get("DB_ENGINE", "django.db.backends.sqlite3"), + "NAME": os.environ.get("DB_NAME", ":memory:"), + "USER": os.environ.get("DB_USER"), + "PASSWORD": os.environ.get("DB_PASSWORD"), + "HOST": os.environ.get("DB_HOST"), } } -if os.environ.get('CONNECTOR'): - CONNECTOR = {'CONNECTOR': os.environ['CONNECTOR']} - DBBACKUP_CONNECTORS = {'default': CONNECTOR} +if os.environ.get("CONNECTOR"): + CONNECTOR = {"CONNECTOR": os.environ["CONNECTOR"]} + DBBACKUP_CONNECTORS = {"default": CONNECTOR} CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + "default": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", } } -SERVER_EMAIL = 'dbbackup@test.org' +SERVER_EMAIL = "dbbackup@test.org" DBBACKUP_GPG_RECIPIENT = "test@test" -DBBACKUP_GPG_ALWAYS_TRUST = True, +DBBACKUP_GPG_ALWAYS_TRUST = (True,) -DBBACKUP_STORAGE = os.environ.get('STORAGE', 'dbbackup.tests.utils.FakeStorage') -DBBACKUP_STORAGE_OPTIONS = dict([keyvalue.split('=') for keyvalue in - os.environ.get('STORAGE_OPTIONS', '').split(',') - if keyvalue]) +DBBACKUP_STORAGE = os.environ.get("STORAGE", "dbbackup.tests.utils.FakeStorage") +DBBACKUP_STORAGE_OPTIONS = dict( + [ + keyvalue.split("=") + for keyvalue in os.environ.get("STORAGE_OPTIONS", "").split(",") + if keyvalue + ] +) LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'root': { - 'handlers': ['console'], - 'level': 'DEBUG' - }, - 'handlers': { - 'console': { - 'level': os.getenv('DJANGO_LOG_LEVEL', 'INFO'), - 'class': 'logging.StreamHandler', - 'formatter': 'simple' + "version": 1, + "disable_existing_loggers": False, + "root": {"handlers": ["console"], "level": "DEBUG"}, + "handlers": { + "console": { + "level": os.getenv("DJANGO_LOG_LEVEL", "INFO"), + "class": "logging.StreamHandler", + "formatter": "simple", } }, - 'formatters': { - 'verbose': { - 'format': "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s", - 'datefmt': "%d/%b/%Y %H:%M:%S" - }, - 'simple': { - 'format': '%(levelname)s %(message)s' + "formatters": { + "verbose": { + "format": "[%(asctime)s] %(levelname)s [%(name)s:%(lineno)s] %(message)s", + "datefmt": "%d/%b/%Y %H:%M:%S", }, + "simple": {"format": "%(levelname)s %(message)s"}, }, - 'loggers': { - 'django.db.backends': { + "loggers": { + "django.db.backends": { # uncomment to see all queries # 'level': 'DEBUG', - 'handlers': ['console'], + "handlers": ["console"], } - } + }, } diff --git a/dbbackup/tests/test_checks.py b/dbbackup/tests/test_checks.py index adc48500..642882b6 100644 --- a/dbbackup/tests/test_checks.py +++ b/dbbackup/tests/test_checks.py @@ -9,7 +9,7 @@ def test_func(*args, **kwargs): - return 'foo' + return "foo" class ChecksTest(TestCase): @@ -20,53 +20,53 @@ def setUp(self): def test_check(self): self.assertFalse(checks.check_settings(DbbackupConfig)) - @patch('dbbackup.checks.settings.HOSTNAME', '') + @patch("dbbackup.checks.settings.HOSTNAME", "") def test_hostname_invalid(self): expected_errors = [checks.W001] errors = checks.check_settings(DbbackupConfig) self.assertEqual(expected_errors, errors) - @patch('dbbackup.checks.settings.STORAGE', '') + @patch("dbbackup.checks.settings.STORAGE", "") def test_hostname_storage(self): expected_errors = [checks.W002] errors = checks.check_settings(DbbackupConfig) self.assertEqual(expected_errors, errors) - @patch('dbbackup.checks.settings.FILENAME_TEMPLATE', test_func) + @patch("dbbackup.checks.settings.FILENAME_TEMPLATE", test_func) def test_filename_template_is_callable(self): self.assertFalse(checks.check_settings(DbbackupConfig)) - @patch('dbbackup.checks.settings.FILENAME_TEMPLATE', '{datetime}.bak') + @patch("dbbackup.checks.settings.FILENAME_TEMPLATE", "{datetime}.bak") def test_filename_template_is_string(self): self.assertFalse(checks.check_settings(DbbackupConfig)) - @patch('dbbackup.checks.settings.FILENAME_TEMPLATE', 'foo.bak') + @patch("dbbackup.checks.settings.FILENAME_TEMPLATE", "foo.bak") def test_filename_template_no_date(self): expected_errors = [checks.W003] errors = checks.check_settings(DbbackupConfig) self.assertEqual(expected_errors, errors) - @patch('dbbackup.checks.settings.MEDIA_FILENAME_TEMPLATE', test_func) + @patch("dbbackup.checks.settings.MEDIA_FILENAME_TEMPLATE", test_func) def test_media_filename_template_is_callable(self): self.assertFalse(checks.check_settings(DbbackupConfig)) - @patch('dbbackup.checks.settings.MEDIA_FILENAME_TEMPLATE', '{datetime}.bak') + @patch("dbbackup.checks.settings.MEDIA_FILENAME_TEMPLATE", "{datetime}.bak") def test_media_filename_template_is_string(self): self.assertFalse(checks.check_settings(DbbackupConfig)) - @patch('dbbackup.checks.settings.MEDIA_FILENAME_TEMPLATE', 'foo.bak') + @patch("dbbackup.checks.settings.MEDIA_FILENAME_TEMPLATE", "foo.bak") def test_media_filename_template_no_date(self): expected_errors = [checks.W004] errors = checks.check_settings(DbbackupConfig) self.assertEqual(expected_errors, errors) - @patch('dbbackup.checks.settings.DATE_FORMAT', 'foo@net.pt') + @patch("dbbackup.checks.settings.DATE_FORMAT", "foo@net.pt") def test_date_format_warning(self): expected_errors = [checks.W005] errors = checks.check_settings(DbbackupConfig) self.assertEqual(expected_errors, errors) - @patch('dbbackup.checks.settings.FAILURE_RECIPIENTS', 'foo@net.pt') + @patch("dbbackup.checks.settings.FAILURE_RECIPIENTS", "foo@net.pt") def test_Failure_recipients_warning(self): expected_errors = [checks.W006] errors = checks.check_settings(DbbackupConfig) diff --git a/dbbackup/tests/test_connectors/test_base.py b/dbbackup/tests/test_connectors/test_base.py index 973ad8fa..43f97aed 100644 --- a/dbbackup/tests/test_connectors/test_base.py +++ b/dbbackup/tests/test_connectors/test_base.py @@ -19,7 +19,7 @@ def test_init(self): def test_settings(self): connector = BaseDBConnector() - connector.settings # pylint: disable=pointless-statement + connector.settings def test_generate_filename(self): connector = BaseDBConnector() @@ -51,40 +51,40 @@ def test_run_command_stdin(self): def test_run_command_with_env(self): connector = BaseCommandDBConnector() # Empty env - stdout, _stderr = connector.run_command("env") + stdout, stderr = connector.run_command("env") self.assertTrue(stdout.read()) # env from self.env connector.env = {"foo": "bar"} - stdout, _stderr = connector.run_command("env") + stdout, stderr = connector.run_command("env") self.assertIn(b"foo=bar\n", stdout.read()) # method overide gloabal env - stdout, _stderr = connector.run_command("env", env={"foo": "ham"}) + stdout, stderr = connector.run_command("env", env={"foo": "ham"}) self.assertIn(b"foo=ham\n", stdout.read()) # get a var from parent env os.environ["bar"] = "foo" - stdout, _stderr = connector.run_command("env") + stdout, stderr = connector.run_command("env") self.assertIn(b"bar=foo\n", stdout.read()) # Conf overides parendt env connector.env = {"bar": "bar"} - stdout, _stderr = connector.run_command("env") + stdout, stderr = connector.run_command("env") self.assertIn(b"bar=bar\n", stdout.read()) # method overides all - stdout, _stderr = connector.run_command("env", env={"bar": "ham"}) + stdout, stderr = connector.run_command("env", env={"bar": "ham"}) self.assertIn(b"bar=ham\n", stdout.read()) def test_run_command_with_parent_env(self): connector = BaseCommandDBConnector(use_parent_env=False) # Empty env - stdout, _stderr = connector.run_command("env") + stdout, stderr = connector.run_command("env") self.assertFalse(stdout.read()) # env from self.env connector.env = {"foo": "bar"} - stdout, _stderr = connector.run_command("env") + stdout, stderr = connector.run_command("env") self.assertEqual(stdout.read(), b"foo=bar\n") # method overide gloabal env - stdout, _stderr = connector.run_command("env", env={"foo": "ham"}) + stdout, stderr = connector.run_command("env", env={"foo": "ham"}) self.assertEqual(stdout.read(), b"foo=ham\n") # no var from parent env os.environ["bar"] = "foo" - stdout, _stderr = connector.run_command("env") + stdout, stderr = connector.run_command("env") self.assertNotIn(b"bar=foo\n", stdout.read()) diff --git a/dbbackup/tests/test_connectors/test_mongodb.py b/dbbackup/tests/test_connectors/test_mongodb.py index 1ebf7bb0..c0330276 100644 --- a/dbbackup/tests/test_connectors/test_mongodb.py +++ b/dbbackup/tests/test_connectors/test_mongodb.py @@ -6,8 +6,10 @@ from dbbackup.db.mongodb import MongoDumpConnector -@patch('dbbackup.db.mongodb.MongoDumpConnector.run_command', - return_value=(BytesIO(b'foo'), BytesIO())) +@patch( + "dbbackup.db.mongodb.MongoDumpConnector.run_command", + return_value=(BytesIO(b"foo"), BytesIO()), +) class MongoDumpConnectorTest(TestCase): def test_create_dump(self, mock_dump_cmd): connector = MongoDumpConnector() @@ -15,34 +17,36 @@ def test_create_dump(self, mock_dump_cmd): # Test dump dump_content = dump.read() self.assertTrue(dump_content) - self.assertEqual(dump_content, b'foo') + self.assertEqual(dump_content, b"foo") # Test cmd self.assertTrue(mock_dump_cmd.called) def test_create_dump_user(self, mock_dump_cmd): connector = MongoDumpConnector() # Without - connector.settings.pop('USER', None) + connector.settings.pop("USER", None) connector.create_dump() - self.assertNotIn(' --user ', mock_dump_cmd.call_args[0][0]) + self.assertNotIn(" --user ", mock_dump_cmd.call_args[0][0]) # With - connector.settings['USER'] = 'foo' + connector.settings["USER"] = "foo" connector.create_dump() - self.assertIn(' --username foo', mock_dump_cmd.call_args[0][0]) + self.assertIn(" --username foo", mock_dump_cmd.call_args[0][0]) def test_create_dump_password(self, mock_dump_cmd): connector = MongoDumpConnector() # Without - connector.settings.pop('PASSWORD', None) + connector.settings.pop("PASSWORD", None) connector.create_dump() - self.assertNotIn(' --password ', mock_dump_cmd.call_args[0][0]) + self.assertNotIn(" --password ", mock_dump_cmd.call_args[0][0]) # With - connector.settings['PASSWORD'] = 'foo' + connector.settings["PASSWORD"] = "foo" connector.create_dump() - self.assertIn(' --password foo', mock_dump_cmd.call_args[0][0]) + self.assertIn(" --password foo", mock_dump_cmd.call_args[0][0]) - @patch('dbbackup.db.mongodb.MongoDumpConnector.run_command', - return_value=(BytesIO(), BytesIO())) + @patch( + "dbbackup.db.mongodb.MongoDumpConnector.run_command", + return_value=(BytesIO(), BytesIO()), + ) def test_restore_dump(self, mock_dump_cmd, mock_restore_cmd): connector = MongoDumpConnector() dump = connector.create_dump() @@ -50,58 +54,66 @@ def test_restore_dump(self, mock_dump_cmd, mock_restore_cmd): # Test cmd self.assertTrue(mock_restore_cmd.called) - @patch('dbbackup.db.mongodb.MongoDumpConnector.run_command', - return_value=(BytesIO(), BytesIO())) + @patch( + "dbbackup.db.mongodb.MongoDumpConnector.run_command", + return_value=(BytesIO(), BytesIO()), + ) def test_restore_dump_user(self, mock_dump_cmd, mock_restore_cmd): connector = MongoDumpConnector() dump = connector.create_dump() # Without - connector.settings.pop('USER', None) + connector.settings.pop("USER", None) connector.restore_dump(dump) - self.assertNotIn(' --username ', mock_restore_cmd.call_args[0][0]) + self.assertNotIn(" --username ", mock_restore_cmd.call_args[0][0]) # With - connector.settings['USER'] = 'foo' + connector.settings["USER"] = "foo" connector.restore_dump(dump) - self.assertIn(' --username foo', mock_restore_cmd.call_args[0][0]) + self.assertIn(" --username foo", mock_restore_cmd.call_args[0][0]) - @patch('dbbackup.db.mongodb.MongoDumpConnector.run_command', - return_value=(BytesIO(), BytesIO())) + @patch( + "dbbackup.db.mongodb.MongoDumpConnector.run_command", + return_value=(BytesIO(), BytesIO()), + ) def test_restore_dump_password(self, mock_dump_cmd, mock_restore_cmd): connector = MongoDumpConnector() dump = connector.create_dump() # Without - connector.settings.pop('PASSWORD', None) + connector.settings.pop("PASSWORD", None) connector.restore_dump(dump) - self.assertNotIn(' --password ', mock_restore_cmd.call_args[0][0]) + self.assertNotIn(" --password ", mock_restore_cmd.call_args[0][0]) # With - connector.settings['PASSWORD'] = 'foo' + connector.settings["PASSWORD"] = "foo" connector.restore_dump(dump) - self.assertIn(' --password foo', mock_restore_cmd.call_args[0][0]) + self.assertIn(" --password foo", mock_restore_cmd.call_args[0][0]) - @patch('dbbackup.db.mongodb.MongoDumpConnector.run_command', - return_value=(BytesIO(), BytesIO())) + @patch( + "dbbackup.db.mongodb.MongoDumpConnector.run_command", + return_value=(BytesIO(), BytesIO()), + ) def test_restore_dump_object_check(self, mock_dump_cmd, mock_restore_cmd): connector = MongoDumpConnector() dump = connector.create_dump() # Without connector.object_check = False connector.restore_dump(dump) - self.assertNotIn('--objcheck', mock_restore_cmd.call_args[0][0]) + self.assertNotIn("--objcheck", mock_restore_cmd.call_args[0][0]) # With connector.object_check = True connector.restore_dump(dump) - self.assertIn(' --objcheck', mock_restore_cmd.call_args[0][0]) + self.assertIn(" --objcheck", mock_restore_cmd.call_args[0][0]) - @patch('dbbackup.db.mongodb.MongoDumpConnector.run_command', - return_value=(BytesIO(), BytesIO())) + @patch( + "dbbackup.db.mongodb.MongoDumpConnector.run_command", + return_value=(BytesIO(), BytesIO()), + ) def test_restore_dump_drop(self, mock_dump_cmd, mock_restore_cmd): connector = MongoDumpConnector() dump = connector.create_dump() # Without connector.drop = False connector.restore_dump(dump) - self.assertNotIn('--drop', mock_restore_cmd.call_args[0][0]) + self.assertNotIn("--drop", mock_restore_cmd.call_args[0][0]) # With connector.drop = True connector.restore_dump(dump) - self.assertIn(' --drop', mock_restore_cmd.call_args[0][0]) + self.assertIn(" --drop", mock_restore_cmd.call_args[0][0]) diff --git a/dbbackup/tests/test_connectors/test_mysql.py b/dbbackup/tests/test_connectors/test_mysql.py index ce1c265e..7eb64148 100644 --- a/dbbackup/tests/test_connectors/test_mysql.py +++ b/dbbackup/tests/test_connectors/test_mysql.py @@ -6,8 +6,10 @@ from dbbackup.db.mysql import MysqlDumpConnector -@patch('dbbackup.db.mysql.MysqlDumpConnector.run_command', - return_value=(BytesIO(b'foo'), BytesIO())) +@patch( + "dbbackup.db.mysql.MysqlDumpConnector.run_command", + return_value=(BytesIO(b"foo"), BytesIO()), +) class MysqlDumpConnectorTest(TestCase): def test_create_dump(self, mock_dump_cmd): connector = MysqlDumpConnector() @@ -15,72 +17,74 @@ def test_create_dump(self, mock_dump_cmd): # Test dump dump_content = dump.read() self.assertTrue(dump_content) - self.assertEqual(dump_content, b'foo') + self.assertEqual(dump_content, b"foo") # Test cmd self.assertTrue(mock_dump_cmd.called) def test_create_dump_host(self, mock_dump_cmd): connector = MysqlDumpConnector() # Without - connector.settings.pop('HOST', None) + connector.settings.pop("HOST", None) connector.create_dump() - self.assertNotIn(' --host=', mock_dump_cmd.call_args[0][0]) + self.assertNotIn(" --host=", mock_dump_cmd.call_args[0][0]) # With - connector.settings['HOST'] = 'foo' + connector.settings["HOST"] = "foo" connector.create_dump() - self.assertIn(' --host=foo', mock_dump_cmd.call_args[0][0]) + self.assertIn(" --host=foo", mock_dump_cmd.call_args[0][0]) def test_create_dump_port(self, mock_dump_cmd): connector = MysqlDumpConnector() # Without - connector.settings.pop('PORT', None) + connector.settings.pop("PORT", None) connector.create_dump() - self.assertNotIn(' --port=', mock_dump_cmd.call_args[0][0]) + self.assertNotIn(" --port=", mock_dump_cmd.call_args[0][0]) # With - connector.settings['PORT'] = 42 + connector.settings["PORT"] = 42 connector.create_dump() - self.assertIn(' --port=42', mock_dump_cmd.call_args[0][0]) + self.assertIn(" --port=42", mock_dump_cmd.call_args[0][0]) def test_create_dump_user(self, mock_dump_cmd): connector = MysqlDumpConnector() # Without - connector.settings.pop('USER', None) + connector.settings.pop("USER", None) connector.create_dump() - self.assertNotIn(' --user=', mock_dump_cmd.call_args[0][0]) + self.assertNotIn(" --user=", mock_dump_cmd.call_args[0][0]) # With - connector.settings['USER'] = 'foo' + connector.settings["USER"] = "foo" connector.create_dump() - self.assertIn(' --user=foo', mock_dump_cmd.call_args[0][0]) + self.assertIn(" --user=foo", mock_dump_cmd.call_args[0][0]) def test_create_dump_password(self, mock_dump_cmd): connector = MysqlDumpConnector() # Without - connector.settings.pop('PASSWORD', None) + connector.settings.pop("PASSWORD", None) connector.create_dump() - self.assertNotIn(' --password=', mock_dump_cmd.call_args[0][0]) + self.assertNotIn(" --password=", mock_dump_cmd.call_args[0][0]) # With - connector.settings['PASSWORD'] = 'foo' + connector.settings["PASSWORD"] = "foo" connector.create_dump() - self.assertIn(' --password=foo', mock_dump_cmd.call_args[0][0]) + self.assertIn(" --password=foo", mock_dump_cmd.call_args[0][0]) def test_create_dump_exclude(self, mock_dump_cmd): connector = MysqlDumpConnector() - connector.settings['NAME'] = 'db' + connector.settings["NAME"] = "db" # Without connector.create_dump() - self.assertNotIn(' --ignore-table=', mock_dump_cmd.call_args[0][0]) + self.assertNotIn(" --ignore-table=", mock_dump_cmd.call_args[0][0]) # With - connector.exclude = ('foo',) + connector.exclude = ("foo",) connector.create_dump() - self.assertIn(' --ignore-table=db.foo', mock_dump_cmd.call_args[0][0]) + self.assertIn(" --ignore-table=db.foo", mock_dump_cmd.call_args[0][0]) # With serveral - connector.exclude = ('foo', 'bar') + connector.exclude = ("foo", "bar") connector.create_dump() - self.assertIn(' --ignore-table=db.foo', mock_dump_cmd.call_args[0][0]) - self.assertIn(' --ignore-table=db.bar', mock_dump_cmd.call_args[0][0]) + self.assertIn(" --ignore-table=db.foo", mock_dump_cmd.call_args[0][0]) + self.assertIn(" --ignore-table=db.bar", mock_dump_cmd.call_args[0][0]) - @patch('dbbackup.db.mysql.MysqlDumpConnector.run_command', - return_value=(BytesIO(), BytesIO())) + @patch( + "dbbackup.db.mysql.MysqlDumpConnector.run_command", + return_value=(BytesIO(), BytesIO()), + ) def test_restore_dump(self, mock_dump_cmd, mock_restore_cmd): connector = MysqlDumpConnector() dump = connector.create_dump() @@ -88,58 +92,66 @@ def test_restore_dump(self, mock_dump_cmd, mock_restore_cmd): # Test cmd self.assertTrue(mock_restore_cmd.called) - @patch('dbbackup.db.mysql.MysqlDumpConnector.run_command', - return_value=(BytesIO(), BytesIO())) + @patch( + "dbbackup.db.mysql.MysqlDumpConnector.run_command", + return_value=(BytesIO(), BytesIO()), + ) def test_restore_dump_host(self, mock_dump_cmd, mock_restore_cmd): connector = MysqlDumpConnector() dump = connector.create_dump() # Without - connector.settings.pop('HOST', None) + connector.settings.pop("HOST", None) connector.restore_dump(dump) - self.assertNotIn(' --host=foo', mock_restore_cmd.call_args[0][0]) + self.assertNotIn(" --host=foo", mock_restore_cmd.call_args[0][0]) # With - connector.settings['HOST'] = 'foo' + connector.settings["HOST"] = "foo" connector.restore_dump(dump) - self.assertIn(' --host=foo', mock_restore_cmd.call_args[0][0]) + self.assertIn(" --host=foo", mock_restore_cmd.call_args[0][0]) - @patch('dbbackup.db.mysql.MysqlDumpConnector.run_command', - return_value=(BytesIO(), BytesIO())) + @patch( + "dbbackup.db.mysql.MysqlDumpConnector.run_command", + return_value=(BytesIO(), BytesIO()), + ) def test_restore_dump_port(self, mock_dump_cmd, mock_restore_cmd): connector = MysqlDumpConnector() dump = connector.create_dump() # Without - connector.settings.pop('PORT', None) + connector.settings.pop("PORT", None) connector.restore_dump(dump) - self.assertNotIn(' --port=', mock_restore_cmd.call_args[0][0]) + self.assertNotIn(" --port=", mock_restore_cmd.call_args[0][0]) # With - connector.settings['PORT'] = 42 + connector.settings["PORT"] = 42 connector.restore_dump(dump) - self.assertIn(' --port=42', mock_restore_cmd.call_args[0][0]) + self.assertIn(" --port=42", mock_restore_cmd.call_args[0][0]) - @patch('dbbackup.db.mysql.MysqlDumpConnector.run_command', - return_value=(BytesIO(), BytesIO())) + @patch( + "dbbackup.db.mysql.MysqlDumpConnector.run_command", + return_value=(BytesIO(), BytesIO()), + ) def test_restore_dump_user(self, mock_dump_cmd, mock_restore_cmd): connector = MysqlDumpConnector() dump = connector.create_dump() # Without - connector.settings.pop('USER', None) + connector.settings.pop("USER", None) connector.restore_dump(dump) - self.assertNotIn(' --user=', mock_restore_cmd.call_args[0][0]) + self.assertNotIn(" --user=", mock_restore_cmd.call_args[0][0]) # With - connector.settings['USER'] = 'foo' + connector.settings["USER"] = "foo" connector.restore_dump(dump) - self.assertIn(' --user=foo', mock_restore_cmd.call_args[0][0]) + self.assertIn(" --user=foo", mock_restore_cmd.call_args[0][0]) - @patch('dbbackup.db.mysql.MysqlDumpConnector.run_command', - return_value=(BytesIO(), BytesIO())) + @patch( + "dbbackup.db.mysql.MysqlDumpConnector.run_command", + return_value=(BytesIO(), BytesIO()), + ) def test_restore_dump_password(self, mock_dump_cmd, mock_restore_cmd): connector = MysqlDumpConnector() dump = connector.create_dump() # Without - connector.settings.pop('PASSWORD', None) + connector.settings.pop("PASSWORD", None) connector.restore_dump(dump) - self.assertNotIn(' --password=', mock_restore_cmd.call_args[0][0]) + self.assertNotIn(" --password=", mock_restore_cmd.call_args[0][0]) # With - connector.settings['PASSWORD'] = 'foo' + connector.settings["PASSWORD"] = "foo" connector.restore_dump(dump) - self.assertIn(' --password=foo', mock_restore_cmd.call_args[0][0]) + self.assertIn(" --password=foo", mock_restore_cmd.call_args[0][0]) diff --git a/dbbackup/tests/test_connectors/test_postgresql.py b/dbbackup/tests/test_connectors/test_postgresql.py index 55c44c58..b3aeaa26 100644 --- a/dbbackup/tests/test_connectors/test_postgresql.py +++ b/dbbackup/tests/test_connectors/test_postgresql.py @@ -11,169 +11,177 @@ ) -@patch('dbbackup.db.postgresql.PgDumpConnector.run_command', - return_value=(BytesIO(b'foo'), BytesIO())) +@patch( + "dbbackup.db.postgresql.PgDumpConnector.run_command", + return_value=(BytesIO(b"foo"), BytesIO()), +) class PgDumpConnectorTest(TestCase): def setUp(self): self.connector = PgDumpConnector() - self.connector.settings['ENGINE'] = 'django.db.backends.postgresql' - self.connector.settings['NAME'] = 'dbname' - self.connector.settings['HOST'] = 'hostname' + self.connector.settings["ENGINE"] = "django.db.backends.postgresql" + self.connector.settings["NAME"] = "dbname" + self.connector.settings["HOST"] = "hostname" def test_user_password_uses_special_characters(self, mock_dump_cmd): - self.connector.settings['PASSWORD'] = '@!' - self.connector.settings['USER'] = '@' + self.connector.settings["PASSWORD"] = "@!" + self.connector.settings["USER"] = "@" self.connector.create_dump() - self.assertIn('postgresql://%40:%40%21@hostname/dbname', mock_dump_cmd.call_args[0][0]) + self.assertIn( + "postgresql://%40:%40%21@hostname/dbname", mock_dump_cmd.call_args[0][0] + ) def test_create_dump(self, mock_dump_cmd): dump = self.connector.create_dump() # Test dump dump_content = dump.read() self.assertTrue(dump_content) - self.assertEqual(dump_content, b'foo') + self.assertEqual(dump_content, b"foo") # Test cmd self.assertTrue(mock_dump_cmd.called) def test_create_dump_without_host_raises_error(self, mock_dump_cmd): - self.connector.settings.pop('HOST', None) + self.connector.settings.pop("HOST", None) with self.assertRaises(DumpError): self.connector.create_dump() def test_password_but_no_user(self, mock_dump_cmd): - self.connector.settings.pop('USER', None) - self.connector.settings['PASSWORD'] = 'hello' + self.connector.settings.pop("USER", None) + self.connector.settings["PASSWORD"] = "hello" self.connector.create_dump() - self.assertIn('postgresql://hostname/dbname', mock_dump_cmd.call_args[0][0]) + self.assertIn("postgresql://hostname/dbname", mock_dump_cmd.call_args[0][0]) def test_create_dump_host(self, mock_dump_cmd): # With - self.connector.settings['HOST'] = 'foo' + self.connector.settings["HOST"] = "foo" self.connector.create_dump() - self.assertIn('postgresql://foo/dbname', mock_dump_cmd.call_args[0][0]) + self.assertIn("postgresql://foo/dbname", mock_dump_cmd.call_args[0][0]) def test_create_dump_port(self, mock_dump_cmd): # Without - self.connector.settings.pop('PORT', None) + self.connector.settings.pop("PORT", None) self.connector.create_dump() - self.assertIn('postgresql://hostname/dbname', mock_dump_cmd.call_args[0][0]) + self.assertIn("postgresql://hostname/dbname", mock_dump_cmd.call_args[0][0]) # With - self.connector.settings['PORT'] = 42 + self.connector.settings["PORT"] = 42 self.connector.create_dump() - self.assertIn('postgresql://hostname:42/dbname', mock_dump_cmd.call_args[0][0]) + self.assertIn("postgresql://hostname:42/dbname", mock_dump_cmd.call_args[0][0]) def test_create_dump_user(self, mock_dump_cmd): # Without - self.connector.settings.pop('USER', None) + self.connector.settings.pop("USER", None) self.connector.create_dump() - self.assertIn('postgresql://hostname/dbname', mock_dump_cmd.call_args[0][0]) + self.assertIn("postgresql://hostname/dbname", mock_dump_cmd.call_args[0][0]) # With - self.connector.settings['USER'] = 'foo' + self.connector.settings["USER"] = "foo" self.connector.create_dump() - self.assertIn('postgresql://foo@hostname/dbname', mock_dump_cmd.call_args[0][0]) + self.assertIn("postgresql://foo@hostname/dbname", mock_dump_cmd.call_args[0][0]) def test_create_dump_exclude(self, mock_dump_cmd): # Without self.connector.create_dump() - self.assertNotIn(' --exclude-table-data=', mock_dump_cmd.call_args[0][0]) + self.assertNotIn(" --exclude-table-data=", mock_dump_cmd.call_args[0][0]) # With - self.connector.exclude = ('foo',) + self.connector.exclude = ("foo",) self.connector.create_dump() - self.assertIn(' --exclude-table-data=foo', mock_dump_cmd.call_args[0][0]) + self.assertIn(" --exclude-table-data=foo", mock_dump_cmd.call_args[0][0]) # With serveral - self.connector.exclude = ('foo', 'bar') + self.connector.exclude = ("foo", "bar") self.connector.create_dump() - self.assertIn(' --exclude-table-data=foo', mock_dump_cmd.call_args[0][0]) - self.assertIn(' --exclude-table-data=bar', mock_dump_cmd.call_args[0][0]) + self.assertIn(" --exclude-table-data=foo", mock_dump_cmd.call_args[0][0]) + self.assertIn(" --exclude-table-data=bar", mock_dump_cmd.call_args[0][0]) def test_create_dump_drop(self, mock_dump_cmd): # Without self.connector.drop = False self.connector.create_dump() - self.assertNotIn(' --clean', mock_dump_cmd.call_args[0][0]) + self.assertNotIn(" --clean", mock_dump_cmd.call_args[0][0]) # With self.connector.drop = True self.connector.create_dump() - self.assertIn(' --clean', mock_dump_cmd.call_args[0][0]) + self.assertIn(" --clean", mock_dump_cmd.call_args[0][0]) - @patch('dbbackup.db.postgresql.PgDumpConnector.run_command', - return_value=(BytesIO(), BytesIO())) + @patch( + "dbbackup.db.postgresql.PgDumpConnector.run_command", + return_value=(BytesIO(), BytesIO()), + ) def test_restore_dump(self, mock_dump_cmd, mock_restore_cmd): dump = self.connector.create_dump() self.connector.restore_dump(dump) # Test cmd self.assertTrue(mock_restore_cmd.called) - @patch('dbbackup.db.postgresql.PgDumpConnector.run_command', - return_value=(BytesIO(), BytesIO())) + @patch( + "dbbackup.db.postgresql.PgDumpConnector.run_command", + return_value=(BytesIO(), BytesIO()), + ) def test_restore_dump_user(self, mock_dump_cmd, mock_restore_cmd): dump = self.connector.create_dump() # Without - self.connector.settings.pop('USER', None) + self.connector.settings.pop("USER", None) self.connector.restore_dump(dump) - self.assertIn( - 'postgresql://hostname/dbname', - mock_restore_cmd.call_args[0][0] - ) + self.assertIn("postgresql://hostname/dbname", mock_restore_cmd.call_args[0][0]) - self.assertNotIn(' --username=', mock_restore_cmd.call_args[0][0]) + self.assertNotIn(" --username=", mock_restore_cmd.call_args[0][0]) # With - self.connector.settings['USER'] = 'foo' + self.connector.settings["USER"] = "foo" self.connector.restore_dump(dump) self.assertIn( - 'postgresql://foo@hostname/dbname', - mock_restore_cmd.call_args[0][0] + "postgresql://foo@hostname/dbname", mock_restore_cmd.call_args[0][0] ) -@patch('dbbackup.db.postgresql.PgDumpBinaryConnector.run_command', - return_value=(BytesIO(b'foo'), BytesIO())) +@patch( + "dbbackup.db.postgresql.PgDumpBinaryConnector.run_command", + return_value=(BytesIO(b"foo"), BytesIO()), +) class PgDumpBinaryConnectorTest(TestCase): def setUp(self): self.connector = PgDumpBinaryConnector() - self.connector.settings['HOST'] = 'hostname' - self.connector.settings['ENGINE'] = 'django.db.backends.postgresql' - self.connector.settings['NAME'] = 'dbname' + self.connector.settings["HOST"] = "hostname" + self.connector.settings["ENGINE"] = "django.db.backends.postgresql" + self.connector.settings["NAME"] = "dbname" def test_create_dump(self, mock_dump_cmd): dump = self.connector.create_dump() # Test dump dump_content = dump.read() self.assertTrue(dump_content) - self.assertEqual(dump_content, b'foo') + self.assertEqual(dump_content, b"foo") # Test cmd self.assertTrue(mock_dump_cmd.called) - self.assertIn('--format=custom', mock_dump_cmd.call_args[0][0]) + self.assertIn("--format=custom", mock_dump_cmd.call_args[0][0]) def test_create_dump_exclude(self, mock_dump_cmd): # Without self.connector.create_dump() - self.assertNotIn(' --exclude-table-data=', mock_dump_cmd.call_args[0][0]) + self.assertNotIn(" --exclude-table-data=", mock_dump_cmd.call_args[0][0]) # With - self.connector.exclude = ('foo',) + self.connector.exclude = ("foo",) self.connector.create_dump() - self.assertIn(' --exclude-table-data=foo', mock_dump_cmd.call_args[0][0]) + self.assertIn(" --exclude-table-data=foo", mock_dump_cmd.call_args[0][0]) # With serveral - self.connector.exclude = ('foo', 'bar') + self.connector.exclude = ("foo", "bar") self.connector.create_dump() - self.assertIn(' --exclude-table-data=foo', mock_dump_cmd.call_args[0][0]) - self.assertIn(' --exclude-table-data=bar', mock_dump_cmd.call_args[0][0]) + self.assertIn(" --exclude-table-data=foo", mock_dump_cmd.call_args[0][0]) + self.assertIn(" --exclude-table-data=bar", mock_dump_cmd.call_args[0][0]) def test_create_dump_drop(self, mock_dump_cmd): # Without self.connector.drop = False self.connector.create_dump() - self.assertNotIn(' --clean', mock_dump_cmd.call_args[0][0]) + self.assertNotIn(" --clean", mock_dump_cmd.call_args[0][0]) # Binary drop at restore level self.connector.drop = True self.connector.create_dump() - self.assertNotIn(' --clean', mock_dump_cmd.call_args[0][0]) + self.assertNotIn(" --clean", mock_dump_cmd.call_args[0][0]) - @patch('dbbackup.db.postgresql.PgDumpBinaryConnector.run_command', - return_value=(BytesIO(), BytesIO())) + @patch( + "dbbackup.db.postgresql.PgDumpBinaryConnector.run_command", + return_value=(BytesIO(), BytesIO()), + ) def test_restore_dump(self, mock_dump_cmd, mock_restore_cmd): dump = self.connector.create_dump() self.connector.restore_dump(dump) @@ -181,79 +189,87 @@ def test_restore_dump(self, mock_dump_cmd, mock_restore_cmd): self.assertTrue(mock_restore_cmd.called) -@patch('dbbackup.db.postgresql.PgDumpGisConnector.run_command', - return_value=(BytesIO(b'foo'), BytesIO())) +@patch( + "dbbackup.db.postgresql.PgDumpGisConnector.run_command", + return_value=(BytesIO(b"foo"), BytesIO()), +) class PgDumpGisConnectorTest(TestCase): def setUp(self): self.connector = PgDumpGisConnector() - self.connector.settings['HOST'] = 'hostname' + self.connector.settings["HOST"] = "hostname" - @patch('dbbackup.db.postgresql.PgDumpGisConnector.run_command', - return_value=(BytesIO(b'foo'), BytesIO())) + @patch( + "dbbackup.db.postgresql.PgDumpGisConnector.run_command", + return_value=(BytesIO(b"foo"), BytesIO()), + ) def test_restore_dump(self, mock_dump_cmd, mock_restore_cmd): dump = self.connector.create_dump() # Without ADMINUSER - self.connector.settings.pop('ADMIN_USER', None) + self.connector.settings.pop("ADMIN_USER", None) self.connector.restore_dump(dump) self.assertTrue(mock_restore_cmd.called) # With - self.connector.settings['ADMIN_USER'] = 'foo' + self.connector.settings["ADMIN_USER"] = "foo" self.connector.restore_dump(dump) self.assertTrue(mock_restore_cmd.called) def test_enable_postgis(self, mock_dump_cmd): - self.connector.settings['ADMIN_USER'] = 'foo' + self.connector.settings["ADMIN_USER"] = "foo" self.connector._enable_postgis() - self.assertIn('"CREATE EXTENSION IF NOT EXISTS postgis;"', mock_dump_cmd.call_args[0][0]) - self.assertIn('--username=foo', mock_dump_cmd.call_args[0][0]) + self.assertIn( + '"CREATE EXTENSION IF NOT EXISTS postgis;"', mock_dump_cmd.call_args[0][0] + ) + self.assertIn("--username=foo", mock_dump_cmd.call_args[0][0]) def test_enable_postgis_host(self, mock_dump_cmd): - self.connector.settings['ADMIN_USER'] = 'foo' + self.connector.settings["ADMIN_USER"] = "foo" # Without - self.connector.settings.pop('HOST', None) + self.connector.settings.pop("HOST", None) self.connector._enable_postgis() - self.assertNotIn(' --host=', mock_dump_cmd.call_args[0][0]) + self.assertNotIn(" --host=", mock_dump_cmd.call_args[0][0]) # With - self.connector.settings['HOST'] = 'foo' + self.connector.settings["HOST"] = "foo" self.connector._enable_postgis() - self.assertIn(' --host=foo', mock_dump_cmd.call_args[0][0]) + self.assertIn(" --host=foo", mock_dump_cmd.call_args[0][0]) def test_enable_postgis_port(self, mock_dump_cmd): - self.connector.settings['ADMIN_USER'] = 'foo' + self.connector.settings["ADMIN_USER"] = "foo" # Without - self.connector.settings.pop('PORT', None) + self.connector.settings.pop("PORT", None) self.connector._enable_postgis() - self.assertNotIn(' --port=', mock_dump_cmd.call_args[0][0]) + self.assertNotIn(" --port=", mock_dump_cmd.call_args[0][0]) # With - self.connector.settings['PORT'] = 42 + self.connector.settings["PORT"] = 42 self.connector._enable_postgis() - self.assertIn(' --port=42', mock_dump_cmd.call_args[0][0]) + self.assertIn(" --port=42", mock_dump_cmd.call_args[0][0]) -@patch('dbbackup.db.base.Popen', **{ - 'return_value.wait.return_value': True, - 'return_value.poll.return_value': False, -}) +@patch( + "dbbackup.db.base.Popen", + **{ + "return_value.wait.return_value": True, + "return_value.poll.return_value": False, + }, +) class PgDumpConnectorRunCommandTest(TestCase): - def test_run_command(self, mock_popen): connector = PgDumpConnector() - connector.settings['HOST'] = 'hostname' + connector.settings["HOST"] = "hostname" connector.create_dump() - self.assertEqual(mock_popen.call_args[0][0][0], 'pg_dump') + self.assertEqual(mock_popen.call_args[0][0][0], "pg_dump") def test_run_command_with_password(self, mock_popen): connector = PgDumpConnector() - connector.settings['HOST'] = 'hostname' - connector.settings['PASSWORD'] = 'foo' + connector.settings["HOST"] = "hostname" + connector.settings["PASSWORD"] = "foo" connector.create_dump() - self.assertEqual(mock_popen.call_args[0][0][0], 'pg_dump') + self.assertEqual(mock_popen.call_args[0][0][0], "pg_dump") def test_run_command_with_password_and_other(self, mock_popen): - connector = PgDumpConnector(env={'foo': 'bar'}) - connector.settings['HOST'] = 'hostname' - connector.settings['PASSWORD'] = 'foo' + connector = PgDumpConnector(env={"foo": "bar"}) + connector.settings["HOST"] = "hostname" + connector.settings["PASSWORD"] = "foo" connector.create_dump() - self.assertEqual(mock_popen.call_args[0][0][0], 'pg_dump') - self.assertIn('foo', mock_popen.call_args[1]['env']) - self.assertEqual('bar', mock_popen.call_args[1]['env']['foo']) + self.assertEqual(mock_popen.call_args[0][0][0], "pg_dump") + self.assertIn("foo", mock_popen.call_args[1]["env"]) + self.assertEqual("bar", mock_popen.call_args[1]["env"]["foo"]) diff --git a/dbbackup/tests/test_connectors/test_sqlite.py b/dbbackup/tests/test_connectors/test_sqlite.py index d54ca095..825721ad 100644 --- a/dbbackup/tests/test_connectors/test_sqlite.py +++ b/dbbackup/tests/test_connectors/test_sqlite.py @@ -14,7 +14,7 @@ def test_write_dump(self): connector._write_dump(dump_file) dump_file.seek(0) for line in dump_file: - self.assertTrue(line.strip().endswith(b';')) + self.assertTrue(line.strip().endswith(b";")) def test_create_dump(self): connector = SqliteConnector() @@ -22,7 +22,7 @@ def test_create_dump(self): self.assertTrue(dump.read()) def test_create_dump_with_unicode(self): - CharModel.objects.create(field='\xe9') + CharModel.objects.create(field="\xe9") connector = SqliteConnector() dump = connector.create_dump() self.assertTrue(dump.read()) @@ -33,14 +33,14 @@ def test_restore_dump(self): connector.restore_dump(dump) -@patch('dbbackup.db.sqlite.open', mock_open(read_data=b'foo'), create=True) +@patch("dbbackup.db.sqlite.open", mock_open(read_data=b"foo"), create=True) class SqliteCPConnectorTest(TestCase): def test_create_dump(self): connector = SqliteCPConnector() dump = connector.create_dump() dump_content = dump.read() self.assertTrue(dump_content) - self.assertEqual(dump_content, b'foo') + self.assertEqual(dump_content, b"foo") def test_restore_dump(self): connector = SqliteCPConnector() diff --git a/dbbackup/tests/test_log.py b/dbbackup/tests/test_log.py index 2a618485..d394a511 100644 --- a/dbbackup/tests/test_log.py +++ b/dbbackup/tests/test_log.py @@ -13,105 +13,105 @@ class LoggerDefaultTestCase(TestCase): @log_capture() def test_root(self, captures): logger = logging.getLogger() - logger.debug('a noise') - logger.info('a message') - logger.warning('a warning') - logger.error('an error') - logger.critical('a critical error') + logger.debug("a noise") + logger.info("a message") + logger.warning("a warning") + logger.error("an error") + logger.critical("a critical error") captures.check( - ('root', 'DEBUG', 'a noise'), - ('root', 'INFO', 'a message'), - ('root', 'WARNING', 'a warning'), - ('root', 'ERROR', 'an error'), - ('root', 'CRITICAL', 'a critical error'), + ("root", "DEBUG", "a noise"), + ("root", "INFO", "a message"), + ("root", "WARNING", "a warning"), + ("root", "ERROR", "an error"), + ("root", "CRITICAL", "a critical error"), ) @log_capture() def test_django(self, captures): - logger = logging.getLogger('django') - logger.debug('a noise') - logger.info('a message') - logger.warning('a warning') - logger.error('an error') - logger.critical('a critical error') + logger = logging.getLogger("django") + logger.debug("a noise") + logger.info("a message") + logger.warning("a warning") + logger.error("an error") + logger.critical("a critical error") if django.VERSION < (1, 9): captures.check( - ('django', 'DEBUG', 'a noise'), - ('django', 'INFO', 'a message'), - ('django', 'WARNING', 'a warning'), - ('django', 'ERROR', 'an error'), - ('django', 'CRITICAL', 'a critical error'), + ("django", "DEBUG", "a noise"), + ("django", "INFO", "a message"), + ("django", "WARNING", "a warning"), + ("django", "ERROR", "an error"), + ("django", "CRITICAL", "a critical error"), ) else: captures.check( - ('django', 'INFO', 'a message'), - ('django', 'WARNING', 'a warning'), - ('django', 'ERROR', 'an error'), - ('django', 'CRITICAL', 'a critical error'), + ("django", "INFO", "a message"), + ("django", "WARNING", "a warning"), + ("django", "ERROR", "an error"), + ("django", "CRITICAL", "a critical error"), ) @log_capture() def test_dbbackup(self, captures): - logger = logging.getLogger('dbbackup') - logger.debug('a noise') - logger.info('a message') - logger.warning('a warning') - logger.error('an error') - logger.critical('a critical error') + logger = logging.getLogger("dbbackup") + logger.debug("a noise") + logger.info("a message") + logger.warning("a warning") + logger.error("an error") + logger.critical("a critical error") captures.check( - ('dbbackup', 'INFO', 'a message'), - ('dbbackup', 'WARNING', 'a warning'), - ('dbbackup', 'ERROR', 'an error'), - ('dbbackup', 'CRITICAL', 'a critical error'), + ("dbbackup", "INFO", "a message"), + ("dbbackup", "WARNING", "a warning"), + ("dbbackup", "ERROR", "an error"), + ("dbbackup", "CRITICAL", "a critical error"), ) @log_capture() def test_dbbackup_storage(self, captures): - logger = logging.getLogger('dbbackup.storage') - logger.debug('a noise') - logger.info('a message') - logger.warning('a warning') - logger.error('an error') - logger.critical('a critical error') + logger = logging.getLogger("dbbackup.storage") + logger.debug("a noise") + logger.info("a message") + logger.warning("a warning") + logger.error("an error") + logger.critical("a critical error") captures.check( - ('dbbackup.storage', 'INFO', 'a message'), - ('dbbackup.storage', 'WARNING', 'a warning'), - ('dbbackup.storage', 'ERROR', 'an error'), - ('dbbackup.storage', 'CRITICAL', 'a critical error'), + ("dbbackup.storage", "INFO", "a message"), + ("dbbackup.storage", "WARNING", "a warning"), + ("dbbackup.storage", "ERROR", "an error"), + ("dbbackup.storage", "CRITICAL", "a critical error"), ) @log_capture() def test_other_module(self, captures): - logger = logging.getLogger('os.path') - logger.debug('a noise') - logger.info('a message') - logger.warning('a warning') - logger.error('an error') - logger.critical('a critical error') + logger = logging.getLogger("os.path") + logger.debug("a noise") + logger.info("a message") + logger.warning("a warning") + logger.error("an error") + logger.critical("a critical error") captures.check( - ('os.path', 'DEBUG', 'a noise'), - ('os.path', 'INFO', 'a message'), - ('os.path', 'WARNING', 'a warning'), - ('os.path', 'ERROR', 'an error'), - ('os.path', 'CRITICAL', 'a critical error'), + ("os.path", "DEBUG", "a noise"), + ("os.path", "INFO", "a message"), + ("os.path", "WARNING", "a warning"), + ("os.path", "ERROR", "an error"), + ("os.path", "CRITICAL", "a critical error"), ) class DbbackupAdminEmailHandlerTest(TestCase): def setUp(self): - self.logger = logging.getLogger('dbbackup') + self.logger = logging.getLogger("dbbackup") - @patch('dbbackup.settings.SEND_EMAIL', True) + @patch("dbbackup.settings.SEND_EMAIL", True) def test_send_mail(self): # Test mail error msg = "Super msg" self.logger.error(msg) - self.assertEqual(mail.outbox[0].subject, '[dbbackup] ERROR: Super msg') + self.assertEqual(mail.outbox[0].subject, "[dbbackup] ERROR: Super msg") # Test don't mail below self.logger.warning(msg) self.assertEqual(len(mail.outbox), 1) - @patch('dbbackup.settings.SEND_EMAIL', False) + @patch("dbbackup.settings.SEND_EMAIL", False) def test_send_mail_is_false(self): msg = "Super msg" self.logger.error(msg) @@ -119,12 +119,12 @@ def test_send_mail_is_false(self): class MailEnabledFilterTest(TestCase): - @patch('dbbackup.settings.SEND_EMAIL', True) + @patch("dbbackup.settings.SEND_EMAIL", True) def test_filter_is_true(self): filter_ = log.MailEnabledFilter() - self.assertTrue(filter_.filter('foo')) + self.assertTrue(filter_.filter("foo")) - @patch('dbbackup.settings.SEND_EMAIL', False) + @patch("dbbackup.settings.SEND_EMAIL", False) def test_filter_is_false(self): filter_ = log.MailEnabledFilter() - self.assertFalse(filter_.filter('foo')) + self.assertFalse(filter_.filter("foo")) diff --git a/dbbackup/tests/test_storage.py b/dbbackup/tests/test_storage.py index f3f22ac7..e9b38558 100644 --- a/dbbackup/tests/test_storage.py +++ b/dbbackup/tests/test_storage.py @@ -5,31 +5,31 @@ from dbbackup.storage import Storage, get_storage from dbbackup.tests.utils import HANDLED_FILES, FakeStorage -DEFAULT_STORAGE_PATH = 'django.core.files.storage.FileSystemStorage' -STORAGE_OPTIONS = {'location': '/tmp'} +DEFAULT_STORAGE_PATH = "django.core.files.storage.FileSystemStorage" +STORAGE_OPTIONS = {"location": "/tmp"} class Get_StorageTest(TestCase): - @patch('dbbackup.settings.STORAGE', DEFAULT_STORAGE_PATH) - @patch('dbbackup.settings.STORAGE_OPTIONS', STORAGE_OPTIONS) + @patch("dbbackup.settings.STORAGE", DEFAULT_STORAGE_PATH) + @patch("dbbackup.settings.STORAGE_OPTIONS", STORAGE_OPTIONS) def test_func(self, *args): self.assertIsInstance(get_storage(), Storage) def test_set_path(self): - fake_storage_path = 'dbbackup.tests.utils.FakeStorage' + fake_storage_path = "dbbackup.tests.utils.FakeStorage" storage = get_storage(fake_storage_path) self.assertIsInstance(storage.storage, FakeStorage) - @patch('dbbackup.settings.STORAGE', DEFAULT_STORAGE_PATH) + @patch("dbbackup.settings.STORAGE", DEFAULT_STORAGE_PATH) def test_set_options(self, *args): storage = get_storage(options=STORAGE_OPTIONS) - self.assertEqual(storage.storage.__module__, 'django.core.files.storage') + self.assertEqual(storage.storage.__module__, "django.core.files.storage") class StorageTest(TestCase): def setUp(self): self.storageCls = Storage - self.storageCls.name = 'foo' + self.storageCls.name = "foo" self.storage = Storage() @@ -38,80 +38,78 @@ def setUp(self): HANDLED_FILES.clean() self.storage = get_storage() # foodb files - HANDLED_FILES['written_files'] += [ - (utils.filename_generate(ext, 'foodb'), None) for ext in - ('db', 'db.gz', 'db.gpg', 'db.gz.gpg') + HANDLED_FILES["written_files"] += [ + (utils.filename_generate(ext, "foodb"), None) + for ext in ("db", "db.gz", "db.gpg", "db.gz.gpg") ] - HANDLED_FILES['written_files'] += [ - (utils.filename_generate(ext, 'hamdb', 'fooserver'), None) for ext in - ('db', 'db.gz', 'db.gpg', 'db.gz.gpg') + HANDLED_FILES["written_files"] += [ + (utils.filename_generate(ext, "hamdb", "fooserver"), None) + for ext in ("db", "db.gz", "db.gpg", "db.gz.gpg") ] # Media file - HANDLED_FILES['written_files'] += [ - (utils.filename_generate(ext, None, None, 'media'), None) for ext in - ('tar', 'tar.gz', 'tar.gpg', 'tar.gz.gpg') + HANDLED_FILES["written_files"] += [ + (utils.filename_generate(ext, None, None, "media"), None) + for ext in ("tar", "tar.gz", "tar.gpg", "tar.gz.gpg") ] - HANDLED_FILES['written_files'] += [ - (utils.filename_generate(ext, 'bardb', 'barserver'), None) for ext in - ('db', 'db.gz', 'db.gpg', 'db.gz.gpg') + HANDLED_FILES["written_files"] += [ + (utils.filename_generate(ext, "bardb", "barserver"), None) + for ext in ("db", "db.gz", "db.gpg", "db.gz.gpg") ] # barserver files - HANDLED_FILES['written_files'] += [ - ('file_without_date', None) - ] + HANDLED_FILES["written_files"] += [("file_without_date", None)] def test_nofilter(self): files = self.storage.list_backups() - self.assertEqual(len(HANDLED_FILES['written_files']) - 1, len(files)) + self.assertEqual(len(HANDLED_FILES["written_files"]) - 1, len(files)) for file in files: - self.assertNotEqual('file_without_date', file) + self.assertNotEqual("file_without_date", file) def test_encrypted(self): files = self.storage.list_backups(encrypted=True) for file in files: - self.assertIn('.gpg', file) + self.assertIn(".gpg", file) def test_compressed(self): files = self.storage.list_backups(compressed=True) for file in files: - self.assertIn('.gz', file) + self.assertIn(".gz", file) def test_not_encrypted(self): files = self.storage.list_backups(encrypted=False) for file in files: - self.assertNotIn('.gpg', file) + self.assertNotIn(".gpg", file) def test_not_compressed(self): files = self.storage.list_backups(compressed=False) for file in files: - self.assertNotIn('.gz', file) + self.assertNotIn(".gz", file) def test_content_type_db(self): - files = self.storage.list_backups(content_type='db') + files = self.storage.list_backups(content_type="db") for file in files: - self.assertIn('.db', file) + self.assertIn(".db", file) def test_database(self): - files = self.storage.list_backups(database='foodb') + files = self.storage.list_backups(database="foodb") for file in files: - self.assertIn('foodb', file) - self.assertNotIn('bardb', file) - self.assertNotIn('hamdb', file) + self.assertIn("foodb", file) + self.assertNotIn("bardb", file) + self.assertNotIn("hamdb", file) def test_servername(self): - files = self.storage.list_backups(servername='fooserver') + files = self.storage.list_backups(servername="fooserver") for file in files: - self.assertIn('fooserver', file) - self.assertNotIn('barserver', file) - files = self.storage.list_backups(servername='barserver') + self.assertIn("fooserver", file) + self.assertNotIn("barserver", file) + files = self.storage.list_backups(servername="barserver") for file in files: - self.assertIn('barserver', file) - self.assertNotIn('fooserver', file) + self.assertIn("barserver", file) + self.assertNotIn("fooserver", file) def test_content_type_media(self): - files = self.storage.list_backups(content_type='media') + files = self.storage.list_backups(content_type="media") for file in files: - self.assertIn('.tar', file) + self.assertIn(".tar", file) # def test_servername(self): # files = self.storage.list_backups(servername='barserver') @@ -122,39 +120,46 @@ def test_content_type_media(self): class StorageGetLatestTest(TestCase): def setUp(self): self.storage = get_storage() - HANDLED_FILES['written_files'] = [(f, None) for f in [ - '2015-02-06-042810.bak', - '2015-02-07-042810.bak', - '2015-02-08-042810.bak', - ]] + HANDLED_FILES["written_files"] = [ + (f, None) + for f in [ + "2015-02-06-042810.bak", + "2015-02-07-042810.bak", + "2015-02-08-042810.bak", + ] + ] def tearDown(self): HANDLED_FILES.clean() def test_func(self): filename = self.storage.get_latest_backup() - self.assertEqual(filename, '2015-02-08-042810.bak') + self.assertEqual(filename, "2015-02-08-042810.bak") class StorageGetMostRecentTest(TestCase): def setUp(self): self.storage = get_storage() - HANDLED_FILES['written_files'] = [(f, None) for f in [ - '2015-02-06-042810.bak', - '2015-02-07-042810.bak', - '2015-02-08-042810.bak', - ]] + HANDLED_FILES["written_files"] = [ + (f, None) + for f in [ + "2015-02-06-042810.bak", + "2015-02-07-042810.bak", + "2015-02-08-042810.bak", + ] + ] def tearDown(self): HANDLED_FILES.clean() def test_func(self): filename = self.storage.get_older_backup() - self.assertEqual(filename, '2015-02-06-042810.bak') + self.assertEqual(filename, "2015-02-06-042810.bak") def keep_only_even_files(filename): from dbbackup.utils import filename_to_date + return filename_to_date(filename).day % 2 == 0 @@ -162,17 +167,20 @@ class StorageCleanOldBackupsTest(TestCase): def setUp(self): self.storage = get_storage() HANDLED_FILES.clean() - HANDLED_FILES['written_files'] = [(f, None) for f in [ - '2015-02-06-042810.bak', - '2015-02-07-042810.bak', - '2015-02-08-042810.bak', - ]] + HANDLED_FILES["written_files"] = [ + (f, None) + for f in [ + "2015-02-06-042810.bak", + "2015-02-07-042810.bak", + "2015-02-08-042810.bak", + ] + ] def test_func(self): self.storage.clean_old_backups(keep_number=1) - self.assertEqual(2, len(HANDLED_FILES['deleted_files'])) + self.assertEqual(2, len(HANDLED_FILES["deleted_files"])) - @patch('dbbackup.settings.CLEANUP_KEEP_FILTER', keep_only_even_files) + @patch("dbbackup.settings.CLEANUP_KEEP_FILTER", keep_only_even_files) def test_keep_filter(self): self.storage.clean_old_backups(keep_number=1) - self.assertListEqual(['2015-02-07-042810.bak'], HANDLED_FILES['deleted_files']) + self.assertListEqual(["2015-02-07-042810.bak"], HANDLED_FILES["deleted_files"]) diff --git a/dbbackup/tests/test_utils.py b/dbbackup/tests/test_utils.py index 4cc915f9..764e7cad 100644 --- a/dbbackup/tests/test_utils.py +++ b/dbbackup/tests/test_utils.py @@ -95,9 +95,9 @@ def func(): self.assertEqual(len(mail.outbox), 1) error_mail = mail.outbox[0] self.assertEqual(["foo@bar"], error_mail.to) - self.assertIn("Exception('Foo')", error_mail.subject) + self.assertIn('Exception("Foo")', error_mail.subject) if django.VERSION >= (1, 7): - self.assertIn("Exception('Foo')", error_mail.body) + self.assertIn('Exception("Foo")', error_mail.body) class Encrypt_FileTest(TestCase): diff --git a/dbbackup/tests/testapp/management/commands/feed.py b/dbbackup/tests/testapp/management/commands/feed.py index bd3d4b4f..e822e1ed 100644 --- a/dbbackup/tests/testapp/management/commands/feed.py +++ b/dbbackup/tests/testapp/management/commands/feed.py @@ -7,5 +7,5 @@ class Command(BaseCommand): help = "Count things" def handle(self, **options): - for st in 'abcde': + for st in "abcde": CharModel.objects.create(field=st) diff --git a/dbbackup/tests/testapp/migrations/0001_initial.py b/dbbackup/tests/testapp/migrations/0001_initial.py index 0189338c..b3fb64c0 100644 --- a/dbbackup/tests/testapp/migrations/0001_initial.py +++ b/dbbackup/tests/testapp/migrations/0001_initial.py @@ -3,36 +3,70 @@ class Migration(migrations.Migration): - dependencies = [ - ] + dependencies = [] operations = [ migrations.CreateModel( - name='CharModel', + name="CharModel", fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - ('field', models.CharField(max_length=10)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("field", models.CharField(max_length=10)), ], ), migrations.CreateModel( - name='FileModel', + name="FileModel", fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - ('field', models.FileField(upload_to='.')), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("field", models.FileField(upload_to=".")), ], ), migrations.CreateModel( - name='ForeignKeyModel', + name="ForeignKeyModel", fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True,)), - ('field', models.ForeignKey(to='testapp.CharModel', on_delete=models.CASCADE)), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ( + "field", + models.ForeignKey(to="testapp.CharModel", on_delete=models.CASCADE), + ), ], ), migrations.CreateModel( - name='ManyToManyModel', + name="ManyToManyModel", fields=[ - ('id', models.AutoField(verbose_name='ID', serialize=False, auto_created=True, primary_key=True)), - ('field', models.ManyToManyField(to='testapp.CharModel')), + ( + "id", + models.AutoField( + verbose_name="ID", + serialize=False, + auto_created=True, + primary_key=True, + ), + ), + ("field", models.ManyToManyField(to="testapp.CharModel")), ], ), ] diff --git a/dbbackup/tests/testapp/models.py b/dbbackup/tests/testapp/models.py index 11e676d4..fcb5efa1 100644 --- a/dbbackup/tests/testapp/models.py +++ b/dbbackup/tests/testapp/models.py @@ -1,8 +1,16 @@ from django.db import models -___all__ = ('CharModel', 'IntegerModel', 'TextModel', 'BooleanModel' - 'DateModel', 'DateTimeModel', 'ForeignKeyModel', 'ManyToManyModel', - 'FileModel', 'TestModel',) +___all__ = ( + "CharModel", + "IntegerModel", + "TextModel", + "BooleanModel" "DateModel", + "DateTimeModel", + "ForeignKeyModel", + "ManyToManyModel", + "FileModel", + "TestModel", +) class CharModel(models.Model): @@ -18,4 +26,4 @@ class ManyToManyModel(models.Model): class FileModel(models.Model): - field = models.FileField(upload_to='.') + field = models.FileField(upload_to=".") diff --git a/dbbackup/tests/utils.py b/dbbackup/tests/utils.py index 082a1da4..09ea789c 100644 --- a/dbbackup/tests/utils.py +++ b/dbbackup/tests/utils.py @@ -9,19 +9,33 @@ from dbbackup.db.base import get_connector -BASE_FILE = os.path.join(settings.BLOB_DIR, 'test.txt') -ENCRYPTED_FILE = os.path.join(settings.BLOB_DIR, 'test.txt.gpg') -COMPRESSED_FILE = os.path.join(settings.BLOB_DIR, 'test.txt.gz') -TARED_FILE = os.path.join(settings.BLOB_DIR, 'test.txt.tar') -ENCRYPTED_COMPRESSED_FILE = os.path.join(settings.BLOB_DIR, 'test.txt.gz.gpg') -TEST_DATABASE = {'ENGINE': 'django.db.backends.sqlite3', 'NAME': '/tmp/foo.db', 'USER': 'foo', 'PASSWORD': 'bar', 'HOST': 'foo', 'PORT': 122} -TEST_MONGODB = {'ENGINE': 'django_mongodb_engine', 'NAME': 'mongo_test', 'USER': 'foo', 'PASSWORD': 'bar', 'HOST': 'foo', 'PORT': 122} -TEST_DATABASE = settings.DATABASES['default'] - -GPG_PRIVATE_PATH = os.path.join(settings.BLOB_DIR, 'gpg/secring.gpg') -GPG_PUBLIC_PATH = os.path.join(settings.BLOB_DIR, 'gpg/pubring.gpg') -GPG_FINGERPRINT = '7438 8D4E 02AF C011 4E2F 1E79 F7D1 BBF0 1F63 FDE9' -DEV_NULL = open(os.devnull, 'w') +BASE_FILE = os.path.join(settings.BLOB_DIR, "test.txt") +ENCRYPTED_FILE = os.path.join(settings.BLOB_DIR, "test.txt.gpg") +COMPRESSED_FILE = os.path.join(settings.BLOB_DIR, "test.txt.gz") +TARED_FILE = os.path.join(settings.BLOB_DIR, "test.txt.tar") +ENCRYPTED_COMPRESSED_FILE = os.path.join(settings.BLOB_DIR, "test.txt.gz.gpg") +TEST_DATABASE = { + "ENGINE": "django.db.backends.sqlite3", + "NAME": "/tmp/foo.db", + "USER": "foo", + "PASSWORD": "bar", + "HOST": "foo", + "PORT": 122, +} +TEST_MONGODB = { + "ENGINE": "django_mongodb_engine", + "NAME": "mongo_test", + "USER": "foo", + "PASSWORD": "bar", + "HOST": "foo", + "PORT": 122, +} +TEST_DATABASE = settings.DATABASES["default"] + +GPG_PRIVATE_PATH = os.path.join(settings.BLOB_DIR, "gpg/secring.gpg") +GPG_PUBLIC_PATH = os.path.join(settings.BLOB_DIR, "gpg/pubring.gpg") +GPG_FINGERPRINT = "7438 8D4E 02AF C011 4E2F 1E79 F7D1 BBF0 1F63 FDE9" +DEV_NULL = open(os.devnull, "w") class handled_files(dict): @@ -36,18 +50,18 @@ def __init__(self): self.clean() def clean(self): - self['written_files'] = [] - self['deleted_files'] = [] + self["written_files"] = [] + self["deleted_files"] = [] HANDLED_FILES = handled_files() class FakeStorage(Storage): - name = 'FakeStorage' + name = "FakeStorage" def exists(self, name): - return name in HANDLED_FILES['written_files'] + return name in HANDLED_FILES["written_files"] def get_available_name(self, name, max_length=None): return name[:max_length] @@ -56,47 +70,47 @@ def get_valid_name(self, name): return name def listdir(self, path): - return ([], [f[0] for f in HANDLED_FILES['written_files']]) + return ([], [f[0] for f in HANDLED_FILES["written_files"]]) def accessed_time(self, name): return timezone.now() + created_time = modified_time = accessed_time - def _open(self, name, mode='rb'): - file_ = [f[1] for f in HANDLED_FILES['written_files'] - if f[0] == name][0] + def _open(self, name, mode="rb"): + file_ = [f[1] for f in HANDLED_FILES["written_files"] if f[0] == name][0] file_.seek(0) return file_ def _save(self, name, content): - HANDLED_FILES['written_files'].append((name, File(content))) + HANDLED_FILES["written_files"].append((name, File(content))) return name def delete(self, name): - HANDLED_FILES['deleted_files'].append(name) + HANDLED_FILES["deleted_files"].append(name) def clean_gpg_keys(): with contextlib.suppress(Exception): - cmd = ("gpg --batch --yes --delete-key '%s'" % GPG_FINGERPRINT) + cmd = "gpg --batch --yes --delete-key '%s'" % GPG_FINGERPRINT subprocess.call(cmd, stdout=DEV_NULL, stderr=DEV_NULL) with contextlib.suppress(Exception): - cmd = ("gpg --batch --yes --delete-secrect-key '%s'" % GPG_FINGERPRINT) + cmd = "gpg --batch --yes --delete-secrect-key '%s'" % GPG_FINGERPRINT subprocess.call(cmd, stdout=DEV_NULL, stderr=DEV_NULL) def add_private_gpg(): - cmd = f'gpg --import {GPG_PRIVATE_PATH}'.split() + cmd = f"gpg --import {GPG_PRIVATE_PATH}".split() subprocess.call(cmd, stdout=DEV_NULL, stderr=DEV_NULL) def add_public_gpg(): - cmd = f'gpg --import {GPG_PUBLIC_PATH}'.split() + cmd = f"gpg --import {GPG_PUBLIC_PATH}".split() subprocess.call(cmd, stdout=DEV_NULL, stderr=DEV_NULL) def callable_for_filename_template(datetime, **kwargs): - return f'{datetime}_foo' + return f"{datetime}_foo" def get_dump(database=TEST_DATABASE): diff --git a/dbbackup/utils.py b/dbbackup/utils.py index ca7ec3d1..30aec8d3 100644 --- a/dbbackup/utils.py +++ b/dbbackup/utils.py @@ -27,21 +27,21 @@ from . import settings FAKE_HTTP_REQUEST = HttpRequest() -FAKE_HTTP_REQUEST.META['SERVER_NAME'] = '' -FAKE_HTTP_REQUEST.META['SERVER_PORT'] = '' -FAKE_HTTP_REQUEST.META['HTTP_HOST'] = settings.HOSTNAME -FAKE_HTTP_REQUEST.path = '/DJANGO-DBBACKUP-EXCEPTION' +FAKE_HTTP_REQUEST.META["SERVER_NAME"] = "" +FAKE_HTTP_REQUEST.META["SERVER_PORT"] = "" +FAKE_HTTP_REQUEST.META["HTTP_HOST"] = settings.HOSTNAME +FAKE_HTTP_REQUEST.path = "/DJANGO-DBBACKUP-EXCEPTION" BYTES = ( - ('PiB', 1125899906842624.0), - ('TiB', 1099511627776.0), - ('GiB', 1073741824.0), - ('MiB', 1048576.0), - ('KiB', 1024.0), - ('B', 1.0) + ("PiB", 1125899906842624.0), + ("TiB", 1099511627776.0), + ("GiB", 1073741824.0), + ("MiB", 1048576.0), + ("KiB", 1024.0), + ("B", 1.0), ) -REG_FILENAME_CLEAN = re.compile(r'-+') +REG_FILENAME_CLEAN = re.compile(r"-+") class EncryptionError(Exception): @@ -66,11 +66,11 @@ def bytes_to_str(byteVal, decimals=1): :rtype: str """ for unit, byte in BYTES: - if (byteVal >= byte): + if byteVal >= byte: if decimals == 0: - return f'{int(round(byteVal / byte, 0))} {unit}' - return f'{round(byteVal / byte, decimals)} {unit}' - return f'{byteVal} B' + return f"{int(round(byteVal / byte, 0))} {unit}" + return f"{round(byteVal / byte, decimals)} {unit}" + return f"{byteVal} B" def handle_size(filehandle): @@ -87,15 +87,22 @@ def handle_size(filehandle): return bytes_to_str(filehandle.tell()) -def mail_admins(subject, message, fail_silently=False, connection=None, - html_message=None): +def mail_admins( + subject, message, fail_silently=False, connection=None, html_message=None +): """Sends a message to the admins, as defined by the DBBACKUP_ADMINS setting.""" if not settings.ADMINS: return - mail = EmailMultiAlternatives(f'{settings.EMAIL_SUBJECT_PREFIX}{subject}', message, settings.SERVER_EMAIL, [a[1] for a in settings.ADMINS], connection=connection) + mail = EmailMultiAlternatives( + f"{settings.EMAIL_SUBJECT_PREFIX}{subject}", + message, + settings.SERVER_EMAIL, + [a[1] for a in settings.ADMINS], + connection=connection, + ) if html_message: - mail.attach_alternative(html_message, 'text/html') + mail.attach_alternative(html_message, "text/html") mail.send(fail_silently=fail_silently) @@ -106,19 +113,21 @@ def email_uncaught_exception(func): (``settings.ADMINS`` if not defined). The message contains a traceback of error. """ + @wraps(func) def wrapper(*args, **kwargs): try: func(*args, **kwargs) except Exception: - logger = logging.getLogger('dbbackup') + logger = logging.getLogger("dbbackup") exc_type, exc_value, tb = sys.exc_info() - tb_str = ''.join(traceback.format_tb(tb)) - msg = f'{exc_type.__name__}: {exc_value}\n{tb_str}' + tb_str = "".join(traceback.format_tb(tb)) + msg = f"{exc_type.__name__}: {exc_value}\n{tb_str}" logger.error(msg) raise finally: connection.close() + return wrapper @@ -137,10 +146,10 @@ def create_spooled_temporary_file(filepath=None, fileobj=None): :rtype: :class:`tempfile.SpooledTemporaryFile` """ spooled_file = tempfile.SpooledTemporaryFile( - max_size=settings.TMP_FILE_MAX_SIZE, - dir=settings.TMP_DIR) + max_size=settings.TMP_FILE_MAX_SIZE, dir=settings.TMP_DIR + ) if filepath: - fileobj = open(filepath, 'r+b') + fileobj = open(filepath, "r+b") if fileobj is not None: fileobj.seek(0) copyfileobj(fileobj, spooled_file, settings.TMP_FILE_READ_SIZE) @@ -161,20 +170,24 @@ def encrypt_file(inputfile, filename): :rtype: :class:`tempfile.SpooledTemporaryFile`, ``str`` """ import gnupg + tempdir = tempfile.mkdtemp(dir=settings.TMP_DIR) try: - filename = f'{filename}.gpg' + filename = f"{filename}.gpg" filepath = os.path.join(tempdir, filename) try: inputfile.seek(0) always_trust = settings.GPG_ALWAYS_TRUST g = gnupg.GPG() - result = g.encrypt_file(inputfile, output=filepath, - recipients=settings.GPG_RECIPIENT, - always_trust=always_trust) + result = g.encrypt_file( + inputfile, + output=filepath, + recipients=settings.GPG_RECIPIENT, + always_trust=always_trust, + ) inputfile.close() if not result: - msg = f'Encryption failed; status: {result.status}' + msg = f"Encryption failed; status: {result.status}" raise EncryptionError(msg) return create_spooled_temporary_file(filepath), filename finally: @@ -205,19 +218,20 @@ def unencrypt_file(inputfile, filename, passphrase=None): import gnupg def get_passphrase(passphrase=passphrase): - return passphrase or getpass('Input Passphrase: ') or None + return passphrase or getpass("Input Passphrase: ") or None temp_dir = tempfile.mkdtemp(dir=settings.TMP_DIR) try: - new_basename = os.path.basename(filename).replace('.gpg', '') + new_basename = os.path.basename(filename).replace(".gpg", "") temp_filename = os.path.join(temp_dir, new_basename) try: inputfile.seek(0) g = gnupg.GPG() - result = g.decrypt_file(file=inputfile, passphrase=get_passphrase(), - output=temp_filename) + result = g.decrypt_file( + file=inputfile, passphrase=get_passphrase(), output=temp_filename + ) if not result: - raise DecryptionError('Decryption failed; status: %s' % result.status) + raise DecryptionError("Decryption failed; status: %s" % result.status) outputfile = create_spooled_temporary_file(temp_filename) finally: if os.path.exists(temp_filename): @@ -241,7 +255,7 @@ def compress_file(inputfile, filename): :rtype: :class:`tempfile.SpooledTemporaryFile`, ``str`` """ outputfile = create_spooled_temporary_file() - new_filename = f'{filename}.gz' + new_filename = f"{filename}.gz" zipfile = gzip.GzipFile(filename=filename, fileobj=outputfile, mode="wb") try: inputfile.seek(0) @@ -269,7 +283,7 @@ def uncompress_file(inputfile, filename): outputfile = create_spooled_temporary_file(fileobj=zipfile) finally: zipfile.close() - new_basename = os.path.basename(filename).replace('.gz', '') + new_basename = os.path.basename(filename).replace(".gz", "") return outputfile, new_basename @@ -289,30 +303,30 @@ def timestamp(value): def filename_details(filepath): # TODO: What was this function made for ? - return '' + return "" PATTERN_MATCHNG = ( - ('%a', r'[A-Z][a-z]+'), - ('%A', r'[A-Z][a-z]+'), - ('%w', r'\d'), - ('%d', r'\d{2}'), - ('%b', r'[A-Z][a-z]+'), - ('%B', r'[A-Z][a-z]+'), - ('%m', r'\d{2}'), - ('%y', r'\d{2}'), - ('%Y', r'\d{4}'), - ('%H', r'\d{2}'), - ('%I', r'\d{2}'), + ("%a", r"[A-Z][a-z]+"), + ("%A", r"[A-Z][a-z]+"), + ("%w", r"\d"), + ("%d", r"\d{2}"), + ("%b", r"[A-Z][a-z]+"), + ("%B", r"[A-Z][a-z]+"), + ("%m", r"\d{2}"), + ("%y", r"\d{2}"), + ("%Y", r"\d{4}"), + ("%H", r"\d{2}"), + ("%I", r"\d{2}"), # ('%p', r'(?AM|PM|am|pm)'), - ('%M', r'\d{2}'), - ('%S', r'\d{2}'), - ('%f', r'\d{6}'), + ("%M", r"\d{2}"), + ("%S", r"\d{2}"), + ("%f", r"\d{6}"), # ('%z', r'\+\d{4}'), # ('%Z', r'(?|UTC|EST|CST)'), - ('%j', r'\d{3}'), - ('%U', r'\d{2}'), - ('%W', r'\d{2}'), + ("%j", r"\d{3}"), + ("%U", r"\d{2}"), + ("%W", r"\d{2}"), # ('%c', r'[A-Z][a-z]+ [A-Z][a-z]{2} \d{2} \d{2}:\d{2}:\d{2} \d{4}'), # ('%x', r'd{2}/d{2}/d{4}'), # ('%X', r'd{2}:d{2}:d{2}'), @@ -333,7 +347,7 @@ def datefmt_to_regex(datefmt): new_string = datefmt for pat, reg in PATTERN_MATCHNG: new_string = new_string.replace(pat, reg) - return re.compile(f'({new_string})') + return re.compile(f"({new_string})") def filename_to_datestring(filename, datefmt=None): @@ -372,7 +386,7 @@ def filename_to_date(filename, datefmt=None): def filename_generate( - extension, database_name='', servername=None, content_type='db', wildcard=None + extension, database_name="", servername=None, content_type="db", wildcard=None ): """ Create a new backup filename. @@ -395,30 +409,30 @@ def filename_generate( :returns: Computed file name :rtype: ``str` """ - if content_type == 'db': - if '/' in database_name: + if content_type == "db": + if "/" in database_name: database_name = os.path.basename(database_name) - if '.' in database_name: - database_name = database_name.split('.')[0] + if "." in database_name: + database_name = database_name.split(".")[0] template = settings.FILENAME_TEMPLATE - elif content_type == 'media': + elif content_type == "media": template = settings.MEDIA_FILENAME_TEMPLATE else: template = settings.FILENAME_TEMPLATE params = { - 'servername': servername or settings.HOSTNAME, - 'datetime': wildcard or datetime.now().strftime(settings.DATE_FORMAT), - 'databasename': database_name, - 'extension': extension, - 'content_type': content_type + "servername": servername or settings.HOSTNAME, + "datetime": wildcard or datetime.now().strftime(settings.DATE_FORMAT), + "databasename": database_name, + "extension": extension, + "content_type": content_type, } if callable(template): filename = template(**params) else: filename = template.format(**params) - filename = REG_FILENAME_CLEAN.sub('-', filename) - filename = filename[1:] if filename.startswith('-') else filename + filename = REG_FILENAME_CLEAN.sub("-", filename) + filename = filename[1:] if filename.startswith("-") else filename return filename diff --git a/docs/conf.py b/docs/conf.py index 7e400a82..5b7df908 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -18,46 +18,42 @@ # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the # documentation root, use os.path.abspath to make it absolute, like shown here. -#sys.path.insert(0, os.path.abspath('.')) +# sys.path.insert(0, os.path.abspath('.')) # -- General configuration ----------------------------------------------------- # If your documentation needs a minimal Sphinx version, state it here. -#needs_sphinx = '1.0' +# needs_sphinx = '1.0' # Add any Sphinx extension module names here, as strings. They can be extensions # coming with Sphinx (named 'sphinx.ext.*') or your custom ones. extensions = [ - 'djcommanddoc', + "djcommanddoc", ] # Add any paths that contain templates here, relative to this directory. -templates_path = ['_templates'] +templates_path = ["_templates"] # The suffix of source filenames. -source_suffix = '.rst' +source_suffix = ".rst" # The encoding of source files. -#source_encoding = 'utf-8-sig' +# source_encoding = 'utf-8-sig' # The master toctree document. -master_doc = 'index' +master_doc = "index" # General information about the project. -project = 'django-dbbackup' -copyright = '2016, Michael Shepanski' +project = "django-dbbackup" +copyright = "2016, Michael Shepanski" # basepath -path = os.path.dirname( - os.path.dirname( - os.path.abspath(__file__) - ) -) +path = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path = [path] + sys.path -sys.path = [os.path.join(path, 'dbbackup')] + sys.path +sys.path = [os.path.join(path, "dbbackup")] + sys.path -os.environ['DJANGO_SETTINGS_MODULE'] = 'dbbackup.tests.settings' +os.environ["DJANGO_SETTINGS_MODULE"] = "dbbackup.tests.settings" # The version info for the project you're documenting, acts as replacement for @@ -71,119 +67,119 @@ # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages. -#language = None +# language = None # There are two options for replacing |today|: either, you set today to some # non-false value, then it is used: -#today = '' +# today = '' # Else, today_fmt is used as the format for a strftime call. -#today_fmt = '%B %d, %Y' +# today_fmt = '%B %d, %Y' # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build'] +exclude_patterns = ["_build"] # The reST default role (used for this markup: `text`) to use for all documents. -#default_role = None +# default_role = None # If true, '()' will be appended to :func: etc. cross-reference text. -#add_function_parentheses = True +# add_function_parentheses = True # If true, the current module name will be prepended to all description # unit titles (such as .. function::). -#add_module_names = True +# add_module_names = True # If true, sectionauthor and moduleauthor directives will be shown in the # output. They are ignored by default. -#show_authors = False +# show_authors = False # The name of the Pygments (syntax highlighting) style to use. -pygments_style = 'sphinx' +pygments_style = "sphinx" # A list of ignored prefixes for module index sorting. -#modindex_common_prefix = [] +# modindex_common_prefix = [] # -- Options for HTML output --------------------------------------------------- -on_rtd = os.environ.get('READTHEDOCS', None) == 'True' +on_rtd = os.environ.get("READTHEDOCS", None) == "True" if on_rtd: - html_theme = 'default' + html_theme = "default" else: - html_theme = 'nature' + html_theme = "nature" # Theme options are theme-specific and customize the look and feel of a theme # further. For a list of options available for each theme, see the # documentation. -#html_theme_options = {} +# html_theme_options = {} # Add any paths that contain custom themes here, relative to this directory. -#html_theme_path = [] +# html_theme_path = [] # The name for this set of Sphinx documents. If None, it defaults to # " v documentation". -#html_title = None +# html_title = None # A shorter title for the navigation bar. Default is the same as html_title. -#html_short_title = None +# html_short_title = None # The name of an image file (relative to this directory) to place at the top # of the sidebar. -#html_logo = None +# html_logo = None # The name of an image file (within the static path) to use as favicon of the # docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 # pixels large. -#html_favicon = None +# html_favicon = None # Add any paths that contain custom static files (such as style sheets) here, # relative to this directory. They are copied after the builtin static files, # so a file named "default.css" will overwrite the builtin "default.css". -html_static_path = ['_static'] +html_static_path = ["_static"] # If not '', a 'Last updated on:' timestamp is inserted at every page bottom, # using the given strftime format. -#html_last_updated_fmt = '%b %d, %Y' +# html_last_updated_fmt = '%b %d, %Y' # If true, SmartyPants will be used to convert quotes and dashes to # typographically correct entities. -#html_use_smartypants = True +# html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +# html_sidebars = {} # Additional templates that should be rendered to pages, maps page names to # template names. -#html_additional_pages = {} +# html_additional_pages = {} # If false, no module index is generated. -#html_domain_indices = True +# html_domain_indices = True # If false, no index is generated. -#html_use_index = True +# html_use_index = True # If true, the index is split into individual pages for each letter. -#html_split_index = False +# html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +# html_show_sourcelink = True # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. -#html_show_sphinx = True +# html_show_sphinx = True # If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. -#html_show_copyright = True +# html_show_copyright = True # If true, an OpenSearch description file will be output, and all pages will # contain a tag referring to it. The value of this option must be the # base URL from which the finished HTML is served. -#html_use_opensearch = '' +# html_use_opensearch = '' # This is the file name suffix for HTML files (e.g. ".xhtml"). -#html_file_suffix = None +# html_file_suffix = None # Output file base name for HTML help builder. -htmlhelp_basename = 'django-dbbackupdoc' +htmlhelp_basename = "django-dbbackupdoc" # -- Options for LaTeX output -------------------------------------------------- @@ -191,10 +187,8 @@ latex_elements = { # The paper size ('letterpaper' or 'a4paper'). # 'papersize': 'letterpaper', - # The font size ('10pt', '11pt' or '12pt'). # 'pointsize': '10pt', - # Additional stuff for the LaTeX preamble. # 'preamble': '', } @@ -202,29 +196,34 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'django-dbbackup.tex', 'django-dbbackup Documentation', - 'Michael Shepanski', 'manual'), + ( + "index", + "django-dbbackup.tex", + "django-dbbackup Documentation", + "Michael Shepanski", + "manual", + ), ] # The name of an image file (relative to this directory) to place at the top of # the title page. -#latex_logo = None +# latex_logo = None # For "manual" documents, if this is true, then toplevel headings are parts, # not chapters. -#latex_use_parts = False +# latex_use_parts = False # If true, show page references after internal links. -#latex_show_pagerefs = False +# latex_show_pagerefs = False # If true, show URL addresses after external links. -#latex_show_urls = False +# latex_show_urls = False # Documents to append as an appendix to all manuals. -#latex_appendices = [] +# latex_appendices = [] # If false, no module index is generated. -#latex_domain_indices = True +# latex_domain_indices = True # -- Options for manual page output -------------------------------------------- @@ -232,12 +231,17 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'django-dbbackup', 'django-dbbackup Documentation', - ['Michael Shepanski'], 1) + ( + "index", + "django-dbbackup", + "django-dbbackup Documentation", + ["Michael Shepanski"], + 1, + ) ] # If true, show URL addresses after external links. -#man_show_urls = False +# man_show_urls = False # -- Options for Texinfo output ------------------------------------------------ @@ -246,16 +250,22 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'django-dbbackup', 'django-dbbackup Documentation', - 'Michael Shepanski', 'django-dbbackup', 'One line description of project.', - 'Miscellaneous'), + ( + "index", + "django-dbbackup", + "django-dbbackup Documentation", + "Michael Shepanski", + "django-dbbackup", + "One line description of project.", + "Miscellaneous", + ), ] # Documents to append as an appendix to all manuals. -#texinfo_appendices = [] +# texinfo_appendices = [] # If false, no module index is generated. -#texinfo_domain_indices = True +# texinfo_domain_indices = True # How to display URL addresses: 'footnote', 'no', or 'inline'. -#texinfo_show_urls = 'footnote' +# texinfo_show_urls = 'footnote' diff --git a/runtests.py b/runtests.py index 8199497f..027821af 100755 --- a/runtests.py +++ b/runtests.py @@ -8,10 +8,11 @@ def main(argv=None): - os.environ['DJANGO_SETTINGS_MODULE'] = 'dbbackup.tests.settings' + os.environ["DJANGO_SETTINGS_MODULE"] = "dbbackup.tests.settings" argv = argv or [] if len(argv) <= 1: from django.test.utils import get_runner + if django.VERSION >= (1, 7): django.setup() TestRunner = get_runner(settings) @@ -21,5 +22,5 @@ def main(argv=None): execute_from_command_line(argv) -if __name__ == '__main__': +if __name__ == "__main__": sys.exit(bool(main(sys.argv))) diff --git a/setup.py b/setup.py index 3d4e6cb1..9231033e 100644 --- a/setup.py +++ b/setup.py @@ -18,7 +18,7 @@ def get_test_requirements(): setup( - name='django-dbbackup', + name="django-dbbackup", version="4.0.0b0", description="Management commands to help backup and restore a project database and media.", author="Archmonger", @@ -27,8 +27,8 @@ def get_test_requirements(): python_requires=">=3.6", install_requires=get_requirements(), tests_require=get_test_requirements(), - license='BSD', - url='https://github.com/django-dbbackup/django-dbbackup', + license="BSD", + url="https://github.com/django-dbbackup/django-dbbackup", keywords=[ "django", "database", @@ -40,26 +40,26 @@ def get_test_requirements(): ], packages=find_packages(), classifiers=[ - 'Development Status :: 4 - Beta', - 'Environment :: Web Environment', - 'Environment :: Console', - 'Framework :: Django :: 2.2', - 'Framework :: Django :: 3.2', - 'Framework :: Django :: 4.0', - 'Intended Audience :: Developers', - 'Intended Audience :: System Administrators', - 'License :: OSI Approved :: BSD License', - 'Natural Language :: English', - 'Operating System :: OS Independent', - 'Programming Language :: Python', - 'Programming Language :: Python :: 3.6', - 'Programming Language :: Python :: 3.7', - 'Programming Language :: Python :: 3.8', - 'Programming Language :: Python :: 3.9', - 'Programming Language :: Python :: 3.10', - 'Topic :: Database', - 'Topic :: System :: Archiving', - 'Topic :: System :: Archiving :: Backup', - 'Topic :: System :: Archiving :: Compression' + "Development Status :: 4 - Beta", + "Environment :: Web Environment", + "Environment :: Console", + "Framework :: Django :: 2.2", + "Framework :: Django :: 3.2", + "Framework :: Django :: 4.0", + "Intended Audience :: Developers", + "Intended Audience :: System Administrators", + "License :: OSI Approved :: BSD License", + "Natural Language :: English", + "Operating System :: OS Independent", + "Programming Language :: Python", + "Programming Language :: Python :: 3.6", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Topic :: Database", + "Topic :: System :: Archiving", + "Topic :: System :: Archiving :: Backup", + "Topic :: System :: Archiving :: Compression", ], ) From 380378a9aa1d19a696d09c06cffc106f4d048523 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 29 Apr 2022 02:35:57 -0700 Subject: [PATCH 15/31] add leniency to codecov --- codecov.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 codecov.yml diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000..9d979f62 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,11 @@ +--- +coverage: + status: + project: + default: + target: 80% # the required coverage value + threshold: 0.5% # the leniency in hitting the target + patch: + default: + target: 80% # the required coverage value + threshold: 0.5% # the leniency in hitting the target From 66eadd15278b7f153766b2cb7851141ad74b52f6 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 29 Apr 2022 02:56:07 -0700 Subject: [PATCH 16/31] remove twine from dev dependencies --- requirements/dev.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 10d8bfcb..efc4bd19 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,4 +2,3 @@ black flake8 pylint rope -twine From 29daf39a35e9b472fc7a8f6019d829e4534e7a7e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 29 Apr 2022 23:17:27 +0000 Subject: [PATCH 17/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- dbbackup/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/dbbackup/__init__.py b/dbbackup/__init__.py index 06d0b9e3..616ceaae 100644 --- a/dbbackup/__init__.py +++ b/dbbackup/__init__.py @@ -2,7 +2,6 @@ import django - VERSION = (4, 0, 0) """The X.Y.Z version. Needed for `docs/conf.py`.""" VERSION_TAG = "a1" From dd44c059a355b60ac8dc8d28d209784f4ffd14d7 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 29 Apr 2022 16:20:13 -0700 Subject: [PATCH 18/31] set black target version to py36 --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 7507676c..c506dea5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,3 @@ [tool.black] -target-version = ['py39'] +target-version = ['py36'] extend-exclude = 'migrations' From 628f719045e8f45a796ffe02344051fd36c81da8 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 29 Apr 2022 16:24:40 -0700 Subject: [PATCH 19/31] use tox.ini for flake8 config --- .flake8 | 5 ----- setup.cfg | 4 ---- tox.ini | 5 +++++ 3 files changed, 5 insertions(+), 9 deletions(-) delete mode 100644 .flake8 delete mode 100644 setup.cfg diff --git a/.flake8 b/.flake8 deleted file mode 100644 index 7268a896..00000000 --- a/.flake8 +++ /dev/null @@ -1,5 +0,0 @@ -[flake8] -ignore = - E501, - E203, - W503 diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 72e9eae7..00000000 --- a/setup.cfg +++ /dev/null @@ -1,4 +0,0 @@ -[flake8] -max-line-length = 100 -include = dbbackup -exclude = tests,settings,venv,docs diff --git a/tox.ini b/tox.ini index 7021ddb3..39e9265e 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,11 @@ [tox] envlist = py{36,37,38,39}-django22,py{36,37,38,39,310}-django{32,40,master},lint,docs,functional +[flake8] +include = dbbackup +exclude = tests, settings, venv, docs +ignore = E501, E203, W503 + [testenv] passenv = * setenv = From 4ea6fc6b458d649c87d74716796729897457fbc2 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 29 Apr 2022 16:30:06 -0700 Subject: [PATCH 20/31] add isort to dev dependencies --- requirements/dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements/dev.txt b/requirements/dev.txt index efc4bd19..278e5858 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -2,3 +2,4 @@ black flake8 pylint rope +isort From c281b57e3da2c9a4d18b386e6a9a2c848f91d80a Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 29 Apr 2022 16:30:35 -0700 Subject: [PATCH 21/31] use lowercase name for django --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 83d6b43d..8053d290 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,2 @@ -Django>=2.2 +django>=2.2 pytz From 242a29dd72cb1339cf124b9cd039373744f83ee4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 29 Apr 2022 23:30:58 +0000 Subject: [PATCH 22/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- requirements/dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/dev.txt b/requirements/dev.txt index 278e5858..897d41be 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,5 +1,5 @@ black flake8 +isort pylint rope -isort From 3194569acd5323c17bf17c73706b388b473f7043 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Fri, 29 Apr 2022 16:33:02 -0700 Subject: [PATCH 23/31] use pyproject for isort config --- .isort.cfg | 3 --- pyproject.toml | 4 ++++ 2 files changed, 4 insertions(+), 3 deletions(-) delete mode 100644 .isort.cfg diff --git a/.isort.cfg b/.isort.cfg deleted file mode 100644 index 437feb47..00000000 --- a/.isort.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[isort] -profile=black -skip=migrations diff --git a/pyproject.toml b/pyproject.toml index c506dea5..8ddb24b4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,3 +1,7 @@ [tool.black] target-version = ['py36'] extend-exclude = 'migrations' + +[tool.isort] +profile = 'black' +skip = 'migrations' From 2a9e1b757c3c8e8ab1db821ece8d22efd5c68569 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Sat, 30 Apr 2022 01:23:38 -0700 Subject: [PATCH 24/31] set default auto field --- dbbackup/settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dbbackup/settings.py b/dbbackup/settings.py index d7a3689b..eec21734 100644 --- a/dbbackup/settings.py +++ b/dbbackup/settings.py @@ -44,6 +44,8 @@ CUSTOM_CONNECTOR_MAPPING = getattr(settings, "DBBACKUP_CONNECTOR_MAPPING", {}) +DEFAULT_AUTO_FIELD = "django.db.models.AutoField" + # Mail SEND_EMAIL = getattr(settings, "DBBACKUP_SEND_EMAIL", True) SERVER_EMAIL = getattr(settings, "DBBACKUP_SERVER_EMAIL", settings.SERVER_EMAIL) From f971a222035b608b68482fa9d4ea4ec6d0f02cdd Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Sat, 30 Apr 2022 01:25:27 -0700 Subject: [PATCH 25/31] clean up build commands --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 055132e8..b369cf59 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,8 +22,8 @@ jobs: - name: Install dependencies run: | python -m pip install --upgrade pip - pip install -r requirements/tests.txt - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + python -m pip install -r requirements/tests.txt + python -m pip install -r requirements.txt - name: Linting run: flake8 # Environments are selected using tox-gh-actions configuration in tox.ini. From 40a44f90bf8d75b6606992291a027343bbda52eb Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Mon, 2 May 2022 22:29:28 -0700 Subject: [PATCH 26/31] recreate tox env --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b369cf59..4ceb22ac 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -28,7 +28,7 @@ jobs: run: flake8 # Environments are selected using tox-gh-actions configuration in tox.ini. - name: Test with tox - run: tox + run: tox -r - name: Upload coverage uses: codecov/codecov-action@v1 with: From 37b982d954aaef2ddd0424bb1828c26f59a199db Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Tue, 3 May 2022 00:20:30 -0700 Subject: [PATCH 27/31] attempt fix for tests --- MANIFEST.in | 1 + README.rst | 2 +- VERSION | 1 + dbbackup/__init__.py | 17 +++++++++++------ docs/conf.py | 2 +- setup.py | 5 +++-- tox.ini | 8 ++++---- 7 files changed, 22 insertions(+), 14 deletions(-) create mode 100644 VERSION diff --git a/MANIFEST.in b/MANIFEST.in index 4288d748..d4f465a7 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,4 +2,5 @@ recursive-include requirements * include requirements.txt include README.rst include LICENSE.txt +include VERSION recursive-include dbbackup/tests/testapp/blobs/ *.gpg *.txt *.gz *.tar diff --git a/README.rst b/README.rst index 249b5103..2ff4efb6 100644 --- a/README.rst +++ b/README.rst @@ -188,7 +188,7 @@ Django3.2 you would run: :: - tox -e py3.9-django3.2 + tox -e py39-django32 The available test environments can be found in ``tox.ini``. diff --git a/VERSION b/VERSION new file mode 100644 index 00000000..a23faae1 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +4.0.0a1 diff --git a/dbbackup/__init__.py b/dbbackup/__init__.py index 616ceaae..994a6ccc 100644 --- a/dbbackup/__init__.py +++ b/dbbackup/__init__.py @@ -1,13 +1,18 @@ """Management commands to help backup and restore a project database and media""" +from pathlib import Path + import django -VERSION = (4, 0, 0) -"""The X.Y.Z version. Needed for `docs/conf.py`.""" -VERSION_TAG = "a1" -"""The alpha/beta/rc tag for the version. For example 'b0'.""" -__version__ = ".".join(map(str, VERSION)) + VERSION_TAG -"""The full version, including alpha/beta/rc tags.""" +project_dir = Path(__file__).parent +with (project_dir.parent / "VERSION").open() as f: + __version__ = f.read().strip() + """The full version, including alpha/beta/rc tags.""" + +VERSION = (x, y, z) = __version__.split(".") +VERSION = ".".join(VERSION[:2]) +"""The X.Y version. Needed for `docs/conf.py`.""" + if django.VERSION < (3, 2): default_app_config = "dbbackup.apps.DbbackupConfig" diff --git a/docs/conf.py b/docs/conf.py index 5b7df908..02c855e2 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -61,7 +61,7 @@ # built documents. # # The short X.Y version. -version = ".".join([str(i) for i in dbbackup.VERSION[:-1]]) +version = dbbackup.VERSION # The full version, including alpha/beta/rc tags. release = dbbackup.__version__ diff --git a/setup.py b/setup.py index db9b351f..1163f414 100644 --- a/setup.py +++ b/setup.py @@ -4,9 +4,10 @@ from setuptools import find_packages, setup -import dbbackup project_dir = Path(__file__).parent +with (project_dir / "VERSION").open() as f: + version = f.read().strip() def get_requirements(): @@ -21,7 +22,7 @@ def get_test_requirements(): setup( name="django-dbbackup", - version=dbbackup.__version__, + version=version, description="Management commands to help backup and restore a project database and media.", author="Archmonger", author_email="archiethemonger@gmail.com", diff --git a/tox.ini b/tox.ini index 39e9265e..aac59d1e 100644 --- a/tox.ini +++ b/tox.ini @@ -12,9 +12,9 @@ setenv = PYTHONDONTWRITEBYTECODE=1 deps = -rrequirements/tests.txt - django22: Django>=2.2,<2.3 - django32: Django>=3.2,<3.3 - django40: Django>=4.0,<4.1 + django22: django>=2.2,<2.3 + django32: django>=3.2,<3.3 + django40: django>=4.0,<4.1 djangomaster: https://github.com/django/django/archive/master.zip commands = {posargs:coverage run runtests.py} @@ -45,7 +45,7 @@ passenv = * whitelist_externals = bash deps = -rrequirements/tests.txt - Django + django mysqlclient psycopg2 commands = {posargs:bash -x functional.sh} From e5950ab8cbfe9ff046fc82f792e65b44c53953ec Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 May 2022 07:21:42 +0000 Subject: [PATCH 28/31] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 1163f414..8966f913 100644 --- a/setup.py +++ b/setup.py @@ -4,7 +4,6 @@ from setuptools import find_packages, setup - project_dir = Path(__file__).parent with (project_dir / "VERSION").open() as f: version = f.read().strip() From 4f4ccc6163e5c1d88500412a63d927b35fc8280b Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Tue, 3 May 2022 00:47:47 -0700 Subject: [PATCH 29/31] switch a version tag to rc --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index a23faae1..3cdeb6b8 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -4.0.0a1 +4.0.0rc1 From 8ebe3fdb4b4635b153adb86d9a09b0defedd7c3e Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Tue, 3 May 2022 00:48:39 -0700 Subject: [PATCH 30/31] switch codecov leniency to 0.3 --- codecov.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/codecov.yml b/codecov.yml index 9d979f62..70fffa1f 100644 --- a/codecov.yml +++ b/codecov.yml @@ -4,8 +4,8 @@ coverage: project: default: target: 80% # the required coverage value - threshold: 0.5% # the leniency in hitting the target + threshold: 0.3% # the leniency in hitting the target patch: default: target: 80% # the required coverage value - threshold: 0.5% # the leniency in hitting the target + threshold: 0.3% # the leniency in hitting the target From 76906f092de61969a968b2dca4436a663a009e54 Mon Sep 17 00:00:00 2001 From: Archmonger <16909269+Archmonger@users.noreply.github.com> Date: Tue, 3 May 2022 01:50:46 -0700 Subject: [PATCH 31/31] move flake8 to bottom of tox.ini --- tox.ini | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tox.ini b/tox.ini index aac59d1e..2d74513b 100644 --- a/tox.ini +++ b/tox.ini @@ -1,11 +1,6 @@ [tox] envlist = py{36,37,38,39}-django22,py{36,37,38,39,310}-django{32,40,master},lint,docs,functional -[flake8] -include = dbbackup -exclude = tests, settings, venv, docs -ignore = E501, E203, W503 - [testenv] passenv = * setenv = @@ -59,3 +54,8 @@ deps = -rrequirements/tests.txt djongo commands = {posargs:bash -x functional.sh} + +[flake8] +include = dbbackup +exclude = tests, settings, venv, docs +ignore = E501, E203, W503