From 689275e8522a9ea091e761d1959b76fe19285c07 Mon Sep 17 00:00:00 2001 From: Samuel Hamard Date: Fri, 6 Sep 2019 14:20:36 +0200 Subject: [PATCH 1/5] Auto-execute all sql files from a glob or a dir --- django_north/management/commands/migrate.py | 32 +++++++++++++++++++-- tests/test_data/sql/schemas/extension.sql | 0 tests/test_data/sql/schemas/grants.sql | 0 tests/test_data/sql/schemas/roles.sql | 0 4 files changed, 29 insertions(+), 3 deletions(-) create mode 100644 tests/test_data/sql/schemas/extension.sql create mode 100644 tests/test_data/sql/schemas/grants.sql create mode 100644 tests/test_data/sql/schemas/roles.sql diff --git a/django_north/management/commands/migrate.py b/django_north/management/commands/migrate.py index 947a701..4e7a345 100644 --- a/django_north/management/commands/migrate.py +++ b/django_north/management/commands/migrate.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +import glob import io import logging import os.path @@ -77,13 +78,38 @@ def migrate(self): def _load_schema_file(self, file_name, load_name=None): file_path = os.path.join( settings.NORTH_MIGRATIONS_ROOT, 'schemas', file_name) + self._load_schema_path(file_path, load_name or file_name) + + def _load_schema_path(self, file_path, load_name): if self.verbosity >= 1: if load_name is None: - load_name = file_name + load_name = load_name self.stdout.write( self.style.MIGRATE_LABEL("Load {}".format(load_name))) self.run_script(file_path) + def _load_schema_fullpath_file(self, file_path): + migration_root = os.path.join( + settings.NORTH_MIGRATIONS_ROOT, 'schemas') + load_name = os.path.relpath(file_path, migration_root) + self._load_schema_file(file_path, load_name) + + def _load_schema_string(self, input_string): + path = os.path.join( + settings.NORTH_MIGRATIONS_ROOT, 'schemas', input_string) + + if os.path.isdir(path): + for file_name in sorted(os.listdir(path)): + file_path = os.path.join(path, file_name) + if os.path.isfile(file_path) and file_path.endswith('.sql'): + self._load_schema_fullpath_file(file_path) + elif os.path.isfile(path): + self._load_schema_fullpath_file(path) + else: + for file_path in sorted(glob.glob(path)): + if os.path.isfile(file_path) and file_path.endswith('.sql'): + self._load_schema_fullpath_file(file_path) + def init_schema(self): init_version = migrations.get_version_for_init() @@ -91,7 +117,7 @@ def init_schema(self): before_files = getattr( settings, 'NORTH_BEFORE_SCHEMA_FILES', []) for file_name in before_files: - self._load_schema_file(file_name) + self._load_schema_string(file_name) # load additional files (deprecated) additional_files = getattr( @@ -115,7 +141,7 @@ def init_schema(self): after_files = getattr( settings, 'NORTH_AFTER_SCHEMA_FILES', []) for file_name in after_files: - self._load_schema_file(file_name) + self._load_schema_string(file_name) # load fixtures try: diff --git a/tests/test_data/sql/schemas/extension.sql b/tests/test_data/sql/schemas/extension.sql new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_data/sql/schemas/grants.sql b/tests/test_data/sql/schemas/grants.sql new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_data/sql/schemas/roles.sql b/tests/test_data/sql/schemas/roles.sql new file mode 100644 index 0000000..e69de29 From 412ea2cebce0c5b94d84852320bf507cd452dc3c Mon Sep 17 00:00:00 2001 From: Samuel Hamard Date: Fri, 6 Sep 2019 14:43:53 +0200 Subject: [PATCH 2/5] Test glob and dirs path for schemas migrations --- tests/test_data/sql/schemas/dir/file_00.sql | 0 tests/test_data/sql/schemas/dir/file_01.sql | 0 tests/test_data/sql/schemas/dir/file_02.sql | 0 tests/test_data/sql/schemas/glob_00.sql | 0 tests/test_data/sql/schemas/glob_01.sql | 0 tests/test_data/sql/schemas/glob_02.sql | 0 tests/test_migrate_command.py | 152 ++++++++++++++++++++ 7 files changed, 152 insertions(+) create mode 100644 tests/test_data/sql/schemas/dir/file_00.sql create mode 100644 tests/test_data/sql/schemas/dir/file_01.sql create mode 100644 tests/test_data/sql/schemas/dir/file_02.sql create mode 100644 tests/test_data/sql/schemas/glob_00.sql create mode 100644 tests/test_data/sql/schemas/glob_01.sql create mode 100644 tests/test_data/sql/schemas/glob_02.sql diff --git a/tests/test_data/sql/schemas/dir/file_00.sql b/tests/test_data/sql/schemas/dir/file_00.sql new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_data/sql/schemas/dir/file_01.sql b/tests/test_data/sql/schemas/dir/file_01.sql new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_data/sql/schemas/dir/file_02.sql b/tests/test_data/sql/schemas/dir/file_02.sql new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_data/sql/schemas/glob_00.sql b/tests/test_data/sql/schemas/glob_00.sql new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_data/sql/schemas/glob_01.sql b/tests/test_data/sql/schemas/glob_01.sql new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_data/sql/schemas/glob_02.sql b/tests/test_data/sql/schemas/glob_02.sql new file mode 100644 index 0000000..e69de29 diff --git a/tests/test_migrate_command.py b/tests/test_migrate_command.py index e596f4b..8cf0e0c 100644 --- a/tests/test_migrate_command.py +++ b/tests/test_migrate_command.py @@ -203,6 +203,158 @@ def test_migrate_init_full(capsys, settings, mocker): ) +def test_migrate_before_schema_glob(capsys, settings, mocker): + root = os.path.dirname(__file__) + settings.NORTH_MIGRATIONS_ROOT = os.path.join(root, 'test_data/sql') + + mocker.patch( + 'django_north.management.migrations.get_version_for_init', + return_value='16.12') + mock_run_script = mocker.patch( + 'django_north.management.commands.migrate.Command.run_script') + command = migrate.Command() + command.verbosity = 2 + + # extensions, roles, fixtures, and grants + settings.NORTH_BEFORE_SCHEMA_FILES = ['glob_*.sql'] + settings.NORTH_AFTER_SCHEMA_FILES = [] + settings.NORTH_ADDITIONAL_SCHEMA_FILES = [] + command.init_schema() + assert len(mock_run_script.call_args_list) == 5 + + base_path = os.path.join(settings.NORTH_MIGRATIONS_ROOT, 'schemas') + assert mock_run_script.call_args_list[0] == mocker.call( + os.path.join(base_path, 'glob_00.sql')) + assert mock_run_script.call_args_list[1] == mocker.call( + os.path.join(base_path, 'glob_01.sql')) + assert mock_run_script.call_args_list[2] == mocker.call( + os.path.join(base_path, 'glob_02.sql')) + captured = capsys.readouterr() + assert captured.out == ( + 'Load glob_00.sql\n' + 'Load glob_01.sql\n' + 'Load glob_02.sql\n' + 'Load schema\n' + ' Applying 16.12...\n' + 'Load fixtures\n' + ' Applying 16.12...\n' + ) + + +def test_migrate_after_schema_glob(capsys, settings, mocker): + root = os.path.dirname(__file__) + settings.NORTH_MIGRATIONS_ROOT = os.path.join(root, 'test_data/sql') + + mocker.patch( + 'django_north.management.migrations.get_version_for_init', + return_value='16.12') + mock_run_script = mocker.patch( + 'django_north.management.commands.migrate.Command.run_script') + command = migrate.Command() + command.verbosity = 2 + + # extensions, roles, fixtures, and grants + settings.NORTH_BEFORE_SCHEMA_FILES = [] + settings.NORTH_AFTER_SCHEMA_FILES = ['glob_*.sql'] + settings.NORTH_ADDITIONAL_SCHEMA_FILES = [] + command.init_schema() + assert len(mock_run_script.call_args_list) == 5 + + base_path = os.path.join(settings.NORTH_MIGRATIONS_ROOT, 'schemas') + assert mock_run_script.call_args_list[1] == mocker.call( + os.path.join(base_path, 'glob_00.sql')) + assert mock_run_script.call_args_list[2] == mocker.call( + os.path.join(base_path, 'glob_01.sql')) + assert mock_run_script.call_args_list[3] == mocker.call( + os.path.join(base_path, 'glob_02.sql')) + captured = capsys.readouterr() + assert captured.out == ( + 'Load schema\n' + ' Applying 16.12...\n' + 'Load glob_00.sql\n' + 'Load glob_01.sql\n' + 'Load glob_02.sql\n' + 'Load fixtures\n' + ' Applying 16.12...\n' + ) + + +def test_migrate_before_schema_dir(capsys, settings, mocker): + root = os.path.dirname(__file__) + settings.NORTH_MIGRATIONS_ROOT = os.path.join(root, 'test_data/sql') + + mocker.patch( + 'django_north.management.migrations.get_version_for_init', + return_value='16.12') + mock_run_script = mocker.patch( + 'django_north.management.commands.migrate.Command.run_script') + command = migrate.Command() + command.verbosity = 2 + + # extensions, roles, fixtures, and grants + settings.NORTH_BEFORE_SCHEMA_FILES = ['dir'] + settings.NORTH_AFTER_SCHEMA_FILES = [] + settings.NORTH_ADDITIONAL_SCHEMA_FILES = [] + command.init_schema() + assert len(mock_run_script.call_args_list) == 5 + + base_path = os.path.join(settings.NORTH_MIGRATIONS_ROOT, 'schemas', 'dir') + assert mock_run_script.call_args_list[0] == mocker.call( + os.path.join(base_path, 'file_00.sql')) + assert mock_run_script.call_args_list[1] == mocker.call( + os.path.join(base_path, 'file_01.sql')) + assert mock_run_script.call_args_list[2] == mocker.call( + os.path.join(base_path, 'file_02.sql')) + captured = capsys.readouterr() + assert captured.out == ( + 'Load dir/file_00.sql\n' + 'Load dir/file_01.sql\n' + 'Load dir/file_02.sql\n' + 'Load schema\n' + ' Applying 16.12...\n' + 'Load fixtures\n' + ' Applying 16.12...\n' + ) + + +def test_migrate_after_schema_dir(capsys, settings, mocker): + root = os.path.dirname(__file__) + settings.NORTH_MIGRATIONS_ROOT = os.path.join(root, 'test_data/sql') + + mocker.patch( + 'django_north.management.migrations.get_version_for_init', + return_value='16.12') + mock_run_script = mocker.patch( + 'django_north.management.commands.migrate.Command.run_script') + command = migrate.Command() + command.verbosity = 2 + + # extensions, roles, fixtures, and grants + settings.NORTH_BEFORE_SCHEMA_FILES = [] + settings.NORTH_AFTER_SCHEMA_FILES = ['dir'] + settings.NORTH_ADDITIONAL_SCHEMA_FILES = [] + command.init_schema() + assert len(mock_run_script.call_args_list) == 5 + + base_path = os.path.join(settings.NORTH_MIGRATIONS_ROOT, 'schemas', 'dir') + assert mock_run_script.call_args_list[1] == mocker.call( + os.path.join(base_path, 'file_00.sql')) + assert mock_run_script.call_args_list[2] == mocker.call( + os.path.join(base_path, 'file_01.sql')) + assert mock_run_script.call_args_list[3] == mocker.call( + os.path.join(base_path, 'file_02.sql')) + captured = capsys.readouterr() + assert captured.out == ( + 'Load schema\n' + ' Applying 16.12...\n' + 'Load dir/file_00.sql\n' + 'Load dir/file_01.sql\n' + 'Load dir/file_02.sql\n' + 'Load fixtures\n' + ' Applying 16.12...\n' + ) + + def test_migrate_init_deprecation(capsys, settings, mocker): root = os.path.dirname(__file__) settings.NORTH_MIGRATIONS_ROOT = os.path.join(root, 'test_data/sql') From 2c6f394d00100c13b5ffb1c1dc15dfdeb75c40b6 Mon Sep 17 00:00:00 2001 From: Samuel Hamard Date: Fri, 6 Sep 2019 14:46:58 +0200 Subject: [PATCH 3/5] Update the documentation for glob in schemas --- docs/usage.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/usage.rst b/docs/usage.rst index 3172a00..c22fde7 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -32,10 +32,12 @@ List of available settings: * ``NORTH_ADDITIONAL_SCHEMA_FILES``: **deprecated** list of sql files to load before the schema. For example: a file of DB roles, some extensions. Default value: ``[]`` -* ``NORTH_BEFORE_SCHEMA_FILES``: list of sql files to load before the schema. +* ``NORTH_BEFORE_SCHEMA_FILES``: list of sql files, dirs and globs to load before the schema. + If it's a dir or a glob, load only sql files in alphabetical order. For example: a file of DB roles, some extensions. Default value: ``[]`` -* ``NORTH_AFTER_SCHEMA_FILES``: list of sql files to load after the schema. +* ``NORTH_AFTER_SCHEMA_FILES``: list of sql files, dirs and globs to load after the schema. + If it's a dir or a glob, load only sql files in alphabetical order. For example: a file of permissions, grants on tables Default value: ``[]`` * ``NORTH_CURRENT_VERSION_DETECTOR``: the current version detector. From 00426741360cd2a6cfac52ad03ada212905d03bf Mon Sep 17 00:00:00 2001 From: Samuel Hamard Date: Fri, 6 Sep 2019 15:00:05 +0200 Subject: [PATCH 4/5] remove usefull conditional --- django_north/management/commands/migrate.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/django_north/management/commands/migrate.py b/django_north/management/commands/migrate.py index 4e7a345..8769123 100644 --- a/django_north/management/commands/migrate.py +++ b/django_north/management/commands/migrate.py @@ -82,8 +82,6 @@ def _load_schema_file(self, file_name, load_name=None): def _load_schema_path(self, file_path, load_name): if self.verbosity >= 1: - if load_name is None: - load_name = load_name self.stdout.write( self.style.MIGRATE_LABEL("Load {}".format(load_name))) self.run_script(file_path) From b13451c8187154436f197684e917335002455efe Mon Sep 17 00:00:00 2001 From: Samuel Hamard Date: Wed, 2 Oct 2019 09:59:32 +0200 Subject: [PATCH 5/5] Use directly the glob to find files. Use only the glob to find files on `NORTH_BEFORE_SCHEMA_FILES` and `NORTH_AFTER_SCHEMA_FILES` --- django_north/management/commands/migrate.py | 22 ++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/django_north/management/commands/migrate.py b/django_north/management/commands/migrate.py index 8769123..1491659 100644 --- a/django_north/management/commands/migrate.py +++ b/django_north/management/commands/migrate.py @@ -92,21 +92,21 @@ def _load_schema_fullpath_file(self, file_path): load_name = os.path.relpath(file_path, migration_root) self._load_schema_file(file_path, load_name) + def _load_schema_dir(self, dir_path): + for file_name in sorted(os.listdir(dir_path)): + file_path = os.path.join(dir_path, file_name) + if os.path.isfile(file_path) and file_path.endswith('.sql'): + self._load_schema_fullpath_file(file_path) + def _load_schema_string(self, input_string): path = os.path.join( settings.NORTH_MIGRATIONS_ROOT, 'schemas', input_string) - if os.path.isdir(path): - for file_name in sorted(os.listdir(path)): - file_path = os.path.join(path, file_name) - if os.path.isfile(file_path) and file_path.endswith('.sql'): - self._load_schema_fullpath_file(file_path) - elif os.path.isfile(path): - self._load_schema_fullpath_file(path) - else: - for file_path in sorted(glob.glob(path)): - if os.path.isfile(file_path) and file_path.endswith('.sql'): - self._load_schema_fullpath_file(file_path) + for file_path in sorted(glob.glob(path)): + if os.path.isdir(file_path): + self._load_schema_dir(file_path) + if os.path.isfile(file_path): + self._load_schema_fullpath_file(file_path) def init_schema(self): init_version = migrations.get_version_for_init()