From 6526a9d984bbed458731d76edabb25ac128bed48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nad=C3=A8ge=20Michel?= Date: Wed, 12 Feb 2020 10:53:33 +0100 Subject: [PATCH 1/3] Add option to run schema file after main schema --- septentrion/cli.py | 20 ++++++++++++++++++- septentrion/configuration.py | 12 ++++++++++++ septentrion/migrate.py | 37 ++++++++++++++++++++++++++++-------- 3 files changed, 60 insertions(+), 9 deletions(-) diff --git a/septentrion/cli.py b/septentrion/cli.py index 8521888..3e89201 100644 --- a/septentrion/cli.py +++ b/septentrion/cli.py @@ -176,7 +176,25 @@ def split_envvar_value(self, rv: str): help="Path to a SQL file relative to /schemas, to be run in " "addition to the migrations, e.g for installing postgres extensions (repeat the " "flag as many times as necessary) (env: SEPTENTRION_ADDITIONAL_SCHEMA_FILE, comma " - "separated values)", + "separated values) DEPRECATED; please use --before-schema-file", +) +@click.option( + "--before-schema-file", + multiple=True, + type=CommaSeparatedMultipleString(), + help="Path to a SQL file relative to /schemas, to be run in " + "addition to the migrations, before the main schemas, e.g for installing postgres " + "extensions (repeat the flag as many times as necessary) (env: " + "SEPTENTRION_BEFORE_SCHEMA_FILE, comma separated values)", +) +@click.option( + "--after-schema-file", + multiple=True, + type=CommaSeparatedMultipleString(), + help="Path to a SQL file relative to /schemas, to be run in " + "addition to the migrations, after the main schemas, e.g for grant files " + "(repeat the flag as many times as necessary) (env: SEPTENTRION_AFTER_SCHEMA_FILE, " + "comma separated values)", ) @click.option( "--ignore-symlinks/--no-ignore-symlinks", diff --git a/septentrion/configuration.py b/septentrion/configuration.py index 12b23ba..cb4a764 100644 --- a/septentrion/configuration.py +++ b/septentrion/configuration.py @@ -39,6 +39,8 @@ "dbname": None, "schema_version": None, "additional_schema_file": [], + "before_schema_file": [], + "after_schema_file": [], } @@ -76,6 +78,16 @@ def parse_configuration_file(content: str) -> Dict: line for line in config["additional_schema_file"].splitlines() if line ] + if "before_schema_file" in config: + config["before_schema_file"] = [ + line for line in config["before_schema_file"].splitlines() if line + ] + + if "after_schema_file" in config: + config["after_schema_file"] = [ + line for line in config["after_schema_file"].splitlines() if line + ] + return config raise exceptions.NoSeptentrionSection diff --git a/septentrion/migrate.py b/septentrion/migrate.py index 28443f9..a5047be 100644 --- a/septentrion/migrate.py +++ b/septentrion/migrate.py @@ -2,6 +2,8 @@ import io import logging import pathlib +import warnings +from typing import List from septentrion import ( configuration, @@ -61,20 +63,34 @@ def migrate( db.write_migration(settings=settings, version=version, name=mig) +def _load_schema_files(settings: configuration.Settings, schema_files: List[str]): + for file_name in schema_files: + if not file_name: + return + file_path = settings.MIGRATIONS_ROOT / "schemas" / file_name + logger.info("Loading %s", file_path) + run_script(settings=settings, path=file_path) + + def init_schema( settings: configuration.Settings, init_version: versions.Version, stylist: style.Stylist = style.noop_stylist, ) -> None: - # load additional files - logger.info("Looking for additional files to run") + # load before files + logger.info("Looking for additional files to run before main schema") + before_files = settings.BEFORE_SCHEMA_FILE + _load_schema_files(settings, before_files) + + # load additional files (deprecated) + logger.info("Looking for additional files to run (deprecated)") additional_files = settings.ADDITIONAL_SCHEMA_FILE - for file_name in additional_files: - if not file_name: - continue - file_path = settings.MIGRATIONS_ROOT / "schemas" / file_name - logger.info("Loading %s", file_path) - run_script(settings=settings, path=file_path) + if additional_files: + warnings.warn( + "ADDITIONAL_SCHEMA_FILES will be deprecated. " + "Use BEFORE_SCHEMA_FILES instead." + ) + _load_schema_files(settings, additional_files) # load schema with stylist.activate("title") as echo: @@ -97,6 +113,11 @@ def init_schema( create_fake_entries(settings=settings, version=init_version) + # load after files + logger.info("Looking for additional files to run after main schema") + after_files = settings.AFTER_SCHEMA_FILE + _load_schema_files(settings, after_files) + # load fixtures try: fixtures_version = core.get_fixtures_version( From 3d84059aa2fcbd43e7ba993717b201c113007e7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nad=C3=A8ge=20Michel?= Date: Wed, 12 Feb 2020 10:54:10 +0100 Subject: [PATCH 2/3] Add test files and tests --- example_migrations/schemas/after_file.sql | 0 example_migrations/schemas/before_file.sql | 0 example_migrations/schemas/extra_file.sql | 0 tests/integration/test_migrate.py | 83 ++++++++++++++++++++++ tests/unit/test_configuration.py | 22 ++++++ 5 files changed, 105 insertions(+) create mode 100644 example_migrations/schemas/after_file.sql create mode 100644 example_migrations/schemas/before_file.sql create mode 100644 example_migrations/schemas/extra_file.sql create mode 100644 tests/integration/test_migrate.py diff --git a/example_migrations/schemas/after_file.sql b/example_migrations/schemas/after_file.sql new file mode 100644 index 0000000..e69de29 diff --git a/example_migrations/schemas/before_file.sql b/example_migrations/schemas/before_file.sql new file mode 100644 index 0000000..e69de29 diff --git a/example_migrations/schemas/extra_file.sql b/example_migrations/schemas/extra_file.sql new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/test_migrate.py b/tests/integration/test_migrate.py new file mode 100644 index 0000000..2bc2f95 --- /dev/null +++ b/tests/integration/test_migrate.py @@ -0,0 +1,83 @@ +import pathlib +from unittest.mock import call + +from septentrion import configuration, versions +from septentrion.core import get_best_schema_version +from septentrion.migrate import init_schema + + +def test_init_schema(mocker): + patch = mocker.patch("septentrion.migrate.run_script",) + + settings = configuration.Settings.from_cli( + { + "host": "", + "port": "", + "username": "", + "dbname": "", + "migrations_root": "example_migrations", + "target_version": versions.Version.from_string("1.1"), + } + ) + + init_schema( + settings=settings, init_version=get_best_schema_version(settings=settings) + ) + + calls = [ + call( + settings=settings, + path=pathlib.Path("example_migrations/schemas/schema_0.1.sql"), + ), + call( + settings=settings, + path=pathlib.Path("example_migrations/fixtures/fixtures_0.1.sql"), + ), + ] + assert calls == patch.call_args_list + + +def test_init_schema_extra_files(mocker): + patch = mocker.patch("septentrion.migrate.run_script",) + + settings = configuration.Settings.from_cli( + { + "host": "", + "port": "", + "username": "", + "dbname": "", + "migrations_root": "example_migrations", + "additional_schema_file": ["extra_file.sql"], + "before_schema_file": ["before_file.sql"], + "after_schema_file": ["after_file.sql"], + "target_version": versions.Version.from_string("1.1"), + } + ) + + init_schema( + settings=settings, init_version=get_best_schema_version(settings=settings) + ) + + calls = [ + call( + settings=settings, + path=pathlib.Path("example_migrations/schemas/before_file.sql"), + ), + call( + settings=settings, + path=pathlib.Path("example_migrations/schemas/extra_file.sql"), + ), + call( + settings=settings, + path=pathlib.Path("example_migrations/schemas/schema_0.1.sql"), + ), + call( + settings=settings, + path=pathlib.Path("example_migrations/schemas/after_file.sql"), + ), + call( + settings=settings, + path=pathlib.Path("example_migrations/fixtures/fixtures_0.1.sql"), + ), + ] + assert calls == patch.call_args_list diff --git a/tests/unit/test_configuration.py b/tests/unit/test_configuration.py index 710d089..53a2b48 100644 --- a/tests/unit/test_configuration.py +++ b/tests/unit/test_configuration.py @@ -31,3 +31,25 @@ def test_settings_from_cli(): @pytest.mark.parametrize("verbosity,expected", [(0, 40), (3, 10), (4, 10)]) def test_log_level(verbosity, expected): assert configuration.log_level(verbosity) == expected + + +def test_get_config_files_settings(): + config = """ + [septentrion] + additional_schema_file= + more.sql + another.sql + before_schema_file= + before.sql + another_before.sql + after_schema_file= + after.sql + another_after.sql + """ + s = configuration.parse_configuration_file(config) + + assert s == { + "additional_schema_file": ["more.sql", "another.sql"], + "before_schema_file": ["before.sql", "another_before.sql"], + "after_schema_file": ["after.sql", "another_after.sql"], + } From 47a3dfc3d0ab4e2c979fce202256c0a580800959 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nad=C3=A8ge=20Michel?= Date: Wed, 12 Feb 2020 10:54:28 +0100 Subject: [PATCH 3/3] Add documentation --- docs/howto/configure.rst | 27 ++++++++++++++++++++++++++- docs/howto/placeholder.rst | 2 -- docs/howto_index.rst | 3 ++- 3 files changed, 28 insertions(+), 4 deletions(-) delete mode 100644 docs/howto/placeholder.rst diff --git a/docs/howto/configure.rst b/docs/howto/configure.rst index 5199ef3..f3986b0 100644 --- a/docs/howto/configure.rst +++ b/docs/howto/configure.rst @@ -1,3 +1,6 @@ +Configure +========= + Configure database connection settings -------------------------------------- @@ -9,4 +12,26 @@ Ensure that your database connection settings are correctly configured. - Or you can set the environment variables `SEPTENTRION_HOST`, `SEPTENTRION_PORT`, `SEPTENTRION_USERNAME`. - configuration file `septentrion.ini` - If you don't want to use environment variables, you can use the `--host`, `--port` or `--username` options - when running the migration. \ No newline at end of file + when running the migration. + + +Configure extra files settings +------------------------------ + +If you need additional schema files to initialize your database +you can use specify to list of files, one for the files that must be run before the main schema, +and one for the files that must be run after. + +In your configuration file you can add: + +.. code-block:: ini + + [septentrion] + ... + before_schema_file= + extra_file.sql + another_file.sql + after_schema_file= + after_schema_file.sql + +You can also use the cli options `--before-schema-file` and `--after-schema-file`. \ No newline at end of file diff --git a/docs/howto/placeholder.rst b/docs/howto/placeholder.rst deleted file mode 100644 index 6f0bed2..0000000 --- a/docs/howto/placeholder.rst +++ /dev/null @@ -1,2 +0,0 @@ -Placeholder ------------ diff --git a/docs/howto_index.rst b/docs/howto_index.rst index e3ca5a8..4ec97a5 100644 --- a/docs/howto_index.rst +++ b/docs/howto_index.rst @@ -6,4 +6,5 @@ How-to... .. toctree:: :maxdepth: 1 - howto/placeholder + howto/configure + howto/file_naming