Merge pull request #57 from peopledoc/after-before-schema-files

Add feature to run schema files after main schema file too

docs/howto/configure.rst

@@ -1,3 +1,6 @@
+Configure
+=========
+
 Configure database connection settings
 --------------------------------------
 
@@ -9,4 +12,26 @@ Ensure that your database connection settings are correctly configured.
  - Or you can set the environment variables `SEPTENTRION_HOST`, `SEPTENTRION_PORT`, `SEPTENTRION_USERNAME`.
  - configuration file `septentrion.ini`
  - If you don't want to use environment variables, you can use the `--host`, `--port` or `--username` options
-   when running the migration.
+   when running the migration.
+
+
+Configure extra files settings
+------------------------------
+
+If you need additional schema files to initialize your database
+you can use specify to list of files, one for the files that must be run before the main schema,
+and one for the files that must be run after.
+
+In your configuration file you can add:
+
+.. code-block:: ini
+
+    [septentrion]
+    ...
+    before_schema_file=
+        extra_file.sql
+        another_file.sql
+    after_schema_file=
+        after_schema_file.sql
+
+You can also use the cli options `--before-schema-file` and `--after-schema-file`.

docs/howto_index.rst

@@ -6,4 +6,5 @@ How-to...
 .. toctree::
     :maxdepth: 1
 
-    howto/placeholder
+    howto/configure
+    howto/file_naming

septentrion/cli.py

@@ -176,7 +176,25 @@ def split_envvar_value(self, rv: str):
     help="Path to a SQL file relative to <migration-root>/schemas, to be run in "
     "addition to the migrations, e.g for installing postgres extensions (repeat the "
     "flag as many times as necessary) (env: SEPTENTRION_ADDITIONAL_SCHEMA_FILE, comma "
-    "separated values)",
+    "separated values) DEPRECATED; please use --before-schema-file",
+)
+@click.option(
+    "--before-schema-file",
+    multiple=True,
+    type=CommaSeparatedMultipleString(),
+    help="Path to a SQL file relative to <migration-root>/schemas, to be run in "
+    "addition to the migrations, before the main schemas, e.g for installing postgres "
+    "extensions (repeat the flag as many times as necessary) (env: "
+    "SEPTENTRION_BEFORE_SCHEMA_FILE, comma separated values)",
+)
+@click.option(
+    "--after-schema-file",
+    multiple=True,
+    type=CommaSeparatedMultipleString(),
+    help="Path to a SQL file relative to <migration-root>/schemas, to be run in "
+    "addition to the migrations, after the main schemas, e.g for grant files "
+    "(repeat the flag as many times as necessary) (env: SEPTENTRION_AFTER_SCHEMA_FILE, "
+    "comma separated values)",
 )
 @click.option(
     "--ignore-symlinks/--no-ignore-symlinks",

septentrion/configuration.py

@@ -39,6 +39,8 @@
     "dbname": None,
     "schema_version": None,
     "additional_schema_file": [],
+    "before_schema_file": [],
+    "after_schema_file": [],
 }
 
 
@@ -76,6 +78,16 @@ def parse_configuration_file(content: str) -> Dict:
                 line for line in config["additional_schema_file"].splitlines() if line
             ]
 
+        if "before_schema_file" in config:
+            config["before_schema_file"] = [
+                line for line in config["before_schema_file"].splitlines() if line
+            ]
+
+        if "after_schema_file" in config:
+            config["after_schema_file"] = [
+                line for line in config["after_schema_file"].splitlines() if line
+            ]
+
         return config
 
     raise exceptions.NoSeptentrionSection

septentrion/migrate.py

@@ -2,6 +2,8 @@
 import io
 import logging
 import pathlib
+import warnings
+from typing import List
 
 from septentrion import (
     configuration,
@@ -61,20 +63,34 @@ def migrate(
                     db.write_migration(settings=settings, version=version, name=mig)
 
 
+def _load_schema_files(settings: configuration.Settings, schema_files: List[str]):
+    for file_name in schema_files:
+        if not file_name:
+            return
+        file_path = settings.MIGRATIONS_ROOT / "schemas" / file_name
+        logger.info("Loading %s", file_path)
+        run_script(settings=settings, path=file_path)
+
+
 def init_schema(
     settings: configuration.Settings,
     init_version: versions.Version,
     stylist: style.Stylist = style.noop_stylist,
 ) -> None:
-    # load additional files
-    logger.info("Looking for additional files to run")
+    # load before files
+    logger.info("Looking for additional files to run before main schema")
+    before_files = settings.BEFORE_SCHEMA_FILE
+    _load_schema_files(settings, before_files)
+
+    # load additional files (deprecated)
+    logger.info("Looking for additional files to run (deprecated)")
     additional_files = settings.ADDITIONAL_SCHEMA_FILE
-    for file_name in additional_files:
-        if not file_name:
-            continue
-        file_path = settings.MIGRATIONS_ROOT / "schemas" / file_name
-        logger.info("Loading %s", file_path)
-        run_script(settings=settings, path=file_path)
+    if additional_files:
+        warnings.warn(
+            "ADDITIONAL_SCHEMA_FILES will be deprecated. "
+            "Use BEFORE_SCHEMA_FILES instead."
+        )
+    _load_schema_files(settings, additional_files)
 
     # load schema
     with stylist.activate("title") as echo:
@@ -97,6 +113,11 @@ def init_schema(
 
     create_fake_entries(settings=settings, version=init_version)
 
+    # load after files
+    logger.info("Looking for additional files to run after main schema")
+    after_files = settings.AFTER_SCHEMA_FILE
+    _load_schema_files(settings, after_files)
+
     # load fixtures
     try:
         fixtures_version = core.get_fixtures_version(

tests/integration/test_migrate.py

@@ -0,0 +1,83 @@
+import pathlib
+from unittest.mock import call
+
+from septentrion import configuration, versions
+from septentrion.core import get_best_schema_version
+from septentrion.migrate import init_schema
+
+
+def test_init_schema(mocker):
+    patch = mocker.patch("septentrion.migrate.run_script",)
+
+    settings = configuration.Settings.from_cli(
+        {
+            "host": "",
+            "port": "",
+            "username": "",
+            "dbname": "",
+            "migrations_root": "example_migrations",
+            "target_version": versions.Version.from_string("1.1"),
+        }
+    )
+
+    init_schema(
+        settings=settings, init_version=get_best_schema_version(settings=settings)
+    )
+
+    calls = [
+        call(
+            settings=settings,
+            path=pathlib.Path("example_migrations/schemas/schema_0.1.sql"),
+        ),
+        call(
+            settings=settings,
+            path=pathlib.Path("example_migrations/fixtures/fixtures_0.1.sql"),
+        ),
+    ]
+    assert calls == patch.call_args_list
+
+
+def test_init_schema_extra_files(mocker):
+    patch = mocker.patch("septentrion.migrate.run_script",)
+
+    settings = configuration.Settings.from_cli(
+        {
+            "host": "",
+            "port": "",
+            "username": "",
+            "dbname": "",
+            "migrations_root": "example_migrations",
+            "additional_schema_file": ["extra_file.sql"],
+            "before_schema_file": ["before_file.sql"],
+            "after_schema_file": ["after_file.sql"],
+            "target_version": versions.Version.from_string("1.1"),
+        }
+    )
+
+    init_schema(
+        settings=settings, init_version=get_best_schema_version(settings=settings)
+    )
+
+    calls = [
+        call(
+            settings=settings,
+            path=pathlib.Path("example_migrations/schemas/before_file.sql"),
+        ),
+        call(
+            settings=settings,
+            path=pathlib.Path("example_migrations/schemas/extra_file.sql"),
+        ),
+        call(
+            settings=settings,
+            path=pathlib.Path("example_migrations/schemas/schema_0.1.sql"),
+        ),
+        call(
+            settings=settings,
+            path=pathlib.Path("example_migrations/schemas/after_file.sql"),
+        ),
+        call(
+            settings=settings,
+            path=pathlib.Path("example_migrations/fixtures/fixtures_0.1.sql"),
+        ),
+    ]
+    assert calls == patch.call_args_list

tests/unit/test_configuration.py

@@ -31,3 +31,25 @@ def test_settings_from_cli():
 @pytest.mark.parametrize("verbosity,expected", [(0, 40), (3, 10), (4, 10)])
 def test_log_level(verbosity, expected):
     assert configuration.log_level(verbosity) == expected
+
+
+def test_get_config_files_settings():
+    config = """
+        [septentrion]
+        additional_schema_file=
+            more.sql
+            another.sql
+        before_schema_file=
+            before.sql
+            another_before.sql
+        after_schema_file=
+            after.sql
+            another_after.sql
+    """
+    s = configuration.parse_configuration_file(config)
+
+    assert s == {
+        "additional_schema_file": ["more.sql", "another.sql"],
+        "before_schema_file": ["before.sql", "another_before.sql"],
+        "after_schema_file": ["after.sql", "another_after.sql"],
+    }