Pathlib

.gitignore

@@ -1,12 +1,12 @@
 __pycache__
-/.coverage
+.coverage
 .DS_Store
-/.mypy_cache
-/.pytest_cache
-/.tox
+.mypy_cache
+.pytest_cache
+.tox
 *.egg-info
-/docs/_build
-/htmlcov
-/dist
-/build
-/.venv
+docs/_build
+htmlcov
+dist
+build
+.venv

.travis.yml

@@ -15,6 +15,7 @@ matrix:
     env:
       TOX_ENV: py36-unit-tests
       COVERAGE_FLAG: unit
+
   - python: 3.6
     env:
       TOX_ENV: py36-integration-tests
@@ -24,31 +25,32 @@ matrix:
       PGHOST: 127.0.0.1
       COVERAGE_FLAG: integration
 
-  - python: 3.7
+  - python: 3.6
     env:
-      TOX_ENV: py37-unit-tests
-      COVERAGE_FLAG: unit
+      TOX_ENV: py36-acceptance-tests
+      PGUSER: postgres
+      PGPORT: 5432
+      PGPASSWORD: ""
+      PGHOST: 127.0.0.1
+      COVERAGE_FLAG: ""
+
   - python: 3.7
     env:
-      TOX_ENV: py37-integration-tests
+      TOX_ENV: py37-unit-tests,py37-integration-tests,py37-acceptance-tests
       PGUSER: postgres
       PGPORT: 5432
       PGPASSWORD: ""
       PGHOST: 127.0.0.1
-      COVERAGE_FLAG: integration
+      COVERAGE_FLAG: ""
 
   - python: 3.8
     env:
-      TOX_ENV: py38-unit-tests
-      COVERAGE_FLAG: unit
-  - python: 3.8
-    env:
-      TOX_ENV: py38-integration-tests
+      TOX_ENV: py38-unit-tests,py38-integration-tests,py38-acceptance-tests
       PGUSER: postgres
       PGPORT: 5432
       PGPASSWORD: ""
       PGHOST: 127.0.0.1
-      COVERAGE_FLAG: integration
+      COVERAGE_FLAG: ""
 
 install:
   - pip install tox codecov
@@ -57,4 +59,7 @@ script:
   - tox -e $TOX_ENV
 
 after_success:
-  - bash <(curl -s https://codecov.io/bash) -c -F $COVERAGE_FLAG
+  - |
+    if [ -n "$COVERAGE_FLAG" ];
+      then bash <(curl -s https://codecov.io/bash) -c -F $COVERAGE_FLAG;
+    fi

CONTRIBUTING.rst

@@ -212,26 +212,9 @@ Try our demo
 
 With a running database:
 
-Launch a worker with:
-
-.. code-block:: console
-
-    (venv) $ export PROCRASTINATE_APP=procrastinate_demo.app.app
-    (venv) $ procrastinate migrate
-    (venv) $ procrastinate worker
-
-Schedule some tasks with:
-
 .. code-block:: console
 
-    (venv) $ python -m procrastinate_demo
-
-Wait, there are ``async`` and ``await`` keywords everywhere!?
--------------------------------------------------------------
-
-Yes, in order to provide both a synchronous **and** asynchronous API, Procrastinate
-needs to be asynchronous at core. Find out more in the documentation, in the Discussions
-section. If you need informations on how to work with asynchronous Python, check out:
-
-- The official documentation: https://docs.python.org/3/library/asyncio.html
-- A more accessible guide by Brad Solomon: https://realpython.com/async-io-python/
+    (venv) $ export PGDATABASE=septentrion PGHOST=localhost PGUSER=postgres
+    (venv) $ createdb
+    (venv) $ export SEPTENTRION_MIGRATIONS_ROOT=example_migrations SEPTENTRION_TARGET_VERSION=1.1
+    (venv) $ septentrion migrate

septentrion/cli.py

@@ -174,6 +174,11 @@ def split_envvar_value(self, rv: str):
     "flag as many times as necessary) (env: SEPTENTRION_ADDITIONAL_SCHEMA_FILE, comma "
     "separated values)",
 )
+@click.option(
+    "--ignore-symlinks/--no-ignore-symlinks",
+    default=configuration.DEFAULTS["ignore_symlinks"],
+    help="Ignore migration files that are symlinks",
+)
 def cli(ctx: click.Context, **kwargs):
     if kwargs.pop("password_flag"):
         password = click.prompt("Database password", hide_input=True)
@@ -210,7 +215,6 @@ def show_migrations(settings: configuration.Settings):
 def migrate_func(settings: configuration.Settings):
     """
     Run unapplied migrations.
-
     """
     migrate.migrate(settings=settings, stylist=style.stylist)
 

septentrion/configuration.py

@@ -4,7 +4,8 @@
 """
 import configparser
 import logging
-from typing import Any, Dict, Tuple
+import pathlib
+from typing import Any, Dict, Tuple, Union
 
 from septentrion import exceptions
 
@@ -28,6 +29,7 @@
     "schema_template": "schema_{}.sql",
     "fixtures_template": "fixtures_{}.sql",
     "non_transactional_keyword": ["CONCURRENTLY", "ALTER TYPE", "VACUUM"],
+    "ignore_symlinks": False,
     # Values that don't have an explicit default need to be present too
     "verbosity": 0,
     "host": None,
@@ -92,21 +94,34 @@ def log_level(verbosity: int) -> int:
     return 40 - 10 * min(verbosity, 3)
 
 
-def clean_key(key: str) -> str:
-    # CLI settings are lowercase
-    return key.upper()
-
-
 class Settings:
     def __init__(self):
         self._settings = {}
         self.update(DEFAULTS)
 
     def __getattr__(self, key: str) -> Any:
-        return self._settings[key]
+        try:
+            return self._settings[key]
+        except KeyError:
+            raise AttributeError(key)
 
     def set(self, key: str, value: Any) -> None:
-        self._settings[clean_key(key)] = value
+        try:
+            method = getattr(self, f"clean_{key.lower()}")
+        except AttributeError:
+            pass
+        else:
+            value = method(value)
+        # TODO: remove the .upper() and fix the tests: from_cli() should be
+        # the only one doing the .upper()
+        self._settings[key.upper()] = value
+
+    def clean_migrations_root(
+        self, migrations_root: Union[str, pathlib.Path]
+    ) -> pathlib.Path:
+        if isinstance(migrations_root, str):
+            migrations_root = pathlib.Path(migrations_root)
+        return migrations_root
 
     def __repr__(self):
         return repr(self._settings)
@@ -118,6 +133,7 @@ def update(self, values: Dict) -> None:
     @classmethod
     def from_cli(cls, cli_settings: Dict):
         settings = cls()
-        settings.update(cli_settings)
+        # CLI settings are lowercase
+        settings.update({key.upper(): value for key, value in cli_settings.items()})
 
         return settings

septentrion/core.py

@@ -3,7 +3,6 @@
 from the existing files (septentrion.files) and from the db (septentrion.db)
 """
 
-import os
 from typing import Any, Dict, Iterable, Optional
 
 from septentrion import configuration, db, exceptions, files, style, utils
@@ -21,6 +20,8 @@ def get_applied_versions(settings: configuration.Settings) -> Iterable[str]:
     return utils.sort_versions(applied_versions & known_versions)
 
 
+# TODO: Refactor: this should just work with version numbers, not sql_tpl and
+# not force_version
 def get_closest_version(
     settings: configuration.Settings,
     target_version: str,
@@ -68,16 +69,21 @@ def get_closest_version(
     return None
 
 
+# TODO: refactor this and the function below
+# TODO: also remove files.get_special_files, it's not really useful
 def get_best_schema_version(settings: configuration.Settings) -> str:
     """
     Get the best candidate to init the DB.
     """
+    schema_files = files.get_special_files(
+        root=settings.MIGRATIONS_ROOT, folder="schemas"
+    )
     version = get_closest_version(
         settings=settings,
         target_version=settings.TARGET_VERSION,
         sql_tpl=settings.SCHEMA_TEMPLATE,
-        existing_files=files.get_known_schemas(settings=settings),
         force_version=settings.SCHEMA_VERSION,
+        existing_files=schema_files,
     )
 
     if version is None:
@@ -90,10 +96,13 @@ def get_fixtures_version(settings: configuration.Settings, target_version: str)
     Get the closest fixtures to use to init a new DB
     to the current target version.
     """
+    fixture_files = files.get_special_files(
+        root=settings.MIGRATIONS_ROOT, folder="fixtures"
+    )
     version = get_closest_version(
         settings=settings,
         target_version=target_version,
-        existing_files=files.get_known_fixtures(settings=settings),
+        existing_files=fixture_files,
         sql_tpl=settings.FIXTURES_TEMPLATE,
     )
 
@@ -136,7 +145,10 @@ def build_migration_plan(settings: configuration.Settings) -> Iterable[Dict[str,
         for mig in migs:
             applied = mig in applied_migrations
             path = migrations_to_apply[mig]
-            is_manual = files.is_manual_migration(os.path.abspath(path))
+            contents = files.file_lines_generator(path)
+            is_manual = files.is_manual_migration(
+                migration_path=path, migration_contents=contents
+            )
             version_plan.append((mig, applied, path, is_manual))
         yield {"version": version, "plan": version_plan}