Merge pull request #585 from NatLibFi/issue584-support-for-multiple-c…

…onfiguration-files-in-a-directory

Support for multiple configuration files in a directory

annif/cli.py

@@ -104,14 +104,15 @@ def set_project_config_file_path(ctx, param, value):
     """Override the default path or the path given in env by CLI option"""
     with ctx.ensure_object(ScriptInfo).load_app().app_context():
         if value:
-            current_app.config['PROJECTS_FILE'] = value
+            current_app.config['PROJECTS_CONFIG_PATH'] = value
 
 
 def common_options(f):
     """Decorator to add common options for all CLI commands"""
     f = click.option(
-        '-p', '--projects', help='Set path to project configuration file',
-        type=click.Path(dir_okay=False, exists=True),
+        '-p', '--projects',
+        help='Set path to project configuration file or directory',
+        type=click.Path(dir_okay=True, exists=True),
         callback=set_project_config_file_path, expose_value=False,
         is_eager=True)(f)
     return click_log.simple_verbosity_option(logger)(f)

annif/config.py

@@ -6,6 +6,7 @@
 import tomli
 import annif
 import annif.util
+from glob import glob
 from annif.exception import ConfigurationException
 
 
@@ -56,43 +57,75 @@ def __getitem__(self, key):
         return self._config[key]
 
 
-def check_config(projects_file):
-    if os.path.exists(projects_file):
-        return projects_file
+class AnnifConfigDirectory:
+    """Class for reading configuration from directory"""
+
+    def __init__(self, directory):
+        files = glob(os.path.join(directory, '*.cfg'))
+        files.extend(glob(os.path.join(directory, '*.toml')))
+        logger.debug(f"Reading configuration files in directory {directory}")
+
+        self._config = dict()
+        for file in files:
+            source_config = parse_config(file)
+            for proj_id in source_config.project_ids:
+                self._check_duplicate_project_ids(proj_id, file)
+                self._config[proj_id] = source_config[proj_id]
+
+    def _check_duplicate_project_ids(self, proj_id, file):
+        if proj_id in self._config:
+            # Error message resembles configparser's DuplicateSection message
+            raise ConfigurationException(
+                f'While reading from "{file}": project ID "{proj_id}" already '
+                'exists in another configuration file in the directory.')
+
+    @property
+    def project_ids(self):
+        return self._config.keys()
+
+    def __getitem__(self, key):
+        return self._config[key]
+
+
+def check_config(projects_config_path):
+    if os.path.exists(projects_config_path):
+        return projects_config_path
     else:
         logger.warning(
-            f'Project configuration file "{projects_file}" is ' +
-            'missing. Please provide one. ' +
+            'Project configuration file or directory ' +
+            f'"{projects_config_path}" is missing. Please provide one. ' +
             'You can set the path to the project configuration ' +
-            'file using the ANNIF_PROJECTS environment ' +
+            'using the ANNIF_PROJECTS environment ' +
             'variable or the command-line option "--projects".')
         return None
 
 
 def find_config():
-    for filename in ('projects.cfg', 'projects.toml'):
-        if os.path.exists(filename):
-            return filename
+    for path in ('projects.cfg', 'projects.toml', 'projects.d'):
+        if os.path.exists(path):
+            return path
 
     logger.warning(
-        'Could not find project configuration file ' +
-        '"projects.cfg" or "projects.toml". ' +
+        'Could not find project configuration ' +
+        '"projects.cfg", "projects.toml" or "projects.d". ' +
         'You can set the path to the project configuration ' +
-        'file using the ANNIF_PROJECTS environment ' +
+        'using the ANNIF_PROJECTS environment ' +
         'variable or the command-line option "--projects".')
     return None
 
 
-def parse_config(projects_file):
-    if projects_file:
-        filename = check_config(projects_file)
+def parse_config(projects_config_path):
+    if projects_config_path:
+        projects_config_path = check_config(projects_config_path)
     else:
-        filename = find_config()
+        projects_config_path = find_config()
 
-    if not filename:  # not found
+    if not projects_config_path:  # not found
         return None
 
-    if filename.endswith('.toml'):  # TOML format
-        return AnnifConfigTOML(filename)
+    if os.path.isdir(projects_config_path):
+        return AnnifConfigDirectory(projects_config_path)
+    elif projects_config_path.endswith('.toml'):  # TOML format
+        return AnnifConfigTOML(projects_config_path)
     else:  # classic CFG/INI style format
-        return AnnifConfigCFG(filename)
+        return AnnifConfigCFG(projects_config_path)

annif/default_config.py

@@ -11,7 +11,7 @@
 class Config(object):
     DEBUG = False
     TESTING = False
-    PROJECTS_FILE = os.environ.get('ANNIF_PROJECTS', default='')
+    PROJECTS_CONFIG_PATH = os.environ.get('ANNIF_PROJECTS', default='')
     DATADIR = os.environ.get('ANNIF_DATADIR', default='data')
     INITIALIZE_PROJECTS = False
 
@@ -26,7 +26,7 @@ class DevelopmentConfig(Config):
 
 class TestingConfig(Config):
     TESTING = True
-    PROJECTS_FILE = 'tests/projects.cfg'
+    PROJECTS_CONFIG_PATH = 'tests/projects.cfg'
     DATADIR = 'tests/data'
 
 
@@ -35,12 +35,16 @@ class TestingInitializeConfig(TestingConfig):
 
 
 class TestingNoProjectsConfig(TestingConfig):
-    PROJECTS_FILE = 'tests/notfound.cfg'
+    PROJECTS_CONFIG_PATH = 'tests/notfound.cfg'
 
 
 class TestingInvalidProjectsConfig(TestingConfig):
-    PROJECTS_FILE = 'tests/projects_invalid.cfg'
+    PROJECTS_CONFIG_PATH = 'tests/projects_invalid.cfg'
 
 
 class TestingTOMLConfig(TestingConfig):
-    PROJECTS_FILE = 'tests/projects.toml'
+    PROJECTS_CONFIG_PATH = 'tests/projects.toml'
+
+
+class TestingDirectoryConfig(TestingConfig):
+    PROJECTS_CONFIG_PATH = 'tests/projects.d'

annif/registry.py

@@ -21,17 +21,17 @@ class AnnifRegistry:
     # processes when using the multiprocessing module.
     _projects = {}
 
-    def __init__(self, projects_file, datadir, init_projects):
+    def __init__(self, projects_config_path, datadir, init_projects):
         self._rid = id(self)
         self._projects[self._rid] = \
-            self._create_projects(projects_file, datadir)
+            self._create_projects(projects_config_path, datadir)
         if init_projects:
             for project in self._projects[self._rid].values():
                 project.initialize()
 
-    def _create_projects(self, projects_file, datadir):
+    def _create_projects(self, projects_config_path, datadir):
         # parse the configuration
-        config = parse_config(projects_file)
+        config = parse_config(projects_config_path)
 
         # handle the case where the config file doesn't exist
         if config is None:
@@ -66,10 +66,11 @@ def get_project(self, project_id, min_access=Access.private):
 
 
 def initialize_projects(app):
-    projects_file = app.config['PROJECTS_FILE']
+    projects_config_path = app.config['PROJECTS_CONFIG_PATH']
     datadir = app.config['DATADIR']
     init_projects = app.config['INITIALIZE_PROJECTS']
-    app.annif_registry = AnnifRegistry(projects_file, datadir, init_projects)
+    app.annif_registry = AnnifRegistry(projects_config_path, datadir,
+                                       init_projects)
 
 
 def get_projects(min_access=Access.private):

tests/projects.d/projects.cfg

@@ -0,0 +1 @@
+../projects.cfg

tests/projects.d/projects.toml

@@ -0,0 +1 @@
+../projects.toml

tests/test_cli.py

@@ -14,7 +14,7 @@
 # Generate a random project name to use in tests
 TEMP_PROJECT = ''.join(
     random.choice('abcdefghiklmnopqrstuvwxyz') for _ in range(8))
-PROJECTS_FILE_OPTION = 'tests/projects_for_config_path_option.cfg'
+PROJECTS_CONFIG_PATH = 'tests/projects_for_config_path_option.cfg'
 
 
 def test_list_projects():
@@ -43,7 +43,7 @@ def test_list_projects_bad_arguments():
 
 def test_list_projects_config_path_option():
     result = runner.invoke(
-        annif.cli.cli, ["list-projects", "--projects", PROJECTS_FILE_OPTION])
+        annif.cli.cli, ["list-projects", "--projects", PROJECTS_CONFIG_PATH])
     assert not result.exception
     assert result.exit_code == 0
     assert 'dummy_for_projects_option' in result.output
@@ -57,7 +57,7 @@ def test_list_projects_config_path_option_nonexistent():
     assert failed_result.exception
     assert failed_result.exit_code != 0
     assert "Error: Invalid value for '-p' / '--projects': " \
-           "File 'nonexistent.cfg' does not exist." in failed_result.output
+           "Path 'nonexistent.cfg' does not exist." in failed_result.output
 
 
 def test_show_project():

tests/test_config.py

@@ -16,12 +16,17 @@ def test_check_config_exists_toml():
     assert cfg == 'tests/projects.toml'
 
 
+def test_check_config_exists_directory():
+    cfg = annif.config.check_config('tests/projects.d')
+    assert cfg == 'tests/projects.d'
+
+
 def test_check_config_not_exists(caplog):
     with caplog.at_level(logging.WARNING):
         cfg = annif.config.check_config('tests/notfound.cfg')
     assert cfg is None
-    assert 'Project configuration file "tests/notfound.cfg" is missing' \
-        in caplog.text
+    assert 'Project configuration file or directory "tests/notfound.cfg" ' + \
+        'is missing' in caplog.text
 
 
 def test_find_config_exists_default(monkeypatch):
@@ -37,7 +42,7 @@ def test_find_config_not_exists_default(monkeypatch, caplog):
     with caplog.at_level(logging.WARNING):
         cfg = annif.config.find_config()
     assert cfg is None
-    assert 'Could not find project configuration file' in caplog.text
+    assert 'Could not find project configuration ' in caplog.text
 
 
 def test_parse_config_cfg_nondefault():
@@ -69,3 +74,21 @@ def test_parse_config_toml_failed(tmpdir):
     with pytest.raises(ConfigurationException) as excinfo:
         annif.config.parse_config(str(conffile))
     assert 'Invalid value' in str(excinfo.value)
+
+
+def test_parse_config_directory():
+    cfg = annif.config.parse_config('tests/projects.d')
+    assert isinstance(cfg, annif.config.AnnifConfigDirectory)
+    assert len(cfg.project_ids) == 16 + 2  # projects.cfg + projects.toml
+    assert cfg['dummy-fi'] is not None
+    assert cfg['dummy-fi-toml'] is not None
+
+
+def test_parse_config_directory_duplicated_project(tmpdir):
+    conffile = tmpdir.join('projects-1.toml')
+    conffile.write("[duplicated]\nkey='value'\n")
+    conffile = tmpdir.join('projects-2.cfg')
+    conffile.write("[duplicated]\nkey=value\n")
+    with pytest.raises(ConfigurationException) as excinfo:
+        annif.config.parse_config(str(tmpdir))
+    assert 'project ID "duplicated" already exists' in str(excinfo.value)

tests/test_project.py

@@ -274,3 +274,13 @@ def test_project_file_toml():
             == 'dummy-fi-toml'
         assert annif.registry.get_project('dummy-en-toml').project_id \
             == 'dummy-en-toml'
+
+
+def test_project_directory():
+    app = annif.create_app(
+        config_name='annif.default_config.TestingDirectoryConfig')
+    with app.app_context():
+        assert len(annif.registry.get_projects()) == 16 + 2
+        assert annif.registry.get_project('dummy-fi').project_id == 'dummy-fi'
+        assert annif.registry.get_project('dummy-fi-toml').project_id \
+            == 'dummy-fi-toml'