Update data finder imports

environment.yml

@@ -69,6 +69,7 @@ dependencies:
   - pytest-env
   - pytest-html !=2.1.0
   - pytest-metadata >=1.5.1
+  - pytest-mock
   - pytest-xdist
   # Python packages needed for building docs
   - autodocsumm>=0.2.2

esmvaltool/diag_scripts/monitor/monitor_base.py

@@ -2,11 +2,11 @@
 
 import logging
 import os
+import re
 
 import cartopy
 import matplotlib.pyplot as plt
 import yaml
-from esmvalcore._data_finder import _replace_tags
 from iris.analysis import MEAN
 from mapgenerator.plotting.timeseries import PlotSeries
 
@@ -15,6 +15,72 @@
 logger = logging.getLogger(__name__)
 
 
+def _replace_tags(paths, variable):
+    """Replace tags in the config-developer's file with actual values."""
+    if isinstance(paths, str):
+        paths = set((paths.strip('/'), ))
+    else:
+        paths = set(path.strip('/') for path in paths)
+    tlist = set()
+    for path in paths:
+        tlist = tlist.union(re.findall(r'{([^}]*)}', path))
+    if 'sub_experiment' in variable:
+        new_paths = []
+        for path in paths:
+            new_paths.extend(
+                (re.sub(r'(\b{ensemble}\b)', r'{sub_experiment}-\1', path),
+                 re.sub(r'({ensemble})', r'{sub_experiment}-\1', path)))
+            tlist.add('sub_experiment')
+        paths = new_paths
+
+    for tag in tlist:
+        original_tag = tag
+        tag, _, _ = _get_caps_options(tag)
+
+        if tag == 'latestversion':  # handled separately later
+            continue
+        if tag in variable:
+            replacewith = variable[tag]
+        else:
+            raise ValueError(f"Dataset key '{tag}' must be specified for "
+                             f"{variable}, check your recipe entry")
+        paths = _replace_tag(paths, original_tag, replacewith)
+    return paths
+
+
+def _replace_tag(paths, tag, replacewith):
+    """Replace tag by replacewith in paths."""
+    _, lower, upper = _get_caps_options(tag)
+    result = []
+    if isinstance(replacewith, (list, tuple)):
+        for item in replacewith:
+            result.extend(_replace_tag(paths, tag, item))
+    else:
+        text = _apply_caps(str(replacewith), lower, upper)
+        result.extend(p.replace('{' + tag + '}', text) for p in paths)
+    return list(set(result))
+
+
+def _get_caps_options(tag):
+    lower = False
+    upper = False
+    if tag.endswith('.lower'):
+        lower = True
+        tag = tag[0:-6]
+    elif tag.endswith('.upper'):
+        upper = True
+        tag = tag[0:-6]
+    return tag, lower, upper
+
+
+def _apply_caps(original, lower, upper):
+    if lower:
+        return original.lower()
+    if upper:
+        return original.upper()
+    return original
+
+
 class MonitorBase():
     """Base class for monitoring diagnostic.
 

setup.py

@@ -75,6 +75,7 @@
         'pytest-env',
         'pytest-html!=2.1.0',
         'pytest-metadata>=1.5.1',
+        'pytest-mock',
         'pytest-xdist',
     ],
     # Documentation dependencies

tests/integration/test_recipes_loading.py

@@ -1,15 +1,13 @@
 """Test recipes are well formed."""
 from pathlib import Path
-from unittest.mock import create_autospec
-
-import pytest
-import yaml
 
 import esmvalcore
 import esmvalcore._config
-import esmvalcore._data_finder
 import esmvalcore._recipe
 import esmvalcore.cmor.check
+import pytest
+import yaml
+
 import esmvaltool
 
 from .test_diagnostic_run import write_config_user_file
@@ -40,27 +38,52 @@ def _get_recipes():
 
 
 @pytest.mark.parametrize('recipe_file', RECIPES, ids=IDS)
-def test_recipe_valid(recipe_file, config_user, monkeypatch):
+def test_recipe_valid(recipe_file, config_user, mocker):
     """Check that recipe files are valid ESMValTool recipes."""
     # Mock input files
-    find_files = create_autospec(esmvalcore._data_finder.find_files,
-                                 spec_set=True)
-    find_files.side_effect = lambda *_, **__: [
-        'test_0001-1849.nc',
-        'test_1850-9999.nc',
-    ]
-    monkeypatch.setattr(esmvalcore._data_finder, 'find_files', find_files)
+    try:
+        # Since ESValCore v2.8.0
+        import esmvalcore.local
+        module = esmvalcore.local
+        method = 'glob'
+        # The patched_datafinder fixture does not return the correct input
+        # directory structure, so make sure it is set to flat for every project
+        from esmvalcore.config import CFG, _config
+        mocker.patch.dict(CFG, drs={})
+        for project in _config.CFG:
+            mocker.patch.dict(_config.CFG[project]['input_dir'], default='/')
+    except ImportError:
+        # Prior to ESMValCore v2.8.0
+        import esmvalcore._data_finder
+        module = esmvalcore._data_finder
+        method = 'find_files'
+
+    mocker.patch.object(
+        module,
+        method,
+        autospec=True,
+        side_effect=lambda *_, **__: [
+            'test_0001-1849.nc',
+            'test_1850-9999.nc',
+        ],
+    )
 
     # Mock vertical levels
-    levels = create_autospec(esmvalcore._recipe.get_reference_levels,
-                             spec_set=True)
-    levels.side_effect = lambda *_, **__: [1, 2]
-    monkeypatch.setattr(esmvalcore._recipe, 'get_reference_levels', levels)
+    mocker.patch.object(
+        esmvalcore._recipe,
+        'get_reference_levels',
+        autospec=True,
+        spec_set=True,
+        side_effect=lambda *_, **__: [1, 2],
+    )
 
     # Mock valid NCL version
-    ncl_version = create_autospec(esmvalcore._recipe_checks.ncl_version,
-                                  spec_set=True)
-    monkeypatch.setattr(esmvalcore._recipe_checks, 'ncl_version', ncl_version)
+    mocker.patch.object(
+        esmvalcore._recipe_checks,
+        'ncl_version',
+        autospec=True,
+        spec_set=True,
+    )
 
     # Mock interpreters installed
     def which(executable):
@@ -70,7 +93,12 @@ def which(executable):
             path = None
         return path
 
-    monkeypatch.setattr(esmvalcore._task, 'which', which)
+    mocker.patch.object(
+        esmvalcore._task,
+        'which',
+        autospec=True,
+        side_effect=which,
+    )
 
     # Create a shapefile for extract_shape preprocessor if needed
     recipe = yaml.safe_load(recipe_file.read_text())