diff --git a/environment.yml b/environment.yml index d8110f0d41..066a641793 100644 --- a/environment.yml +++ b/environment.yml @@ -69,6 +69,7 @@ dependencies: - pytest-env - pytest-html !=2.1.0 - pytest-metadata >=1.5.1 + - pytest-mock - pytest-xdist # Python packages needed for building docs - autodocsumm>=0.2.2 diff --git a/environment_osx.yml b/environment_osx.yml index a2da023883..a9cc65c3a8 100644 --- a/environment_osx.yml +++ b/environment_osx.yml @@ -69,6 +69,7 @@ dependencies: - pytest-env - pytest-html !=2.1.0 - pytest-metadata >=1.5.1 + - pytest-mock - pytest-xdist # Python packages needed for building docs - autodocsumm>=0.2.2 diff --git a/esmvaltool/diag_scripts/monitor/monitor_base.py b/esmvaltool/diag_scripts/monitor/monitor_base.py index a2a7bf5524..265443d022 100644 --- a/esmvaltool/diag_scripts/monitor/monitor_base.py +++ b/esmvaltool/diag_scripts/monitor/monitor_base.py @@ -2,11 +2,11 @@ import logging import os +import re import cartopy import matplotlib.pyplot as plt import yaml -from esmvalcore._data_finder import _replace_tags from iris.analysis import MEAN from mapgenerator.plotting.timeseries import PlotSeries @@ -15,6 +15,72 @@ logger = logging.getLogger(__name__) +def _replace_tags(paths, variable): + """Replace tags in the config-developer's file with actual values.""" + if isinstance(paths, str): + paths = set((paths.strip('/'), )) + else: + paths = set(path.strip('/') for path in paths) + tlist = set() + for path in paths: + tlist = tlist.union(re.findall(r'{([^}]*)}', path)) + if 'sub_experiment' in variable: + new_paths = [] + for path in paths: + new_paths.extend( + (re.sub(r'(\b{ensemble}\b)', r'{sub_experiment}-\1', path), + re.sub(r'({ensemble})', r'{sub_experiment}-\1', path))) + tlist.add('sub_experiment') + paths = new_paths + + for tag in tlist: + original_tag = tag + tag, _, _ = _get_caps_options(tag) + + if tag == 'latestversion': # handled separately later + continue + if tag in variable: + replacewith = variable[tag] + else: + raise ValueError(f"Dataset key '{tag}' must be specified for " + f"{variable}, check your recipe entry") + paths = _replace_tag(paths, original_tag, replacewith) + return paths + + +def _replace_tag(paths, tag, replacewith): + """Replace tag by replacewith in paths.""" + _, lower, upper = _get_caps_options(tag) + result = [] + if isinstance(replacewith, (list, tuple)): + for item in replacewith: + result.extend(_replace_tag(paths, tag, item)) + else: + text = _apply_caps(str(replacewith), lower, upper) + result.extend(p.replace('{' + tag + '}', text) for p in paths) + return list(set(result)) + + +def _get_caps_options(tag): + lower = False + upper = False + if tag.endswith('.lower'): + lower = True + tag = tag[0:-6] + elif tag.endswith('.upper'): + upper = True + tag = tag[0:-6] + return tag, lower, upper + + +def _apply_caps(original, lower, upper): + if lower: + return original.lower() + if upper: + return original.upper() + return original + + class MonitorBase(): """Base class for monitoring diagnostic. diff --git a/setup.py b/setup.py index 47c58d6116..bf0c37d176 100755 --- a/setup.py +++ b/setup.py @@ -75,6 +75,7 @@ 'pytest-env', 'pytest-html!=2.1.0', 'pytest-metadata>=1.5.1', + 'pytest-mock', 'pytest-xdist', ], # Documentation dependencies diff --git a/tests/integration/test_recipes_loading.py b/tests/integration/test_recipes_loading.py index d88d541bd1..5aef20197a 100644 --- a/tests/integration/test_recipes_loading.py +++ b/tests/integration/test_recipes_loading.py @@ -1,15 +1,13 @@ """Test recipes are well formed.""" from pathlib import Path -from unittest.mock import create_autospec - -import pytest -import yaml import esmvalcore import esmvalcore._config -import esmvalcore._data_finder import esmvalcore._recipe import esmvalcore.cmor.check +import pytest +import yaml + import esmvaltool from .test_diagnostic_run import write_config_user_file @@ -40,27 +38,52 @@ def _get_recipes(): @pytest.mark.parametrize('recipe_file', RECIPES, ids=IDS) -def test_recipe_valid(recipe_file, config_user, monkeypatch): +def test_recipe_valid(recipe_file, config_user, mocker): """Check that recipe files are valid ESMValTool recipes.""" # Mock input files - find_files = create_autospec(esmvalcore._data_finder.find_files, - spec_set=True) - find_files.side_effect = lambda *_, **__: [ - 'test_0001-1849.nc', - 'test_1850-9999.nc', - ] - monkeypatch.setattr(esmvalcore._data_finder, 'find_files', find_files) + try: + # Since ESValCore v2.8.0 + import esmvalcore.local + module = esmvalcore.local + method = 'glob' + # The patched_datafinder fixture does not return the correct input + # directory structure, so make sure it is set to flat for every project + from esmvalcore.config import CFG, _config + mocker.patch.dict(CFG, drs={}) + for project in _config.CFG: + mocker.patch.dict(_config.CFG[project]['input_dir'], default='/') + except ImportError: + # Prior to ESMValCore v2.8.0 + import esmvalcore._data_finder + module = esmvalcore._data_finder + method = 'find_files' + + mocker.patch.object( + module, + method, + autospec=True, + side_effect=lambda *_, **__: [ + 'test_0001-1849.nc', + 'test_1850-9999.nc', + ], + ) # Mock vertical levels - levels = create_autospec(esmvalcore._recipe.get_reference_levels, - spec_set=True) - levels.side_effect = lambda *_, **__: [1, 2] - monkeypatch.setattr(esmvalcore._recipe, 'get_reference_levels', levels) + mocker.patch.object( + esmvalcore._recipe, + 'get_reference_levels', + autospec=True, + spec_set=True, + side_effect=lambda *_, **__: [1, 2], + ) # Mock valid NCL version - ncl_version = create_autospec(esmvalcore._recipe_checks.ncl_version, - spec_set=True) - monkeypatch.setattr(esmvalcore._recipe_checks, 'ncl_version', ncl_version) + mocker.patch.object( + esmvalcore._recipe_checks, + 'ncl_version', + autospec=True, + spec_set=True, + ) # Mock interpreters installed def which(executable): @@ -70,7 +93,12 @@ def which(executable): path = None return path - monkeypatch.setattr(esmvalcore._task, 'which', which) + mocker.patch.object( + esmvalcore._task, + 'which', + autospec=True, + side_effect=which, + ) # Create a shapefile for extract_shape preprocessor if needed recipe = yaml.safe_load(recipe_file.read_text())