Skip to content

Update data finder imports #2958

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
valeriupredoi merged 8 commits into from
Dec 16, 2022
Merged

Update data finder imports #2958

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
00a316a
Update patched data finder
bouweandela Dec 9, 2022
46ab609
Avoid importing private functions from esmvalcore._data_finder
bouweandela Dec 9, 2022
fdb0b5b
Add pytest-mock as a dependency
bouweandela Dec 9, 2022
af4bfbe
Merge branch 'main' into update-data-finder-imports
bouweandela Dec 9, 2022
69e72c8
Add pytest-mock
bouweandela Dec 13, 2022
edeef8f
Merge branch 'main' into update-data-finder-imports
bouweandela Dec 15, 2022
9017b55
turn on GA dev test for sanity
valeriupredoi Dec 15, 2022
2cc10be
dont run GA tests on feature branch
valeriupredoi Dec 16, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions environment.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ dependencies:
- pytest-env
- pytest-html !=2.1.0
- pytest-metadata >=1.5.1
- pytest-mock
- pytest-xdist
# Python packages needed for building docs
- autodocsumm>=0.2.2
Expand Down
1 change: 1 addition & 0 deletions environment_osx.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ dependencies:
- pytest-env
- pytest-html !=2.1.0
- pytest-metadata >=1.5.1
- pytest-mock
- pytest-xdist
# Python packages needed for building docs
- autodocsumm>=0.2.2
Expand Down
68 changes: 67 additions & 1 deletion esmvaltool/diag_scripts/monitor/monitor_base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,11 @@

import logging
import os
import re

import cartopy
import matplotlib.pyplot as plt
import yaml
from esmvalcore._data_finder import _replace_tags
from iris.analysis import MEAN
from mapgenerator.plotting.timeseries import PlotSeries

Expand All @@ -15,6 +15,72 @@
logger = logging.getLogger(__name__)


def _replace_tags(paths, variable):
"""Replace tags in the config-developer's file with actual values."""
if isinstance(paths, str):
paths = set((paths.strip('/'), ))
else:
paths = set(path.strip('/') for path in paths)
tlist = set()
for path in paths:
tlist = tlist.union(re.findall(r'{([^}]*)}', path))
if 'sub_experiment' in variable:
new_paths = []
for path in paths:
new_paths.extend(
(re.sub(r'(\b{ensemble}\b)', r'{sub_experiment}-\1', path),
re.sub(r'({ensemble})', r'{sub_experiment}-\1', path)))
tlist.add('sub_experiment')
paths = new_paths

for tag in tlist:
original_tag = tag
tag, _, _ = _get_caps_options(tag)

if tag == 'latestversion': # handled separately later
continue
if tag in variable:
replacewith = variable[tag]
else:
raise ValueError(f"Dataset key '{tag}' must be specified for "
f"{variable}, check your recipe entry")
paths = _replace_tag(paths, original_tag, replacewith)
return paths


def _replace_tag(paths, tag, replacewith):
"""Replace tag by replacewith in paths."""
_, lower, upper = _get_caps_options(tag)
result = []
if isinstance(replacewith, (list, tuple)):
for item in replacewith:
result.extend(_replace_tag(paths, tag, item))
else:
text = _apply_caps(str(replacewith), lower, upper)
result.extend(p.replace('{' + tag + '}', text) for p in paths)
return list(set(result))


def _get_caps_options(tag):
lower = False
upper = False
if tag.endswith('.lower'):
lower = True
tag = tag[0:-6]
elif tag.endswith('.upper'):
upper = True
tag = tag[0:-6]
return tag, lower, upper


def _apply_caps(original, lower, upper):
if lower:
return original.lower()
if upper:
return original.upper()
return original


class MonitorBase():
"""Base class for monitoring diagnostic.

Expand Down
1 change: 1 addition & 0 deletions setup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@
'pytest-env',
'pytest-html!=2.1.0',
'pytest-metadata>=1.5.1',
'pytest-mock',
'pytest-xdist',
],
# Documentation dependencies
Expand Down
70 changes: 49 additions & 21 deletions tests/integration/test_recipes_loading.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
"""Test recipes are well formed."""
from pathlib import Path
from unittest.mock import create_autospec

import pytest
import yaml

import esmvalcore
import esmvalcore._config
import esmvalcore._data_finder
import esmvalcore._recipe
import esmvalcore.cmor.check
import pytest
import yaml

import esmvaltool

from .test_diagnostic_run import write_config_user_file
Expand Down Expand Up @@ -40,27 +38,52 @@ def _get_recipes():


@pytest.mark.parametrize('recipe_file', RECIPES, ids=IDS)
def test_recipe_valid(recipe_file, config_user, monkeypatch):
def test_recipe_valid(recipe_file, config_user, mocker):
"""Check that recipe files are valid ESMValTool recipes."""
# Mock input files
find_files = create_autospec(esmvalcore._data_finder.find_files,
spec_set=True)
find_files.side_effect = lambda *_, **__: [
'test_0001-1849.nc',
'test_1850-9999.nc',
]
monkeypatch.setattr(esmvalcore._data_finder, 'find_files', find_files)
try:
# Since ESValCore v2.8.0
import esmvalcore.local
module = esmvalcore.local
method = 'glob'
# The patched_datafinder fixture does not return the correct input
# directory structure, so make sure it is set to flat for every project
from esmvalcore.config import CFG, _config
mocker.patch.dict(CFG, drs={})
for project in _config.CFG:
mocker.patch.dict(_config.CFG[project]['input_dir'], default='/')
except ImportError:
# Prior to ESMValCore v2.8.0
import esmvalcore._data_finder
module = esmvalcore._data_finder
method = 'find_files'

mocker.patch.object(
module,
method,
autospec=True,
side_effect=lambda *_, **__: [
'test_0001-1849.nc',
'test_1850-9999.nc',
],
)

# Mock vertical levels
levels = create_autospec(esmvalcore._recipe.get_reference_levels,
spec_set=True)
levels.side_effect = lambda *_, **__: [1, 2]
monkeypatch.setattr(esmvalcore._recipe, 'get_reference_levels', levels)
mocker.patch.object(
esmvalcore._recipe,
'get_reference_levels',
autospec=True,
spec_set=True,
side_effect=lambda *_, **__: [1, 2],
)

# Mock valid NCL version
ncl_version = create_autospec(esmvalcore._recipe_checks.ncl_version,
spec_set=True)
monkeypatch.setattr(esmvalcore._recipe_checks, 'ncl_version', ncl_version)
mocker.patch.object(
esmvalcore._recipe_checks,
'ncl_version',
autospec=True,
spec_set=True,
)

# Mock interpreters installed
def which(executable):
Expand All @@ -70,7 +93,12 @@ def which(executable):
path = None
return path

monkeypatch.setattr(esmvalcore._task, 'which', which)
mocker.patch.object(
esmvalcore._task,
'which',
autospec=True,
side_effect=which,
)

# Create a shapefile for extract_shape preprocessor if needed
recipe = yaml.safe_load(recipe_file.read_text())
Expand Down