Skip to content

Remove deprecated CMOR fix/check code #2552

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
valeriupredoi merged 2 commits into from
Oct 11, 2024
Merged

Remove deprecated CMOR fix/check code #2552

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
db808bb
Remove automatic_fixes from checks
schlunma Oct 11, 2024
b759d56
Remove cmor checks from fixes
schlunma Oct 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 0 additions & 53 deletions esmvalcore/cmor/check.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,6 @@
from __future__ import annotations

import logging
import warnings
from collections import namedtuple
from collections.abc import Callable
from enum import IntEnum
Expand All @@ -20,15 +19,13 @@
from iris.coords import Coord
from iris.cube import Cube

from esmvalcore.cmor._fixes.fix import GenericFix
from esmvalcore.cmor._utils import (
_get_alternative_generic_lev_coord,
_get_generic_lev_coord_names,
_get_new_generic_level_coord,
_get_simplified_calendar,
)
from esmvalcore.cmor.table import CoordinateInfo, get_var_info
from esmvalcore.exceptions import ESMValCoreDeprecationWarning
from esmvalcore.iris_helpers import has_unstructured_grid


Expand Down Expand Up @@ -70,18 +67,6 @@ class CMORCheck:
fail_on_error: bool
If true, CMORCheck stops on the first error. If false, it collects
all possible errors before stopping.
automatic_fixes: bool
If True, CMORCheck will try to apply automatic fixes for any
detected error, if possible.

.. deprecated:: 2.10.0
This option has been deprecated in ESMValCore version 2.10.0 and is
scheduled for removal in version 2.12.0. Please use the functions
:func:`~esmvalcore.preprocessor.fix_metadata`,
:func:`~esmvalcore.preprocessor.fix_data`, or
:meth:`esmvalcore.dataset.Dataset.load` (which automatically
includes the first two functions) instead. Fixes and CMOR checks
have been clearly separated in ESMValCore version 2.10.0.
check_level: CheckLevels
Level of strictness of the checks.

Expand All @@ -104,7 +89,6 @@ def __init__(
frequency=None,
fail_on_error=False,
check_level=CheckLevels.DEFAULT,
automatic_fixes=False,
):
self._cube = cube
self._failerr = fail_on_error
Expand All @@ -118,26 +102,6 @@ def __init__(
if not frequency:
frequency = self._cmor_var.frequency
self.frequency = frequency
self.automatic_fixes = automatic_fixes

# Deprecate automatic_fixes (remove in v2.12)
if automatic_fixes:
msg = (
"The option `automatic_fixes` has been deprecated in "
"ESMValCore version 2.10.0 and is scheduled for removal in "
"version 2.12.0. Please use the functions "
"esmvalcore.preprocessor.fix_metadata(), "
"esmvalcore.preprocessor.fix_data(), or "
"esmvalcore.dataset.Dataset.load() (which automatically "
"includes the first two functions) instead. Fixes and CMOR "
"checks have been clearly separated in ESMValCore version "
"2.10.0."
)
warnings.warn(msg, ESMValCoreDeprecationWarning)

# TODO: remove in v2.12

self._generic_fix = GenericFix(var_info, frequency=frequency)

@cached_property
def _unstructured_grid(self) -> bool:
Expand Down Expand Up @@ -171,10 +135,6 @@ def check_metadata(self, logger: Optional[logging.Logger] = None) -> Cube:
if logger is not None:
self._logger = logger

# TODO: remove in v2.12
if self.automatic_fixes:
[self._cube] = self._generic_fix.fix_metadata([self._cube])

self._check_var_metadata()
self._check_fill_value()
self._check_multiple_coords_same_stdname()
Expand Down Expand Up @@ -220,10 +180,6 @@ def check_data(self, logger: Optional[logging.Logger] = None) -> Cube:
if logger is not None:
self._logger = logger

# TODO: remove in v2.12
if self.automatic_fixes:
self._cube = self._generic_fix.fix_data(self._cube)

self._check_coords_data()

self.report_debug_messages()
Expand Down Expand Up @@ -345,7 +301,6 @@ def _check_var_metadata(self):

def _get_effective_units(self):
"""Get effective units."""
# TODO: remove entire function in v2.12
if self._cmor_var.units.lower() == "psu":
units = "1.0"
else:
Expand Down Expand Up @@ -606,12 +561,6 @@ def _check_coords_data(self):
except iris.exceptions.CoordinateNotFoundError:
continue

# TODO: remove in v2.12
if self.automatic_fixes:
(self._cube, coord) = self._generic_fix._fix_coord_direction(
self._cube, coordinate, coord
)

self._check_coord_monotonicity_and_direction(
coordinate, coord, var_name
)
Expand Down Expand Up @@ -935,7 +884,6 @@ def _get_cmor_checker(
frequency: None | str = None,
fail_on_error: bool = False,
check_level: CheckLevels = CheckLevels.DEFAULT,
automatic_fixes: bool = False, # TODO: remove in v2.12
) -> Callable[[Cube], CMORCheck]:
"""Get a CMOR checker."""
var_info = get_var_info(project, mip, short_name)
Expand All @@ -947,7 +895,6 @@ def _checker(cube: Cube) -> CMORCheck:
frequency=frequency,
fail_on_error=fail_on_error,
check_level=check_level,
automatic_fixes=automatic_fixes,
)

return _checker
Expand Down
79 changes: 0 additions & 79 deletions esmvalcore/cmor/fix.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
from __future__ import annotations

import logging
import warnings
from collections import defaultdict
from collections.abc import Sequence
from pathlib import Path
Expand All @@ -17,8 +16,6 @@
from iris.cube import Cube, CubeList

from esmvalcore.cmor._fixes.fix import Fix
from esmvalcore.cmor.check import CheckLevels, _get_cmor_checker
from esmvalcore.exceptions import ESMValCoreDeprecationWarning

if TYPE_CHECKING:
from ..config import Session
Expand Down Expand Up @@ -109,7 +106,6 @@ def fix_metadata(
dataset: str,
mip: str,
frequency: Optional[str] = None,
check_level: CheckLevels = CheckLevels.DEFAULT,
session: Optional[Session] = None,
**extra_facets,
) -> CubeList:
Expand All @@ -132,17 +128,6 @@ def fix_metadata(
Variable's MIP.
frequency:
Variable's data frequency, if available.
check_level:
Level of strictness of the checks.

.. deprecated:: 2.10.0
This option has been deprecated in ESMValCore version 2.10.0 and is
scheduled for removal in version 2.12.0. Please use the functions
:func:`~esmvalcore.preprocessor.cmor_check_metadata`,
:func:`~esmvalcore.preprocessor.cmor_check_data`, or
:meth:`~esmvalcore.cmor.check.cmor_check` instead. This function
will no longer perform CMOR checks. Fixes and CMOR checks have been
clearly separated in ESMValCore version 2.10.0.
session:
Current session which includes configuration and directory information.
**extra_facets:
Expand All @@ -155,20 +140,6 @@ def fix_metadata(
Fixed cubes.

"""
# Deprecate CMOR checks (remove in v2.12)
if check_level != CheckLevels.DEFAULT:
msg = (
"The option `check_level` has been deprecated in ESMValCore "
"version 2.10.0 and is scheduled for removal in version 2.12.0. "
"Please use the functions "
"esmvalcore.preprocessor.cmor_check_metadata, "
"esmvalcore.preprocessor.cmor_check_data, or "
"esmvalcore.cmor.check.cmor_check instead. This function will no "
"longer perform CMOR checks. Fixes and CMOR checks have been "
"clearly separated in ESMValCore version 2.10.0."
)
warnings.warn(msg, ESMValCoreDeprecationWarning)

# Update extra_facets with variable information given as regular arguments
# to this function
extra_facets.update(
Expand Down Expand Up @@ -207,18 +178,6 @@ def fix_metadata(
# returns a single cube
cube = cube_list[0]

# Perform CMOR checks
# TODO: remove in v2.12
checker = _get_cmor_checker(
project,
mip,
short_name,
frequency,
fail_on_error=False,
check_level=check_level,
)
cube = checker(cube).check_metadata()

cube.attributes.pop("source_file", None)
fixed_cubes.append(cube)

Expand All @@ -232,7 +191,6 @@ def fix_data(
dataset: str,
mip: str,
frequency: Optional[str] = None,
check_level: CheckLevels = CheckLevels.DEFAULT,
session: Optional[Session] = None,
**extra_facets,
) -> Cube:
Expand All @@ -257,17 +215,6 @@ def fix_data(
Variable's MIP.
frequency:
Variable's data frequency, if available.
check_level:
Level of strictness of the checks.

.. deprecated:: 2.10.0
This option has been deprecated in ESMValCore version 2.10.0 and is
scheduled for removal in version 2.12.0. Please use the functions
:func:`~esmvalcore.preprocessor.cmor_check_metadata`,
:func:`~esmvalcore.preprocessor.cmor_check_data`, or
:meth:`~esmvalcore.cmor.check.cmor_check` instead. This function
will no longer perform CMOR checks. Fixes and CMOR checks have been
clearly separated in ESMValCore version 2.10.0.
session:
Current session which includes configuration and directory information.
**extra_facets:
Expand All @@ -280,20 +227,6 @@ def fix_data(
Fixed cube.

"""
# Deprecate CMOR checks (remove in v2.12)
if check_level != CheckLevels.DEFAULT:
msg = (
"The option `check_level` has been deprecated in ESMValCore "
"version 2.10.0 and is scheduled for removal in version 2.12.0. "
"Please use the functions "
"esmvalcore.preprocessor.cmor_check_metadata, "
"esmvalcore.preprocessor.cmor_check_data, or "
"esmvalcore.cmor.check.cmor_check instead. This function will no "
"longer perform CMOR checks. Fixes and CMOR checks have been "
"clearly separated in ESMValCore version 2.10.0."
)
warnings.warn(msg, ESMValCoreDeprecationWarning)

# Update extra_facets with variable information given as regular arguments
# to this function
extra_facets.update(
Expand All @@ -317,16 +250,4 @@ def fix_data(
):
cube = fix.fix_data(cube)

# Perform CMOR checks
# TODO: remove in v2.12
checker = _get_cmor_checker(
project,
mip,
short_name,
frequency,
fail_on_error=False,
check_level=check_level,
)
cube = checker(cube).check_data()

return cube
2 changes: 0 additions & 2 deletions esmvalcore/dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -761,7 +761,6 @@ def _load(self) -> Cube:
),
}
settings["fix_metadata"] = {
"check_level": self.session["check_level"],
"session": self.session,
**self.facets,
}
Expand All @@ -778,7 +777,6 @@ def _load(self) -> Cube:
"timerange": self.facets["timerange"],
}
settings["fix_data"] = {
"check_level": self.session["check_level"],
"session": self.session,
**self.facets,
}
Expand Down
9 changes: 1 addition & 8 deletions tests/integration/cmor/_fixes/obs4mips/test_airs_2_0.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,14 +20,7 @@ def test_fix_metadata_hur():
]
)

fixed_cubes = fix_metadata(
cubes,
"hur",
"obs4MIPs",
"AIRS-2-0",
"Amon",
check_level=5,
)
fixed_cubes = fix_metadata(cubes, "hur", "obs4MIPs", "AIRS-2-0", "Amon")

assert len(fixed_cubes) == 1
fixed_cube = fixed_cubes[0]
Expand Down
46 changes: 1 addition & 45 deletions tests/integration/cmor/test_fix.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,7 @@
from iris.coords import AuxCoord, DimCoord
from iris.cube import Cube, CubeList

from esmvalcore.cmor.check import CheckLevels, CMORCheckError
from esmvalcore.exceptions import ESMValCoreDeprecationWarning
from esmvalcore.cmor.check import CMORCheckError
from esmvalcore.preprocessor import (
cmor_check_data,
cmor_check_metadata,
Expand All @@ -18,25 +17,6 @@
)


# TODO: remove in v2.12
@pytest.fixture(autouse=True)
def disable_fix_cmor_checker(mocker):
"""Disable the CMOR checker in fixes (will be default in v2.12)."""

class MockChecker:
def __init__(self, cube):
self._cube = cube

def check_metadata(self):
return self._cube

def check_data(self):
return self._cube

mock = mocker.patch("esmvalcore.cmor.fix._get_cmor_checker")
mock.return_value = MockChecker


class TestGenericFix:
"""Tests for ``GenericFix``."""

Expand Down Expand Up @@ -887,27 +867,3 @@ def test_fix_data_amon_tas(self):

assert self.mock_debug.call_count == 0
assert self.mock_warning.call_count == 0

def test_deprecate_check_level_fix_metadata(self):
"""Test deprecation of check level in ``fix_metadata``."""
with pytest.warns(ESMValCoreDeprecationWarning):
fix_metadata(
self.cubes_4d,
"ta",
"CMIP6",
"MODEL",
"Amon",
check_level=CheckLevels.RELAXED,
)

def test_deprecate_check_level_fix_data(self):
"""Test deprecation of check level in ``fix_data``."""
with pytest.warns(ESMValCoreDeprecationWarning):
fix_metadata(
self.cubes_4d,
"ta",
"CMIP6",
"MODEL",
"Amon",
check_level=CheckLevels.RELAXED,
)
Loading