Dry run release v2.7.1 bugfix

doc/changelog.rst

@@ -2,6 +2,28 @@ Changelog
 =========
 
 
+.. _changelog-v2-7-1:
+
+
+v2.7.1
+------
+Highlights
+~~~~~~~~~~
+
+This is a bugfix release where we unpin `cf-units` to allow for latest `iris=3.4.0` to be installed.
+
+Installation
+~~~~~~~~~~~~
+
+- Unpin cf-units (`#1770 <https://github.com/ESMValGroup/ESMValCore/pull/1770>`__) `Bouwe Andela <https://github.com/bouweandela>`__
+
+Bug fixes
+~~~~~~~~~
+
+- Improve error handling if an esgf index node is offline (`#1834 <https://github.com/ESMValGroup/ESMValCore/pull/1834>`__) `Bouwe Andela <https://github.com/bouweandela>`__
+
+
+
 .. _changelog-v2-7-0:
 
 

environment.yml

@@ -2,20 +2,19 @@
 name: esmvaltool
 channels:
   - conda-forge
+  - nodefaults
 
 dependencies:
   - cartopy
-  - cf-units>=3.0.0,<3.1.0  # github.com/ESMValGroup/ESMValCore/issues/1655
+  - cf-units
   - cftime
   - compilers
-  # 1.8.18/py39, they seem weary to build manylinux wheels
-  # and pypi ver built with older gdal
   - fiona
   - esmpy!=8.1.0  # see github.com/ESMValGroup/ESMValCore/issues/1208
   - geopy
   - iris>=3.2.1
   - nested-lookup
-  - netcdf4!=1.6.1  # github.com/ESMValGroup/ESMValCore/issues/1723
+  - netcdf4
   - pandas
   - pillow
   - pip!=21.3

esmvalcore/cmor/check.py

@@ -945,7 +945,7 @@ def _simplify_calendar(calendar):
         calendar_aliases = {
             'all_leap': '366_day',
             'noleap': '365_day',
-            'standard': 'gregorian',
+            'gregorian': 'standard',
         }
         return calendar_aliases.get(calendar, calendar)
 

esmvalcore/esgf/_search.py

@@ -110,7 +110,12 @@ def _search_index_nodes(facets):
             )
             FIRST_ONLINE_INDEX_NODE = url
             return results
-        except requests.exceptions.Timeout as error:
+        except (
+            requests.exceptions.ConnectionError,
+            requests.exceptions.HTTPError,
+            requests.exceptions.Timeout,
+        ) as error:
+            logger.debug("Unable to connect to %s due to %s", url, error)
             errors.append(error)
 
     raise FileNotFoundError("Failed to search ESGF, unable to connect:\n" +

esmvalcore/preprocessor/_multimodel.py

@@ -96,7 +96,7 @@ def _unify_time_coordinates(cubes):
     are no mismatches in the time arrays.
 
     If cubes have different time units, it will reset the calendar to a
-    default gregorian calendar with unit "days since 1850-01-01".
+    the "standard" calendar with unit "days since 1850-01-01".
 
     Might not work for (sub)daily data, because different calendars may have
     different number of days in the year.

pyproject.toml

@@ -3,6 +3,7 @@ requires = ["setuptools >= 40.6.0", "wheel", "setuptools_scm>=6.2"]
 build-backend = "setuptools.build_meta"
 
 [tool.setuptools_scm]
+version_scheme = "release-branch-semver"
 
 [tool.pylint.main]
 jobs = 0

setup.py

@@ -28,8 +28,7 @@
     # Use with pip install . to install from source
     'install': [
         'cartopy',
-        # see https://github.com/SciTools/cf-units/issues/218
-        'cf-units>=3.0.0,<3.1.0,!=3.0.1.post0',  # ESMValCore/issues/1655
+        'cf-units',
         'dask[array]',
         'esgf-pyclient>=0.3.1',
         'esmpy!=8.1.0',  # see github.com/ESMValGroup/ESMValCore/issues/1208
@@ -42,7 +41,7 @@
         'jinja2',
         'nc-time-axis',  # needed by iris.plot
         'nested-lookup',
-        'netCDF4!=1.6.1',  # github.com/ESMValGroup/ESMValCore/issues/1723
+        'netCDF4',
         'numpy',
         'pandas',
         'pillow',
@@ -59,7 +58,8 @@
     ],
     # Test dependencies
     'test': [
-        'flake8<5.0',  # github.com/ESMValGroup/ESMValCore/issues/1696
+        'flake8',
+        'mypy<0.990',
         'pytest>=3.9,!=6.0.0rc1,!=6.0.0',
         'pytest-cov>=2.10.1',
         'pytest-env',

tests/integration/cmor/_fixes/cmip5/test_access1_0.py

@@ -13,45 +13,54 @@
 from esmvalcore.iris_helpers import date2num
 
 
-class TestAllVars(unittest.TestCase):
-    """Test all vars fixes."""
+@pytest.fixture
+def cube():
+    """Cube for testing."""
+    test_cube = Cube([1.0, 2.0], var_name='co2', units='J')
+    reference_dates = [
+        datetime(300, 1, 16, 12),  # e.g. piControl
+        datetime(1850, 1, 16, 12)  # e.g. historical
+    ]
+    esgf_time_units = Unit(
+        'days since 0001-01-01',
+        calendar='proleptic_gregorian',
+    )
+    time_points = date2num(reference_dates, esgf_time_units)
+    test_cube.add_dim_coord(
+        DimCoord(time_points, 'time', 'time', 'time', esgf_time_units),
+        data_dim=0,
+    )
+    return test_cube
+
 
-    def setUp(self):
-        """Prepare tests."""
-        self.cube = Cube([1.0, 2.0], var_name='co2', units='J')
-        reference_dates = [
-            datetime(300, 1, 16, 12),  # e.g. piControl
-            datetime(1850, 1, 16, 12)  # e.g. historical
-        ]
-        esgf_time_units = Unit('days since 0001-01-01',
-                               calendar='proleptic_gregorian')
-        time_points = date2num(reference_dates, esgf_time_units)
-        self.cube.add_dim_coord(
-            DimCoord(time_points, 'time', 'time', 'time', esgf_time_units),
-            data_dim=0)
-        self.fix = AllVars(None)
+class TestAllVars:
+    """Test all vars fixes."""
 
-    def test_get(self):
+    @staticmethod
+    def test_get():
         """Test getting of fix."""
-        self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'ACCESS1-0', 'Amon', 'tas'),
-            [AllVars(None)])
+        assert (Fix.get_fixes('CMIP5', 'ACCESS1-0', 'Amon', 'tas')
+                == [AllVars(None)])
 
-    def test_fix_metadata(self):
+    @staticmethod
+    def test_fix_metadata(cube):
         """Test fix for bad calendar."""
-        cube = self.fix.fix_metadata([self.cube])[0]
+        fix = AllVars(None)
+        cube = fix.fix_metadata([cube])[0]
         time = cube.coord('time')
         dates = num2date(time.points, time.units.name, time.units.calendar)
-        self.assertEqual(time.units.calendar, 'gregorian')
-        u = Unit('days since 300-01-01 12:00:00', calendar='gregorian')
-        self.assertEqual(dates[0], u.num2date(15))
-        u = Unit('days since 1850-01-01 12:00:00', calendar='gregorian')
-        self.assertEqual(dates[1], u.num2date(15))
+        assert time.units.calendar in ('standard', 'gregorian')
+        u = Unit('days since 300-01-01 12:00:00', calendar='standard')
+        assert dates[0] == u.num2date(15)
+        u = Unit('days since 1850-01-01 12:00:00', calendar='standard')
+        assert dates[1] == u.num2date(15)
 
-    def test_fix_metadata_if_not_time(self):
+    @staticmethod
+    def test_fix_metadata_if_not_time(cube):
         """Test calendar fix do not fail if no time coord present."""
-        self.cube.remove_coord('time')
-        self.fix.fix_metadata([self.cube])
+        cube.remove_coord('time')
+        fix = AllVars(None)
+        fix.fix_metadata([cube])
 
 
 def test_get_cl_fix():

tests/integration/cmor/_fixes/cmip5/test_access1_3.py

@@ -1,7 +1,7 @@
 """Test fixes for ACCESS1-3."""
-import unittest
 from datetime import datetime
 
+import pytest
 from cf_units import Unit, num2date
 from iris.coords import DimCoord
 from iris.cube import Cube
@@ -12,45 +12,54 @@
 from esmvalcore.iris_helpers import date2num
 
 
-class TestAllVars(unittest.TestCase):
-    """Test fixes for all vars."""
+@pytest.fixture
+def cube():
+    """Cube for testing."""
+    test_cube = Cube([1.0, 2.0], var_name='co2', units='J')
+    reference_dates = [
+        datetime(300, 1, 16, 12),  # e.g. piControl
+        datetime(1850, 1, 16, 12)  # e.g. historical
+    ]
+    esgf_time_units = Unit(
+        'days since 0001-01-01',
+        calendar='proleptic_gregorian',
+    )
+    time_points = date2num(reference_dates, esgf_time_units)
+    test_cube.add_dim_coord(
+        DimCoord(time_points, 'time', 'time', 'time', esgf_time_units),
+        data_dim=0,
+    )
+    return test_cube
+
 
-    def setUp(self):
-        """Prepare tests."""
-        self.cube = Cube([1.0, 2.0], var_name='co2', units='J')
-        reference_dates = [
-            datetime(300, 1, 16, 12),  # e.g. piControl
-            datetime(1850, 1, 16, 12)  # e.g. historical
-        ]
-        esgf_time_units = Unit('days since 0001-01-01',
-                               calendar='proleptic_gregorian')
-        time_points = date2num(reference_dates, esgf_time_units)
-        self.cube.add_dim_coord(
-            DimCoord(time_points, 'time', 'time', 'time', esgf_time_units),
-            data_dim=0)
-        self.fix = AllVars(None)
+class TestAllVars:
+    """Test fixes for all vars."""
 
-    def test_get(self):
+    @staticmethod
+    def test_get():
         """Test getting of fix."""
-        self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'ACCESS1-3', 'Amon', 'tas'),
-            [AllVars(None)])
+        assert (Fix.get_fixes('CMIP5', 'ACCESS1-3', 'Amon', 'tas')
+                == [AllVars(None)])
 
-    def test_fix_metadata(self):
+    @staticmethod
+    def test_fix_metadata(cube):
         """Test fix for bad calendar."""
-        cube = self.fix.fix_metadata([self.cube])[0]
+        fix = AllVars(None)
+        cube = fix.fix_metadata([cube])[0]
         time = cube.coord('time')
         dates = num2date(time.points, time.units.name, time.units.calendar)
-        self.assertEqual(time.units.calendar, 'gregorian')
-        u = Unit('days since 300-01-01 12:00:00', calendar='gregorian')
-        self.assertEqual(dates[0], u.num2date(15))
-        u = Unit('days since 1850-01-01 12:00:00', calendar='gregorian')
-        self.assertEqual(dates[1], u.num2date(15))
+        assert time.units.calendar in ('standard', 'gregorian')
+        u = Unit('days since 300-01-01 12:00:00', calendar='standard')
+        assert dates[0] == u.num2date(15)
+        u = Unit('days since 1850-01-01 12:00:00', calendar='standard')
+        assert dates[1] == u.num2date(15)
 
-    def test_fix_metadata_if_not_time(self):
+    @staticmethod
+    def test_fix_metadata_if_not_time(cube):
         """Test calendar fix do not fail if no time coord present."""
-        self.cube.remove_coord('time')
-        self.fix.fix_metadata([self.cube])
+        cube.remove_coord('time')
+        fix = AllVars(None)
+        fix.fix_metadata([cube])
 
 
 def test_get_cl_fix():

tests/integration/cmor/_fixes/cmip5/test_fgoals_g2.py

@@ -1,6 +1,5 @@
 """Test FGOALS-g2 fixes."""
-import unittest
-
+import pytest
 from cf_units import Unit
 from iris.coords import DimCoord
 from iris.cube import Cube
@@ -9,47 +8,54 @@
 from esmvalcore.cmor.fix import Fix
 
 
-class TestAll(unittest.TestCase):
+@pytest.fixture
+def cube():
+    """Cube for testing."""
+    test_cube = Cube([[1.0, 2.0]], var_name='co2', units='J')
+    test_cube.add_dim_coord(
+        DimCoord(
+            [0.0, 1.0],
+            standard_name='time',
+            units=Unit('days since 0001-01', calendar='gregorian')),
+        1)
+    test_cube.add_dim_coord(
+        DimCoord(
+            [180],
+            standard_name='longitude',
+            units=Unit('degrees')),
+        0)
+    return test_cube
+
+
+class TestAll:
     """Test fixes for all vars."""
 
-    def setUp(self):
-        """Prepare tests."""
-        self.cube = Cube([[1.0, 2.0]], var_name='co2', units='J')
-        self.cube.add_dim_coord(
-            DimCoord(
-                [0.0, 1.0],
-                standard_name='time',
-                units=Unit('days since 0001-01', calendar='gregorian')),
-            1)
-        self.cube.add_dim_coord(
-            DimCoord(
-                [180],
-                standard_name='longitude',
-                units=Unit('degrees')),
-            0)
-        self.fix = AllVars(None)
-
-    def test_get(self):
+    @staticmethod
+    def test_get():
         """Test fix get."""
-        self.assertListEqual(
-            Fix.get_fixes('CMIP5', 'FGOALS-G2', 'Amon', 'tas'),
-            [AllVars(None)])
+        assert (Fix.get_fixes('CMIP5', 'FGOALS-G2', 'Amon', 'tas')
+                == [AllVars(None)])
 
-    def test_fix_metadata(self):
+    @staticmethod
+    def test_fix_metadata(cube):
         """Test calendar fix."""
-        cube = self.fix.fix_metadata([self.cube])[0]
+        fix = AllVars(None)
+        cube = fix.fix_metadata([cube])[0]
 
         time = cube.coord('time')
-        self.assertEqual(time.units.origin,
-                         'day since 1-01-01 00:00:00.000000')
-        self.assertEqual(time.units.calendar, 'gregorian')
+        assert time.units.origin == 'day since 1-01-01 00:00:00.000000'
+        assert time.units.calendar in ('standard', 'gregorian')
 
-    def test_fix_metadata_dont_fail_if_not_longitude(self):
+    @staticmethod
+    def test_fix_metadata_dont_fail_if_not_longitude(cube):
         """Test calendar fix."""
-        self.cube.remove_coord('longitude')
-        self.fix.fix_metadata([self.cube])
+        cube.remove_coord('longitude')
+        fix = AllVars(None)
+        fix.fix_metadata([cube])
 
-    def test_fix_metadata_dont_fail_if_not_time(self):
+    @staticmethod
+    def test_fix_metadata_dont_fail_if_not_time(cube):
         """Test calendar fix."""
-        self.cube.remove_coord('time')
-        self.fix.fix_metadata([self.cube])
+        cube.remove_coord('time')
+        fix = AllVars(None)
+        fix.fix_metadata([cube])