diff --git a/environment.yml b/environment.yml index 5873831080..20602765ed 100644 --- a/environment.yml +++ b/environment.yml @@ -6,7 +6,7 @@ channels: dependencies: - cartopy - - cf-units>=3.0.0,<3.1.0 # github.com/ESMValGroup/ESMValCore/issues/1655 + - cf-units - cftime - dask - compilers @@ -22,7 +22,7 @@ dependencies: - jinja2 - nc-time-axis - nested-lookup - - netcdf4!=1.6.1 # github.com/ESMValGroup/ESMValCore/issues/1723 + - netcdf4<1.6.1 # github.com/ESMValGroup/ESMValCore/issues/1723 - numpy - pandas - pillow diff --git a/esmvalcore/cmor/check.py b/esmvalcore/cmor/check.py index 734d401726..5712ecc22b 100644 --- a/esmvalcore/cmor/check.py +++ b/esmvalcore/cmor/check.py @@ -945,7 +945,7 @@ def _simplify_calendar(calendar): calendar_aliases = { 'all_leap': '366_day', 'noleap': '365_day', - 'standard': 'gregorian', + 'gregorian': 'standard', } return calendar_aliases.get(calendar, calendar) diff --git a/esmvalcore/preprocessor/_multimodel.py b/esmvalcore/preprocessor/_multimodel.py index 9c9fa9662e..1b994ecbad 100644 --- a/esmvalcore/preprocessor/_multimodel.py +++ b/esmvalcore/preprocessor/_multimodel.py @@ -96,7 +96,7 @@ def _unify_time_coordinates(cubes): are no mismatches in the time arrays. If cubes have different time units, it will reset the calendar to a - default gregorian calendar with unit "days since 1850-01-01". + the "standard" calendar with unit "days since 1850-01-01". Might not work for (sub)daily data, because different calendars may have different number of days in the year. diff --git a/setup.py b/setup.py index 291b9955d0..98401f48b7 100755 --- a/setup.py +++ b/setup.py @@ -29,7 +29,7 @@ 'install': [ 'cartopy', # see https://github.com/SciTools/cf-units/issues/218 - 'cf-units>=3.0.0,<3.1.0,!=3.0.1.post0', # ESMValCore/issues/1655 + 'cf-units', 'dask[array]', 'esgf-pyclient>=0.3.1', 'esmpy!=8.1.0', # see github.com/ESMValGroup/ESMValCore/issues/1208 @@ -42,7 +42,7 @@ 'jinja2', 'nc-time-axis', # needed by iris.plot 'nested-lookup', - 'netCDF4!=1.6.1', # github.com/ESMValGroup/ESMValCore/issues/1723 + 'netCDF4<1.6.1', # github.com/ESMValGroup/ESMValCore/issues/1723 'numpy', 'pandas', 'pillow', diff --git a/tests/integration/cmor/_fixes/cmip5/test_access1_0.py b/tests/integration/cmor/_fixes/cmip5/test_access1_0.py index 5a89460a9d..1312762aab 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_access1_0.py +++ b/tests/integration/cmor/_fixes/cmip5/test_access1_0.py @@ -13,45 +13,54 @@ from esmvalcore.iris_helpers import date2num -class TestAllVars(unittest.TestCase): - """Test all vars fixes.""" +@pytest.fixture +def cube(): + """Cube for testing.""" + test_cube = Cube([1.0, 2.0], var_name='co2', units='J') + reference_dates = [ + datetime(300, 1, 16, 12), # e.g. piControl + datetime(1850, 1, 16, 12) # e.g. historical + ] + esgf_time_units = Unit( + 'days since 0001-01-01', + calendar='proleptic_gregorian', + ) + time_points = date2num(reference_dates, esgf_time_units) + test_cube.add_dim_coord( + DimCoord(time_points, 'time', 'time', 'time', esgf_time_units), + data_dim=0, + ) + return test_cube + - def setUp(self): - """Prepare tests.""" - self.cube = Cube([1.0, 2.0], var_name='co2', units='J') - reference_dates = [ - datetime(300, 1, 16, 12), # e.g. piControl - datetime(1850, 1, 16, 12) # e.g. historical - ] - esgf_time_units = Unit('days since 0001-01-01', - calendar='proleptic_gregorian') - time_points = date2num(reference_dates, esgf_time_units) - self.cube.add_dim_coord( - DimCoord(time_points, 'time', 'time', 'time', esgf_time_units), - data_dim=0) - self.fix = AllVars(None) +class TestAllVars: + """Test all vars fixes.""" - def test_get(self): + @staticmethod + def test_get(): """Test getting of fix.""" - self.assertListEqual( - Fix.get_fixes('CMIP5', 'ACCESS1-0', 'Amon', 'tas'), - [AllVars(None)]) + assert (Fix.get_fixes('CMIP5', 'ACCESS1-0', 'Amon', 'tas') + == [AllVars(None)]) - def test_fix_metadata(self): + @staticmethod + def test_fix_metadata(cube): """Test fix for bad calendar.""" - cube = self.fix.fix_metadata([self.cube])[0] + fix = AllVars(None) + cube = fix.fix_metadata([cube])[0] time = cube.coord('time') dates = num2date(time.points, time.units.name, time.units.calendar) - self.assertEqual(time.units.calendar, 'gregorian') - u = Unit('days since 300-01-01 12:00:00', calendar='gregorian') - self.assertEqual(dates[0], u.num2date(15)) - u = Unit('days since 1850-01-01 12:00:00', calendar='gregorian') - self.assertEqual(dates[1], u.num2date(15)) + assert time.units.calendar in ('standard', 'gregorian') + u = Unit('days since 300-01-01 12:00:00', calendar='standard') + assert dates[0] == u.num2date(15) + u = Unit('days since 1850-01-01 12:00:00', calendar='standard') + assert dates[1] == u.num2date(15) - def test_fix_metadata_if_not_time(self): + @staticmethod + def test_fix_metadata_if_not_time(cube): """Test calendar fix do not fail if no time coord present.""" - self.cube.remove_coord('time') - self.fix.fix_metadata([self.cube]) + cube.remove_coord('time') + fix = AllVars(None) + fix.fix_metadata([cube]) def test_get_cl_fix(): diff --git a/tests/integration/cmor/_fixes/cmip5/test_access1_3.py b/tests/integration/cmor/_fixes/cmip5/test_access1_3.py index d28286078f..68d59201de 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_access1_3.py +++ b/tests/integration/cmor/_fixes/cmip5/test_access1_3.py @@ -1,7 +1,7 @@ """Test fixes for ACCESS1-3.""" -import unittest from datetime import datetime +import pytest from cf_units import Unit, num2date from iris.coords import DimCoord from iris.cube import Cube @@ -12,45 +12,54 @@ from esmvalcore.iris_helpers import date2num -class TestAllVars(unittest.TestCase): - """Test fixes for all vars.""" +@pytest.fixture +def cube(): + """Cube for testing.""" + test_cube = Cube([1.0, 2.0], var_name='co2', units='J') + reference_dates = [ + datetime(300, 1, 16, 12), # e.g. piControl + datetime(1850, 1, 16, 12) # e.g. historical + ] + esgf_time_units = Unit( + 'days since 0001-01-01', + calendar='proleptic_gregorian', + ) + time_points = date2num(reference_dates, esgf_time_units) + test_cube.add_dim_coord( + DimCoord(time_points, 'time', 'time', 'time', esgf_time_units), + data_dim=0, + ) + return test_cube + - def setUp(self): - """Prepare tests.""" - self.cube = Cube([1.0, 2.0], var_name='co2', units='J') - reference_dates = [ - datetime(300, 1, 16, 12), # e.g. piControl - datetime(1850, 1, 16, 12) # e.g. historical - ] - esgf_time_units = Unit('days since 0001-01-01', - calendar='proleptic_gregorian') - time_points = date2num(reference_dates, esgf_time_units) - self.cube.add_dim_coord( - DimCoord(time_points, 'time', 'time', 'time', esgf_time_units), - data_dim=0) - self.fix = AllVars(None) +class TestAllVars: + """Test fixes for all vars.""" - def test_get(self): + @staticmethod + def test_get(): """Test getting of fix.""" - self.assertListEqual( - Fix.get_fixes('CMIP5', 'ACCESS1-3', 'Amon', 'tas'), - [AllVars(None)]) + assert (Fix.get_fixes('CMIP5', 'ACCESS1-3', 'Amon', 'tas') + == [AllVars(None)]) - def test_fix_metadata(self): + @staticmethod + def test_fix_metadata(cube): """Test fix for bad calendar.""" - cube = self.fix.fix_metadata([self.cube])[0] + fix = AllVars(None) + cube = fix.fix_metadata([cube])[0] time = cube.coord('time') dates = num2date(time.points, time.units.name, time.units.calendar) - self.assertEqual(time.units.calendar, 'gregorian') - u = Unit('days since 300-01-01 12:00:00', calendar='gregorian') - self.assertEqual(dates[0], u.num2date(15)) - u = Unit('days since 1850-01-01 12:00:00', calendar='gregorian') - self.assertEqual(dates[1], u.num2date(15)) + assert time.units.calendar in ('standard', 'gregorian') + u = Unit('days since 300-01-01 12:00:00', calendar='standard') + assert dates[0] == u.num2date(15) + u = Unit('days since 1850-01-01 12:00:00', calendar='standard') + assert dates[1] == u.num2date(15) - def test_fix_metadata_if_not_time(self): + @staticmethod + def test_fix_metadata_if_not_time(cube): """Test calendar fix do not fail if no time coord present.""" - self.cube.remove_coord('time') - self.fix.fix_metadata([self.cube]) + cube.remove_coord('time') + fix = AllVars(None) + fix.fix_metadata([cube]) def test_get_cl_fix(): diff --git a/tests/integration/cmor/_fixes/cmip5/test_fgoals_g2.py b/tests/integration/cmor/_fixes/cmip5/test_fgoals_g2.py index 2cfffcdb3f..abda3d180b 100644 --- a/tests/integration/cmor/_fixes/cmip5/test_fgoals_g2.py +++ b/tests/integration/cmor/_fixes/cmip5/test_fgoals_g2.py @@ -1,6 +1,5 @@ """Test FGOALS-g2 fixes.""" -import unittest - +import pytest from cf_units import Unit from iris.coords import DimCoord from iris.cube import Cube @@ -9,47 +8,54 @@ from esmvalcore.cmor.fix import Fix -class TestAll(unittest.TestCase): +@pytest.fixture +def cube(): + """Cube for testing.""" + test_cube = Cube([[1.0, 2.0]], var_name='co2', units='J') + test_cube.add_dim_coord( + DimCoord( + [0.0, 1.0], + standard_name='time', + units=Unit('days since 0001-01', calendar='gregorian')), + 1) + test_cube.add_dim_coord( + DimCoord( + [180], + standard_name='longitude', + units=Unit('degrees')), + 0) + return test_cube + + +class TestAll: """Test fixes for all vars.""" - def setUp(self): - """Prepare tests.""" - self.cube = Cube([[1.0, 2.0]], var_name='co2', units='J') - self.cube.add_dim_coord( - DimCoord( - [0.0, 1.0], - standard_name='time', - units=Unit('days since 0001-01', calendar='gregorian')), - 1) - self.cube.add_dim_coord( - DimCoord( - [180], - standard_name='longitude', - units=Unit('degrees')), - 0) - self.fix = AllVars(None) - - def test_get(self): + @staticmethod + def test_get(): """Test fix get.""" - self.assertListEqual( - Fix.get_fixes('CMIP5', 'FGOALS-G2', 'Amon', 'tas'), - [AllVars(None)]) + assert (Fix.get_fixes('CMIP5', 'FGOALS-G2', 'Amon', 'tas') + == [AllVars(None)]) - def test_fix_metadata(self): + @staticmethod + def test_fix_metadata(cube): """Test calendar fix.""" - cube = self.fix.fix_metadata([self.cube])[0] + fix = AllVars(None) + cube = fix.fix_metadata([cube])[0] time = cube.coord('time') - self.assertEqual(time.units.origin, - 'day since 1-01-01 00:00:00.000000') - self.assertEqual(time.units.calendar, 'gregorian') + assert time.units.origin == 'day since 1-01-01 00:00:00.000000' + assert time.units.calendar in ('standard', 'gregorian') - def test_fix_metadata_dont_fail_if_not_longitude(self): + @staticmethod + def test_fix_metadata_dont_fail_if_not_longitude(cube): """Test calendar fix.""" - self.cube.remove_coord('longitude') - self.fix.fix_metadata([self.cube]) + cube.remove_coord('longitude') + fix = AllVars(None) + fix.fix_metadata([cube]) - def test_fix_metadata_dont_fail_if_not_time(self): + @staticmethod + def test_fix_metadata_dont_fail_if_not_time(cube): """Test calendar fix.""" - self.cube.remove_coord('time') - self.fix.fix_metadata([self.cube]) + cube.remove_coord('time') + fix = AllVars(None) + fix.fix_metadata([cube]) diff --git a/tests/sample_data/multimodel_statistics/test_multimodel.py b/tests/sample_data/multimodel_statistics/test_multimodel.py index 480169dc61..aa04b3e283 100644 --- a/tests/sample_data/multimodel_statistics/test_multimodel.py +++ b/tests/sample_data/multimodel_statistics/test_multimodel.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Optional +import cf_units import iris import numpy as np import pytest @@ -24,7 +25,7 @@ marks=pytest.mark.skip( reason='Cannot calculate statistics with single cube in list')), '365_day', - 'gregorian', + 'standard' if cf_units.__version__ >= '3.1' else 'gregorian', pytest.param( 'proleptic_gregorian', marks=pytest.mark.xfail( diff --git a/tests/sample_data/multimodel_statistics/timeseries_daily_standard-full-mean.nc b/tests/sample_data/multimodel_statistics/timeseries_daily_standard-full-mean.nc new file mode 100644 index 0000000000..1e02c34a69 Binary files /dev/null and b/tests/sample_data/multimodel_statistics/timeseries_daily_standard-full-mean.nc differ diff --git a/tests/sample_data/multimodel_statistics/timeseries_daily_standard-overlap-mean.nc b/tests/sample_data/multimodel_statistics/timeseries_daily_standard-overlap-mean.nc new file mode 100644 index 0000000000..1e02c34a69 Binary files /dev/null and b/tests/sample_data/multimodel_statistics/timeseries_daily_standard-overlap-mean.nc differ diff --git a/tests/unit/preprocessor/_multimodel/test_multimodel.py b/tests/unit/preprocessor/_multimodel/test_multimodel.py index 7d1f6bc2bc..00ced5a348 100644 --- a/tests/unit/preprocessor/_multimodel/test_multimodel.py +++ b/tests/unit/preprocessor/_multimodel/test_multimodel.py @@ -22,7 +22,7 @@ FREQUENCY_OPTIONS = ('daily', 'monthly', 'yearly') # hourly -CALENDAR_OPTIONS = ('360_day', '365_day', 'gregorian', 'proleptic_gregorian', +CALENDAR_OPTIONS = ('360_day', '365_day', 'standard', 'proleptic_gregorian', 'julian') @@ -35,7 +35,7 @@ def assert_array_allclose(this, other): def timecoord(frequency, - calendar='gregorian', + calendar='standard', offset='days since 1850-01-01', num=3): """Return a time coordinate with the given time points and calendar.""" @@ -58,7 +58,7 @@ def timecoord(frequency, def generate_cube_from_dates( dates, - calendar='gregorian', + calendar='standard', offset='days since 1850-01-01', fill_val=1, len_data=3, @@ -350,19 +350,19 @@ def test_unsupported_statistics_fail(statistic, error): @pytest.mark.parametrize('calendar1, calendar2, expected', ( - ('360_day', '360_day', '360_day'), - ('365_day', '365_day', '365_day'), - ('365_day', '360_day', 'gregorian'), - ('360_day', '365_day', 'gregorian'), - ('gregorian', '365_day', 'gregorian'), - ('proleptic_gregorian', 'julian', 'gregorian'), - ('julian', '365_day', 'gregorian'), + ('360_day', '360_day', ('360_day',)), + ('365_day', '365_day', ('365_day',)), + ('365_day', '360_day', ('standard', 'gregorian')), + ('360_day', '365_day', ('standard', 'gregorian')), + ('standard', '365_day', ('standard', 'gregorian')), + ('proleptic_gregorian', 'julian', ('standard', 'gregorian')), + ('julian', '365_day', ('standard', 'gregorian')), )) def test_get_consistent_time_unit(calendar1, calendar2, expected): """Test same calendar returned or default if calendars differ. Expected behaviour: If the calendars are the same, return that one. - If the calendars are not the same, return 'gregorian'. + If the calendars are not the same, return 'standard'. """ cubes = ( generate_cube_from_dates('monthly', calendar=calendar1), @@ -370,7 +370,7 @@ def test_get_consistent_time_unit(calendar1, calendar2, expected): ) result = mm._get_consistent_time_unit(cubes) - assert result.calendar == expected + assert result.calendar in expected @pytest.mark.parametrize('span', SPAN_OPTIONS) @@ -395,7 +395,7 @@ def test_align(span): calendars = set(cube.coord('time').units.calendar for cube in result_cubes) assert len(calendars) == 1 - assert list(calendars)[0] == 'gregorian' + assert list(calendars)[0] in ('standard', 'gregorian') shapes = set(cube.shape for cube in result_cubes) @@ -528,7 +528,7 @@ def test_edge_case_different_time_offsets(span): time_coord = result_cube.coord('time') - assert time_coord.units.calendar == 'gregorian' + assert time_coord.units.calendar in ('standard', 'gregorian') assert time_coord.units.origin == 'days since 1850-01-01' desired = np.array((14., 45., 73.)) @@ -640,7 +640,7 @@ def test_unify_time_coordinates(): calendar='360_day', offset='days since 1850-01-01') cube2 = generate_cube_from_dates('monthly', - calendar='gregorian', + calendar='standard', offset='days since 1943-05-16') mm._unify_time_coordinates([cube1, cube2]) @@ -787,7 +787,7 @@ def test_ignore_tas_scalar_height_coord(): def test_daily_inconsistent_calendars(): """Determine behaviour for inconsistent calendars. - Deviating calendars should be converted to gregorian. Missing data + Deviating calendars should be converted to standard. Missing data inside original bounds is filled with nearest neighbour Missing data outside original bounds is masked. """ @@ -804,7 +804,7 @@ def test_daily_inconsistent_calendars(): leapcube = generate_cube_from_dates( leapdates, - calendar='gregorian', + calendar='standard', offset='days since 1850-01-01', fill_val=1, ) @@ -821,7 +821,7 @@ def test_daily_inconsistent_calendars(): # span=full aligned_cubes = mm._align(cubes, span='full') for cube in aligned_cubes: - assert cube.coord('time').units.calendar == "gregorian" + assert cube.coord('time').units.calendar in ("standard", "gregorian") assert cube.shape == (367, ) assert cube[59].coord('time').points == 789 # 29 Feb 1852 np.ma.is_masked(aligned_cubes[1][366].data) # outside original range @@ -834,7 +834,7 @@ def test_daily_inconsistent_calendars(): # span=overlap aligned_cubes = mm._align(cubes, span='overlap') for cube in aligned_cubes: - assert cube.coord('time').units.calendar == "gregorian" + assert cube.coord('time').units.calendar in ("standard", "gregorian") assert cube.shape == (365, ) assert cube[59].coord('time').points == 790 # 1 March 1852