From d1bb52707ce0f234af14ef4ca8f88e5e2d09dcde Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 25 Jul 2022 16:14:12 +0200 Subject: [PATCH 01/12] First prototype of CESM2 on-the-fly CMORizer --- .../_config/extra_facets/cesm-mappings.yml | 17 +++ esmvalcore/cmor/_fixes/cesm/__init__.py | 0 esmvalcore/cmor/_fixes/cesm/_base_fixes.py | 31 ++++ esmvalcore/cmor/_fixes/cesm/cesm2.py | 139 ++++++++++++++++++ esmvalcore/config-developer.yml | 10 ++ 5 files changed, 197 insertions(+) create mode 100644 esmvalcore/_config/extra_facets/cesm-mappings.yml create mode 100644 esmvalcore/cmor/_fixes/cesm/__init__.py create mode 100644 esmvalcore/cmor/_fixes/cesm/_base_fixes.py create mode 100644 esmvalcore/cmor/_fixes/cesm/cesm2.py diff --git a/esmvalcore/_config/extra_facets/cesm-mappings.yml b/esmvalcore/_config/extra_facets/cesm-mappings.yml new file mode 100644 index 0000000000..9d0369e9f5 --- /dev/null +++ b/esmvalcore/_config/extra_facets/cesm-mappings.yml @@ -0,0 +1,17 @@ +# Extra facets for native CESM model output + +# All extra facets for CESM are optional but might be necessary for some +# variables. + +# Notes: +# - ... + +# A complete list of supported keys is given in the documentation (see +# ESMValCore/doc/quickstart/find_data.rst). +--- + +CESM2: + + '*': + tas: + raw_name: TREFHT diff --git a/esmvalcore/cmor/_fixes/cesm/__init__.py b/esmvalcore/cmor/_fixes/cesm/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/esmvalcore/cmor/_fixes/cesm/_base_fixes.py b/esmvalcore/cmor/_fixes/cesm/_base_fixes.py new file mode 100644 index 0000000000..ed02fc4be4 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cesm/_base_fixes.py @@ -0,0 +1,31 @@ +"""Fix base classes for CESM on-the-fly CMORizer.""" + +import logging + +from iris import NameConstraint +from iris.exceptions import ConstraintMismatchError + +from ..fix import Fix + +logger = logging.getLogger(__name__) + + +class CesmFix(Fix): + """Base class for all CESM fixes.""" + + def get_cube(self, cubes, var_name=None): + """Extract single cube.""" + # If no var_name given, use the CMOR short_name + if var_name is None: + var_name = self.extra_facets.get('raw_name', + self.vardef.short_name) + + # Try to extract the variable + try: + return cubes.extract_cube(NameConstraint(var_name=var_name)) + except ConstraintMismatchError: + raise ValueError( + f"No variable of {var_name} necessary for the extraction/" + f"derivation the CMOR variable '{self.vardef.short_name}' is " + f"available in the input file" + ) diff --git a/esmvalcore/cmor/_fixes/cesm/cesm2.py b/esmvalcore/cmor/_fixes/cesm/cesm2.py new file mode 100644 index 0000000000..0e5b02129e --- /dev/null +++ b/esmvalcore/cmor/_fixes/cesm/cesm2.py @@ -0,0 +1,139 @@ +"""On-the-fly CMORizer for CESM2.""" + +import logging + +from iris.cube import CubeList + +from ..shared import ( + add_scalar_height_coord, + add_scalar_lambda550nm_coord, + add_scalar_typesi_coord, +) +from ._base_fixes import CesmFix + +logger = logging.getLogger(__name__) + + +INVALID_UNITS = { + 'fraction': '1', +} + + +class AllVars(CesmFix): + """Fixes for all variables.""" + + def fix_metadata(self, cubes): + """Fix metadata.""" + cube = self.get_cube(cubes) + + # Fix time + if 'time' in self.vardef.dimensions: + self._fix_time(cube) + + # Fix latitude + if 'latitude' in self.vardef.dimensions: + self._fix_lat(cube) + + # Fix longitude + if 'longitude' in self.vardef.dimensions: + self._fix_lon(cube) + + # Fix scalar coordinates + self._fix_scalar_coords(cube) + + # Fix metadata of variable + self._fix_var_metadata(cube) + + return CubeList([cube]) + + @staticmethod + def _fix_time(cube): + """Fix time coordinate of cube.""" + time_coord = cube.coord('time') + time_coord.var_name = 'time' + time_coord.standard_name = 'time' + time_coord.long_name = 'time' + + # Move time points to center of time period given by time bounds + # (currently the points are located at the end of the interval) + if time_coord.bounds is not None: + time_coord.points = time_coord.bounds.mean(axis=-1) + + # Add bounds if possible (not possible if cube only contains single + # time point) + if not time_coord.has_bounds(): + try: + time_coord.guess_bounds() + except ValueError: + pass + + @staticmethod + def _fix_lat(cube): + """Fix latitude coordinate of cube.""" + lat = cube.coord('latitude') + lat.var_name = 'lat' + lat.standard_name = 'latitude' + lat.long_name = 'latitude' + lat.convert_units('degrees_north') + + # Add bounds if possible (not possible if cube only contains single + # lat point) + if not lat.has_bounds(): + try: + lat.guess_bounds() + except ValueError: + pass + + @staticmethod + def _fix_lon(cube): + """Fix longitude coordinate of cube.""" + lon = cube.coord('longitude') + lon.var_name = 'lon' + lon.standard_name = 'longitude' + lon.long_name = 'longitude' + lon.convert_units('degrees_east') + + # Add bounds if possible (not possible if cube only contains single + # lon point) + if not lon.has_bounds(): + try: + lon.guess_bounds() + except ValueError: + pass + + def _fix_scalar_coords(self, cube): + """Fix scalar coordinates.""" + if 'height2m' in self.vardef.dimensions: + add_scalar_height_coord(cube, 2.0) + if 'height10m' in self.vardef.dimensions: + add_scalar_height_coord(cube, 10.0) + if 'lambda550nm' in self.vardef.dimensions: + add_scalar_lambda550nm_coord(cube) + if 'typesi' in self.vardef.dimensions: + add_scalar_typesi_coord(cube, 'sea_ice') + + def _fix_var_metadata(self, cube): + """Fix metadata of variable.""" + if self.vardef.standard_name == '': + cube.standard_name = None + else: + cube.standard_name = self.vardef.standard_name + cube.var_name = self.vardef.short_name + cube.long_name = self.vardef.long_name + + # Fix units + if 'invalid_units' in cube.attributes: + invalid_units = cube.attributes.pop('invalid_units') + new_units = INVALID_UNITS.get(invalid_units, invalid_units) + try: + cube.units = new_units + except ValueError as exc: + raise ValueError( + f"Failed to fix invalid units '{invalid_units}' for " + f"variable '{self.vardef.short_name}'") from exc + if cube.units != self.vardef.units: + cube.convert_units(self.vardef.units) + + # Fix attributes + if self.vardef.positive != '': + cube.attributes['positive'] = self.vardef.positive diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index 8c7dedcec0..6ae69db940 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -174,3 +174,13 @@ ICON: output_file: '{dataset}_{version}_{component}_{grid}_{mip}_{exp}_{ensemble}_{short_name}_{var_type}' cmor_type: 'CMIP6' cmor_default_table_prefix: 'CMIP6_' + +CESM: + cmor_strict: false + input_dir: + default: '{dataset}' + input_file: + default: '{simulation}.*.nc' + output_file: '{project}_{dataset}_{simulation}_{mip}_{short_name}' + cmor_type: 'CMIP6' + cmor_default_table_prefix: 'CMIP6_' From f1b5834d17605b182388e3fb4daa7654a6821458 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 2 Sep 2022 13:32:30 +0200 Subject: [PATCH 02/12] Used new NativeDatasetFix class for CESM2 CMORizer --- esmvalcore/cmor/_fixes/cesm/_base_fixes.py | 31 ----- esmvalcore/cmor/_fixes/cesm/cesm2.py | 130 +++------------------ 2 files changed, 13 insertions(+), 148 deletions(-) delete mode 100644 esmvalcore/cmor/_fixes/cesm/_base_fixes.py diff --git a/esmvalcore/cmor/_fixes/cesm/_base_fixes.py b/esmvalcore/cmor/_fixes/cesm/_base_fixes.py deleted file mode 100644 index ed02fc4be4..0000000000 --- a/esmvalcore/cmor/_fixes/cesm/_base_fixes.py +++ /dev/null @@ -1,31 +0,0 @@ -"""Fix base classes for CESM on-the-fly CMORizer.""" - -import logging - -from iris import NameConstraint -from iris.exceptions import ConstraintMismatchError - -from ..fix import Fix - -logger = logging.getLogger(__name__) - - -class CesmFix(Fix): - """Base class for all CESM fixes.""" - - def get_cube(self, cubes, var_name=None): - """Extract single cube.""" - # If no var_name given, use the CMOR short_name - if var_name is None: - var_name = self.extra_facets.get('raw_name', - self.vardef.short_name) - - # Try to extract the variable - try: - return cubes.extract_cube(NameConstraint(var_name=var_name)) - except ConstraintMismatchError: - raise ValueError( - f"No variable of {var_name} necessary for the extraction/" - f"derivation the CMOR variable '{self.vardef.short_name}' is " - f"available in the input file" - ) diff --git a/esmvalcore/cmor/_fixes/cesm/cesm2.py b/esmvalcore/cmor/_fixes/cesm/cesm2.py index 0e5b02129e..3dc95f7bf9 100644 --- a/esmvalcore/cmor/_fixes/cesm/cesm2.py +++ b/esmvalcore/cmor/_fixes/cesm/cesm2.py @@ -4,136 +4,32 @@ from iris.cube import CubeList -from ..shared import ( - add_scalar_height_coord, - add_scalar_lambda550nm_coord, - add_scalar_typesi_coord, -) -from ._base_fixes import CesmFix +from ..native_datasets import NativeDatasetFix logger = logging.getLogger(__name__) -INVALID_UNITS = { - 'fraction': '1', -} - - -class AllVars(CesmFix): +class AllVars(NativeDatasetFix): """Fixes for all variables.""" + # Dictionary to map invalid units in the data to valid entries + INVALID_UNITS = { + 'fraction': '1', + } + def fix_metadata(self, cubes): """Fix metadata.""" cube = self.get_cube(cubes) - # Fix time - if 'time' in self.vardef.dimensions: - self._fix_time(cube) - - # Fix latitude - if 'latitude' in self.vardef.dimensions: - self._fix_lat(cube) - - # Fix longitude - if 'longitude' in self.vardef.dimensions: - self._fix_lon(cube) + # Fix time, latitude, and longitude coordinates + self.fix_regular_time(cube) + self.fix_regular_lat(cube) + self.fix_regular_lon(cube) # Fix scalar coordinates - self._fix_scalar_coords(cube) + self.fix_scalar_coords(cube) # Fix metadata of variable - self._fix_var_metadata(cube) + self.fix_var_metadata(cube) return CubeList([cube]) - - @staticmethod - def _fix_time(cube): - """Fix time coordinate of cube.""" - time_coord = cube.coord('time') - time_coord.var_name = 'time' - time_coord.standard_name = 'time' - time_coord.long_name = 'time' - - # Move time points to center of time period given by time bounds - # (currently the points are located at the end of the interval) - if time_coord.bounds is not None: - time_coord.points = time_coord.bounds.mean(axis=-1) - - # Add bounds if possible (not possible if cube only contains single - # time point) - if not time_coord.has_bounds(): - try: - time_coord.guess_bounds() - except ValueError: - pass - - @staticmethod - def _fix_lat(cube): - """Fix latitude coordinate of cube.""" - lat = cube.coord('latitude') - lat.var_name = 'lat' - lat.standard_name = 'latitude' - lat.long_name = 'latitude' - lat.convert_units('degrees_north') - - # Add bounds if possible (not possible if cube only contains single - # lat point) - if not lat.has_bounds(): - try: - lat.guess_bounds() - except ValueError: - pass - - @staticmethod - def _fix_lon(cube): - """Fix longitude coordinate of cube.""" - lon = cube.coord('longitude') - lon.var_name = 'lon' - lon.standard_name = 'longitude' - lon.long_name = 'longitude' - lon.convert_units('degrees_east') - - # Add bounds if possible (not possible if cube only contains single - # lon point) - if not lon.has_bounds(): - try: - lon.guess_bounds() - except ValueError: - pass - - def _fix_scalar_coords(self, cube): - """Fix scalar coordinates.""" - if 'height2m' in self.vardef.dimensions: - add_scalar_height_coord(cube, 2.0) - if 'height10m' in self.vardef.dimensions: - add_scalar_height_coord(cube, 10.0) - if 'lambda550nm' in self.vardef.dimensions: - add_scalar_lambda550nm_coord(cube) - if 'typesi' in self.vardef.dimensions: - add_scalar_typesi_coord(cube, 'sea_ice') - - def _fix_var_metadata(self, cube): - """Fix metadata of variable.""" - if self.vardef.standard_name == '': - cube.standard_name = None - else: - cube.standard_name = self.vardef.standard_name - cube.var_name = self.vardef.short_name - cube.long_name = self.vardef.long_name - - # Fix units - if 'invalid_units' in cube.attributes: - invalid_units = cube.attributes.pop('invalid_units') - new_units = INVALID_UNITS.get(invalid_units, invalid_units) - try: - cube.units = new_units - except ValueError as exc: - raise ValueError( - f"Failed to fix invalid units '{invalid_units}' for " - f"variable '{self.vardef.short_name}'") from exc - if cube.units != self.vardef.units: - cube.convert_units(self.vardef.units) - - # Fix attributes - if self.vardef.positive != '': - cube.attributes['positive'] = self.vardef.positive From cb70031a36181de70e3efd35ac509fbe45309e60 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Thu, 8 Sep 2022 16:09:00 +0200 Subject: [PATCH 03/12] Added tests for input/output filenames for ICON and EMAC CMORizer --- .../_config/extra_facets/ipslcm-mappings.yml | 6 +- esmvalcore/config-developer.yml | 2 +- tests/integration/data_finder.yml | 217 +++++++++++++++++- tests/integration/test_data_finder.py | 18 ++ 4 files changed, 235 insertions(+), 8 deletions(-) diff --git a/esmvalcore/_config/extra_facets/ipslcm-mappings.yml b/esmvalcore/_config/extra_facets/ipslcm-mappings.yml index eb99044a6a..cacedeb370 100644 --- a/esmvalcore/_config/extra_facets/ipslcm-mappings.yml +++ b/esmvalcore/_config/extra_facets/ipslcm-mappings.yml @@ -111,7 +111,7 @@ IPSL-CM6: # -> general variables precip: {ipsl_varname: precip, <<: *atmvars} slp: {ipsl_varname: slp, <<: *atmvars} - + # -> Turbulent fluxes taux: {ipsl_varname: taux, <<: *atmvars} tauy: {ipsl_varname: tauy, <<: *atmvars} @@ -136,8 +136,8 @@ IPSL-CM6: SWdnSFC: {ipsl_varname: SWdnSFC, <<: *atmvars} LWdnSFcclr: {ipsl_varname: LWdnSFcclr, <<: *atmvars} SWdnSFcclr: {ipsl_varname: SWdnSFcclr, <<: *atmvars} - - + + # ================================================= Lmon: # =============================================== diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index 7d2ca06cac..ff282bd4f0 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -133,7 +133,7 @@ EMAC: default: '{exp}/{channel}' input_file: default: '{exp}*{channel}{postproc_flag}.nc' - output_file: '{dataset}_{exp}_{channel}_{mip}_{short_name}' + output_file: '{project}_{dataset}_{exp}_{channel}_{mip}_{short_name}' cmor_type: 'CMIP6' ignore_warnings: load: diff --git a/tests/integration/data_finder.yml b/tests/integration/data_finder.yml index c375585da3..25ee1ea267 100644 --- a/tests/integration/data_finder.yml +++ b/tests/integration/data_finder.yml @@ -26,6 +26,77 @@ get_output_file: preproc_dir: /test output_file: /test/test_diag/test/CMIP5_HadGEM2-ES_Amon_historical-rcp85_r1i1p1_ta_1960-1980.nc + # EMAC + + - variable: + variable_group: test + short_name: tas + original_short_name: tas + dataset: EMAC + project: EMAC + cmor_table: CMIP6 + frequency: mon + mip: Amon + exp: amip + timerange: '1960/1980' + diagnostic: test_diag + preprocessor: test_preproc + preproc_dir: this/is/a/path + output_file: this/is/a/path/test_diag/test/EMAC_EMAC_amip_Amon_Amon_tas_1960-1980.nc + + - variable: + variable_group: test + short_name: tas + original_short_name: tas + dataset: EMAC + project: EMAC + cmor_table: CMIP6 + frequency: mon + mip: Amon + exp: piControl + channel: CH + postproc_flag: -p-mm + timerange: '199001/199002' + diagnostic: test_diag + preprocessor: test_preproc + preproc_dir: this/is/a/path + output_file: this/is/a/path/test_diag/test/EMAC_EMAC_piControl_CH_Amon_tas_199001-199002.nc + + # ICON + + - variable: + variable_group: test + short_name: tas + original_short_name: tas + dataset: ICON + project: ICON + cmor_table: CMIP6 + frequency: mon + mip: Amon + exp: amip + timerange: '1960/1980' + diagnostic: test_diag + preprocessor: test_preproc + preproc_dir: this/is/a/path + output_file: this/is/a/path/test_diag/test/ICON_ICON_amip_atm_2d_ml_Amon_tas_1960-1980.nc + + - variable: + variable_group: test + short_name: tas + original_short_name: tas + dataset: ICON + project: ICON + cmor_table: CMIP6 + frequency: mon + mip: Amon + exp: amip + var_type: custom_var_type + timerange: '20000101/20000102' + diagnostic: test_diag + preprocessor: test_preproc + preproc_dir: this/is/a/path + output_file: this/is/a/path/test_diag/test/ICON_ICON_amip_custom_var_type_Amon_tas_20000101-20000102.nc + get_input_filelist: - drs: default @@ -460,9 +531,9 @@ get_input_filelist: file_patterns: - simulation_*_t2m.nc - simulation_*_histmth.nc - found_files: + found_files: - thredds/tgcc/store/p86caub/IPSLCM6/PROD/historical/simulation/ATM/Output/MO/simulation_18500101_18591231_1M_histmth.nc - + - drs: default variable: <<: *ipsl_variable @@ -475,9 +546,9 @@ get_input_filelist: file_patterns: - simulation_*_t2m.nc - simulation_*_histmth.nc - found_files: + found_files: - thredds/tgcc/store/p86caub/IPSLCM6/PROD/historical/simulation/ATM/Analyse/TS_MO/simulation_18500101_20141231_1M_t2m.nc - + # Test fx files - drs: default @@ -779,3 +850,141 @@ get_input_filelist: file_patterns: - OBS6_ERA-Interim_reanaly_42_Omon_deptho[_.]*nc found_files: [] + + # EMAC + + - drs: default + variable: + variable_group: test + short_name: tas + original_short_name: tas + dataset: EMAC + project: EMAC + cmor_table: CMIP6 + frequency: mon + mip: Amon + exp: amip + timerange: '200002/200003' + diagnostic: test_diag + preprocessor: test_preproc + available_files: + - amip/Amon/amip___________200001_Amon.nc + - amip/Amon/amip___________200002_Amon.nc + - amip/Amon/amip___________200003_Amon.nc + - amip/Amon/amip___________200001_Amon-p-mm.nc + - amip/Amon/amip___________200002_Amon-p-mm.nc + - amip/Amon/amip___________200003_Amon-p-mm.nc + - amip/rad/amip___________200001_rad.nc + - amip/rad/amip___________200002_rad.nc + - amip/rad/amip___________200003_rad.nc + - amip/rad/amip___________200001_rad-p-mm.nc + - amip/rad/amip___________200002_rad-p-mm.nc + - amip/rad/amip___________200003_rad-p-mm.nc + dirs: + - amip/Amon + file_patterns: + - amip*Amon.nc + found_files: + - amip/Amon/amip___________200002_Amon.nc + - amip/Amon/amip___________200003_Amon.nc + + - drs: default + variable: + variable_group: test + short_name: tas + original_short_name: tas + dataset: EMAC + project: EMAC + cmor_table: CMIP6 + frequency: mon + mip: Amon + exp: amip + channel: rad + postproc_flag: -p-mm + timerange: '200001/200002' + diagnostic: test_diag + preprocessor: test_preproc + available_files: + - amip/Amon/amip___________200001_Amon.nc + - amip/Amon/amip___________200002_Amon.nc + - amip/Amon/amip___________200003_Amon.nc + - amip/Amon/amip___________200001_Amon-p-mm.nc + - amip/Amon/amip___________200002_Amon-p-mm.nc + - amip/Amon/amip___________200003_Amon-p-mm.nc + - amip/rad/amip___________200001_rad.nc + - amip/rad/amip___________200002_rad.nc + - amip/rad/amip___________200003_rad.nc + - amip/rad/amip___________200001_rad-p-mm.nc + - amip/rad/amip___________200002_rad-p-mm.nc + - amip/rad/amip___________200003_rad-p-mm.nc + dirs: + - amip/rad + file_patterns: + - amip*rad-p-mm.nc + found_files: + - amip/rad/amip___________200001_rad-p-mm.nc + - amip/rad/amip___________200002_rad-p-mm.nc + + # ICON + + - drs: default + variable: + variable_group: test + short_name: tas + original_short_name: tas + dataset: ICON + project: ICON + cmor_table: CMIP6 + frequency: mon + mip: Amon + exp: amip + timerange: '200002/200003' + diagnostic: test_diag + preprocessor: test_preproc + available_files: + - amip/amip_atm_2d_ml_20000101T000000Z.nc + - amip/amip_atm_2d_ml_20000201T000000Z.nc + - amip/amip_atm_2d_ml_20000301T000000Z.nc + - amip/outdata/amip_atm_2d_ml_20000401T000000Z.nc + - amip/outdata/amip_atm_2d_ml_20000501T000000Z.nc + - amip/outdata/amip_atm_2d_ml_20000601T000000Z.nc + dirs: + - amip + - amip/outdata + file_patterns: + - amip_atm_2d_ml*.nc + found_files: + - amip/amip_atm_2d_ml_20000201T000000Z.nc + - amip/amip_atm_2d_ml_20000301T000000Z.nc + + - drs: default + variable: + variable_group: test + short_name: tas + original_short_name: tas + dataset: ICON + project: ICON + cmor_table: CMIP6 + frequency: mon + mip: Amon + exp: amip + var_type: var + timerange: '200003/200005' + diagnostic: test_diag + preprocessor: test_preproc + available_files: + - amip/amip_var_20000101T000000Z.nc + - amip/amip_var_20000201T000000Z.nc + - amip/amip_var_20000301T000000Z.nc + - amip/outdata/amip_var_20000401T000000Z.nc + - amip/outdata/amip_var_20000501T000000Z.nc + - amip/outdata/amip_var_20000601T000000Z.nc + dirs: + - amip + - amip/outdata + file_patterns: + - amip_var*.nc + found_files: + - amip/amip_var_20000301T000000Z.nc + - amip/outdata/amip_var_20000401T000000Z.nc + - amip/outdata/amip_var_20000501T000000Z.nc diff --git a/tests/integration/test_data_finder.py b/tests/integration/test_data_finder.py index 07028b2610..966e5a268d 100644 --- a/tests/integration/test_data_finder.py +++ b/tests/integration/test_data_finder.py @@ -25,6 +25,20 @@ CONFIG = yaml.safe_load(file) +def _augment_with_extra_facets(variable): + """Augment variable dict with extra facets.""" + extra_facets = esmvalcore._config.get_extra_facets( + variable['project'], + variable['dataset'], + variable['mip'], + variable['short_name'], + (), + ) + for (key, val) in extra_facets.items(): + if key not in variable: + variable[key] = val + + def print_path(path): """Print path.""" txt = path @@ -68,6 +82,7 @@ def create_tree(path, filenames=None, symlinks=None): @pytest.mark.parametrize('cfg', CONFIG['get_output_file']) def test_get_output_file(cfg): """Test getting output name for preprocessed files.""" + _augment_with_extra_facets(cfg['variable']) output_file = get_output_file(cfg['variable'], cfg['preproc_dir']) assert output_file == cfg['output_file'] @@ -88,6 +103,9 @@ def test_get_input_filelist(root, cfg): create_tree(root, cfg.get('available_files'), cfg.get('available_symlinks')) + # Augment variable dict with extra facets + _augment_with_extra_facets(cfg['variable']) + # Find files rootpath = {cfg['variable']['project']: [root]} drs = {cfg['variable']['project']: cfg['drs']} From 4025fbe2af93914ada7115d22d8ad5a4d20dd3be Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 9 Sep 2022 12:40:55 +0200 Subject: [PATCH 04/12] Adapted input files and dirs for CESM --- esmvalcore/config-developer.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index 79370bcda0..8b0a842cac 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -180,9 +180,12 @@ ICON: CESM: cmor_strict: false input_dir: - default: '{dataset}' + default: + - '/' # run directory + - '{case}/{gcomp}/hist/' # short-term archiving + - '{case}/{gcomp}/proc/{tdir}/{tperiod}' # short-term archiving input_file: - default: '{simulation}.*.nc' - output_file: '{project}_{dataset}_{simulation}_{mip}_{short_name}' + default: '{case}.{scomp}.{type}.{string}*.nc' + output_file: '{project}_{dataset}_{case}_{gcomp}_{scomp}_{mip}_{short_name}' cmor_type: 'CMIP6' cmor_default_table_prefix: 'CMIP6_' From c124ebf917057fd435280a0884682d397aa90e9a Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 9 Sep 2022 13:33:47 +0200 Subject: [PATCH 05/12] Added tests for input/output filenames of CESM --- .../_config/extra_facets/cesm-mappings.yml | 12 ++++ .../_config/extra_facets/icon-mappings.yml | 1 + .../_config/extra_facets/ipslcm-mappings.yml | 4 +- esmvalcore/config-developer.yml | 4 +- tests/integration/data_finder.yml | 68 +++++++++++++++++++ 5 files changed, 85 insertions(+), 4 deletions(-) diff --git a/esmvalcore/_config/extra_facets/cesm-mappings.yml b/esmvalcore/_config/extra_facets/cesm-mappings.yml index 9d0369e9f5..57d3cc8d2c 100644 --- a/esmvalcore/_config/extra_facets/cesm-mappings.yml +++ b/esmvalcore/_config/extra_facets/cesm-mappings.yml @@ -13,5 +13,17 @@ CESM2: '*': + # Optional facets + # It is necessary to define them here to allow multiple file/dir name + # conventions, see + # https://www.cesm.ucar.edu/models/cesm2/naming_conventions.html + '*': + string: '' + tdir: '' + tperiod: '' + + # Default facets for variables tas: raw_name: TREFHT + gcomp: atm + scomp: cam diff --git a/esmvalcore/_config/extra_facets/icon-mappings.yml b/esmvalcore/_config/extra_facets/icon-mappings.yml index 1b0eef09ce..5806247677 100644 --- a/esmvalcore/_config/extra_facets/icon-mappings.yml +++ b/esmvalcore/_config/extra_facets/icon-mappings.yml @@ -6,6 +6,7 @@ --- ICON: + '*': # Cell measures areacella: diff --git a/esmvalcore/_config/extra_facets/ipslcm-mappings.yml b/esmvalcore/_config/extra_facets/ipslcm-mappings.yml index cacedeb370..179681af95 100644 --- a/esmvalcore/_config/extra_facets/ipslcm-mappings.yml +++ b/esmvalcore/_config/extra_facets/ipslcm-mappings.yml @@ -6,12 +6,12 @@ # ESMValTool to use key 'ipsl_varname' for building the filename, # while for format 'Output' it specifies to use key 'group' # -# Specifying 'igcm_dir' here allows to avoid having to specifiy it in +# Specifying 'igcm_dir' here allows to avoid having to specify it in # datasets definitions # # Key 'use_cdo' allows to choose whether CDO will be invoked for # selecting a variable in a multi-variable file. This generally allows -# for smaller overal load time. But because CDO has a licence which is +# for smaller overall load time. But because CDO has a licence which is # not compliant with ESMValtool licence policy, the default # configuration is to avoid using it. You may use customized settings # by installing a modified version of this file as diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index 8b0a842cac..0503fd965b 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -183,9 +183,9 @@ CESM: default: - '/' # run directory - '{case}/{gcomp}/hist/' # short-term archiving - - '{case}/{gcomp}/proc/{tdir}/{tperiod}' # short-term archiving + - '{case}/{gcomp}/proc/{tdir}/{tperiod}' # postprocessed data input_file: - default: '{case}.{scomp}.{type}.{string}*.nc' + default: '{case}.{scomp}.{type}.{string}*nc' output_file: '{project}_{dataset}_{case}_{gcomp}_{scomp}_{mip}_{short_name}' cmor_type: 'CMIP6' cmor_default_table_prefix: 'CMIP6_' diff --git a/tests/integration/data_finder.yml b/tests/integration/data_finder.yml index 25ee1ea267..a52d64a95c 100644 --- a/tests/integration/data_finder.yml +++ b/tests/integration/data_finder.yml @@ -988,3 +988,71 @@ get_input_filelist: - amip/amip_var_20000301T000000Z.nc - amip/outdata/amip_var_20000401T000000Z.nc - amip/outdata/amip_var_20000501T000000Z.nc + + # CESM2 + + - drs: default + variable: + variable_group: test + short_name: tas + original_short_name: tas + dataset: CESM2 + project: CESM + cmor_table: CMIP6 + frequency: mon + mip: Amon + exp: amip + case: f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1 + type: h0 + timerange: '2000/2002' + diagnostic: test_diag + preprocessor: test_preproc + available_files: + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.2000.nc + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.2001.nc + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/atm/hist/f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.2002.nc + dirs: + - '' + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/atm/hist + file_patterns: + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.*nc + found_files: + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.2000.nc + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.2001.nc + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/atm/hist/f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.2002.nc + + - drs: default + variable: + variable_group: test + short_name: tas + original_short_name: tas + dataset: CESM2 + project: CESM + cmor_table: CMIP6 + frequency: mon + mip: Amon + exp: amip + case: f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1 + type: h0 + tdir: tseries + tperiod: month_1 + string: TREFHT + timerange: '2015/2025' + diagnostic: test_diag + preprocessor: test_preproc + available_files: + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.2000.nc + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.2001.nc + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/atm/hist/f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.2002.nc + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/atm/proc/tseries/month_1/f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.TREFHT.201001-201912.nc + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/atm/proc/tseries/month_1/f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.TREFHT.202001-202912.nc + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/atm/proc/tseries/month_1/f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.TREFHT.203001-203912.nc + dirs: + - '' + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/atm/hist + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/atm/proc/tseries/month_1 + file_patterns: + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.TREFHT*nc + found_files: + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/atm/proc/tseries/month_1/f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.TREFHT.201001-201912.nc + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/atm/proc/tseries/month_1/f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.TREFHT.202001-202912.nc From 13d8c55272e6eeec842bb35e5800b4d45e5a2047 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Fri, 9 Sep 2022 17:47:08 +0200 Subject: [PATCH 06/12] Added tests for CESM2 on-the-fly CMORizer --- .../integration/cmor/_fixes/cesm/__init__.py | 0 .../cmor/_fixes/cesm/test_cesm2.py | 304 ++++++++++++++++++ .../cmor/_fixes/test_data/cesm2_native.nc | Bin 0 -> 67544 bytes 3 files changed, 304 insertions(+) create mode 100644 tests/integration/cmor/_fixes/cesm/__init__.py create mode 100644 tests/integration/cmor/_fixes/cesm/test_cesm2.py create mode 100644 tests/integration/cmor/_fixes/test_data/cesm2_native.nc diff --git a/tests/integration/cmor/_fixes/cesm/__init__.py b/tests/integration/cmor/_fixes/cesm/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/cmor/_fixes/cesm/test_cesm2.py b/tests/integration/cmor/_fixes/cesm/test_cesm2.py new file mode 100644 index 0000000000..449d4329f2 --- /dev/null +++ b/tests/integration/cmor/_fixes/cesm/test_cesm2.py @@ -0,0 +1,304 @@ +"""Tests for the CESM2 on-the-fly CMORizer.""" +import iris +import numpy as np +import pytest +from cf_units import Unit +from iris.coords import DimCoord +from iris.cube import Cube, CubeList + +import esmvalcore.cmor._fixes.cesm.cesm2 +from esmvalcore._config import get_extra_facets +from esmvalcore.cmor.fix import Fix +from esmvalcore.cmor.table import get_var_info + +# Note: test_data_path is defined in tests/integration/cmor/_fixes/conftest.py + + +@pytest.fixture +def cubes_2d(test_data_path): + """2D sample cubes.""" + nc_path = test_data_path / 'cesm2_native.nc' + return iris.load(str(nc_path)) + + +def _get_fix(mip, short_name, fix_name): + """Load a fix from :mod:`esmvalcore.cmor._fixes.cesm.cesm2`.""" + extra_facets = get_extra_facets('CESM', 'CESM2', mip, short_name, ()) + vardef = get_var_info(project='CESM', mip=mip, short_name=short_name) + cls = getattr(esmvalcore.cmor._fixes.cesm.cesm2, fix_name) + fix = cls(vardef, extra_facets=extra_facets) + return fix + + +def get_fix(mip, short_name): + """Load a variable fix from esmvalcore.cmor._fixes.cesm.cesm.""" + fix_name = short_name[0].upper() + short_name[1:] + return _get_fix(mip, short_name, fix_name) + + +def get_allvars_fix(mip, short_name): + """Load the AllVars fix from esmvalcore.cmor._fixes.cesm.cesm.""" + return _get_fix(mip, short_name, 'AllVars') + + +def fix_metadata(cubes, mip, short_name): + """Fix metadata of cubes.""" + fix = get_fix(mip, short_name) + cubes = fix.fix_metadata(cubes) + fix = get_allvars_fix(mip, short_name) + cubes = fix.fix_metadata(cubes) + return cubes + + +def check_tas_metadata(cubes): + """Check tas metadata.""" + assert len(cubes) == 1 + cube = cubes[0] + assert cube.var_name == 'tas' + assert cube.standard_name == 'air_temperature' + assert cube.long_name == 'Near-Surface Air Temperature' + assert cube.units == 'K' + assert 'positive' not in cube.attributes + return cube + + +def check_time(cube): + """Check time coordinate of cube.""" + assert cube.coords('time', dim_coords=True) + time = cube.coord('time', dim_coords=True) + assert time.var_name == 'time' + assert time.standard_name == 'time' + assert time.long_name == 'time' + assert time.units == Unit('days since 1979-01-01 00:00:00', + calendar='365_day') + assert time.shape == (12,) + assert time.bounds.shape == (12, 2) + assert time.attributes == {} + + +def check_lat(cube): + """Check latitude coordinate of cube.""" + assert cube.coords('latitude', dim_coords=True) + lat = cube.coord('latitude', dim_coords=True) + assert lat.var_name == 'lat' + assert lat.standard_name == 'latitude' + assert lat.long_name == 'latitude' + assert lat.units == 'degrees_north' + np.testing.assert_allclose( + lat.points, + [59.4444082891668, 19.8757191474409, -19.8757191474409, + -59.4444082891668], + ) + np.testing.assert_allclose( + lat.bounds, + [[90.0, 39.384861047478], + [39.384861047478, 0.0], + [0.0, -39.384861047478], + [-39.384861047478, -90.0]], + ) + assert lat.attributes == {} + + +def check_lon(cube): + """Check longitude coordinate of cube.""" + assert cube.coords('longitude', dim_coords=True) + lon = cube.coord('longitude', dim_coords=True) + assert lon.var_name == 'lon' + assert lon.standard_name == 'longitude' + assert lon.long_name == 'longitude' + assert lon.units == 'degrees_east' + np.testing.assert_allclose( + lon.points, + [0.0, 45.0, 90.0, 135.0, 180.0, 225.0, 270.0, 315.0], + ) + np.testing.assert_allclose( + lon.bounds, + [[-22.5, 22.5], [22.5, 67.5], [67.5, 112.5], [112.5, 157.5], + [157.5, 202.5], [202.5, 247.5], [247.5, 292.5], [292.5, 337.5]], + ) + assert lon.attributes == {} + + +def check_heightxm(cube, height_value): + """Check scalar heightxm coordinate of cube.""" + assert cube.coords('height') + height = cube.coord('height') + assert height.var_name == 'height' + assert height.standard_name == 'height' + assert height.long_name == 'height' + assert height.units == 'm' + assert height.attributes == {'positive': 'up'} + np.testing.assert_allclose(height.points, [height_value]) + assert height.bounds is None + + +# Test with single-dimension cubes + + +def test_only_time(monkeypatch): + """Test fix.""" + fix = get_allvars_fix('Amon', 'tas') + + # We know that tas has dimensions time, plev19, latitude, longitude, but + # the CESM2 CMORizer is designed to check for the presence of each + # dimension individually. To test this, remove all but one dimension of tas + # to create an artificial, but realistic test case. + monkeypatch.setattr(fix.vardef, 'dimensions', ['time']) + + # Create cube with only a single dimension + time_coord = DimCoord([0.0, 1.0], var_name='time', standard_name='time', + long_name='time', units='days since 1850-01-01') + cubes = CubeList([ + Cube([1, 1], var_name='TREFHT', units='K', + dim_coords_and_dims=[(time_coord, 0)]), + ]) + fixed_cubes = fix.fix_metadata(cubes) + + # Check cube metadata + cube = check_tas_metadata(fixed_cubes) + + # Check cube data + assert cube.shape == (2,) + np.testing.assert_equal(cube.data, [1, 1]) + + # Check time metadata + assert cube.coords('time') + new_time_coord = cube.coord('time', dim_coords=True) + assert new_time_coord.var_name == 'time' + assert new_time_coord.standard_name == 'time' + assert new_time_coord.long_name == 'time' + assert new_time_coord.units == 'days since 1850-01-01' + + # Check time data + np.testing.assert_allclose(new_time_coord.points, [0.0, 1.0]) + np.testing.assert_allclose(new_time_coord.bounds, + [[-0.5, 0.5], [0.5, 1.5]]) + + +def test_only_latitude(monkeypatch): + """Test fix.""" + fix = get_allvars_fix('Amon', 'tas') + + # We know that tas has dimensions time, plev19, latitude, longitude, but + # the CESM2 CMORizer is designed to check for the presence of each + # dimension individually. To test this, remove all but one dimension of tas + # to create an artificial, but realistic test case. + monkeypatch.setattr(fix.vardef, 'dimensions', ['latitude']) + + # Create cube with only a single dimension + lat_coord = DimCoord([0.0, 10.0], var_name='lat', standard_name='latitude', + units='degrees') + cubes = CubeList([ + Cube([1, 1], var_name='TREFHT', units='K', + dim_coords_and_dims=[(lat_coord, 0)]), + ]) + fixed_cubes = fix.fix_metadata(cubes) + + # Check cube metadata + cube = check_tas_metadata(fixed_cubes) + + # Check cube data + assert cube.shape == (2,) + np.testing.assert_equal(cube.data, [1, 1]) + + # Check latitude metadata + assert cube.coords('latitude', dim_coords=True) + new_lat_coord = cube.coord('latitude') + assert new_lat_coord.var_name == 'lat' + assert new_lat_coord.standard_name == 'latitude' + assert new_lat_coord.long_name == 'latitude' + assert new_lat_coord.units == 'degrees_north' + + # Check latitude data + np.testing.assert_allclose(new_lat_coord.points, [0.0, 10.0]) + np.testing.assert_allclose(new_lat_coord.bounds, + [[-5.0, 5.0], [5.0, 15.0]]) + + +def test_only_longitude(monkeypatch): + """Test fix.""" + fix = get_allvars_fix('Amon', 'tas') + + # We know that tas has dimensions time, plev19, latitude, longitude, but + # the CESM2 CMORizer is designed to check for the presence of each + # dimension individually. To test this, remove all but one dimension of tas + # to create an artificial, but realistic test case. + monkeypatch.setattr(fix.vardef, 'dimensions', ['longitude']) + + # Create cube with only a single dimension + lon_coord = DimCoord([0.0, 180.0], var_name='lon', + standard_name='longitude', units='degrees') + cubes = CubeList([ + Cube([1, 1], var_name='TREFHT', units='K', + dim_coords_and_dims=[(lon_coord, 0)]), + ]) + fixed_cubes = fix.fix_metadata(cubes) + + # Check cube metadata + cube = check_tas_metadata(fixed_cubes) + + # Check cube data + assert cube.shape == (2,) + np.testing.assert_equal(cube.data, [1, 1]) + + # Check longitude metadata + assert cube.coords('longitude', dim_coords=True) + new_lon_coord = cube.coord('longitude') + assert new_lon_coord.var_name == 'lon' + assert new_lon_coord.standard_name == 'longitude' + assert new_lon_coord.long_name == 'longitude' + assert new_lon_coord.units == 'degrees_east' + + # Check longitude data + np.testing.assert_allclose(new_lon_coord.points, [0.0, 180.0]) + np.testing.assert_allclose(new_lon_coord.bounds, + [[-90.0, 90.0], [90.0, 270.0]]) + + +# Test 2D variables in extra_facets/cesm-mappings.yml + + +def test_get_tas_fix(): + """Test getting of fix.""" + fix = Fix.get_fixes('CESM', 'CESM2', 'Amon', 'tas') + assert fix == [ + esmvalcore.cmor._fixes.cesm.cesm2.AllVars(None), + ] + + +def test_tas_fix(cubes_2d): + """Test fix.""" + fix = get_allvars_fix('Amon', 'tas') + fixed_cubes = fix.fix_metadata(cubes_2d) + + fixed_cube = check_tas_metadata(fixed_cubes) + + check_time(fixed_cube) + check_lat(fixed_cube) + check_lon(fixed_cube) + check_heightxm(fixed_cube, 2.0) + + assert fixed_cube.shape == (12, 4, 8) + + +# Test fix invalid units (using INVALID_UNITS) + + +def test_fix_invalid_units(monkeypatch): + """Test fix.""" + fix = get_allvars_fix('Amon', 'tas') + + # We know that tas has units 'K', but to check if the invalid units + # 'fraction' are correctly handled, we change tas' units to '1'. This is an + # artificial, but realistic test case. + monkeypatch.setattr(fix.vardef, 'units', '1') + cube = Cube(1.0, attributes={'invalid_units': 'fraction'}) + fix.fix_var_metadata(cube) + + assert cube.var_name == 'tas' + assert cube.standard_name == 'air_temperature' + assert cube.long_name == 'Near-Surface Air Temperature' + assert cube.units == '1' + assert 'positive' not in cube.attributes + + np.testing.assert_allclose(cube.data, 1.0) diff --git a/tests/integration/cmor/_fixes/test_data/cesm2_native.nc b/tests/integration/cmor/_fixes/test_data/cesm2_native.nc new file mode 100644 index 0000000000000000000000000000000000000000..61a4ecf359d387c3e2592c1b06a6c6dc5e55c7c2 GIT binary patch literal 67544 zcmeHQ31E{&_Mh|y3c?D43K9etkTy+Afr6429xVk5B1J_-Luk@A&?YrW5j+bNdU3T$ z5D^ej5ET_s)Kx)LJn&#q7sPc}Sq~IE)>}MQ|8Hi#mp0QD=~_Vd{|%(?n|bri@#cLq zznSlSGgaB6vf8&jqisZ793%IO=Jv-LM37lybm;NUk_)m&O^RsW-VkxN2#oBk@v{5K zi^w*TuKtsxZ6laSq)T@eg&~pM7^SY#_)KKuzYZ*pN3~9Xi5_=}!{c*0yzErGm**IP zzgZ5C*Xec*Pf2h&e0lc#;i;xHQ*uI~J%7;fWK(jADLKABM#H4y@gGmhl{e7p^?96m zKGqSd$qT}mQ6$C-*Rdn_n;rZwW8X-^*g163fOZO9M;6(9huTW-EqC2+mx5sigOEss z!C;8j;!;IXR+w8SV>(uly9}1vj-~z*$yl@yQe-R%sVjROL%DOvva%YD#&JQg$cTtZ zBl!h7l!!=fzVx2-zrkCkEFC#HdlZ%mi^k<;@3&)_ELIz)X$QM;`CYkLxw)Avr?+Go zo-?pvcmrdr-!J$MtF^jwrW9#x1Cfdd7>B4P(q277ZQ3=Cb-fW|Xd-PZ+cs0n_d=cl zIudkD^V&Fo$BSe!QX3&>L~gFoLar-3`%z=YXHLi+bHM~_E{w>@Wbv0VY~Dx~EpM1G zVtl5EpP|K{TA@xVKS0S!BlRV z^nN=bvvD2JhtBY$f!7n=yI)-{RGwe6tV!7ug-%2;q1YlYR8gaN92t8+(gX4k45h@?8(lx7hx^f8>rP+QUDb!kXvu9C%4O%yJ8l-S+Hm2vN4C9RWULzN z>90@Ul$DFR#i$lqWwe|rPhO;nY&MpJw+VOsH3c|R_!uL4`QsPAGjeiH7Ru`@&O&YE zWy{%U?8tolR_p}$u`)W^OgRjR#;Fkw_fz7C$?W-!hajQG4f_2iQYAAi=;+ zLvq^JpFS=w*a-BY%X?+dq2dYu)&wfU^uH~VfN1|VVNwli*|0VXYp<~O4pX48h7WJj zhH4e2Qeh4K#F{1JTT}}@@_#D<(eFg_euwvY_I_RF$do!W0ym}(@Vw}GsnaxI|8qd} zyt!X*n3Q_~WRK$mO%z)@T;97yD@;5-*%A0`chHc?1c6vmM>&g~_9n3iW20Zb{<;vD z=)=>AKr}&MLS%|z1S-fc$GO9)GT&TDXcUZ%>&gQnBO>RGl^ous@LPG`MuTBwC{aqB zD2Haehw6;AK$%#zT7cW_u{&Kh9|E*6T!r9=%0-kXz%!NQd{;w&>AXXCNnv&~7()6f%--S!Ku7Q{gsVuFlFiy`^w$X97P?a=Es$9sy3f6OZKaCL zR%?3PHk6BLciNNcBe+__mG!aSy1eiriOi@LT4jV(DMwCKWcG~q=o2C%nsw@$0~#{> zmfv@N17vm?PaGq1h}1L?dx_tIeJ>Y)0Ai*57OXn&@alosSq*^LTI7TQu@rs_LP(C? z7zbhk|KVeRJxbF+?9C0|pTZ;Su5~1EY{*P&M7EY^ifzu3G;62;tf2z2ux4$jNvxrc zgf({0PkV<}1$g4RJ24Uv{Z0%AD*UlRWOl90w;q|jl{kJ-_?#H~pL4_@6q&NRcIvgY zf8yXQek$+YBQcqd9Kh7i-E+p|PG+ZLs=PmW zH~UJ@Xt*GUhtsrMT%(t#$=w(&fYMaMMLHOd!dGb#sTZekT6UsIpBgSYtR~VCGs*_x z!v~?KfuGW$aBa@qs5~?PQG6;7?-#kFaH=n1S2bM2PNDdejy48`!YLiuQ~7<(OL*! zYk)m#pA}2ne1V}!qXP^Vj$S8(mQ`peMB~#;Tk|$ca783^ra9yK!b-bO_H`0g%qTl7+kk|WB?&;k>Y#>OFhWrrt%wnUiFF});|d1^*d zUZ2fnw|VSVm#x?#%uV>*UrW*HE44c!aUoyWQx&;g1@+N-XwhhJ$f1XO{A^CL^vCwl8 zG7jUtDSu9UXbX0%9e~WV*$#lZ6rN-=4}W-+#t$f8_oY&4PpD=FSKkrfOBdM@*yebR zUZ*S1VN4!6WN4x}8UGv2=3(N0B-%=e$3M?jiGrM#;k;ewP9J(qHNmUv7Q-{Sc*a#vR5J zwBz=0nxOfKj??@^otmF$rRFEPLGu%RR`U~mU-J` z+cp=nC*A;6x#bXjq#V+3oeR6(SqR_Ft%CvM>!8ym%iw`4{jhOp0EWF@2kvX@pvTz( zxM`#x*xCS$?Hqt>ZV1Bc^#MqY4!}Qi>R@+q0H)nt2|o>}gvwJYVZt+&aAZUcWUQ-( zZJ$R{n#eps>s^S9JN=|(@ayW0;FZV$lBssLDS4np^z0x-!Qfb>u5z`QE}7oA%L_fM;Y zy=PX!lmnIU(LFU#HLwQa2Gl^!ni_chk}8NByb#vjH4hSRDTBd=a)_M206KkjJv?=7 zIc%I!2WO3}gK1lrLH7l9U|Z{loRxKO(S$lEyV?&Q!~|f|jsUo?4#49Zf-v;YL0I-= z04{O*;hKL2py0(yNPW5j2F6#w)DD%f9P>Tcvj+0I)xaGIH86H*H9TOefR1J5aCXvs zn7L^Il&@L<1rz7OhvOH(8;|*+Q=%WT66&BA*0t-yez^G7Iyi)K|M(_9tcnc4Kh^}` z-O>QOGbRX`?Sk;xR{_|0u^+Z>48VwwE8wn!3nA~qg;3tL0+wD>4c1$#VZ(*hu;Ne^ zbh*C^4I*3I5PlCFAhNU-2s^QVgR=J1MoBRL*S|aY?-|fZYWv+>%S_8FRohv z7xk!u7)KSPH`_lsfs${M(MOEv78T?K7j)iC+GYPfJxHEgdA!uP|M z!?sNU@O>SCC#DAAh70}h5w>m5a{{pUbU$427eCyCI(OFde&}*#0OC*=i!ZK&t@Hda z?B5HZ%ZhTila<3YKbFIqZ!6$G|EYjC7FL4gPz4;icM0UayA%@Am%^~WF9xHt1}=Ge z5p3AD7#6-)12_E;gfUB(!+&G{i1oYlsUWgZUmmOnb=> zt5f}un_Ua(_xs_lf0e`3P4j_mnGZWrr;CPGz(a=?f_-cSEV_3g^etTkKeSl{1=vqn z9IQ3+kH@WX4p{BZBeT9|Qj0LE_%fWv`# zobQMCo<*H_t`0W03&7{)sCR9G;K>a@=PiCH-d76`cMD)UR=|qVh0xEk5EgcXmm?isD*#h+PsZQ%Xn2zRRg*JVqRtJWk}Ceo?5c#b%p7SFu;d|5$SJP+0k z!^xynOoX)Jgpv5KM*bLX%>09a#RwT(vVE+!T+!f;quZ3CA+&I087&MVc}SEDr@v&< zL;mI{-WIx3Ce3Y1Cs`=|8;;i=S8Yna(Db<8TC`wYr*5iE=@-Lf8uhWNe936m&A;}c zT4(5gdAsYLt2=+`ux((m|v*d}}iF244;oF|SExvn0?rZk~}P3bIt zOCPD2$7oZU%5Ul0z@~I154esAbK53A?wNH<@3S$A?pjF#ztyHxQvlXbfmm3xHq<26 zP)At9#=c<5yo+0@N_sLakN}4PF{kkx3{kX7-0)l87|~f6qC!TLxNt5O!Kwz!TrM$w zvbOc#tMI{%cLnrbg)dJ1y~o*mc}_Hp(c2Rm2>B-&(YS1Q{FcvYjf|I!J6cXBKgopr zSM_7P#5yZs8F81T_|xxx(>g0}r{A>B%Ev4|gkoDB>oTj%v)N@7i6t2_%bqP$Kl$mM3 zzKnV+qGT1X*8j1|N{FU1)XOPc-K$GLmw+w-T>`oUbP4F+_(phvoBkZlwKYjD z#iKchde1N=UUP_atP+nVhiZD4De*$Fjm|x%fw#-@fQg3IowW{8$2EBhL806s39d|Jid|OfV4e) zyF_96SBU$+V&ZZ558wXa!45vRXO6Jw9*0r*i;OIf)0pchF&c*&lLrn5SuBf59l`oWsH z^%(v(4Zm;bfTVoi{tj}ld#h}$ZsQkqVo&m0Gv)5BMFant$iCuoM8kdlerS7#S>sqb zk8!CqlRS{Vbtc=$6U~x`j10W)Z4ayEjn`Y+a0hO>!ONm}Ku>A1{np_J=Q1O|rN2C3 z;`syd^Vux^x~)uZKz`x8XUpX=02qTi^A|&^*=by#Fk8Ocqxpc_&qnY!Z_60pv4EZT z{fOJx3cTP(-MqjbniKC~r}C@S@}P*A6`Ss3Wjxz{avRfkze{+6oyOl9eq_M?)ET?x z`rgm8pZV)SwC?9`GY=I^eTi-6X@E2g-(|4wXT5n{P`^aPtNOgdzU4*70R9a8?fDTO zGM83;9)9Z8Q@=T!G*NpjFyJJ)hUV6qRKprFOn6~R5Y~8MwGXRkSfhujRai5G$ums1 z!rCHCL0Chr4Qs=&<_gopFog zC3~|BEQnC}B4;t4Ctw_fdwv)j_h~8Erwuap;~kk7d$QsZ+~mM#DlRE7*&RuFcI>Wn zr%OPW!0$%_;m3)g51`ZjPk`}^VGraMJg><`?00q1|IGvDt?wW3p!S7uak@ zkE7UDV)f>EoF%^Dt~}ulV->o+KECtP@FK@78*Z7DY(fa;H5IvWyOXepbYz{xWQmEM z0*?{*e6qTVt?p9aaJ1Zc8|ycRqh z0h{>cxm}1HO(jly%HU*ks@a%Sh$t{=deZcP=Ap@{$w^ncJu{Oy+DwAX-0>+%`6fq7 zvI&m`$(?K+IT}p)=Al;pD=tVLVuJBwCJs&Ka3Q%zN=Oqwg1-EM!Zg+;~@X6iDICvHr zWF;n>%&l@9W$4=+ue;QfC)JTp2t+m`#z!Kg*EUJ=N@lXgY_VE~`qd(LL1;@js?iLG zYi5JpmU!nSMA}BnEw9iq$Ki50QX=sHK27DEKBujyUg`Kp%)HdovA-~zM)Lkb^cB3D zNHO~g!IUy|h!s1DSt+82aEhKmZVHvidsaTPjl@ZgNZ#izaTj=OC53b9bKoC1^Bj^2 zifoAKN<=TkcYsUE^EwKW#6uvwHeYcPc0x%!k)-@tR||5QAb!{<=3|}Uz1p0b zVx8den$2leaGM94tyxaqfn7Gq>MbZ7Xtt)L&MdZ$o-`rHnwLDpoNOK_3a5SYRg5#^ zBCFl)w3d2`gq)08(FlDa&*b(LBqbvzH4h$?G-~jW?Bs0o#gTaGrK}&KQ!CEPoo>4| z-{ZJSq{1hUyn@AUm#@&89Fldz!6q!C*2(j=43VmgNJEc^%4dK)%_fGSkOIrwv_4PV z#V3;dE-@%Y;wc#7a4$ZF;IXSN98!FR9)}J07A!5E?(j4^;fqf;xg*MxYV71^{i_?C z;mI)ocSL{vGk*C6{S!;&eI7|d$CgDmvK*!_j^an~sK4pULtd$P96I>nAx&ZG75JW zY09%sGcG2bXnd|KyIkIJ-2U#p^opCrqD6Gi2iLS`X;}C(*zM1BIfoq-m+sS?e`YiB z@^rT1hMazEJ(lD&w&utwBiXrl*_A#2{j^c68ZXae@0T9P69;v6WmV~4*jc%_?P*(t zgYClzr?d0_QCq|&iO^Rz|8p_(iNwDB^4nX*a#I(!DrN1RY&Ayc#y$(?-@{%Jaev&o z=qWY@<925k&AIVu)(b7fv=IJsX-L@UB)ksV$+TWxIqd z`S}uK7x9>g$Wll6@GP8cTI~tJr%5yHuF)6i(`Gd-6&hFT2r-l zNm!Og9;?riYf|fG$(!!+6@#*69w+jz9Aj!5fC1818dIp_Z~lyQ2?ids%W`|Q;?t0F zT)_a(D<0ot5d=8*?dAL4lqMFT2yqg zt*F$&nmQDopF&1`4pCkLW?Zx*ezY9+JIIuZod^bSd*@mIRbrUr-v{pA-g=GG<$6(0 z_B(AHcgi2Dac=9e_GZ~QR~_xiiRVbX;eObJ$K(@J`TK5^QjZ5`;YOU=IgI@-MK^x@ zs8Rjn)98tj2M^ajF%2u))RS`EPrv*Ue;z$0;_mRLh2u&-o3)-w`-j zPXx{nn(eOe#i;DAG~s(4xNvCq@*PU@8nnmBZ-oB#BktDgo(2X_R%0B?*pZ){{Kx8^ z25idow_f9H_?D8U=T3 zY<6rpQyE)upMPV&e2=@>I2{l3z;S-g?_h90J7asUY=GInp#gq5X$^qO@y#{>Z9GOJ zd6R?toRdGc_g?Q@D%)VjZ)gM0O(&~oN;*m#@D&v?AJotkKa<1jbCjHXsgLrXvQ|oc z>(Qj%na?Sjnw*BuEk0;Q`Z1xkVw&F*ZH1+I-r(_rjw1cjH0k@?K71YTbr;z@MyJPP zv*WsXo+BR7FQ1GzI@yFTxbQ~1B&j%s&)E6$dh5*Cxj|}c&Dg=1bhcPC66x!Atq@$4+!2*NJQL>F9NreXQ?7vU&4#C}pu3;A zH*bz+U&`$Y7*6+iG_jp;PkxmlttG#%?%9W};{gfMHnh#kW%Jp?=x4ZS$yaVasJvhS zv+@`q_vRfnt<9}$C6CcvY6Ekwd4TQaF^c3eV4lDBkTez@!=vG%&ntTlv48O^oOAJV zzc{o1PXph2`mz`6OQ?2x5 zk5vL;>*oEQk+<>3ls|#3yKdeYQ(TrXeq>mdQ(mgivi08%@E1bwm*ia?mSyGZ$1_Vl zYA%0p{bylWKKj=2%#tsg@7R6%=&&q5dapjq)^E#hnE0Rw?Dom=%<^>UQlu3?!2d*dQIgEwE2E*uc6pIgJ&@7_~$j3#&Qyv$x zmI)bBG;4YB+iWhHhLnxX9hzF&X3Vy;Ss>YxBs43pJUsH#igxscZtZ|hj`p}X_;h8M zvDjgAwd6x8YA&K&xbz_ve*xXl&)+~d^oL$SW4P}A$Py6Ge|DexDm=LOoW6!AGIfY3 zxSEb)HSpeDxG1i0W4Jz`Gl*}u_*E1?Bz6^7NOXoM?Si%8vBl2(pXdC+|GR%m?ecXrRWKw29_8D;)8n*YpA&U2y+{H-CXWHi65Ji)k7<~CVtJp)Q zM;fBYU?V2aoq~+EW#WFU2$^m~@t0n7nxr9$UWv!@U|hB7GoNlCYmOLvM}R5O0h5L( zvg<$w&w@^xG(?ewM+~lwB7=^cG(^#Z7VWa*0udcOX^0}Dk7$<{!|4c0Lljwm#JKqS zMIoIr=ia>}#_q*I!=5jm{HCq-)%Q45&CZGO>}iND$bG1_%g1FkNh(R%BwOhfV7_YDm|>kr-wm}NB$Y9bR{Xpr*@#{ zlKkPZb)PN)T>`oUbP4DZ&?TTtK$pPpT>{O`9|-5E<=?1H7m2UV7d9(=nnhANk1Bk1 zKCnaKt2pxog|FiMJqn+un3Vs1g|FiI0}7w&N%w!G@KwD2xx!cR`4NRr_?psZ(cuy$ zzFLk@eEmX~fGz=D0=fir37h~4v=lGIY2z7>XU0?pwCb)2NKyr7L|#8>gc9)+*sh5ZU& z#R~@%zB*2Rr0`X|@VUZQ@xl>>ui^z3BWpzUBTDs;Q}`-g=&0~jywFYI6CR-SjT)c) z#8=Bfe4XkN&?TTtK$n0nffFHtmg0qe%6O;Z1+&6e@j{xySMfrI!dLM^w!&BO!bF9y z;)N*+U&Ra46uvr67b<)eFO(>J6)((I_=E?jt;;k%`H8QVgZMhtC7??{mw+w-T>>XU z0xiW0mCAUh;)P`jU&RY66uycV)+&4zFKkr!Dqh&E@KwCvhzFH3A>r|J3E&*Kvx&(9yoCpathZodwn#%H#Hl7h*#S5P( zd=)QzuJBd7@U_CHd?^1T3SY$wKPh|_FR)lyVoF!V3sDMR9jD_IzKR#(6~2lWIx2j^ z1604x8lU{cr}9#`PIU?B63``}OF);viIhM~@j^Fcyi@T)cZILw1*5`O@j@Skua1-b z6uycV5){6Q7t9J@#S5tlU&RY)3SY$w=PP^_FJvfu!UNRiqclGGiLc6m_&U`kpi4lP zfGz=D0w+QOEyWAj%6O;Zg&c*i;)RI{U&RZP6}~!7PEq(OUYM%zRlG1w;j4JTuJBd7 zP^j=#yila@RlHE5@Cgr4Tl+LV`H8Q}f%rPrC7??{mw+w-T>>XU0?pwCb)2NK%+|&; z;;VR}OyR3|p;FJrc;pi4lPfG&X(Apvv3Ib9F_ZJTA=JLh=*hIByn=y^Ml zJ~BM}pIeX~T=Da++mL#5e(AFe>1}sz?Q|p3d6BWZkG#Ik@{8Skrwi%a!Xe*$^cwni zP1}7X(z&k`bbs^JZI+w!et6#W=d^U+yfqIS`lY2mInG{sXaBVHRWsu3*9=HY&wl!m zk(R`?^xprvJh@X+TKb-}b{+Sc)6y+B{InuwU|Rb4m8oOS8{^l0P#}A#CmhMPQHtfP*qAA%>i@$I1cWF|xVH^H>nUf6*@OKD*l-nd~1H{kiW!QR;D^%>$N@B^7JF?rgVKK)^f!s?LL0*`B=*Wcji@Zz8q^g>yG;( zH@q5aIpvYqF|jvCSk}fLeDuMU5tcW*F5fbERfJ_mvG45nR!3Ogo(wzY-Wp+X-`&e> zx-G);UhU_vyu2;~#dGAKnB5&RR74{Qg0Ub=x(2*PI))l(+42Zr_BUC6 zeg6h?&~jkM+-IIh4qDnDT=Dkql%VC)4QHIae_)WOJN$5J(DK8g2g|-16tsNN1NZdA zGqDr!Hx7SJ{8i#_1OA@H-~0HBz$1Zs;V&J3Ep_JAhzM_wW*Hl!S;zaL+2#kMS=nEr kneoYJ_W90ew&TTUHf>KdGw+LLd)^}S`1zAEpVp`U2Z;hS=l}o! literal 0 HcmV?d00001 From bfd2ed50595b7d996074cb723ddbf39f55b3bf73 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 12 Sep 2022 09:42:16 +0200 Subject: [PATCH 07/12] Added tests for output filenames of CESM2 --- esmvalcore/config-developer.yml | 2 +- tests/integration/data_finder.yml | 71 +++++++++++++++++++++++++------ 2 files changed, 58 insertions(+), 15 deletions(-) diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index 0503fd965b..f671c1882d 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -186,6 +186,6 @@ CESM: - '{case}/{gcomp}/proc/{tdir}/{tperiod}' # postprocessed data input_file: default: '{case}.{scomp}.{type}.{string}*nc' - output_file: '{project}_{dataset}_{case}_{gcomp}_{scomp}_{mip}_{short_name}' + output_file: '{project}_{dataset}_{case}_{gcomp}_{scomp}_{type}_{mip}_{short_name}' cmor_type: 'CMIP6' cmor_default_table_prefix: 'CMIP6_' diff --git a/tests/integration/data_finder.yml b/tests/integration/data_finder.yml index a52d64a95c..64727db0cd 100644 --- a/tests/integration/data_finder.yml +++ b/tests/integration/data_finder.yml @@ -97,6 +97,49 @@ get_output_file: preproc_dir: this/is/a/path output_file: this/is/a/path/test_diag/test/ICON_ICON_amip_custom_var_type_Amon_tas_20000101-20000102.nc + # CESM2 + + - variable: + variable_group: test + short_name: tas + original_short_name: tas + dataset: CESM2 + project: CESM + cmor_table: CMIP6 + frequency: mon + mip: Amon + exp: amip + case: f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1 + type: h0 + timerange: '2000/2002' + diagnostic: test_diag + preprocessor: test_preproc + preproc_dir: this/is/a/path + output_file: this/is/a/path/test_diag/test/CESM_CESM2_f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1_atm_cam_h0_Amon_tas_2000-2002.nc + + - variable: + variable_group: test + short_name: tas + original_short_name: tas + dataset: CESM2 + project: CESM + cmor_table: CMIP6 + frequency: mon + mip: Amon + exp: amip + case: f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001 + gcomp: gcomp + scomp: scomp + type: h1 + tdir: tseries + tperiod: month_1 + string: TREFHT + timerange: '2015/2025' + diagnostic: test_diag + preprocessor: test_preproc + preproc_dir: this/is/a/path + output_file: this/is/a/path/test_diag/test/CESM_CESM2_f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_gcomp_scomp_h1_Amon_tas_2015-2025.nc + get_input_filelist: - drs: default @@ -1001,7 +1044,6 @@ get_input_filelist: cmor_table: CMIP6 frequency: mon mip: Amon - exp: amip case: f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1 type: h0 timerange: '2000/2002' @@ -1031,9 +1073,10 @@ get_input_filelist: cmor_table: CMIP6 frequency: mon mip: Amon - exp: amip case: f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1 - type: h0 + gcomp: gcomp + scomp: scomp + type: h1 tdir: tseries tperiod: month_1 string: TREFHT @@ -1041,18 +1084,18 @@ get_input_filelist: diagnostic: test_diag preprocessor: test_preproc available_files: - - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.2000.nc - - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.2001.nc - - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/atm/hist/f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.2002.nc - - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/atm/proc/tseries/month_1/f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.TREFHT.201001-201912.nc - - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/atm/proc/tseries/month_1/f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.TREFHT.202001-202912.nc - - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/atm/proc/tseries/month_1/f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.TREFHT.203001-203912.nc + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.scomp.h1.2000.nc + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.scomp.h1.2001.nc + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/gcomp/hist/f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.scomp.h1.2002.nc + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/gcomp/proc/tseries/month_1/f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.scomp.h1.TREFHT.201001-201912.nc + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/gcomp/proc/tseries/month_1/f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.scomp.h1.TREFHT.202001-202912.nc + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/gcomp/proc/tseries/month_1/f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.scomp.h1.TREFHT.203001-203912.nc dirs: - '' - - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/atm/hist - - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/atm/proc/tseries/month_1 + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/gcomp/hist + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/gcomp/proc/tseries/month_1 file_patterns: - - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.TREFHT*nc + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.scomp.h1.TREFHT*nc found_files: - - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/atm/proc/tseries/month_1/f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.TREFHT.201001-201912.nc - - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/atm/proc/tseries/month_1/f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.cam.h0.TREFHT.202001-202912.nc + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/gcomp/proc/tseries/month_1/f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.scomp.h1.TREFHT.201001-201912.nc + - f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1/gcomp/proc/tseries/month_1/f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001_cosp1.scomp.h1.TREFHT.202001-202912.nc From 59b1fc4f46b9b61fef2dca4b71dfde10dcad433a Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 12 Sep 2022 10:32:09 +0200 Subject: [PATCH 08/12] Added doc for CESM2 on-the-fly CMORizer --- doc/quickstart/configure.rst | 8 +- doc/quickstart/find_data.rst | 77 +++++++++++++++++++ .../_config/extra_facets/cesm-mappings.yml | 11 +-- .../_config/extra_facets/emac-mappings.yml | 7 +- .../_config/extra_facets/icon-mappings.yml | 11 ++- esmvalcore/config-developer.yml | 2 +- 6 files changed, 100 insertions(+), 16 deletions(-) diff --git a/doc/quickstart/configure.rst b/doc/quickstart/configure.rst index 0016982308..691ac97dd5 100644 --- a/doc/quickstart/configure.rst +++ b/doc/quickstart/configure.rst @@ -639,10 +639,12 @@ Example: ICON: cmor_strict: false input_dir: - default: '{version}_{component}_{exp}_{grid}_{ensemble}' + default: + - '{exp}' + - '{exp}/outdata' input_file: - default: '{version}_{component}_{exp}_{grid}_{ensemble}_{var_type}*.nc' - output_file: '{dataset}_{version}_{component}_{grid}_{mip}_{exp}_{ensemble}_{short_name}_{var_type}' + default: '{exp}_{var_type}*.nc' + output_file: '{project}_{dataset}_{exp}_{var_type}_{mip}_{short_name}' cmor_type: 'CMIP6' cmor_default_table_prefix: 'CMIP6_' diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index bc72ad65ea..72b67c46dc 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -148,6 +148,83 @@ The following models are natively supported by ESMValCore. In contrast to the native observational datasets listed above, they use dedicated projects instead of the project ``native6``. +.. _read_cesm: + +CESM +^^^^ + +ESMValTool is able to read native `CESM `__ model +output. + +The default naming conventions for input directories and files for CESM are + +* input directories: 3 different types supported: + * ``/`` (run directory) + * ``[case]/[gcomp]/hist`` (short-term archiving) + * ``[case]/[gcomp]/proc/[tdir]/[tperiod]`` (post-processed data) +* input files: ``[case].[scomp].[type].[string]*nc`` + +as configured in the :ref:`config-developer file ` (using the +default DRS ``drs: default`` in the :ref:`user configuration file`). +More information about CESM naming conventions are given `here +`__. + +.. note:: + + The ``[string]`` entry in the input file names above does not only + correspond to the (optional) ``$string`` entry for `CESM model output files + `__, + but can also be used to read `post-processed files + `__. + In the latter case, ``[string]`` corresponds to the combination + ``$SSTRING.$TSTRING``. + +Thus, example dataset entries could look like this: + +.. code-block:: yaml + + datasets: + - {project: CESM, dataset: CESM2, case: f.e21.FHIST_BGC.f09_f09_mg17.CMIP6-AMIP.001, type: h0, mip: Amon, short_name: tas, start_year: 2000, end_year: 2014} + - {project: CESM, dataset: CESM2, case: f.e21.F1850_BGC.f09_f09_mg17.CFMIP-hadsst-piForcing.001, type: h0, gcomp: atm, scomp: cam, mip: Amon, short_name: tas, start_year: 2000, end_year: 2014} + +Variable-specific defaults for the facet ``gcomp`` and ``scomp`` are given in +the extra facets (see next paragraph) for some variables, but this can be +overwritten in the recipe. + +Similar to any other fix, the CESM fix allows the use of :ref:`extra +facets`. +By default, the file :download:`cesm-mappings.yml +` is used for that +purpose. +For some variables, extra facets are necessary; otherwise ESMValTool cannot +read them properly. +Supported keys for extra facets are: + +==================== ====================================== ================================= +Key Description Default value if not specified +==================== ====================================== ================================= +``gcomp`` Generic component-model name No default (needs to be specified + in extra facets or recipe if + default DRS is used) +``raw_name`` Variable name of the variable in the CMOR variable name of the + raw input file corresponding variable +``scomp`` Specific component-model name No default (needs to be specified + in extra facets or recipe if + default DRS is used) +``string`` Short string which is used to further ``''`` (empty string) + identify the history file type + (corresponds to ``$string`` or + ``$SSTRING.$TSTRING`` in the CESM file + name conventions (see note above) +``tdir`` Entry to distinguish time averages ``''`` (empty string) + from time series from diagnostic plot + sets (only used for post-processed + data) +``tperiod`` Time period over which the data was ``''`` (empty string) + processed (only used for + post-processed data) +==================== ====================================== ================================= + .. _read_emac: EMAC diff --git a/esmvalcore/_config/extra_facets/cesm-mappings.yml b/esmvalcore/_config/extra_facets/cesm-mappings.yml index 57d3cc8d2c..be0bcf7a53 100644 --- a/esmvalcore/_config/extra_facets/cesm-mappings.yml +++ b/esmvalcore/_config/extra_facets/cesm-mappings.yml @@ -1,10 +1,11 @@ # Extra facets for native CESM model output -# All extra facets for CESM are optional but might be necessary for some -# variables. - # Notes: -# - ... +# - All facets can also be specified in the recipes. The values given here are +# only defaults. +# - The facets ``gcomp``, ``scomp``, ``string``, ``tdir``, and ``tperiod`` have +# to be specified in the recipe if they are not given here and default DRS is +# used. # A complete list of supported keys is given in the documentation (see # ESMValCore/doc/quickstart/find_data.rst). @@ -13,7 +14,7 @@ CESM2: '*': - # Optional facets + # Optional facets for every variable # It is necessary to define them here to allow multiple file/dir name # conventions, see # https://www.cesm.ucar.edu/models/cesm2/naming_conventions.html diff --git a/esmvalcore/_config/extra_facets/emac-mappings.yml b/esmvalcore/_config/extra_facets/emac-mappings.yml index ca14f32166..a80b852861 100644 --- a/esmvalcore/_config/extra_facets/emac-mappings.yml +++ b/esmvalcore/_config/extra_facets/emac-mappings.yml @@ -1,13 +1,10 @@ # Extra facets for native EMAC model output -# All extra facets for EMAC are optional but might be necessary for some -# variables. - # Notes: # - All facets can also be specified in the recipes. The values given here are # only defaults. # - The facets ``channel`` and ``postproc_flag`` have to be specified in the -# recipe if they are not given here. +# recipe if they are not given here and default DRS is used. # - If ``raw_name`` is omitted and no derivation in the EMAC fix is given, the # CMOR short_name is used by default. To support single and multiple raw # names for a variable, ``raw_name`` can be given as str and list. In the @@ -26,6 +23,8 @@ # A complete list of supported keys is given in the documentation (see # ESMValCore/doc/quickstart/find_data.rst). --- + +# Optional facets for every variable '*': '*': '*': diff --git a/esmvalcore/_config/extra_facets/icon-mappings.yml b/esmvalcore/_config/extra_facets/icon-mappings.yml index 5806247677..4a6fce075b 100644 --- a/esmvalcore/_config/extra_facets/icon-mappings.yml +++ b/esmvalcore/_config/extra_facets/icon-mappings.yml @@ -1,8 +1,13 @@ # Extra facets for native ICON model output -# All extra facets for ICON are optional but might be necessary for some -# variables. A complete list of supported keys is given in the documentation -# (see ESMValCore/doc/quickstart/find_data.rst). +# Notes: +# - All facets can also be specified in the recipes. The values given here are +# only defaults. +# - The facet ``var_type`` has to be specified in the recipe if it is not given +# here and default DRS is used. + +# A complete list of supported keys is given in the documentation (see +# ESMValCore/doc/quickstart/find_data.rst). --- ICON: diff --git a/esmvalcore/config-developer.yml b/esmvalcore/config-developer.yml index f671c1882d..aec3c3df52 100644 --- a/esmvalcore/config-developer.yml +++ b/esmvalcore/config-developer.yml @@ -182,7 +182,7 @@ CESM: input_dir: default: - '/' # run directory - - '{case}/{gcomp}/hist/' # short-term archiving + - '{case}/{gcomp}/hist' # short-term archiving - '{case}/{gcomp}/proc/{tdir}/{tperiod}' # postprocessed data input_file: default: '{case}.{scomp}.{type}.{string}*nc' From a5eb230a9ac816cf72146cce609fdd1edcd0a7cc Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 12 Sep 2022 10:46:56 +0200 Subject: [PATCH 09/12] Fixed typo in doc --- doc/quickstart/find_data.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index 72b67c46dc..6f4224c712 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -215,7 +215,7 @@ Key Description Default value if not identify the history file type (corresponds to ``$string`` or ``$SSTRING.$TSTRING`` in the CESM file - name conventions (see note above) + name conventions; see note above) ``tdir`` Entry to distinguish time averages ``''`` (empty string) from time series from diagnostic plot sets (only used for post-processed From 23eb429448f315f93bde117e0207ef8987665a32 Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Mon, 12 Sep 2022 13:43:06 +0200 Subject: [PATCH 10/12] Added warning about supported variables for CESM on-the-fly CMORizer --- doc/quickstart/find_data.rst | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index 6f4224c712..2ef865400e 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -156,6 +156,11 @@ CESM ESMValTool is able to read native `CESM `__ model output. +.. warning:: + + Currently, 3D variables (data that uses a vertical dimension) are not + supported, yet. + The default naming conventions for input directories and files for CESM are * input directories: 3 different types supported: @@ -196,8 +201,8 @@ facets`. By default, the file :download:`cesm-mappings.yml ` is used for that purpose. -For some variables, extra facets are necessary; otherwise ESMValTool cannot -read them properly. +Right now, this file only contains default facets for a single variable +(`tas`); for other variables, these entries need to be defined in the recipe. Supported keys for extra facets are: ==================== ====================================== ================================= From 5815f99c2de270152db151146a2cd29b7c8aa10a Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Thu, 15 Sep 2022 12:04:50 +0200 Subject: [PATCH 11/12] Made clear that CESM on-the-fly CMORizer is prototype --- doc/quickstart/find_data.rst | 9 ++++--- esmvalcore/cmor/_fixes/cesm/cesm2.py | 18 +++++++++++++- .../cmor/_fixes/cesm/test_cesm2.py | 24 +++++++++---------- 3 files changed, 35 insertions(+), 16 deletions(-) diff --git a/doc/quickstart/find_data.rst b/doc/quickstart/find_data.rst index 2ef865400e..8146377e74 100644 --- a/doc/quickstart/find_data.rst +++ b/doc/quickstart/find_data.rst @@ -158,8 +158,11 @@ output. .. warning:: - Currently, 3D variables (data that uses a vertical dimension) are not - supported, yet. + The support for native CESM output is still experimental. + Currently, only one variable (`tas`) is fully supported. Other 2D variables + might be supported by specifying appropriate facets in the recipe or extra + facets files (see text below). + 3D variables (data that uses a vertical dimension) are not supported, yet. The default naming conventions for input directories and files for CESM are @@ -201,7 +204,7 @@ facets`. By default, the file :download:`cesm-mappings.yml ` is used for that purpose. -Right now, this file only contains default facets for a single variable +Currently, this file only contains default facets for a single variable (`tas`); for other variables, these entries need to be defined in the recipe. Supported keys for extra facets are: diff --git a/esmvalcore/cmor/_fixes/cesm/cesm2.py b/esmvalcore/cmor/_fixes/cesm/cesm2.py index 3dc95f7bf9..70683a697d 100644 --- a/esmvalcore/cmor/_fixes/cesm/cesm2.py +++ b/esmvalcore/cmor/_fixes/cesm/cesm2.py @@ -1,4 +1,19 @@ -"""On-the-fly CMORizer for CESM2.""" +"""On-the-fly CMORizer for CESM2. + +Warning +------- +The support for native CESM output is still experimental. Currently, only one +variable (`tas`) is fully supported. Other 2D variables might be supported by +specifying appropriate facets in the recipe or extra facets files (see +doc/quickstart/find_data.rst for details). 3D variables are currently not +supported. + +To add support for more variables, expand the extra facets file +(esmvalcore/_config/extra_facets/cesm-mappings.yml) and/or add classes to this +file for variables that need more complex fixes (see +esmvalcore/cmor/_fixes/emac/emac.py for examples). + +""" import logging @@ -22,6 +37,7 @@ def fix_metadata(self, cubes): cube = self.get_cube(cubes) # Fix time, latitude, and longitude coordinates + # Note: 3D variables are currently not supported self.fix_regular_time(cube) self.fix_regular_lat(cube) self.fix_regular_lon(cube) diff --git a/tests/integration/cmor/_fixes/cesm/test_cesm2.py b/tests/integration/cmor/_fixes/cesm/test_cesm2.py index 449d4329f2..111fd99376 100644 --- a/tests/integration/cmor/_fixes/cesm/test_cesm2.py +++ b/tests/integration/cmor/_fixes/cesm/test_cesm2.py @@ -139,10 +139,10 @@ def test_only_time(monkeypatch): """Test fix.""" fix = get_allvars_fix('Amon', 'tas') - # We know that tas has dimensions time, plev19, latitude, longitude, but - # the CESM2 CMORizer is designed to check for the presence of each - # dimension individually. To test this, remove all but one dimension of tas - # to create an artificial, but realistic test case. + # We know that tas has dimensions time, latitude, longitude, but the CESM2 + # CMORizer is designed to check for the presence of each dimension + # individually. To test this, remove all but one dimension of tas to create + # an artificial, but realistic test case. monkeypatch.setattr(fix.vardef, 'dimensions', ['time']) # Create cube with only a single dimension @@ -179,10 +179,10 @@ def test_only_latitude(monkeypatch): """Test fix.""" fix = get_allvars_fix('Amon', 'tas') - # We know that tas has dimensions time, plev19, latitude, longitude, but - # the CESM2 CMORizer is designed to check for the presence of each - # dimension individually. To test this, remove all but one dimension of tas - # to create an artificial, but realistic test case. + # We know that tas has dimensions time, latitude, longitude, but the CESM2 + # CMORizer is designed to check for the presence of each dimension + # individually. To test this, remove all but one dimension of tas to create + # an artificial, but realistic test case. monkeypatch.setattr(fix.vardef, 'dimensions', ['latitude']) # Create cube with only a single dimension @@ -219,10 +219,10 @@ def test_only_longitude(monkeypatch): """Test fix.""" fix = get_allvars_fix('Amon', 'tas') - # We know that tas has dimensions time, plev19, latitude, longitude, but - # the CESM2 CMORizer is designed to check for the presence of each - # dimension individually. To test this, remove all but one dimension of tas - # to create an artificial, but realistic test case. + # We know that tas has dimensions time, latitude, longitude, but the CESM2 + # CMORizer is designed to check for the presence of each dimension + # individually. To test this, remove all but one dimension of tas to create + # an artificial, but realistic test case. monkeypatch.setattr(fix.vardef, 'dimensions', ['longitude']) # Create cube with only a single dimension From b2af170a0db0ef30c820d4f7547a7af6db78a2ab Mon Sep 17 00:00:00 2001 From: Manuel Schlund Date: Thu, 15 Sep 2022 12:05:17 +0200 Subject: [PATCH 12/12] Updated ICON example in native dataset fix doc --- doc/develop/fixing_data.rst | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/doc/develop/fixing_data.rst b/doc/develop/fixing_data.rst index fb29ac26fb..ede188291b 100644 --- a/doc/develop/fixing_data.rst +++ b/doc/develop/fixing_data.rst @@ -415,27 +415,34 @@ To allow ESMValCore to locate the data files, use the following steps: ICON: ... input_dir: - default: '{version}_{component}_{exp}_{grid}_{ensemble}' + default: + - '{exp}' + - '{exp}/outdata' input_file: - default: '{version}_{component}_{exp}_{grid}_{ensemble}_{var_type}*.nc' + default: '{exp}_{var_type}*.nc' ... - To find your ICON data that is for example located in - ``{rootpath}/42-0_atm_amip_R2B5_r1i1/42-0_atm_amip_R2B5_r1i1_2d_1979.nc`` - (``{rootpath}`` is ESMValTool ``rootpath`` for the project ``ICON`` - defined in your :ref:`user configuration file`), use the following dataset - entry in your recipe: + To find your ICON data that is for example located in files like + ``{rootpath}/amip/amip_atm_2d_ml_20000101T000000Z.nc`` (``{rootpath}`` is + ESMValTool ``rootpath`` for the project ``ICON`` defined in your + :ref:`user configuration file`), use the following dataset entry in your + recipe: .. code-block:: yaml datasets: - - {project: ICON, dataset: ICON, version: 42-0, component: atm, exp: amip, grid: R2B5, ensemble: r1i1, var_type: 2d} + - {project: ICON, dataset: ICON, exp: amip} Please note the duplication of the name ``ICON`` in ``project`` and ``dataset``, which is necessary to comply with ESMValTool's data finding and CMORizing functionalities. For other native models, ``dataset`` could also refer to a subversion of the model. + Note that it is possible to predefine facets in an :ref:`extra facets file + `. + In this ICON example, the facet ``var_type`` is :download:`predefined + ` for many + variables. .. _add_new_fix_native_datasets_fix_data: