diff --git a/doc/recipe/preprocessor.rst b/doc/recipe/preprocessor.rst index 566a4cd3f0..1884bd5032 100644 --- a/doc/recipe/preprocessor.rst +++ b/doc/recipe/preprocessor.rst @@ -245,6 +245,17 @@ available tables of the specified project. a given dataset) fx files are found in more than one table, ``mip`` needs to be specified, otherwise an error is raised. +Additionally, it is possible to search across all ensembles and experiments (or +any other keys) when specifying the fx variable, by using the ``*`` character, +which is useful for some projects where the location of the fx files is not +consistent. This makes it possible to search for fx files under multiple +ensemble members or experiments. For example: ``ensemble: '*'``. Note that the +``*`` character must be quoted since ``*`` is a special charcter in YAML. This +functionality is only supported for time invariant fx variables (i.e. frequency +``fx``). Note also that if multiple folders of matching fx files are found, +ESMValTool will default to ensemble r0i0p0 if it exists and then first folder +found only if it does not. + Internally, the required ``fx_variables`` are automatically loaded by the preprocessor step ``add_fx_variables`` which also checks them against CMOR standards and adds them either as ``cell_measure`` (see `CF conventions on cell @@ -1507,6 +1518,7 @@ The ``_volume.py`` module contains the following preprocessor functions: * ``extract_transect``: Extract data along a line of constant latitude or longitude. * ``extract_trajectory``: Extract data along a specified trajectory. +* ``extract_surface``: Extract the surface layer from a cube. ``extract_volume`` @@ -1590,6 +1602,24 @@ Note that this function uses the expensive ``interpolate`` method from See also :func:`esmvalcore.preprocessor.extract_trajectory`. +``extract_surface`` +------------------- + +This function extracts the surface layer from a dataset. + +The surface layer is defined as the minimum +of the absolute value of the Z-dimension. +This is typically the case for ocean models, but might not be the +case for atmospheric models. + +The same functionality exists in the ``extract_levels`` preprocessor, +and this function should be used for more complex datasets +if extract_surface fails. However, ``extract_levels`` also +has a higher computational cost, and may be slower. + +See also :func:`esmvalcore.preprocessor.extract_levels`. + + .. _cycles: Cycles diff --git a/esmvalcore/_data_finder.py b/esmvalcore/_data_finder.py index cc9cf4c52e..04abb4e069 100644 --- a/esmvalcore/_data_finder.py +++ b/esmvalcore/_data_finder.py @@ -93,7 +93,7 @@ def get_start_end_year(filename): for cube in cubes: logger.debug(cube) try: - time = cube.coord('time') + time = cube.coord("time") except iris.exceptions.CoordinateNotFoundError: continue start_year = time.cell(0).point.year @@ -101,8 +101,8 @@ def get_start_end_year(filename): break if start_year is None or end_year is None: - raise ValueError(f'File {filename} dates do not match a recognized' - 'pattern and time can not be read from the file') + raise ValueError(f"File {filename} dates do not match a recognized" + "pattern and time can not be read from the file") logger.debug("Found start_year %s and end_year %s", start_year, end_year) return int(start_year), int(end_year) @@ -124,7 +124,7 @@ def select_files(filenames, start_year, end_year): def _replace_tags(paths, variable): """Replace tags in the config-developer's file with actual values.""" if isinstance(paths, str): - paths = set((paths.strip('/'),)) + paths = set((paths.strip('/'), )) else: paths = set(path.strip('/') for path in paths) tlist = set() @@ -133,10 +133,9 @@ def _replace_tags(paths, variable): if 'sub_experiment' in variable: new_paths = [] for path in paths: - new_paths.extend(( - re.sub(r'(\b{ensemble}\b)', r'{sub_experiment}-\1', path), - re.sub(r'({ensemble})', r'{sub_experiment}-\1', path) - )) + new_paths.extend( + (re.sub(r'(\b{ensemble}\b)', r'{sub_experiment}-\1', path), + re.sub(r'({ensemble})', r'{sub_experiment}-\1', path))) tlist.add('sub_experiment') paths = new_paths logger.debug(tlist) @@ -145,7 +144,7 @@ def _replace_tags(paths, variable): original_tag = tag tag, _, _ = _get_caps_options(tag) - if tag == 'latestversion': # handled separately later + if tag == "latestversion": # handled separately later continue if tag in variable: replacewith = variable[tag] @@ -172,10 +171,10 @@ def _replace_tag(paths, tag, replacewith): def _get_caps_options(tag): lower = False upper = False - if tag.endswith('.lower'): + if tag.endswith(".lower"): lower = True tag = tag[0:-6] - elif tag.endswith('.upper'): + elif tag.endswith(".upper"): upper = True tag = tag[0:-6] return tag, lower, upper @@ -195,16 +194,16 @@ def _resolve_latestversion(dirname_template): This implementation avoid globbing on centralized clusters with very large data root dirs (i.e. ESGF nodes like Jasmin/DKRZ). """ - if '{latestversion}' not in dirname_template: + if "{latestversion}" not in dirname_template: return dirname_template # Find latest version - part1, part2 = dirname_template.split('{latestversion}') + part1, part2 = dirname_template.split("{latestversion}") part2 = part2.lstrip(os.sep) if os.path.exists(part1): versions = os.listdir(part1) versions.sort(reverse=True) - for version in ['latest'] + versions: + for version in ["latest"] + versions: dirname = os.path.join(part1, version, part2) if os.path.isdir(dirname): return dirname @@ -212,6 +211,52 @@ def _resolve_latestversion(dirname_template): return dirname_template +def _resolve_wildcards_and_version(dirname, basepath, project, drs): + """Resolve wildcards and latestversion tag.""" + if "{latestversion}" in dirname: + dirname_version_wildcard = dirname.replace("{latestversion}", "*") + + # Find all directories that match the template + all_dirs = sorted(glob.glob(dirname_version_wildcard)) + + # Sort directories by version + all_dirs_dict = {} + for directory in all_dirs: + version = dir_to_var( + directory, basepath, project, drs)['latestversion'] + all_dirs_dict.setdefault(version, []) + all_dirs_dict[version].append(directory) + + # Select latest version + if not all_dirs_dict: + dirnames = [] + elif 'latest' in all_dirs_dict: + dirnames = all_dirs_dict['latest'] + else: + all_versions = sorted(list(all_dirs_dict)) + dirnames = all_dirs_dict[all_versions[-1]] + + # No {latestversion} tag + else: + dirnames = sorted(glob.glob(dirname)) + + # No directories found + if not dirnames: + logger.debug("Unable to resolve %s", dirname) + return dirname + + # Exactly one directory found + if len(dirnames) == 1: + return dirnames[0] + + # Warn if multiple directories have been found and prioritize r0i0p0 + logger.warning("Multiple directories for fx variables found: %s", dirnames) + r0i0p0_matches = [d for d in dirnames if "r0i0p0" in d] + if r0i0p0_matches: + return r0i0p0_matches[0] + return dirnames[0] + + def _select_drs(input_type, drs, project): """Select the directory structure of input path.""" cfg = get_project_config(project) @@ -219,12 +264,12 @@ def _select_drs(input_type, drs, project): if isinstance(input_path, str): return input_path - structure = drs.get(project, 'default') + structure = drs.get(project, "default") if structure in input_path: return input_path[structure] raise KeyError( - 'drs {} for {} project not specified in config-developer file'.format( + "drs {} for {} project not specified in config-developer file".format( structure, project)) @@ -248,16 +293,24 @@ def get_rootpath(rootpath, project): def _find_input_dirs(variable, rootpath, drs): """Return a the full paths to input directories.""" - project = variable['project'] + project = variable["project"] root = get_rootpath(rootpath, project) - path_template = _select_drs('input_dir', drs, project) + path_template = _select_drs("input_dir", drs, project) dirnames = [] for dirname_template in _replace_tags(path_template, variable): for base_path in root: dirname = os.path.join(base_path, dirname_template) - dirname = _resolve_latestversion(dirname) + if variable["frequency"] == "fx" and "*" in dirname: + dirname = _resolve_wildcards_and_version(dirname, base_path, + project, drs) + var_from_dir = dir_to_var(dirname, base_path, project, drs) + for (key, val) in variable.items(): + if val == '*': + variable[key] = var_from_dir.get(key, '*') + else: + dirname = _resolve_latestversion(dirname) matches = glob.glob(dirname) matches = [match for match in matches if os.path.isdir(match)] if matches: @@ -272,18 +325,18 @@ def _find_input_dirs(variable, rootpath, drs): def _get_filenames_glob(variable, drs): """Return patterns that can be used to look for input files.""" - path_template = _select_drs('input_file', drs, variable['project']) + path_template = _select_drs("input_file", drs, variable["project"]) filenames_glob = _replace_tags(path_template, variable) return filenames_glob def _find_input_files(variable, rootpath, drs): - short_name = variable['short_name'] - variable['short_name'] = variable['original_short_name'] + short_name = variable["short_name"] + variable["short_name"] = variable["original_short_name"] input_dirs = _find_input_dirs(variable, rootpath, drs) filenames_glob = _get_filenames_glob(variable, drs) files = find_files(input_dirs, filenames_glob) - variable['short_name'] = short_name + variable["short_name"] = short_name return (files, input_dirs, filenames_glob) @@ -291,34 +344,38 @@ def get_input_filelist(variable, rootpath, drs): """Return the full path to input files.""" # change ensemble to fixed r0i0p0 for fx variables # this is needed and is not a duplicate effort - if variable['project'] == 'CMIP5' and variable['frequency'] == 'fx': + if all([ + variable['project'] == 'CMIP5', variable['frequency'] == 'fx', + variable.get('ensemble') != '*' + ]): variable['ensemble'] = 'r0i0p0' (files, dirnames, filenames) = _find_input_files(variable, rootpath, drs) + # do time gating only for non-fx variables - if variable['frequency'] != 'fx': - files = select_files(files, variable['start_year'], - variable['end_year']) + if variable["frequency"] != "fx": + files = select_files(files, variable["start_year"], + variable["end_year"]) return (files, dirnames, filenames) def get_output_file(variable, preproc_dir): """Return the full path to the output (preprocessed) file.""" - cfg = get_project_config(variable['project']) + cfg = get_project_config(variable["project"]) # Join different experiment names - if isinstance(variable.get('exp'), (list, tuple)): + if isinstance(variable.get("exp"), (list, tuple)): variable = dict(variable) - variable['exp'] = '-'.join(variable['exp']) + variable["exp"] = "-".join(variable["exp"]) outfile = os.path.join( preproc_dir, - variable['diagnostic'], - variable['variable_group'], - _replace_tags(cfg['output_file'], variable)[0], + variable["diagnostic"], + variable["variable_group"], + _replace_tags(cfg["output_file"], variable)[0], ) - if variable['frequency'] != 'fx': - outfile += '_{start_year}-{end_year}'.format(**variable) - outfile += '.nc' + if variable["frequency"] != "fx": + outfile += "_{start_year}-{end_year}".format(**variable) + outfile += ".nc" return outfile @@ -326,11 +383,46 @@ def get_statistic_output_file(variable, preproc_dir): """Get multi model statistic filename depending on settings.""" template = os.path.join( preproc_dir, - '{diagnostic}', - '{variable_group}', - '{dataset}_{mip}_{short_name}_{start_year}-{end_year}.nc', + "{diagnostic}", + "{variable_group}", + "{dataset}_{mip}_{short_name}_{start_year}-{end_year}.nc", ) outfile = template.format(**variable) return outfile + + +def dir_to_var(dirname, basepath, project, drs): + """Convert directory path to variable :obj:`dict`.""" + if dirname != os.sep: + dirname = dirname.rstrip(os.sep) + if basepath != os.sep: + basepath = basepath.rstrip(os.sep) + path_template = _select_drs("input_dir", drs, project).rstrip(os.sep) + rel_dir = os.path.relpath(dirname, basepath) + keys = path_template.split(os.sep) + vals = rel_dir.split(os.sep) + if len(keys) != len(vals): + raise ValueError( + f"Cannot extract tags '{path_template}' from directory " + f"'{rel_dir}' (root: '{basepath}') with different numbers of " + f"elements") + variable = {} + for (idx, full_key) in enumerate(keys): + matches = re.findall(r'.*\{(.*)\}.*', full_key) + if len(matches) != 1: + continue + key = matches[0] + regex = rf"{full_key.replace(key, '(.*)')}" + regex = regex.replace('{', '').replace('}', '') + matches = re.findall(regex, vals[idx]) + while '' in matches: + matches.remove('') + if len(matches) != 1: + raise ValueError( + f"Regex pattern '{regex}' for '{full_key}' cannot be " + f"(uniquely) matched to element '{vals[idx]}' in directory " + f"'{dirname}'") + variable[key] = matches[0] + return variable diff --git a/esmvalcore/_recipe.py b/esmvalcore/_recipe.py index 0fcb039973..a78b7916d6 100644 --- a/esmvalcore/_recipe.py +++ b/esmvalcore/_recipe.py @@ -336,7 +336,8 @@ def _add_fxvar_keys(fx_info, variable): fx_variable['variable_group'] = fx_info['short_name'] # add special ensemble for CMIP5 only - if fx_variable['project'] == 'CMIP5': + if (fx_variable['project'] == 'CMIP5' and + fx_variable.get('ensemble') != '*'): fx_variable['ensemble'] = 'r0i0p0' # add missing cmor info @@ -427,11 +428,14 @@ def _get_fx_files(variable, fx_info, config_user): f"table '{mip}' for '{var_project}'") fx_info = _add_fxvar_keys(fx_info, variable) fx_files = _get_input_files(fx_info, config_user)[0] - + + # Flag a warning if no files are found if not fx_files: - logger.warning("Missing data for fx variable %s of dataset %s", - fx_info['short_name'], fx_info['dataset']) + outs = ', '.join([str(i)+': '+str(v) for i, v in fx_info.items()]) + logger.error("Missing data for fx variable '%s', '%s'", + fx_info['short_name'], outs) + assert 0 # If frequency = fx, only allow a single file if fx_files: @@ -796,6 +800,11 @@ def _get_preprocessor_products(variables, profile, order, ancestor_products, else: missing_vars.add(ex.message) continue + + # Update output filename in case wildcards have been resolved + if '*' in variable['filename']: + variable['filename'] = get_output_file(variable, + config_user['preproc_dir']) product = PreprocessorFile( attributes=variable, settings=settings, diff --git a/esmvalcore/cmor/_fixes/cmip6/bcc_csm2_mr.py b/esmvalcore/cmor/_fixes/cmip6/bcc_csm2_mr.py index 38f96526e1..607ccae36b 100644 --- a/esmvalcore/cmor/_fixes/cmip6/bcc_csm2_mr.py +++ b/esmvalcore/cmor/_fixes/cmip6/bcc_csm2_mr.py @@ -1,5 +1,6 @@ """Fixes for BCC-CSM2-MR model.""" from ..common import ClFixHybridPressureCoord, OceanFixGrid +from ..fix import Fix Cl = ClFixHybridPressureCoord @@ -15,5 +16,36 @@ Siconc = OceanFixGrid +uo = OceanFixGrid + +#class Omon(Fix): +# """Fixes for ocean variables.""" +# +# def fix_metadata(self, cubes): +# """Fix ocean depth coordinate. +# +# Parameters +# ---------- +# cubes: iris CubeList +# List of cubes to fix +# +# Returns +# ------- +# iris.cube.CubeList +# +# """ +# cubes = OceanFixGrid.fix_metadata(cubes) +# +# for cube in cubes: +# if cube.coords('latitude'): +# cube.coord('latitude').var_name = 'lat' +# if cube.coords('longitude'): +# cube.coord('longitude').var_name = 'lon' +# +# if cube.coords(axis='Z'): +# z_coord = cube.coord(axis='Z') +# if z_coord.var_name == 'olevel': +# fix_ocean_depth_coord(cube) +# return cubes Sos = OceanFixGrid diff --git a/esmvalcore/cmor/_fixes/cmip6/cesm2_fv2.py b/esmvalcore/cmor/_fixes/cmip6/cesm2_fv2.py index 80f2e58849..f85b1a509a 100644 --- a/esmvalcore/cmor/_fixes/cmip6/cesm2_fv2.py +++ b/esmvalcore/cmor/_fixes/cmip6/cesm2_fv2.py @@ -2,6 +2,10 @@ from .cesm2 import Cl as BaseCl from .cesm2 import Fgco2 as BaseFgco2 from .cesm2 import Tas as BaseTas +from ..fix import Fix +from ..shared import fix_ocean_depth_coord +import numpy as np +import cf_units from ..common import SiconcFixScalarCoord @@ -21,3 +25,42 @@ Tas = BaseTas + + +class Omon(Fix): + """Fixes for ocean variables.""" + + def fix_metadata(self, cubes): + """Fix ocean depth coordinate. + + Parameters + ---------- + cubes: iris CubeList + List of cubes to fix + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + if cube.coords('latitude'): + cube.coord('latitude').var_name = 'lat' + if cube.coords('longitude'): + cube.coord('longitude').var_name = 'lon' + + if cube.coords(axis='Z'): + z_coord = cube.coord(axis='Z') + if str(z_coord.units).lower() in ['cm', 'centimeters'] and np.max(z_coord.points)>10000.: + z_coord.units = cf_units.Unit('m') + z_coord.points = z_coord.points /100. + if str(z_coord.units).lower() in ['cm', 'centimeters'] and np.max(z_coord.points)<10000.: + z_coord.units = cf_units.Unit('m') + #z_coord.points = z_coord.points /100. + + + #z_coord = cube.coord(axis='Z') + #if z_coord.var_name == 'olevel': + fix_ocean_depth_coord(cube) + return cubes + diff --git a/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm.py b/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm.py index d0014f308a..ad486687ac 100644 --- a/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm.py +++ b/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm.py @@ -4,6 +4,10 @@ from .cesm2 import Cl as BaseCl from .cesm2 import Fgco2 as BaseFgco2 from .cesm2 import Tas as BaseTas +from ..fix import Fix +from ..shared import fix_ocean_depth_coord +import numpy as np +import cf_units from ..common import SiconcFixScalarCoord @@ -56,3 +60,39 @@ def fix_file(self, filepath, output_dir): Tas = BaseTas + + +class Omon(Fix): + """Fixes for ocean variables.""" + + def fix_metadata(self, cubes): + """Fix ocean depth coordinate. + + Parameters + ---------- + cubes: iris CubeList + List of cubes to fix + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + if cube.coords('latitude'): + cube.coord('latitude').var_name = 'lat' + if cube.coords('longitude'): + cube.coord('longitude').var_name = 'lon' + + if cube.coords(axis='Z'): + z_coord = cube.coord(axis='Z') + if str(z_coord.units).lower() in ['cm', 'centimeters'] and np.max(z_coord.points)>10000.: + z_coord.units = cf_units.Unit('m') + z_coord.points = z_coord.points /100. + if str(z_coord.units).lower() in ['cm', 'centimeters'] and np.max(z_coord.points)<10000.: + z_coord.units = cf_units.Unit('m') + + fix_ocean_depth_coord(cube) + return cubes + + diff --git a/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm_fv2.py b/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm_fv2.py index bc8068af8a..757f41cb27 100644 --- a/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm_fv2.py +++ b/esmvalcore/cmor/_fixes/cmip6/cesm2_waccm_fv2.py @@ -1,3 +1,5 @@ +"""Fixes for cesm2-waccm-fv2.""" +from iris.cube import CubeList """Fixes for CESM2-WACCM-FV2 model.""" from .cesm2 import Tas as BaseTas from .cesm2 import Fgco2 as BaseFgco2 @@ -6,15 +8,65 @@ from .cesm2_waccm import Clw as BaseClw from ..common import SiconcFixScalarCoord +from ..fix import Fix +from ..shared import fix_ocean_depth_coord -Cl = BaseCl +import numpy as np +import cf_units +class AllVars(Fix): + """Fixes for thetao.""" -Cli = BaseCli + def fix_metadata(self, cubes): + """ + Fix cell_area coordinate. + Parameters + ---------- + cubes: iris CubeList + List of cubes to fix -Clw = BaseClw + Returns + ------- + iris.cube.CubeList + """ + cube = self.get_cube_from_list(cubes) + if cube.coords('latitude'): + cube.coord('latitude').var_name = 'lat' + if cube.coords('longitude'): + cube.coord('longitude').var_name = 'lon' + return CubeList([cube]) + + +class Omon(Fix): + """Fixes for ocean variables.""" + + def fix_metadata(self, cubes): + """Fix ocean depth coordinate. + + Parameters + ---------- + cubes: iris CubeList + List of cubes to fix + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + if cube.coords(axis='Z'): + z_coord = cube.coord(axis='Z') + if str(z_coord.units).lower() in ['cm', 'centimeters'] and np.max(z_coord.points)>10000.: + z_coord.units = cf_units.Unit('m') + z_coord.points = z_coord.points /100. + if str(z_coord.units).lower() in ['cm', 'centimeters'] and np.max(z_coord.points)<10000.: + z_coord.units = cf_units.Unit('m') +# z_coord.points = z_coord.points /100. + + fix_ocean_depth_coord(cube) + return cubes Fgco2 = BaseFgco2 diff --git a/esmvalcore/cmor/_fixes/cmip6/cnrm_cm6_1.py b/esmvalcore/cmor/_fixes/cmip6/cnrm_cm6_1.py index 1e2762bdd4..ef03d017f1 100644 --- a/esmvalcore/cmor/_fixes/cmip6/cnrm_cm6_1.py +++ b/esmvalcore/cmor/_fixes/cmip6/cnrm_cm6_1.py @@ -3,8 +3,7 @@ from ..common import ClFixHybridPressureCoord from ..fix import Fix -from ..shared import add_aux_coords_from_cubes, get_bounds_cube - +from ..shared import add_aux_coords_from_cubes, get_bounds_cube, fix_ocean_depth_coord class Cl(ClFixHybridPressureCoord): """Fixes for ``cl``.""" @@ -77,3 +76,28 @@ def fix_metadata(self, cubes): Clw = Cl + +class Omon(Fix): + """Fixes for ocean variables.""" + + def fix_metadata(self, cubes): + """Fix ocean depth coordinate. + + Parameters + ---------- + cubes: iris CubeList + List of cubes to fix + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + if cube.coords(axis='Z'): + z_coord = cube.coord(axis='Z') + if z_coord.var_name in ['olevel', 'lev']: + fix_ocean_depth_coord(cube) + return cubes + + diff --git a/esmvalcore/cmor/_fixes/cmip6/fgoals_f3_l.py b/esmvalcore/cmor/_fixes/cmip6/fgoals_f3_l.py index 15dd31ede9..a10f9cade3 100644 --- a/esmvalcore/cmor/_fixes/cmip6/fgoals_f3_l.py +++ b/esmvalcore/cmor/_fixes/cmip6/fgoals_f3_l.py @@ -64,3 +64,39 @@ def fix_data(self, cube): if cube.units == "%" and da.max(cube.core_data()).compute() <= 1.: cube.data = cube.core_data() * 100. return cube + +class Omon(Fix): + """Fixes for ocean variables.""" + + def fix_metadata(self, cubes): + """Fix ocean depth coordinate. + + Parameters + ---------- + cubes: iris CubeList + List of cubes to fix + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + if not cube.coord('latitude').bounds: + cube.coord('latitude').guess_bounds() + if not cube.coord('longitude').bounds: + cube.coord('longitude').guess_bounds() + + if cube.coords('latitude'): + cube.coord('latitude').var_name = 'lat' + if cube.coords('longitude'): + cube.coord('longitude').var_name = 'lon' + + if cube.coords(axis='Z'): + z_coord = cube.coord(axis='Z') + if str(z_coords.units) == 'cm' and np.max(z_points)>10000.: + z_coord.units = cf_units.Unit('m') + fix_ocean_depth_coord(cube) + return cubes + + diff --git a/esmvalcore/cmor/_fixes/cmip6/gfdl_cm4.py b/esmvalcore/cmor/_fixes/cmip6/gfdl_cm4.py index 35baf68a1f..041b956ea5 100644 --- a/esmvalcore/cmor/_fixes/cmip6/gfdl_cm4.py +++ b/esmvalcore/cmor/_fixes/cmip6/gfdl_cm4.py @@ -4,7 +4,7 @@ from ..common import ClFixHybridPressureCoord, SiconcFixScalarCoord from ..fix import Fix from ..shared import add_aux_coords_from_cubes, add_scalar_height_coord - +from ..shared import fix_ocean_depth_coord class Cl(ClFixHybridPressureCoord): """Fixes for ``cl``.""" @@ -108,3 +108,34 @@ def fix_metadata(self, cubes): cube = self.get_cube_from_list(cubes) add_scalar_height_coord(cube, 10.0) return cubes + + +class Omon(Fix): + """Fixes for ocean variables.""" + + def fix_metadata(self, cubes): + """Fix ocean depth coordinate. + + Parameters + ---------- + cubes: iris CubeList + List of cubes to fix + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + if cube.coords('latitude'): + cube.coord('latitude').var_name = 'lat' + if cube.coords('longitude'): + cube.coord('longitude').var_name = 'lon' + + if cube.coords(axis='Z'): +# z_coord = cube.coord(axis='Z') +# if z_coord.var_name == 'olevel': + fix_ocean_depth_coord(cube) + return cubes + + diff --git a/esmvalcore/cmor/_fixes/cmip6/iitm_esm.py b/esmvalcore/cmor/_fixes/cmip6/iitm_esm.py new file mode 100644 index 0000000000..3fcd4107c2 --- /dev/null +++ b/esmvalcore/cmor/_fixes/cmip6/iitm_esm.py @@ -0,0 +1,40 @@ +"""Fixes for IITM-ESM model.""" +import iris + +from ..fix import Fix +from ..shared import fix_ocean_depth_coord + + + +class Omon(Fix): + """Fixes for ocean variables.""" + + def fix_metadata(self, cubes): + """Fix ocean depth coordinate. + + Parameters + ---------- + cubes: iris CubeList + List of cubes to fix + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + if cube.coords('latitude'): + cube.coord('latitude').var_name = 'lat' + cube.coord('latitude').guess_bounds() + + if cube.coords('longitude'): + cube.coord('longitude').var_name = 'lon' + cube.coord('longitude').guess_bounds() + if cube.coords(axis='Z'): + z_coord = cube.coord(axis='Z') + if z_coord.var_name == 'olevel': + fix_ocean_depth_coord(cube) + return cubes + + + diff --git a/esmvalcore/cmor/_fixes/cmip6/ipsl_cm5a2_inca.py b/esmvalcore/cmor/_fixes/cmip6/ipsl_cm5a2_inca.py new file mode 100644 index 0000000000..ccc45d8b1f --- /dev/null +++ b/esmvalcore/cmor/_fixes/cmip6/ipsl_cm5a2_inca.py @@ -0,0 +1,78 @@ +"""Fixes for IPSL-CM6A-LR model.""" +from iris.cube import CubeList + +from ..fix import Fix +from ..shared import fix_ocean_depth_coord + + +class AllVars(Fix): + """Fixes for thetao.""" + + def fix_metadata(self, cubes): + """ + Fix cell_area coordinate. + + Parameters + ---------- + cubes: iris CubeList + List of cubes to fix + + Returns + ------- + iris.cube.CubeList + + """ + cube = self.get_cube_from_list(cubes) + if cube.coords('latitude'): + cube.coord('latitude').var_name = 'lat' + if cube.coords('longitude'): + cube.coord('longitude').var_name = 'lon' + return CubeList([cube]) + + +class Clcalipso(Fix): + """Fixes for ``clcalipso``.""" + + def fix_metadata(self, cubes): + """Fix ``alt40`` coordinate. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes + + Returns + ------- + iris.cube.CubeList + + """ + cube = self.get_cube_from_list(cubes) + alt_40_coord = cube.coord('height') + alt_40_coord.long_name = 'altitude' + alt_40_coord.standard_name = 'altitude' + alt_40_coord.var_name = 'alt40' + return CubeList([cube]) + + +class Omon(Fix): + """Fixes for ocean variables.""" + + def fix_metadata(self, cubes): + """Fix ocean depth coordinate. + + Parameters + ---------- + cubes: iris CubeList + List of cubes to fix + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + if cube.coords(axis='Z'): + #z_coord = cube.coord(axis='Z') + #if z_coord.var_name == 'olevel': + fix_ocean_depth_coord(cube) + return cubes diff --git a/esmvalcore/cmor/_fixes/cmip6/ipsl_cm6a_inca.py b/esmvalcore/cmor/_fixes/cmip6/ipsl_cm6a_inca.py new file mode 100644 index 0000000000..88898a6e1a --- /dev/null +++ b/esmvalcore/cmor/_fixes/cmip6/ipsl_cm6a_inca.py @@ -0,0 +1,78 @@ +"""Fixes for IPSL-CM6A-INCA model.""" +from iris.cube import CubeList + +from ..fix import Fix +from ..shared import fix_ocean_depth_coord + + +class AllVars(Fix): + """Fixes for thetao.""" + + def fix_metadata(self, cubes): + """ + Fix cell_area coordinate. + + Parameters + ---------- + cubes: iris CubeList + List of cubes to fix + + Returns + ------- + iris.cube.CubeList + + """ + cube = self.get_cube_from_list(cubes) + if cube.coords('latitude'): + cube.coord('latitude').var_name = 'lat' + if cube.coords('longitude'): + cube.coord('longitude').var_name = 'lon' + return CubeList([cube]) + + +class Clcalipso(Fix): + """Fixes for ``clcalipso``.""" + + def fix_metadata(self, cubes): + """Fix ``alt40`` coordinate. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes + + Returns + ------- + iris.cube.CubeList + + """ + cube = self.get_cube_from_list(cubes) + alt_40_coord = cube.coord('height') + alt_40_coord.long_name = 'altitude' + alt_40_coord.standard_name = 'altitude' + alt_40_coord.var_name = 'alt40' + return CubeList([cube]) + + +class Omon(Fix): + """Fixes for ocean variables.""" + + def fix_metadata(self, cubes): + """Fix ocean depth coordinate. + + Parameters + ---------- + cubes: iris CubeList + List of cubes to fix + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + if cube.coords(axis='Z'): + z_coord = cube.coord(axis='Z') + if z_coord.var_name == 'olevel': + fix_ocean_depth_coord(cube) + return cubes diff --git a/esmvalcore/cmor/_fixes/cmip6/ipsl_cm6a_lr_inca.py b/esmvalcore/cmor/_fixes/cmip6/ipsl_cm6a_lr_inca.py new file mode 100644 index 0000000000..bfc99c923c --- /dev/null +++ b/esmvalcore/cmor/_fixes/cmip6/ipsl_cm6a_lr_inca.py @@ -0,0 +1,78 @@ +"""Fixes for IPSL-CM6A-LR-INCA model.""" +from iris.cube import CubeList + +from ..fix import Fix +from ..shared import fix_ocean_depth_coord + + +class AllVars(Fix): + """Fixes for thetao.""" + + def fix_metadata(self, cubes): + """ + Fix cell_area coordinate. + + Parameters + ---------- + cubes: iris CubeList + List of cubes to fix + + Returns + ------- + iris.cube.CubeList + + """ + cube = self.get_cube_from_list(cubes) + if cube.coords('latitude'): + cube.coord('latitude').var_name = 'lat' + if cube.coords('longitude'): + cube.coord('longitude').var_name = 'lon' + return CubeList([cube]) + + +class Clcalipso(Fix): + """Fixes for ``clcalipso``.""" + + def fix_metadata(self, cubes): + """Fix ``alt40`` coordinate. + + Parameters + ---------- + cubes : iris.cube.CubeList + Input cubes + + Returns + ------- + iris.cube.CubeList + + """ + cube = self.get_cube_from_list(cubes) + alt_40_coord = cube.coord('height') + alt_40_coord.long_name = 'altitude' + alt_40_coord.standard_name = 'altitude' + alt_40_coord.var_name = 'alt40' + return CubeList([cube]) + + +class Omon(Fix): + """Fixes for ocean variables.""" + + def fix_metadata(self, cubes): + """Fix ocean depth coordinate. + + Parameters + ---------- + cubes: iris CubeList + List of cubes to fix + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + if cube.coords(axis='Z'): + z_coord = cube.coord(axis='Z') + if z_coord.var_name == 'olevel': + fix_ocean_depth_coord(cube) + return cubes diff --git a/esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py b/esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py index 2a07905002..02902fe22a 100644 --- a/esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py +++ b/esmvalcore/cmor/_fixes/cmip6/noresm2_lm.py @@ -77,3 +77,28 @@ def fix_metadata(self, cubes): longitude.bounds = np.round(longitude.bounds, 4) return cubes + +class Omon(Fix): + """Fixes for ocean variables.""" + + def fix_metadata(self, cubes): + """Fix ocean depth coordinate. + + Parameters + ---------- + cubes: iris CubeList + List of cubes to fix + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + if cube.coords(axis='Z'): + z_coord = cube.coord(axis='Z') + if z_coord.var_name == 'olevel': + fix_ocean_depth_coord(cube) + return cubes + + diff --git a/esmvalcore/cmor/_fixes/cmip6/noresm2_mm.py b/esmvalcore/cmor/_fixes/cmip6/noresm2_mm.py index 3a9a97faa0..d620b59505 100644 --- a/esmvalcore/cmor/_fixes/cmip6/noresm2_mm.py +++ b/esmvalcore/cmor/_fixes/cmip6/noresm2_mm.py @@ -8,3 +8,28 @@ Clw = ClFixHybridPressureCoord + +class Omon(Fix): + """Fixes for ocean variables.""" + + def fix_metadata(self, cubes): + """Fix ocean depth coordinate. + + Parameters + ---------- + cubes: iris CubeList + List of cubes to fix + + Returns + ------- + iris.cube.CubeList + + """ + for cube in cubes: + if cube.coords(axis='Z'): + z_coord = cube.coord(axis='Z') + if z_coord.var_name == 'olevel': + fix_ocean_depth_coord(cube) + return cubes + + diff --git a/esmvalcore/cmor/tables/cordex/Tables/CORDEX_fx b/esmvalcore/cmor/tables/cordex/Tables/CORDEX_fx index 30ecaf0997..0b7a026048 100644 --- a/esmvalcore/cmor/tables/cordex/Tables/CORDEX_fx +++ b/esmvalcore/cmor/tables/cordex/Tables/CORDEX_fx @@ -223,3 +223,26 @@ valid_max: 30.0 !---------------------------------- ! +!============ +variable_entry: sftof +!============ +modeling_realm: ocean +!---------------------------------- +! Variable attributes: +!---------------------------------- +standard_name: sea_area_fraction +units: % +cell_measures: area: areacello +long_name: Sea Area Fraction +comment: This is the area fraction at the ocean surface. +!---------------------------------- +! Additional variable information: +!---------------------------------- +dimensions: longitude latitude +out_name: sftof +type: real +valid_min: 0.0 +valid_max: 100.0 +!---------------------------------- +! + diff --git a/esmvalcore/preprocessor/__init__.py b/esmvalcore/preprocessor/__init__.py index 8905099618..5d5d004b49 100644 --- a/esmvalcore/preprocessor/__init__.py +++ b/esmvalcore/preprocessor/__init__.py @@ -69,6 +69,7 @@ extract_trajectory, extract_transect, extract_volume, + extract_surface, volume_statistics, ) from ._weighting import weighting_landsea_fraction @@ -101,6 +102,7 @@ 'resample_hours', 'resample_time', # Level extraction + 'extract_surface', 'extract_levels', # Weighting 'weighting_landsea_fraction', diff --git a/esmvalcore/preprocessor/_volume.py b/esmvalcore/preprocessor/_volume.py index 424eabe0f8..0826a788bc 100644 --- a/esmvalcore/preprocessor/_volume.py +++ b/esmvalcore/preprocessor/_volume.py @@ -279,6 +279,78 @@ def volume_statistics(cube, operator): return _create_cube_time(src_cube, result, times) +def extract_surface(cube): + """ + Extact the surface layer from a 3D cube. + + Extracts the entire layer along the z axis where the z coordinate is + the minimum of the absolute value of the z-axis dimensions points. + + Requires a cube with a z axis. + + This assumes that the surface is the closest layer to zero. + This assumption is usually true in the ocean, but may not be the + case in all models for all z-axes. + + In the case of temporally or spatially varying depth grid, + (ie the depth array is 3D or 4D), the surface layer is determined + using the mean of the depth points along the time, latitude and + longitude axes. This preprocessor may also behave strangely + if the z axis data are not monotonic, or unusual in some other way. + + The preprocessor extract_layer can also do this, but it may be + much slower and more memory intensie as it regrids. + It is more robust to variability in the grid definitions. + + Parameters + ---------- + cube: iris.cube.Cube + input cube. + + Returns + ------- + iris.cube.Cube + collapsed cube. + + """ + # Get the z axis. + zcoord = cube.coord(axis='Z') + positive = zcoord.attributes['positive'] + + # Get the Z points dimension, usually 0 or 1. + zcoord_dim = cube.coord_dims(zcoord)[0] + + # make a list of axes for the non-z axes + # something like [0, 2, 3]. or [1, 2] + axes = list(range(cube.data.ndim)) + axes = axes.remove(zcoord_dim) + + # Get a list of points. + points = zcoord.points + if points.ndim == cube.data.ndim: + points = zcoord.points.mean(axis=axes) + + # Calculate the surface layer index: + surf = np.abs(points).argmin() + + # Get the z axis dimension in the cude: + if zcoord_dim in [0, (0,)]: + return cube[surf] + + if zcoord_dim in [1, (1,)]: + return cube[:, surf] + + if zcoord_dim in [2, (2,)]: + return cube[:, :, surf] + + if zcoord_dim in [3, (3,)]: + return cube[:, :, :, surf] + + logger.error('Z coordinate is strange: positive is %s , Z dim is along axis: %s, ' + 'surface level: %s', positive, zcoord_dim, surf) + return cube + + def depth_integration(cube): """ Determine the total sum over the vertical component. @@ -467,5 +539,10 @@ def extract_trajectory(cube, latitudes, longitudes, number_points=2): latitudes = np.linspace(minlat, maxlat, num=number_points) points = [('latitude', latitudes), ('longitude', longitudes)] + #times = cube.coord('time') + #if len(times.points): interpolated_cube = interpolate(cube, points) # Very slow! + #interpolated_cube.coord('latitude').guess_bounds() + #interpolated_cube.coord('longitude').guess_bounds() + return interpolated_cube diff --git a/tests/integration/data_finder.yml b/tests/integration/data_finder.yml index 9ce5ea7ce4..198eae6a9c 100644 --- a/tests/integration/data_finder.yml +++ b/tests/integration/data_finder.yml @@ -421,8 +421,98 @@ get_input_filelist: ensemble: r1i1p1 diagnostic: test_diag preprocessor: test_preproc + available_files: + - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r0i0p0/v20110330/sftlf/sftlf_fx_HadGEM2-ES_historical_r0i0p0.nc + - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r1i1p1/v20110330/sftlf/sftlf_fx_HadGEM2-ES_historical_r1i1p1.nc + dirs: + - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r0i0p0/v20110330/sftlf + file_patterns: + - sftlf_fx_HadGEM2-ES_historical_r0i0p0*.nc + found_files: + - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r0i0p0/v20110330/sftlf/sftlf_fx_HadGEM2-ES_historical_r0i0p0.nc + + # CMIP6 wildcard specification for fx variable + + - drs: BADC + variable: + variable_group: test + short_name: sftlf + original_short_name: sftlf + dataset: HadGEM3-GC31-LL + activity: CMIP + project: CMIP6 + cmor_table: CMIP6 + institute: [MOHC] + frequency: fx + modeling_realm: [atmos] + mip: fx + exp: '*' + grid: gn + ensemble: '*' + diagnostic: test_diag + preprocessor: test_preproc + available_files: + - CMIP/MOHC/HadGEM3-GC31-LL/historical/r1i1p1f1/fx/sftlf/gn/v2020/sftlf_fx_HadGEM3-GC31-LL_historical_r1i1p1f1_gn.nc + - CMIP/MOHC/HadGEM3-GC31-LL/piControl/r1i1p1f1/fx/sftlf/gn/latest/sftlf_fx_HadGEM3-GC31-LL_piControl_r1i1p1f1_gn.nc + dirs: + - CMIP/MOHC/HadGEM3-GC31-LL/piControl/r1i1p1f1/fx/sftlf/gn/latest + file_patterns: + - sftlf_fx_HadGEM3-GC31-LL_piControl_r1i1p1f1_gn*.nc + found_files: + - CMIP/MOHC/HadGEM3-GC31-LL/piControl/r1i1p1f1/fx/sftlf/gn/latest/sftlf_fx_HadGEM3-GC31-LL_piControl_r1i1p1f1_gn.nc + + # CORDEX wildcard fx specification + + - drs: BADC + variable: + variable_group: test + short_name: sftlf + original_short_name: sftlf + dataset: RCA4 + driver: NCC-NorESM1-M + rcm_version: v1 + project: CORDEX + cmor_table: cordex + institute: [SMHI] + frequency: fx + modeling_realm: [atmos] + mip: fx + exp: '*' + domain: EUR-11 + ensemble: '*' + diagnostic: test_diag + preprocessor: test_preproc + available_files: + - EUR-11/SMHI/NCC-NorESM1-M/historical/r0i0p0/RCA4/v1/fx/sftlf/v20100101/sftlf_EUR-11_NCC-NorESM1-M_historical_r0i0p0_RCA4_v1_fx.nc + - EUR-11/SMHI/NCC-NorESM1-M/historical/r0i0p0/RCA4/v1/fx/sftlf/v20180820/sftlf_EUR-11_NCC-NorESM1-M_historical_r0i0p0_RCA4_v1_fx.nc + - EUR-11/SMHI/NCC-NorESM1-M/rcp85/r0i0p0/RCA4/v1/fx/sftlf/v20100101/sftlf_EUR-11_NCC-NorESM1-M_rcp85_r0i0p0_RCA4_v1_fx.nc + - EUR-11/SMHI/NCC-NorESM1-M/rcp85/r0i0p0/RCA4/v1/fx/sftlf/v20180820/sftlf_EUR-11_NCC-NorESM1-M_rcp85_r0i0p0_RCA4_v1_fx.nc + dirs: + - EUR-11/SMHI/NCC-NorESM1-M/historical/r0i0p0/RCA4/v1/fx/sftlf/v20180820 + file_patterns: + - sftlf_EUR-11_NCC-NorESM1-M_historical_r0i0p0_RCA4_v1_fx*.nc + found_files: + - EUR-11/SMHI/NCC-NorESM1-M/historical/r0i0p0/RCA4/v1/fx/sftlf/v20180820/sftlf_EUR-11_NCC-NorESM1-M_historical_r0i0p0_RCA4_v1_fx.nc + + - drs: DKRZ + variable: + variable_group: test + short_name: sftlf + original_short_name: sftlf + dataset: HadGEM2-ES + project: CMIP5 + cmor_table: CMIP5 + institute: [INPE, MOHC] + frequency: fx + modeling_realm: [atmos] + mip: fx + exp: historical + ensemble: '*' + diagnostic: test_diag + preprocessor: test_preproc available_files: - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r1i1p1/v20110330/sftlf/sftlf_fx_HadGEM2-ES_historical_r1i1p1.nc + - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r2i1p1/v20110330/sftlf/sftlf_fx_HadGEM2-ES_historical_r2i1p1.nc - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r0i0p0/v20110330/sftlf/sftlf_fx_HadGEM2-ES_historical_r0i0p0.nc dirs: - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r0i0p0/v20110330/sftlf @@ -431,6 +521,140 @@ get_input_filelist: found_files: - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r0i0p0/v20110330/sftlf/sftlf_fx_HadGEM2-ES_historical_r0i0p0.nc + - drs: DKRZ + variable: + variable_group: test + short_name: sftlf + original_short_name: sftlf + dataset: HadGEM2-ES + project: CMIP5 + cmor_table: CMIP5 + institute: [INPE, MOHC] + frequency: fx + modeling_realm: [atmos] + mip: fx + exp: historical + ensemble: '*' + diagnostic: test_diag + preprocessor: test_preproc + available_files: + - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r2i1p1/v20110330/sftlf/sftlf_fx_HadGEM2-ES_historical_r2i1p1.nc + - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r1i1p1/v20110330/sftlf/sftlf_fx_HadGEM2-ES_historical_r1i1p1.nc + - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r3i1p1/v20110330/sftlf/sftlf_fx_HadGEM2-ES_historical_r3i1p1.nc + dirs: + - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r1i1p1/v20110330/sftlf + file_patterns: + - sftlf_fx_HadGEM2-ES_historical_r1i1p1*.nc + found_files: + - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r1i1p1/v20110330/sftlf/sftlf_fx_HadGEM2-ES_historical_r1i1p1.nc + + - drs: DKRZ + variable: + variable_group: test + short_name: areacella + original_short_name: areacella + dataset: CanESM2 + project: CMIP5 + cmor_table: CMIP5 + institute: [CCCma] + frequency: fx + modeling_realm: [atmos] + mip: fx + exp: historical + ensemble: '*' + diagnostic: test_diag + preprocessor: test_preproc + available_files: + - CCCma/CanESM2/historical/fx/atmos/fx/r0i0p0/v20120410/sftlf/sftlf_fx_CanESM2_historical_r0i0p0.nc + - CCCma/CanESM2/historical/fx/atmos/fx/r1i1p1/v20000101/areacella/areacella_fx_CanESM2_historical_r1i1p1.nc + - CCCma/CanESM2/historical/fx/atmos/fx/r1i1p1/v20120410/areacella/areacella_fx_CanESM2_historical_r1i1p1.nc + dirs: + - CCCma/CanESM2/historical/fx/atmos/fx/r1i1p1/v20120410/areacella + file_patterns: + - areacella_fx_CanESM2_historical_r1i1p1*.nc + found_files: + - CCCma/CanESM2/historical/fx/atmos/fx/r1i1p1/v20120410/areacella/areacella_fx_CanESM2_historical_r1i1p1.nc + + - drs: DKRZ + variable: + variable_group: test + short_name: areacella + original_short_name: areacella + dataset: CanESM2 + project: CMIP5 + cmor_table: CMIP5 + institute: [CCCma] + frequency: fx + modeling_realm: [atmos] + mip: fx + exp: historical + ensemble: '*' + diagnostic: test_diag + preprocessor: test_preproc + available_files: [] + dirs: [] + file_patterns: + - areacella_fx_CanESM2_historical_**.nc + found_files: [] + + - drs: DKRZ + variable: + variable_group: test + short_name: areacella + original_short_name: areacella + dataset: CanESM2 + project: CMIP5 + cmor_table: CMIP5 + institute: [CCCma] + frequency: fx + modeling_realm: [atmos] + mip: fx + exp: historical + ensemble: '*' + diagnostic: test_diag + preprocessor: test_preproc + available_files: + - CCCma/CanESM2/historical/fx/atmos/fx/r0i0p0/v20120410/sftlf/sftlf_fx_CanESM2_historical_r0i0p0.nc + - CCCma/CanESM2/historical/fx/atmos/fx/r0i0p0/v20200101/sftlf/sftlf_fx_CanESM2_historical_r0i0p0.nc + - CCCma/CanESM2/historical/fx/atmos/fx/r1i1p1/v20120410/areacella/areacella_fx_CanESM2_historical_r1i1p1.nc + - CCCma/CanESM2/historical/fx/atmos/fx/r2i1p1/v20200101/areacella/areacella_fx_CanESM2_historical_r2i1p1.nc + - CCCma/CanESM2/historical/fx/atmos/fx/r2i1p1/latest/areacella/areacella_fx_CanESM2_historical_r2i1p1.nc + - CCCma/CanESM2/historical/fx/atmos/fx/r2i1p2/v20300101/areacella/areacella_fx_CanESM2_historical_r2i1p2.nc + dirs: + - CCCma/CanESM2/historical/fx/atmos/fx/r2i1p1/latest/areacella + file_patterns: + - areacella_fx_CanESM2_historical_r2i1p1*.nc + found_files: + - CCCma/CanESM2/historical/fx/atmos/fx/r2i1p1/latest/areacella/areacella_fx_CanESM2_historical_r2i1p1.nc + + - drs: ETHZ + variable: + variable_group: test + short_name: areacella + original_short_name: areacella + dataset: CanESM2 + project: CMIP5 + cmor_table: CMIP5 + institute: [CCCma] + frequency: fx + modeling_realm: [atmos] + mip: fx + exp: historical + ensemble: '*' + diagnostic: test_diag + preprocessor: test_preproc + available_files: + - historical/fx/sftlf/CanESM2/r0i0p0/sftlf_fx_CanESM2_historical_r0i0p0.nc + - historical/fx/sftlf/CanESM2/r1i1p1/sftlf_fx_CanESM2_historical_r1i1p1.nc + - historical/fx/areacella/CanESM2/r2i2p2/areacella_fx_CanESM2_historical_r2i2p2.nc + - historical/fx/areacella/CanESM2/r1i1p1/areacella_fx_CanESM2_historical_r1i1p1.nc + dirs: + - historical/fx/areacella/CanESM2/r1i1p1 + file_patterns: + - areacella_fx_CanESM2_historical_r1i1p1*.nc + found_files: + - historical/fx/areacella/CanESM2/r1i1p1/areacella_fx_CanESM2_historical_r1i1p1.nc + - drs: DKRZ variable: variable_group: test @@ -448,8 +672,8 @@ get_input_filelist: diagnostic: test_diag preprocessor: test_preproc available_files: - - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r1i1p1/v20110330/sftlf/sftlf_fx_HadGEM2-ES_historical_r0i0p0.nc - - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r1i1p1/v20110330/areacella/areacella_fx_HadGEM2-ES_historical_r0i0p0.nc + - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r0i0p0/v20110330/sftlf/sftlf_fx_HadGEM2-ES_historical_r0i0p0.nc + - MOHC/HadGEM2-ES/historical/fx/atmos/fx/r0i0p0/v20110330/areacella/areacella_fx_HadGEM2-ES_historical_r0i0p0.nc dirs: [] file_patterns: - orog_fx_HadGEM2-ES_historical_r0i0p0*.nc @@ -684,3 +908,79 @@ get_input_filelist: file_patterns: - OBS6_ERA-Interim_reanaly_42_Omon_deptho[_.]*nc found_files: [] + + +dir_to_var: + - dirname: /this/is/root/ + basepath: /this/is/root/ + project: CMIP5 + drs: default + variable: {} + - dirname: / + basepath: / + project: CMIP5 + drs: default + variable: {} + - dirname: /this/is/root/TYPE/PROJECT/EXP/data + basepath: /this/is/root/ + project: CMIP5 + drs: BSC + variable: + type: TYPE + project: PROJECT + exp: EXP + dataset.lower: data + - dirname: /this/is/root/TEST/NAME + basepath: /this/is/root/ + project: CMIP5 + drs: default + variable: null + - dirname: /this/is/root/INST/DATA/EXP/FREQ/REALM/MIP/ENS/VERSION/NAME + basepath: /this/is/root + project: CMIP5 + drs: DKRZ + variable: + institute: INST + dataset: DATA + exp: EXP + frequency: FREQ + modeling_realm: REALM + mip: MIP + ensemble: ENS + latestversion: VERSION + short_name: NAME + - dirname: /this/is/root/EXP/MIP/NAME/DATA/ENS/GRID/ + basepath: /this/is/root/ + project: CMIP6 + drs: ETHZ + variable: + dataset: DATA + exp: EXP + mip: MIP + ensemble: ENS + short_name: NAME + grid: GRID + - dirname: /this/is/root/THIS/FAILS + basepath: /this/is/root/ + project: CMIP6 + drs: ETHZ + variable: null + - dirname: /this/is/root/Tier2/DATA + basepath: /this/is/root/ + project: OBS + drs: default + variable: + tier: '2' + dataset: DATA + - dirname: /Tier1/DATA + basepath: / + project: OBS + drs: default + variable: + tier: '1' + dataset: DATA + - dirname: /this/is/root/ThisFails/DATA + basepath: /this/is/root/ + project: OBS + drs: default + variable: null diff --git a/tests/integration/test_data_finder.py b/tests/integration/test_data_finder.py index b265ee59ab..5ce667ac49 100644 --- a/tests/integration/test_data_finder.py +++ b/tests/integration/test_data_finder.py @@ -7,7 +7,11 @@ import yaml import esmvalcore._config -from esmvalcore._data_finder import get_input_filelist, get_output_file +from esmvalcore._data_finder import ( + dir_to_var, + get_input_filelist, + get_output_file, +) from esmvalcore.cmor.table import read_cmor_tables # Initialize with standard config developer file @@ -101,3 +105,15 @@ def test_get_input_filelist(root, cfg): assert sorted(input_filelist) == sorted(ref_files) assert sorted(dirnames) == sorted(ref_dirs) assert sorted(filenames) == sorted(ref_patterns) + + +@pytest.mark.parametrize('cfg', CONFIG['dir_to_var']) +def test_dir_to_var(cfg): + """Test converting directory path to variable :obj:`dict`.""" + drs = {cfg['project']: cfg['drs']} + if cfg['variable'] is None: + with pytest.raises(ValueError): + dir_to_var(cfg['dirname'], cfg['basepath'], cfg['project'], drs) + return + output = dir_to_var(cfg['dirname'], cfg['basepath'], cfg['project'], drs) + assert output == cfg['variable'] diff --git a/tests/unit/preprocessor/_volume/test_volume.py b/tests/unit/preprocessor/_volume/test_volume.py index 82a755660d..052e52d448 100644 --- a/tests/unit/preprocessor/_volume/test_volume.py +++ b/tests/unit/preprocessor/_volume/test_volume.py @@ -12,6 +12,7 @@ extract_trajectory, extract_transect, extract_volume, + extract_surface, calculate_volume) @@ -21,6 +22,7 @@ class Test(tests.Test): def setUp(self): """Prepare tests""" coord_sys = iris.coord_systems.GeogCS(iris.fileformats.pp.EARTH_RADIUS) + data0 = np.ones((2, 2)) data1 = np.ones((3, 2, 2)) data2 = np.ma.ones((2, 3, 2, 2)) data3 = np.ma.ones((4, 3, 2, 2)) @@ -63,6 +65,9 @@ def setUp(self): units='degrees_north', coord_system=coord_sys) + coords_spec2 = [(lats2, 0), (lons2, 1)] + self.grid_2d = iris.cube.Cube(data0, dim_coords_and_dims=coords_spec2) + coords_spec3 = [(zcoord, 0), (lats2, 1), (lons2, 2)] self.grid_3d = iris.cube.Cube(data1, dim_coords_and_dims=coords_spec3) @@ -188,6 +193,13 @@ def test_extract_trajectory(self): expected = np.ones((3, 2)) self.assert_array_equal(result.data, expected) + def test_extract_surface(self): + """Tests to extract the surface from a 3D and 4D cube.""" + result = extract_surface(self.grid_3d) + self.assert_array_equal(result.data, self.grid_2d.data) + result2 = extract_surface(self.grid_4d) + self.assert_array_equal(self.grid_4d[:, 0, :, :].data, result2.data) + if __name__ == '__main__': unittest.main()