diff --git a/.cirrus.yml b/.cirrus.yml index 0a7c972821..08cf67f7fc 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -41,7 +41,7 @@ env: # Pip package to be upgraded/installed. PIP_CACHE_PACKAGES: "pip setuptools wheel nox" # Git commit hash for iris test data. - IRIS_TEST_DATA_REF: "fffb9b14b9cb472c5eb2ebb7fd19acb7f6414a30" + IRIS_TEST_DATA_REF: "v2.1" # Base directory for the iris-test-data. IRIS_TEST_DATA_DIR: ${HOME}/iris-test-data diff --git a/lib/iris/experimental/ugrid.py b/lib/iris/experimental/ugrid.py index 8326bdccb4..45b1b96ed2 100644 --- a/lib/iris/experimental/ugrid.py +++ b/lib/iris/experimental/ugrid.py @@ -11,9 +11,12 @@ """ from abc import ABC, abstractmethod -from collections import Iterable, namedtuple +from collections import namedtuple +from collections.abc import Iterable +from contextlib import contextmanager from functools import wraps import re +import threading import dask.array as da import numpy as np @@ -33,10 +36,16 @@ from ..config import get_logger from ..coords import _DimensionalMetadata, AuxCoord from ..exceptions import ConnectivityNotFoundError, CoordinateNotFoundError +from ..fileformats import cf, netcdf +from ..fileformats._pyke_rules.compiled_krb.fc_rules_cf_fc import ( + get_names, + get_attr_units, +) from ..util import guess_coord_axis __all__ = [ + "CFUGridReader", "Connectivity", "ConnectivityMetadata", "Mesh", @@ -52,6 +61,8 @@ "MeshMetadata", "MeshCoord", "MeshCoordMetadata", + "ParseUGridOnLoad", + "PARSE_UGRID_ON_LOAD", ] @@ -947,7 +958,7 @@ def normalise(location, axis): if result not in self.AXES: emsg = f"Invalid axis specified for {location} coordinate {coord.name()!r}, got {axis!r}." raise ValueError(emsg) - return f"{location}_{axis}" + return f"{location}_{result}" if not isinstance(node_coords_and_axes, Iterable): node_coords_and_axes = [node_coords_and_axes] @@ -1893,16 +1904,57 @@ def xml_element(self, doc): # def to_AuxCoords(self, location): # # factory method # # return the lazy AuxCoord(...), AuxCoord(...) - # - # def to_MeshCoord(self, location, axis): - # # factory method - # # return MeshCoord(..., location=location, axis=axis) - # # use Connectivity.indices_by_src() for fetching indices, passing in the lazy_indices() result as an argument. - # - # def to_MeshCoords(self, location): - # # factory method - # # return MeshCoord(..., location=location, axis="x"), MeshCoord(..., location=location, axis="y") - # # use Connectivity.indices_by_src for fetching indices, passing in the lazy_indices() result as an argument. + + def to_MeshCoord(self, location, axis): + """ + Generate a :class:`MeshCoord` that references the current + :class:`Mesh`, and passing through the ``location`` and ``axis`` + arguments. + + .. seealso:: + + :meth:`to_MeshCoords` for generating a series of mesh coords. + + Args: + + * location (str) + The ``location`` argument for :class:`MeshCoord` instantiation. + + * axis (str) + The ``axis`` argument for :class:`MeshCoord` instantiation. + + Returns: + A :class:`MeshCoord` referencing the current :class:`Mesh`. + + """ + return MeshCoord(mesh=self, location=location, axis=axis) + + def to_MeshCoords(self, location): + """ + Generate a tuple of :class:`MeshCoord`'s, each referencing the current + :class:`Mesh`, one for each :attr:`AXES` value, passing through the + ``location`` argument. + + .. seealso:: + + :meth:`to_MeshCoord` for generating a single mesh coord. + + Args: + + * location (str) + The ``location`` argument for :class:`MeshCoord` instantiation. + + Returns: + tuple of :class:`MeshCoord`'s referencing the current :class:`Mesh`. + One for each value in :attr:`AXES`, using the value for the + ``axis`` argument. + + """ + # factory method + result = [ + self.to_MeshCoord(location=location, axis=ax) for ax in self.AXES + ] + return tuple(result) def dimension_names_reset(self, node=False, edge=False, face=False): """ @@ -3159,3 +3211,549 @@ def equal(self, other, lenient=None): _service_collection, _method, ) + + +############################################################################### +# LOADING + + +class ParseUGridOnLoad(threading.local): + def __init__(self): + """ + A flag for dictating whether to use the experimental UGRID-aware + version of Iris NetCDF loading. Object is thread-safe. + + Use via the run-time switch :const:`PARSE_UGRID_ON_LOAD`. + Use :meth:`context` to temporarily activate. + + .. seealso:: + + The UGRID Conventions, + https://ugrid-conventions.github.io/ugrid-conventions/ + + """ + self._state = False + + def __bool__(self): + return self._state + + @contextmanager + def context(self): + """ + Temporarily activate experimental UGRID-aware NetCDF loading. + + Use the standard Iris loading API while within the context manager. If + the loaded file(s) include any UGRID content, this will be parsed and + attached to the resultant cube(s) accordingly. + + Use via the run-time switch :const:`PARSE_UGRID_ON_LOAD`. + + For example:: + + with PARSE_UGRID_ON_LOAD.context(): + my_cube_list = iris.load([my_file_path, my_file_path2], + constraint=my_constraint, + callback=my_callback) + + """ + try: + self._state = True + yield + finally: + self._state = False + + +#: Run-time switch for experimental UGRID-aware NetCDF loading. See :class:`ParseUGridOnLoad`. +PARSE_UGRID_ON_LOAD = ParseUGridOnLoad() + + +############ +# CF Overrides. +# These are not included in __all__ since they are not [currently] needed +# outside this module. + + +class CFUGridConnectivityVariable(cf.CFVariable): + """ + A CF_UGRID connectivity variable points to an index variable identifying + for every element (edge/face/volume) the indices of its corner nodes. The + connectivity array will thus be a matrix of size n-elements x n-corners. + For the indexing one may use either 0- or 1-based indexing; the convention + used should be specified using a ``start_index`` attribute to the index + variable. + + For face elements: the corner nodes should be specified in anticlockwise + direction as viewed from above. For volume elements: use the + additional attribute ``volume_shape_type`` which points to a flag variable + that specifies for every volume its shape. + + Identified by a CF-netCDF variable attribute equal to any one of the values + in :attr:`~iris.experimental.ugrid.Connectivity.UGRID_CF_ROLES`. + + .. seealso:: + + The UGRID Conventions, https://ugrid-conventions.github.io/ugrid-conventions/ + + """ + + cf_identity = NotImplemented + cf_identities = Connectivity.UGRID_CF_ROLES + + @classmethod + def identify(cls, variables, ignore=None, target=None, warn=True): + result = {} + ignore, target = cls._identify_common(variables, ignore, target) + + # Identify all CF-UGRID connectivity variables. + for nc_var_name, nc_var in target.items(): + # Check for connectivity variable references, iterating through + # the valid cf roles. + for identity in cls.cf_identities: + nc_var_att = getattr(nc_var, identity, None) + + if nc_var_att is not None: + # UGRID only allows for one of each connectivity cf role. + name = nc_var_att.strip() + if name not in ignore: + if name not in variables: + if warn: + message = ( + f"Missing CF-UGRID connectivity variable " + f"{name}, referenced by netCDF variable " + f"{nc_var_name}" + ) + logger.debug(message) + else: + # Restrict to non-string type i.e. not a + # CFLabelVariable. + if not cf._is_str_dtype(variables[name]): + result[name] = CFUGridConnectivityVariable( + name, variables[name] + ) + elif warn: + message = ( + f"Ignoring variable {name}, identified " + f"as a CF-UGRID connectivity - is a " + f"CF-netCDF label variable." + ) + logger.debug(message) + + return result + + +class CFUGridAuxiliaryCoordinateVariable(cf.CFVariable): + """ + A CF-UGRID auxiliary coordinate variable is a CF-netCDF auxiliary + coordinate variable representing the element (node/edge/face/volume) + locations (latitude, longitude or other spatial coordinates, and optional + elevation or other coordinates). These auxiliary coordinate variables will + have length n-elements. + + For elements other than nodes, these auxiliary coordinate variables may + have in turn a ``bounds`` attribute that specifies the bounding coordinates + of the element (thereby duplicating the data in the ``node_coordinates`` + variables). + + Identified by the CF-netCDF variable attribute + 'node_'/'edge_'/'face_'/'volume_coordinates'. + + .. seealso:: + + The UGRID Conventions, https://ugrid-conventions.github.io/ugrid-conventions/ + + """ + + cf_identity = NotImplemented + cf_identities = [ + "node_coordinates", + "edge_coordinates", + "face_coordinates", + "volume_coordinates", + ] + + @classmethod + def identify(cls, variables, ignore=None, target=None, warn=True): + result = {} + ignore, target = cls._identify_common(variables, ignore, target) + + # Identify any CF-UGRID-relevant auxiliary coordinate variables. + for nc_var_name, nc_var in target.items(): + # Check for UGRID auxiliary coordinate variable references. + for identity in cls.cf_identities: + nc_var_att = getattr(nc_var, identity, None) + + if nc_var_att is not None: + for name in nc_var_att.split(): + if name not in ignore: + if name not in variables: + if warn: + message = ( + f"Missing CF-netCDF auxiliary " + f"coordinate variable {name}, " + f"referenced by netCDF variable " + f"{nc_var_name}" + ) + logger.debug(message) + else: + # Restrict to non-string type i.e. not a + # CFLabelVariable. + if not cf._is_str_dtype(variables[name]): + result[ + name + ] = CFUGridAuxiliaryCoordinateVariable( + name, variables[name] + ) + elif warn: + message = ( + f"Ignoring variable {name}, " + f"identified as a CF-netCDF auxiliary " + f"coordinate - is a CF-netCDF label " + f"variable." + ) + logger.debug(message) + + return result + + +class CFUGridMeshVariable(cf.CFVariable): + """ + A CF-UGRID mesh variable is a dummy variable for storing topology + information as attributes. The mesh variable has the ``cf_role`` + 'mesh_topology'. + + The UGRID conventions describe define the mesh topology as the + interconnection of various geometrical elements of the mesh. The pure + interconnectivity is independent of georeferencing the individual + geometrical elements, but for the practical applications for which the + UGRID CF extension is defined, coordinate data will always be added. + + Identified by the CF-netCDF variable attribute 'mesh'. + + .. seealso:: + + The UGRID Conventions, https://ugrid-conventions.github.io/ugrid-conventions/ + + """ + + cf_identity = "mesh" + + @classmethod + def identify(cls, variables, ignore=None, target=None, warn=True): + result = {} + ignore, target = cls._identify_common(variables, ignore, target) + + # Identify all CF-UGRID mesh variables. + for nc_var_name, nc_var in target.items(): + # Check for mesh variable references. + nc_var_att = getattr(nc_var, cls.cf_identity, None) + + if nc_var_att is not None: + # UGRID only allows for 1 mesh per variable. + name = nc_var_att.strip() + if name not in ignore: + if name not in variables: + if warn: + message = ( + f"Missing CF-UGRID mesh variable {name}, " + f"referenced by netCDF variable {nc_var_name}" + ) + logger.debug(message) + else: + # Restrict to non-string type i.e. not a + # CFLabelVariable. + if not cf._is_str_dtype(variables[name]): + result[name] = CFUGridMeshVariable( + name, variables[name] + ) + elif warn: + message = ( + f"Ignoring variable {name}, identified as a " + f"CF-UGRID mesh - is a CF-netCDF label " + f"variable." + ) + logger.debug(message) + + return result + + +class CFUGridGroup(cf.CFGroup): + """ + Represents a collection of 'NetCDF Climate and Forecast (CF) Metadata + Conventions' variables and netCDF global attributes. + + Specialisation of :class:`~iris.fileformats.cf.CFGroup` that includes extra + collections for CF-UGRID-specific variable types. + + """ + + @property + def connectivities(self): + """Collection of CF-UGRID connectivity variables.""" + return self._cf_getter(CFUGridConnectivityVariable) + + @property + def ugrid_coords(self): + """Collection of CF-UGRID-relevant auxiliary coordinate variables.""" + return self._cf_getter(CFUGridAuxiliaryCoordinateVariable) + + @property + def meshes(self): + """Collection of CF-UGRID mesh variables.""" + return self._cf_getter(CFUGridMeshVariable) + + @property + def non_data_variable_names(self): + """ + :class:`set` of the names of the CF-netCDF/CF-UGRID variables that are + not the data pay-load. + + """ + extra_variables = (self.connectivities, self.ugrid_coords, self.meshes) + extra_result = set() + for variable in extra_variables: + extra_result |= set(variable) + return super().non_data_variable_names | extra_result + + +class CFUGridReader(cf.CFReader): + """ + This class allows the contents of a netCDF file to be interpreted according + to the 'NetCDF Climate and Forecast (CF) Metadata Conventions'. + + Specialisation of :class:`~iris.fileformats.cf.CFReader` that can also + handle CF-UGRID-specific variable types. + + """ + + _variable_types = cf.CFReader._variable_types + ( + CFUGridConnectivityVariable, + CFUGridAuxiliaryCoordinateVariable, + CFUGridMeshVariable, + ) + + CFGroup = CFUGridGroup + + +############ +# Object construction. +# Helper functions, supporting netcdf.load_cubes ONLY, expected to +# altered/moved when pyke is removed. + + +def _build_aux_coord(coord_var, file_path): + """ + Construct a :class:`~iris.coords.AuxCoord` from a given + :class:`CFUGridAuxiliaryCoordinateVariable`, and guess its mesh axis. + + todo: integrate with standard loading API post-pyke. + + """ + assert isinstance(coord_var, CFUGridAuxiliaryCoordinateVariable) + attributes = {} + attr_units = get_attr_units(coord_var, attributes) + points_data = netcdf._get_cf_var_data(coord_var, file_path) + + # Bounds will not be loaded: + # Bounds may be present, but the UGRID conventions state this would + # always be duplication of the same info provided by the mandatory + # connectivities. + + # Fetch climatological - not allowed for a Mesh, but loading it will + # mean an informative error gets raised. + climatological = False + # TODO: use CF_ATTR_CLIMATOLOGY once re-integrated post-pyke. + attr_climatology = getattr(coord_var, "climatology", None) + if attr_climatology is not None: + climatology_vars = coord_var.cf_group.climatology + climatological = attr_climatology in climatology_vars + + standard_name, long_name, var_name = get_names(coord_var, None, attributes) + coord = AuxCoord( + points_data, + standard_name=standard_name, + long_name=long_name, + var_name=var_name, + units=attr_units, + attributes=attributes, + # TODO: coord_system + climatological=climatological, + ) + + axis = guess_coord_axis(coord) + if axis is None: + if var_name[-2] == "_": + # Fall back on UGRID var_name convention. + axis = var_name[-1] + else: + message = f"Cannot guess axis for UGRID coord: {var_name} ." + raise ValueError(message) + + return coord, axis + + +def _build_connectivity(connectivity_var, file_path, location_dims): + """ + Construct a :class:`Connectivity` from a given + :class:`CFUGridConnectivityVariable`, and identify the name of its first + dimension. + + todo: integrate with standard loading API post-pyke. + + """ + assert isinstance(connectivity_var, CFUGridConnectivityVariable) + attributes = {} + attr_units = get_attr_units(connectivity_var, attributes) + indices_data = netcdf._get_cf_var_data(connectivity_var, file_path) + + cf_role = connectivity_var.cf_role + start_index = connectivity_var.start_index + + dim_names = connectivity_var.dimensions + # Connectivity arrays must have two dimensions. + assert len(dim_names) == 2 + if dim_names[1] in location_dims: + src_dim = 1 + else: + src_dim = 0 + + standard_name, long_name, var_name = get_names( + connectivity_var, None, attributes + ) + + connectivity = Connectivity( + indices=indices_data, + cf_role=cf_role, + standard_name=standard_name, + long_name=long_name, + var_name=var_name, + units=attr_units, + attributes=attributes, + start_index=start_index, + src_dim=src_dim, + ) + + return connectivity, dim_names[0] + + +def _build_mesh(cf, mesh_var, file_path): + """ + Construct a :class:`Mesh` from a given :class:`CFUGridMeshVariable`. + + todo: integrate with standard loading API post-pyke. + + """ + assert isinstance(mesh_var, CFUGridMeshVariable) + attributes = {} + attr_units = get_attr_units(mesh_var, attributes) + + topology_dimension = mesh_var.topology_dimension + + node_dimension = None + edge_dimension = getattr(mesh_var, "edge_dimension", None) + face_dimension = getattr(mesh_var, "face_dimension", None) + + node_coord_args = [] + edge_coord_args = [] + face_coord_args = [] + for coord_var in mesh_var.cf_group.ugrid_coords.values(): + coord_and_axis = _build_aux_coord(coord_var, file_path) + coord = coord_and_axis[0] + + if coord.var_name in mesh_var.node_coordinates.split(): + node_coord_args.append(coord_and_axis) + node_dimension = coord_var.dimensions[0] + elif ( + coord.var_name in getattr(mesh_var, "edge_coordinates", "").split() + ): + edge_coord_args.append(coord_and_axis) + elif ( + coord.var_name in getattr(mesh_var, "face_coordinates", "").split() + ): + face_coord_args.append(coord_and_axis) + # TODO: support volume_coordinates. + else: + message = ( + f"Invalid UGRID coord: {coord.var_name} . Must be either a" + f"node_, edge_ or face_coordinate." + ) + raise ValueError(message) + + if node_dimension is None: + message = ( + "'node_dimension' could not be identified from mesh node " + "coordinates." + ) + raise ValueError(message) + + # Used for detecting transposed connectivities. + location_dims = (edge_dimension, face_dimension) + connectivity_args = [] + for connectivity_var in mesh_var.cf_group.connectivities.values(): + connectivity, first_dim_name = _build_connectivity( + connectivity_var, file_path, location_dims + ) + assert connectivity.var_name == getattr(mesh_var, connectivity.cf_role) + connectivity_args.append(connectivity) + + # If the mesh_var has not supplied the dimension name, it is safe to + # fall back on the connectivity's first dimension's name. + if edge_dimension is None and connectivity.src_location == "edge": + edge_dimension = first_dim_name + if face_dimension is None and connectivity.src_location == "face": + face_dimension = first_dim_name + + standard_name, long_name, var_name = get_names(mesh_var, None, attributes) + + mesh = Mesh( + topology_dimension=topology_dimension, + node_coords_and_axes=node_coord_args, + connectivities=connectivity_args, + edge_coords_and_axes=edge_coord_args, + face_coords_and_axes=face_coord_args, + standard_name=standard_name, + long_name=long_name, + var_name=var_name, + units=attr_units, + attributes=attributes, + node_dimension=node_dimension, + edge_dimension=edge_dimension, + face_dimension=face_dimension, + ) + assert mesh.cf_role == mesh_var.cf_role + + mesh_elements = ( + list(mesh.all_coords) + list(mesh.all_connectivities) + [mesh] + ) + mesh_elements = filter(None, mesh_elements) + for iris_object in mesh_elements: + netcdf._add_unused_attributes( + iris_object, cf.cf_group[iris_object.var_name] + ) + + return mesh + + +def _build_mesh_coords(mesh, cf_var): + """ + Construct a tuple of :class:`MeshCoord` using from a given :class:`Mesh` + and :class:`~iris.fileformats.cf.CFVariable`. + + todo: integrate with standard loading API post-pyke. + + """ + # Identify the cube's mesh dimension, for attaching MeshCoords. + locations_dimensions = { + "node": mesh.node_dimension, + "edge": mesh.edge_dimension, + "face": mesh.face_dimension, + } + mesh_dim_name = locations_dimensions[cf_var.location] + # (Only expecting 1 mesh dimension per cf_var). + mesh_dim = cf_var.dimensions.index(mesh_dim_name) + + mesh_coords = mesh.to_MeshCoords(location=cf_var.location) + return mesh_coords, mesh_dim + + +# END of loading section. +############################################################################### diff --git a/lib/iris/fileformats/cf.py b/lib/iris/fileformats/cf.py index 47ff6291b0..acda1def52 100644 --- a/lib/iris/fileformats/cf.py +++ b/lib/iris/fileformats/cf.py @@ -949,6 +949,28 @@ def cell_measures(self): """Collection of CF-netCDF measure variables.""" return self._cf_getter(CFMeasureVariable) + @property + def non_data_variable_names(self): + """ + :class:`set` of the names of the CF-netCDF variables that are not + the data pay-load. + + """ + non_data_variables = ( + self.ancillary_variables, + self.auxiliary_coordinates, + self.bounds, + self.climatology, + self.coordinates, + self.grid_mappings, + self.labels, + self.cell_measures, + ) + result = set() + for variable in non_data_variables: + result |= set(variable) + return result + def keys(self): """Return the names of all the CF-netCDF variables in the group.""" return self._cf_variables.keys() @@ -1008,22 +1030,26 @@ class CFReader: """ + # All CF variable types EXCEPT for the "special cases" of + # CFDataVariable, CFCoordinateVariable and _CFFormulaTermsVariable. + _variable_types = ( + CFAncillaryDataVariable, + CFAuxiliaryCoordinateVariable, + CFBoundaryVariable, + CFClimatologyVariable, + CFGridMappingVariable, + CFLabelVariable, + CFMeasureVariable, + ) + + # TODO: remove once iris.experimental.ugrid.CFUGridReader is folded in. + CFGroup = CFGroup + def __init__(self, filename, warn=False, monotonic=False): self._filename = os.path.expanduser(filename) - # All CF variable types EXCEPT for the "special cases" of - # CFDataVariable, CFCoordinateVariable and _CFFormulaTermsVariable. - self._variable_types = ( - CFAncillaryDataVariable, - CFAuxiliaryCoordinateVariable, - CFBoundaryVariable, - CFClimatologyVariable, - CFGridMappingVariable, - CFLabelVariable, - CFMeasureVariable, - ) #: Collection of CF-netCDF variables associated with this netCDF file - self.cf_group = CFGroup() + self.cf_group = self.CFGroup() self._dataset = netCDF4.Dataset(self._filename, mode="r") @@ -1098,15 +1124,7 @@ def _translate(self): # Determine the CF data variables. data_variable_names = ( - set(netcdf_variable_names) - - set(self.cf_group.ancillary_variables) - - set(self.cf_group.auxiliary_coordinates) - - set(self.cf_group.bounds) - - set(self.cf_group.climatology) - - set(self.cf_group.coordinates) - - set(self.cf_group.grid_mappings) - - set(self.cf_group.labels) - - set(self.cf_group.cell_measures) + set(netcdf_variable_names) - self.cf_group.non_data_variable_names ) for name in data_variable_names: @@ -1118,17 +1136,28 @@ def _build_cf_groups(self): """Build the first order relationships between CF-netCDF variables.""" def _build(cf_variable): + # TODO: isinstance(cf_variable, UGridMeshVariable) + # UGridMeshVariable currently in experimental.ugrid - circular import. + is_mesh_var = cf_variable.cf_identity == "mesh" + ugrid_coord_names = [] + ugrid_coords = getattr(self.cf_group, "ugrid_coords", None) + if ugrid_coords is not None: + ugrid_coord_names = list(ugrid_coords.keys()) + coordinate_names = list(self.cf_group.coordinates.keys()) - cf_group = CFGroup() + cf_group = self.CFGroup() # Build CF variable relationships. for variable_type in self._variable_types: - # Prevent grid mapping variables being mis-identified as - # CF coordinate variables. - if issubclass(variable_type, CFGridMappingVariable): - ignore = None - else: - ignore = coordinate_names + ignore = [] + # Avoid UGridAuxiliaryCoordinateVariables also being + # processed as CFAuxiliaryCoordinateVariables. + if not is_mesh_var: + ignore += ugrid_coord_names + # Prevent grid mapping variables being mis-identified as CF coordinate variables. + if not issubclass(variable_type, CFGridMappingVariable): + ignore += coordinate_names + match = variable_type.identify( self._dataset.variables, ignore=ignore, @@ -1137,7 +1166,8 @@ def _build(cf_variable): ) # Sanity check dimensionality coverage. for cf_name, cf_var in match.items(): - if cf_var.spans(cf_variable): + # No span check is necessary if variable is attached to a mesh. + if is_mesh_var or cf_var.spans(cf_variable): cf_group[cf_name] = self.cf_group[cf_name] else: # Register the ignored variable. diff --git a/lib/iris/fileformats/netcdf.py b/lib/iris/fileformats/netcdf.py index bb7a870d58..35626b15ca 100644 --- a/lib/iris/fileformats/netcdf.py +++ b/lib/iris/fileformats/netcdf.py @@ -51,6 +51,9 @@ # Show Pyke inference engine statistics. DEBUG = False +# Configure the logger. +logger = iris.config.get_logger(__name__, fmt="[%(cls)s.%(funcName)s]") + # Pyke CF related file names. _PYKE_RULE_BASE = "fc_rules_cf" _PYKE_FACT_BASE = "facts_cf" @@ -551,6 +554,22 @@ def _set_attributes(attributes, key, value): attributes[str(key)] = value +def _add_unused_attributes(iris_object, cf_var): + """ + Populate the attributes of a cf element with the "unused" attributes + from the associated CF-netCDF variable. That is, all those that aren't CF + reserved terms. + + """ + + def attribute_predicate(item): + return item[0] not in _CF_ATTRS + + tmpvar = filter(attribute_predicate, cf_var.cf_attrs_unused()) + for attr_name, attr_value in tmpvar: + _set_attributes(iris_object.attributes, attr_name, attr_value) + + def _get_actual_dtype(cf_var): # Figure out what the eventual data type will be after any scale/offset # transforms. @@ -606,22 +625,12 @@ def _load_cube(engine, cf, cf_var, filename): # Run pyke inference engine with forward chaining rules. engine.activate(_PYKE_RULE_BASE) - # Having run the rules, now populate the attributes of all the cf elements with the - # "unused" attributes from the associated CF-netCDF variable. - # That is, all those that aren't CF reserved terms. - def attribute_predicate(item): - return item[0] not in _CF_ATTRS - - def add_unused_attributes(iris_object, cf_var): - tmpvar = filter(attribute_predicate, cf_var.cf_attrs_unused()) - for attr_name, attr_value in tmpvar: - _set_attributes(iris_object.attributes, attr_name, attr_value) - + # Having run the rules, now add the "unused" attributes to each cf element. def fix_attributes_all_elements(role_name): elements_and_names = engine.cube_parts.get(role_name, []) for iris_object, cf_var_name in elements_and_names: - add_unused_attributes(iris_object, cf.cf_group[cf_var_name]) + _add_unused_attributes(iris_object, cf.cf_group[cf_var_name]) # Populate the attributes of all coordinates, cell-measures and ancillary-vars. fix_attributes_all_elements("coordinates") @@ -629,7 +638,7 @@ def fix_attributes_all_elements(role_name): fix_attributes_all_elements("cell_measures") # Also populate attributes of the top-level cube itself. - add_unused_attributes(cube, cf_var) + _add_unused_attributes(cube, cf_var) # Work out reference names for all the coords. names = { @@ -786,9 +795,19 @@ def load_cubes(filenames, callback=None): Function which can be passed on to :func:`iris.io.run_callback`. Returns: - Generator of loaded NetCDF :class:`iris.cubes.Cube`. + Generator of loaded NetCDF :class:`iris.cube.Cube`. """ + # TODO: rationalise UGRID/mesh handling once experimental.ugrid is folded + # into standard behaviour. + # Deferred import to avoid circular imports. + from iris.experimental.ugrid import ( + PARSE_UGRID_ON_LOAD, + CFUGridReader, + _build_mesh, + _build_mesh_coords, + ) + # Initialise the pyke inference engine. engine = _pyke_kb_engine() @@ -797,15 +816,53 @@ def load_cubes(filenames, callback=None): for filename in filenames: # Ingest the netCDF file. - cf = iris.fileformats.cf.CFReader(filename) + meshes = {} + if PARSE_UGRID_ON_LOAD: + cf = CFUGridReader(filename) + + # Mesh instances are shared between file phenomena. + # TODO: more sophisticated Mesh sharing between files. + # TODO: access external Mesh cache? + mesh_vars = cf.cf_group.meshes + meshes = { + name: _build_mesh(cf, var, filename) + for name, var in mesh_vars.items() + } + else: + cf = iris.fileformats.cf.CFReader(filename) # Process each CF data variable. data_variables = list(cf.cf_group.data_variables.values()) + list( cf.cf_group.promoted.values() ) for cf_var in data_variables: + # cf_var-specific mesh handling, if a mesh is present. + # Build the mesh_coords *before* loading the cube - avoids + # mesh-related attributes being picked up by + # _add_unused_attributes(). + mesh_name = None + mesh = None + mesh_coords, mesh_dim = [], None + if PARSE_UGRID_ON_LOAD: + mesh_name = getattr(cf_var, "mesh", None) + if mesh_name is not None: + try: + mesh = meshes[mesh_name] + except KeyError: + message = ( + f"File does not contain mesh: '{mesh_name}' - " + f"referenced by variable: '{cf_var.cf_name}' ." + ) + logger.debug(message) + if mesh is not None: + mesh_coords, mesh_dim = _build_mesh_coords(mesh, cf_var) + cube = _load_cube(engine, cf, cf_var, filename) + # Attach the mesh (if present) to the cube. + for mesh_coord in mesh_coords: + cube.add_aux_coord(mesh_coord, mesh_dim) + # Process any associated formula terms and attach # the corresponding AuxCoordFactory. try: @@ -1435,11 +1492,11 @@ def _add_aux_factories(self, cube, cf_var_cube, dimension_names): or cf_var.standard_name != std_name ): # TODO: We need to resolve this corner-case where - # the dimensionless vertical coordinate containing the - # formula_terms is a dimension coordinate of the - # associated cube and a new alternatively named - # dimensionless vertical coordinate is required with - # new formula_terms and a renamed dimension. + # the dimensionless vertical coordinate containing + # the formula_terms is a dimension coordinate of + # the associated cube and a new alternatively named + # dimensionless vertical coordinate is required + # with new formula_terms and a renamed dimension. if cf_name in dimension_names: msg = ( "Unable to create dimensonless vertical " diff --git a/lib/iris/tests/integration/experimental/test_ugrid_load.py b/lib/iris/tests/integration/experimental/test_ugrid_load.py new file mode 100644 index 0000000000..bcd56a02ab --- /dev/null +++ b/lib/iris/tests/integration/experimental/test_ugrid_load.py @@ -0,0 +1,64 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +""" +Integration tests for NetCDF-UGRID file loading. + +todo: fold these tests into netcdf tests when experimental.ugrid is folded into + standard behaviour. + +""" + +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests + +from iris import load, load_cube, NameConstraint +from iris.experimental.ugrid import PARSE_UGRID_ON_LOAD + + +def ugrid_load(*args, **kwargs): + with PARSE_UGRID_ON_LOAD.context(): + return load(*args, **kwargs) + + +def ugrid_load_cube(*args, **kwargs): + with PARSE_UGRID_ON_LOAD.context(): + return load_cube(*args, **kwargs) + + +@tests.skip_data +class TestBasic(tests.IrisTest): + def test_2D_1t_face_half_levels(self): + # TODO: remove constraint once file no longer has orphan connectivities. + conv_rain = NameConstraint(var_name="conv_rain") + cube = ugrid_load_cube( + tests.get_data_path( + [ + "NetCDF", + "unstructured_grid", + "lfric_ngvat_2D_1t_face_half_levels_main_conv_rain.nc", + ] + ), + constraint=conv_rain, + ) + self.assertCML( + cube, ("experimental", "ugrid", "2D_1t_face_half_levels.cml") + ) + + # def test_3D_1t_face_half_levels(self): + # pass + # + # def test_3D_1t_face_full_levels(self): + # pass + # + # def test_multiple_time_values(self): + # pass + # + # etc ... + + +# class TestPseudoLevels(tests.IrisTest): +# pass diff --git a/lib/iris/tests/results/experimental/ugrid/2D_1t_face_half_levels.cml b/lib/iris/tests/results/experimental/ugrid/2D_1t_face_half_levels.cml new file mode 100644 index 0000000000..be79f3ff57 --- /dev/null +++ b/lib/iris/tests/results/experimental/ugrid/2D_1t_face_half_levels.cml @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/test_netcdf.py b/lib/iris/tests/test_netcdf.py index 2d1b4a53d5..a2ac49c23c 100644 --- a/lib/iris/tests/test_netcdf.py +++ b/lib/iris/tests/test_netcdf.py @@ -16,11 +16,9 @@ import os.path import shutil import stat -from subprocess import check_call import tempfile from unittest import mock -from cf_units import as_unit import netCDF4 as nc import numpy as np import numpy.ma as ma @@ -29,10 +27,9 @@ import iris.analysis.trajectory import iris.fileformats._pyke_rules.compiled_krb.fc_rules_cf_fc as pyke_rules import iris.fileformats.netcdf -from iris.fileformats.netcdf import load_cubes as nc_load_cubes import iris.std_names import iris.util -from iris.coords import AncillaryVariable, CellMeasure +import iris.coords import iris.coord_systems as icoord_systems import iris.tests.stock as stock from iris._lazy_data import is_lazy_data @@ -251,153 +248,6 @@ def test_cell_methods(self): self.assertCML(cubes, ("netcdf", "netcdf_cell_methods.cml")) - def test_ancillary_variables(self): - # Note: using a CDL string as a test data reference, rather than a binary file. - ref_cdl = """ - netcdf cm_attr { - dimensions: - axv = 3 ; - variables: - int64 qqv(axv) ; - qqv:long_name = "qq" ; - qqv:units = "1" ; - qqv:ancillary_variables = "my_av" ; - int64 axv(axv) ; - axv:units = "1" ; - axv:long_name = "x" ; - double my_av(axv) ; - my_av:units = "1" ; - my_av:long_name = "refs" ; - my_av:custom = "extra-attribute"; - data: - axv = 1, 2, 3; - my_av = 11., 12., 13.; - } - """ - self.tmpdir = tempfile.mkdtemp() - cdl_path = os.path.join(self.tmpdir, "tst.cdl") - nc_path = os.path.join(self.tmpdir, "tst.nc") - # Write CDL string into a temporary CDL file. - with open(cdl_path, "w") as f_out: - f_out.write(ref_cdl) - # Use ncgen to convert this into an actual (temporary) netCDF file. - command = "ncgen -o {} {}".format(nc_path, cdl_path) - check_call(command, shell=True) - # Load with iris.fileformats.netcdf.load_cubes, and check expected content. - cubes = list(nc_load_cubes(nc_path)) - self.assertEqual(len(cubes), 1) - avs = cubes[0].ancillary_variables() - self.assertEqual(len(avs), 1) - expected = AncillaryVariable( - np.ma.array([11.0, 12.0, 13.0]), - long_name="refs", - var_name="my_av", - units="1", - attributes={"custom": "extra-attribute"}, - ) - self.assertEqual(avs[0], expected) - - def test_status_flags(self): - # Note: using a CDL string as a test data reference, rather than a binary file. - ref_cdl = """ - netcdf cm_attr { - dimensions: - axv = 3 ; - variables: - int64 qqv(axv) ; - qqv:long_name = "qq" ; - qqv:units = "1" ; - qqv:ancillary_variables = "my_av" ; - int64 axv(axv) ; - axv:units = "1" ; - axv:long_name = "x" ; - byte my_av(axv) ; - my_av:long_name = "qq status_flag" ; - my_av:flag_values = 1b, 2b ; - my_av:flag_meanings = "a b" ; - data: - axv = 11, 21, 31; - my_av = 1b, 1b, 2b; - } - """ - self.tmpdir = tempfile.mkdtemp() - cdl_path = os.path.join(self.tmpdir, "tst.cdl") - nc_path = os.path.join(self.tmpdir, "tst.nc") - # Write CDL string into a temporary CDL file. - with open(cdl_path, "w") as f_out: - f_out.write(ref_cdl) - # Use ncgen to convert this into an actual (temporary) netCDF file. - command = "ncgen -o {} {}".format(nc_path, cdl_path) - check_call(command, shell=True) - # Load with iris.fileformats.netcdf.load_cubes, and check expected content. - cubes = list(nc_load_cubes(nc_path)) - self.assertEqual(len(cubes), 1) - avs = cubes[0].ancillary_variables() - self.assertEqual(len(avs), 1) - expected = AncillaryVariable( - np.ma.array([1, 1, 2], dtype=np.int8), - long_name="qq status_flag", - var_name="my_av", - units="no_unit", - attributes={ - "flag_values": np.array([1, 2], dtype=np.int8), - "flag_meanings": "a b", - }, - ) - self.assertEqual(avs[0], expected) - - def test_cell_measures(self): - # Note: using a CDL string as a test data reference, rather than a binary file. - ref_cdl = """ - netcdf cm_attr { - dimensions: - axv = 3 ; - ayv = 2 ; - variables: - int64 qqv(ayv, axv) ; - qqv:long_name = "qq" ; - qqv:units = "1" ; - qqv:cell_measures = "area: my_areas" ; - int64 ayv(ayv) ; - ayv:units = "1" ; - ayv:long_name = "y" ; - int64 axv(axv) ; - axv:units = "1" ; - axv:long_name = "x" ; - double my_areas(ayv, axv) ; - my_areas:units = "m2" ; - my_areas:long_name = "standardised cell areas" ; - my_areas:custom = "extra-attribute"; - data: - axv = 11, 12, 13; - ayv = 21, 22; - my_areas = 110., 120., 130., 221., 231., 241.; - } - """ - self.tmpdir = tempfile.mkdtemp() - cdl_path = os.path.join(self.tmpdir, "tst.cdl") - nc_path = os.path.join(self.tmpdir, "tst.nc") - # Write CDL string into a temporary CDL file. - with open(cdl_path, "w") as f_out: - f_out.write(ref_cdl) - # Use ncgen to convert this into an actual (temporary) netCDF file. - command = "ncgen -o {} {}".format(nc_path, cdl_path) - check_call(command, shell=True) - # Load with iris.fileformats.netcdf.load_cubes, and check expected content. - cubes = list(nc_load_cubes(nc_path)) - self.assertEqual(len(cubes), 1) - cms = cubes[0].cell_measures() - self.assertEqual(len(cms), 1) - expected = CellMeasure( - np.ma.array([[110.0, 120.0, 130.0], [221.0, 231.0, 241.0]]), - measure="area", - var_name="my_areas", - long_name="standardised cell areas", - units="m2", - attributes={"custom": "extra-attribute"}, - ) - self.assertEqual(cms[0], expected) - def test_deferred_loading(self): # Test exercising CF-netCDF deferred loading and deferred slicing. # shape (31, 161, 320) @@ -450,55 +300,6 @@ def test_deferred_loading(self): cube[0][(0, 2), (1, 3)], ("netcdf", "netcdf_deferred_mix_1.cml") ) - def test_default_units(self): - # Note: using a CDL string as a test data reference, rather than a binary file. - ref_cdl = """ - netcdf cm_attr { - dimensions: - axv = 3 ; - ayv = 2 ; - variables: - int64 qqv(ayv, axv) ; - qqv:long_name = "qq" ; - qqv:ancillary_variables = "my_av" ; - qqv:cell_measures = "area: my_areas" ; - int64 ayv(ayv) ; - ayv:long_name = "y" ; - int64 axv(axv) ; - axv:units = "1" ; - axv:long_name = "x" ; - double my_av(axv) ; - my_av:long_name = "refs" ; - double my_areas(ayv, axv) ; - my_areas:long_name = "areas" ; - data: - axv = 11, 12, 13; - ayv = 21, 22; - my_areas = 110., 120., 130., 221., 231., 241.; - } - """ - self.tmpdir = tempfile.mkdtemp() - cdl_path = os.path.join(self.tmpdir, "tst.cdl") - nc_path = os.path.join(self.tmpdir, "tst.nc") - # Write CDL string into a temporary CDL file. - with open(cdl_path, "w") as f_out: - f_out.write(ref_cdl) - # Use ncgen to convert this into an actual (temporary) netCDF file. - command = "ncgen -o {} {}".format(nc_path, cdl_path) - check_call(command, shell=True) - # Load with iris.fileformats.netcdf.load_cubes, and check expected content. - cubes = list(nc_load_cubes(nc_path)) - self.assertEqual(len(cubes), 1) - self.assertEqual(cubes[0].units, as_unit("unknown")) - self.assertEqual(cubes[0].coord("y").units, as_unit("unknown")) - self.assertEqual(cubes[0].coord("x").units, as_unit(1)) - self.assertEqual( - cubes[0].ancillary_variable("refs").units, as_unit("unknown") - ) - self.assertEqual( - cubes[0].cell_measure("areas").units, as_unit("unknown") - ) - def test_units(self): # Test exercising graceful cube and coordinate units loading. cube0, cube1 = sorted( diff --git a/lib/iris/tests/unit/experimental/ugrid/test_CFUGridAuxiliaryCoordinateVariable.py b/lib/iris/tests/unit/experimental/ugrid/test_CFUGridAuxiliaryCoordinateVariable.py new file mode 100644 index 0000000000..a0e5c5e3f2 --- /dev/null +++ b/lib/iris/tests/unit/experimental/ugrid/test_CFUGridAuxiliaryCoordinateVariable.py @@ -0,0 +1,245 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +""" +Unit tests for the :class:`iris.experimental.ugrid.CFUGridAuxiliaryCoordinateVariable` class. + +todo: fold these tests into cf tests when experimental.ugrid is folded into + standard behaviour. + +""" +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests + +import numpy as np + +from iris.experimental.ugrid import ( + CFUGridAuxiliaryCoordinateVariable, + logger, +) +from iris.tests.unit.experimental.ugrid.test_CFUGridReader import ( + netcdf_ugrid_variable, +) + + +def named_variable(name): + # Don't need to worry about dimensions or dtype for these tests. + return netcdf_ugrid_variable(name, "", np.int) + + +class TestIdentify(tests.IrisTest): + def setUp(self): + self.cf_identities = [ + "node_coordinates", + "edge_coordinates", + "face_coordinates", + "volume_coordinates", + ] + + def test_cf_identities(self): + subject_name = "ref_subject" + ref_subject = named_variable(subject_name) + vars_common = { + subject_name: ref_subject, + "ref_not_subject": named_variable("ref_not_subject"), + } + # ONLY expecting ref_subject, excluding ref_not_subject. + expected = { + subject_name: CFUGridAuxiliaryCoordinateVariable( + subject_name, ref_subject + ) + } + + for identity in self.cf_identities: + ref_source = named_variable("ref_source") + setattr(ref_source, identity, subject_name) + vars_all = dict({"ref_source": ref_source}, **vars_common) + result = CFUGridAuxiliaryCoordinateVariable.identify(vars_all) + self.assertDictEqual(expected, result) + + def test_duplicate_refs(self): + subject_name = "ref_subject" + ref_subject = named_variable(subject_name) + ref_source_vars = { + name: named_variable(name) + for name in ("ref_source_1", "ref_source_2") + } + for var in ref_source_vars.values(): + setattr(var, self.cf_identities[0], subject_name) + vars_all = dict( + { + subject_name: ref_subject, + "ref_not_subject": named_variable("ref_not_subject"), + }, + **ref_source_vars, + ) + + # ONLY expecting ref_subject, excluding ref_not_subject. + expected = { + subject_name: CFUGridAuxiliaryCoordinateVariable( + subject_name, ref_subject + ) + } + result = CFUGridAuxiliaryCoordinateVariable.identify(vars_all) + self.assertDictEqual(expected, result) + + def test_two_coords(self): + subject_names = ("ref_subject_1", "ref_subject_2") + ref_subject_vars = { + name: named_variable(name) for name in subject_names + } + + ref_source_vars = { + name: named_variable(name) + for name in ("ref_source_1", "ref_source_2") + } + for ix, var in enumerate(ref_source_vars.values()): + setattr(var, self.cf_identities[ix], subject_names[ix]) + vars_all = dict( + {"ref_not_subject": named_variable("ref_not_subject")}, + **ref_subject_vars, + **ref_source_vars, + ) + + # Not expecting ref_not_subject. + expected = { + name: CFUGridAuxiliaryCoordinateVariable(name, var) + for name, var in ref_subject_vars.items() + } + result = CFUGridAuxiliaryCoordinateVariable.identify(vars_all) + self.assertDictEqual(expected, result) + + def test_two_part_ref(self): + subject_names = ("ref_subject_1", "ref_subject_2") + ref_subject_vars = { + name: named_variable(name) for name in subject_names + } + + ref_source = named_variable("ref_source") + setattr(ref_source, self.cf_identities[0], " ".join(subject_names)) + vars_all = { + "ref_not_subject": named_variable("ref_not_subject"), + "ref_source": ref_source, + **ref_subject_vars, + } + + expected = { + name: CFUGridAuxiliaryCoordinateVariable(name, var) + for name, var in ref_subject_vars.items() + } + result = CFUGridAuxiliaryCoordinateVariable.identify(vars_all) + self.assertDictEqual(expected, result) + + def test_string_type_ignored(self): + subject_name = "ref_subject" + ref_source = named_variable("ref_source") + setattr(ref_source, self.cf_identities[0], subject_name) + vars_all = { + subject_name: netcdf_ugrid_variable(subject_name, "", np.bytes_), + "ref_not_subject": named_variable("ref_not_subject"), + "ref_source": ref_source, + } + + result = CFUGridAuxiliaryCoordinateVariable.identify(vars_all) + self.assertDictEqual({}, result) + + def test_ignore(self): + subject_names = ("ref_subject_1", "ref_subject_2") + ref_subject_vars = { + name: named_variable(name) for name in subject_names + } + + ref_source_vars = { + name: named_variable(name) + for name in ("ref_source_1", "ref_source_2") + } + for ix, var in enumerate(ref_source_vars.values()): + setattr(var, self.cf_identities[0], subject_names[ix]) + vars_all = dict( + {"ref_not_subject": named_variable("ref_not_subject")}, + **ref_subject_vars, + **ref_source_vars, + ) + + # ONLY expect the subject variable that hasn't been ignored. + expected_name = subject_names[0] + expected = { + expected_name: CFUGridAuxiliaryCoordinateVariable( + expected_name, ref_subject_vars[expected_name] + ) + } + result = CFUGridAuxiliaryCoordinateVariable.identify( + vars_all, ignore=subject_names[1] + ) + self.assertDictEqual(expected, result) + + def test_target(self): + subject_names = ("ref_subject_1", "ref_subject_2") + ref_subject_vars = { + name: named_variable(name) for name in subject_names + } + + source_names = ("ref_source_1", "ref_source_2") + ref_source_vars = {name: named_variable(name) for name in source_names} + for ix, var in enumerate(ref_source_vars.values()): + setattr(var, self.cf_identities[0], subject_names[ix]) + vars_all = dict( + {"ref_not_subject": named_variable("ref_not_subject")}, + **ref_subject_vars, + **ref_source_vars, + ) + + # ONLY expect the variable referenced by the named ref_source_var. + expected_name = subject_names[0] + expected = { + expected_name: CFUGridAuxiliaryCoordinateVariable( + expected_name, ref_subject_vars[expected_name] + ) + } + result = CFUGridAuxiliaryCoordinateVariable.identify( + vars_all, target=source_names[0] + ) + self.assertDictEqual(expected, result) + + def test_warn(self): + subject_name = "ref_subject" + ref_source = named_variable("ref_source") + setattr(ref_source, self.cf_identities[0], subject_name) + vars_all = { + "ref_not_subject": named_variable("ref_not_subject"), + "ref_source": ref_source, + } + + # Missing warning. + with self.assertLogs(logger, level="DEBUG") as log: + result = CFUGridAuxiliaryCoordinateVariable.identify( + vars_all, warn=False + ) + self.assertEqual(0, len(log.output)) + self.assertDictEqual({}, result) + + # Default is warn=True + result = CFUGridAuxiliaryCoordinateVariable.identify(vars_all) + self.assertIn( + f"Missing CF-netCDF auxiliary coordinate variable {subject_name}", + log.output[0], + ) + self.assertDictEqual({}, result) + + # String variable warning. + with self.assertLogs(logger, level="DEBUG") as log: + vars_all[subject_name] = netcdf_ugrid_variable( + subject_name, "", np.bytes_ + ) + result = CFUGridAuxiliaryCoordinateVariable.identify( + vars_all, warn=False + ) + self.assertDictEqual({}, result) + + # Default is warn=True + result = CFUGridAuxiliaryCoordinateVariable.identify(vars_all) + self.assertIn("is a CF-netCDF label variable", log.output[0]) + self.assertDictEqual({}, result) diff --git a/lib/iris/tests/unit/experimental/ugrid/test_CFUGridConnectivityVariable.py b/lib/iris/tests/unit/experimental/ugrid/test_CFUGridConnectivityVariable.py new file mode 100644 index 0000000000..5cf40026cb --- /dev/null +++ b/lib/iris/tests/unit/experimental/ugrid/test_CFUGridConnectivityVariable.py @@ -0,0 +1,230 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +""" +Unit tests for the :class:`iris.experimental.ugrid.CFUGridConnectivityVariable` class. + +todo: fold these tests into cf tests when experimental.ugrid is folded into + standard behaviour. + +""" +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests + +import numpy as np + +from iris.experimental.ugrid import ( + CFUGridConnectivityVariable, + Connectivity, + logger, +) +from iris.tests.unit.experimental.ugrid.test_CFUGridReader import ( + netcdf_ugrid_variable, +) + + +def named_variable(name): + # Don't need to worry about dimensions or dtype for these tests. + return netcdf_ugrid_variable(name, "", np.int) + + +class TestIdentify(tests.IrisTest): + def test_cf_identities(self): + subject_name = "ref_subject" + ref_subject = named_variable(subject_name) + vars_common = { + subject_name: ref_subject, + "ref_not_subject": named_variable("ref_not_subject"), + } + # ONLY expecting ref_subject, excluding ref_not_subject. + expected = { + subject_name: CFUGridConnectivityVariable( + subject_name, ref_subject + ) + } + + for identity in Connectivity.UGRID_CF_ROLES: + ref_source = named_variable("ref_source") + setattr(ref_source, identity, subject_name) + vars_all = dict({"ref_source": ref_source}, **vars_common) + result = CFUGridConnectivityVariable.identify(vars_all) + self.assertDictEqual(expected, result) + + def test_duplicate_refs(self): + subject_name = "ref_subject" + ref_subject = named_variable(subject_name) + ref_source_vars = { + name: named_variable(name) + for name in ("ref_source_1", "ref_source_2") + } + for var in ref_source_vars.values(): + setattr(var, Connectivity.UGRID_CF_ROLES[0], subject_name) + vars_all = dict( + { + subject_name: ref_subject, + "ref_not_subject": named_variable("ref_not_subject"), + }, + **ref_source_vars, + ) + + # ONLY expecting ref_subject, excluding ref_not_subject. + expected = { + subject_name: CFUGridConnectivityVariable( + subject_name, ref_subject + ) + } + result = CFUGridConnectivityVariable.identify(vars_all) + self.assertDictEqual(expected, result) + + def test_two_cf_roles(self): + subject_names = ("ref_subject_1", "ref_subject_2") + ref_subject_vars = { + name: named_variable(name) for name in subject_names + } + + ref_source_vars = { + name: named_variable(name) + for name in ("ref_source_1", "ref_source_2") + } + for ix, var in enumerate(ref_source_vars.values()): + setattr(var, Connectivity.UGRID_CF_ROLES[ix], subject_names[ix]) + vars_all = dict( + {"ref_not_subject": named_variable("ref_not_subject")}, + **ref_subject_vars, + **ref_source_vars, + ) + + # Not expecting ref_not_subject. + expected = { + name: CFUGridConnectivityVariable(name, var) + for name, var in ref_subject_vars.items() + } + result = CFUGridConnectivityVariable.identify(vars_all) + self.assertDictEqual(expected, result) + + def test_two_part_ref_ignored(self): + # Not expected to handle more than one variable for a connectivity + # cf role - invalid UGRID. + subject_name = "ref_subject" + ref_source = named_variable("ref_source") + setattr( + ref_source, Connectivity.UGRID_CF_ROLES[0], subject_name + " foo" + ) + vars_all = { + subject_name: named_variable(subject_name), + "ref_not_subject": named_variable("ref_not_subject"), + "ref_source": ref_source, + } + + result = CFUGridConnectivityVariable.identify(vars_all) + self.assertDictEqual({}, result) + + def test_string_type_ignored(self): + subject_name = "ref_subject" + ref_source = named_variable("ref_source") + setattr(ref_source, Connectivity.UGRID_CF_ROLES[0], subject_name) + vars_all = { + subject_name: netcdf_ugrid_variable(subject_name, "", np.bytes_), + "ref_not_subject": named_variable("ref_not_subject"), + "ref_source": ref_source, + } + + result = CFUGridConnectivityVariable.identify(vars_all) + self.assertDictEqual({}, result) + + def test_ignore(self): + subject_names = ("ref_subject_1", "ref_subject_2") + ref_subject_vars = { + name: named_variable(name) for name in subject_names + } + + ref_source_vars = { + name: named_variable(name) + for name in ("ref_source_1", "ref_source_2") + } + for ix, var in enumerate(ref_source_vars.values()): + setattr(var, Connectivity.UGRID_CF_ROLES[0], subject_names[ix]) + vars_all = dict( + {"ref_not_subject": named_variable("ref_not_subject")}, + **ref_subject_vars, + **ref_source_vars, + ) + + # ONLY expect the subject variable that hasn't been ignored. + expected_name = subject_names[0] + expected = { + expected_name: CFUGridConnectivityVariable( + expected_name, ref_subject_vars[expected_name] + ) + } + result = CFUGridConnectivityVariable.identify( + vars_all, ignore=subject_names[1] + ) + self.assertDictEqual(expected, result) + + def test_target(self): + subject_names = ("ref_subject_1", "ref_subject_2") + ref_subject_vars = { + name: named_variable(name) for name in subject_names + } + + source_names = ("ref_source_1", "ref_source_2") + ref_source_vars = {name: named_variable(name) for name in source_names} + for ix, var in enumerate(ref_source_vars.values()): + setattr(var, Connectivity.UGRID_CF_ROLES[0], subject_names[ix]) + vars_all = dict( + {"ref_not_subject": named_variable("ref_not_subject")}, + **ref_subject_vars, + **ref_source_vars, + ) + + # ONLY expect the variable referenced by the named ref_source_var. + expected_name = subject_names[0] + expected = { + expected_name: CFUGridConnectivityVariable( + expected_name, ref_subject_vars[expected_name] + ) + } + result = CFUGridConnectivityVariable.identify( + vars_all, target=source_names[0] + ) + self.assertDictEqual(expected, result) + + def test_warn(self): + subject_name = "ref_subject" + ref_source = named_variable("ref_source") + setattr(ref_source, Connectivity.UGRID_CF_ROLES[0], subject_name) + vars_all = { + "ref_not_subject": named_variable("ref_not_subject"), + "ref_source": ref_source, + } + + # Missing warning. + with self.assertLogs(logger, level="DEBUG") as log: + result = CFUGridConnectivityVariable.identify(vars_all, warn=False) + self.assertEqual(0, len(log.output)) + self.assertDictEqual({}, result) + + # Default is warn=True + result = CFUGridConnectivityVariable.identify(vars_all) + self.assertIn( + f"Missing CF-UGRID connectivity variable {subject_name}", + log.output[0], + ) + self.assertDictEqual({}, result) + + # String variable warning. + with self.assertLogs(logger, level="DEBUG") as log: + vars_all[subject_name] = netcdf_ugrid_variable( + subject_name, "", np.bytes_ + ) + result = CFUGridConnectivityVariable.identify(vars_all, warn=False) + self.assertDictEqual({}, result) + + # Default is warn=True + result = CFUGridConnectivityVariable.identify(vars_all) + self.assertIn("is a CF-netCDF label variable", log.output[0]) + self.assertDictEqual({}, result) diff --git a/lib/iris/tests/unit/experimental/ugrid/test_CFUGridGroup.py b/lib/iris/tests/unit/experimental/ugrid/test_CFUGridGroup.py new file mode 100644 index 0000000000..1210c4b83c --- /dev/null +++ b/lib/iris/tests/unit/experimental/ugrid/test_CFUGridGroup.py @@ -0,0 +1,99 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +""" +Unit tests for the :class:`iris.experimental.ugrid.CFUGridGroup` class. + +todo: fold these tests into cf tests when experimental.ugrid is folded into + standard behaviour. + +""" +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests + +from iris.experimental.ugrid import ( + CFUGridGroup, + CFUGridAuxiliaryCoordinateVariable, + CFUGridConnectivityVariable, + CFUGridMeshVariable, +) +from iris.fileformats.cf import CFCoordinateVariable, CFDataVariable + +from unittest.mock import MagicMock + + +class Tests(tests.IrisTest): + def setUp(self): + self.cf_group = CFUGridGroup() + + def test_inherited(self): + coord_var = MagicMock(spec=CFCoordinateVariable, cf_name="coord_var") + self.cf_group[coord_var.cf_name] = coord_var + self.assertEqual( + coord_var, self.cf_group.coordinates[coord_var.cf_name] + ) + + def test_connectivities(self): + conn_var = MagicMock( + spec=CFUGridConnectivityVariable, cf_name="conn_var" + ) + self.cf_group[conn_var.cf_name] = conn_var + self.assertEqual( + conn_var, self.cf_group.connectivities[conn_var.cf_name] + ) + + def test_ugrid_coords(self): + coord_var = MagicMock( + spec=CFUGridAuxiliaryCoordinateVariable, cf_name="coord_var" + ) + self.cf_group[coord_var.cf_name] = coord_var + self.assertEqual( + coord_var, self.cf_group.ugrid_coords[coord_var.cf_name] + ) + + def test_meshes(self): + mesh_var = MagicMock(spec=CFUGridMeshVariable, cf_name="mesh_var") + self.cf_group[mesh_var.cf_name] = mesh_var + self.assertEqual(mesh_var, self.cf_group.meshes[mesh_var.cf_name]) + + def test_non_data_names(self): + data_var = MagicMock(spec=CFDataVariable, cf_name="data_var") + coord_var = MagicMock(spec=CFCoordinateVariable, cf_name="coord_var") + conn_var = MagicMock( + spec=CFUGridConnectivityVariable, cf_name="conn_var" + ) + ugrid_coord_var = MagicMock( + spec=CFUGridAuxiliaryCoordinateVariable, cf_name="ugrid_coord_var" + ) + mesh_var = MagicMock(spec=CFUGridMeshVariable, cf_name="mesh_var") + mesh_var2 = MagicMock(spec=CFUGridMeshVariable, cf_name="mesh_var2") + duplicate_name_var = MagicMock( + spec=CFUGridMeshVariable, cf_name="coord_var" + ) + + for var in ( + data_var, + coord_var, + conn_var, + ugrid_coord_var, + mesh_var, + mesh_var2, + duplicate_name_var, + ): + self.cf_group[var.cf_name] = var + + expected_names = [ + var.cf_name + for var in ( + coord_var, + conn_var, + ugrid_coord_var, + mesh_var, + mesh_var2, + ) + ] + expected = set(expected_names) + self.assertEqual(expected, self.cf_group.non_data_variable_names) diff --git a/lib/iris/tests/unit/experimental/ugrid/test_CFUGridMeshVariable.py b/lib/iris/tests/unit/experimental/ugrid/test_CFUGridMeshVariable.py new file mode 100644 index 0000000000..79601b8604 --- /dev/null +++ b/lib/iris/tests/unit/experimental/ugrid/test_CFUGridMeshVariable.py @@ -0,0 +1,222 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +""" +Unit tests for the :class:`iris.experimental.ugrid.CFUGridMeshVariable` class. + +todo: fold these tests into cf tests when experimental.ugrid is folded into + standard behaviour. + +""" +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests + +import numpy as np + +from iris.experimental.ugrid import ( + CFUGridMeshVariable, + logger, +) +from iris.tests.unit.experimental.ugrid.test_CFUGridReader import ( + netcdf_ugrid_variable, +) + + +def named_variable(name): + # Don't need to worry about dimensions or dtype for these tests. + return netcdf_ugrid_variable(name, "", np.int) + + +class TestIdentify(tests.IrisTest): + def setUp(self): + self.cf_identity = "mesh" + + def test_cf_identity(self): + subject_name = "ref_subject" + ref_subject = named_variable(subject_name) + ref_source = named_variable("ref_source") + setattr(ref_source, self.cf_identity, subject_name) + vars_all = { + subject_name: ref_subject, + "ref_not_subject": named_variable("ref_not_subject"), + "ref_source": ref_source, + } + + # ONLY expecting ref_subject, excluding ref_not_subject. + expected = { + subject_name: CFUGridMeshVariable(subject_name, ref_subject) + } + result = CFUGridMeshVariable.identify(vars_all) + self.assertDictEqual(expected, result) + + def test_duplicate_refs(self): + subject_name = "ref_subject" + ref_subject = named_variable(subject_name) + ref_source_vars = { + name: named_variable(name) + for name in ("ref_source_1", "ref_source_2") + } + for var in ref_source_vars.values(): + setattr(var, self.cf_identity, subject_name) + vars_all = dict( + { + subject_name: ref_subject, + "ref_not_subject": named_variable("ref_not_subject"), + }, + **ref_source_vars, + ) + + # ONLY expecting ref_subject, excluding ref_not_subject. + expected = { + subject_name: CFUGridMeshVariable(subject_name, ref_subject) + } + result = CFUGridMeshVariable.identify(vars_all) + self.assertDictEqual(expected, result) + + def test_two_refs(self): + subject_names = ("ref_subject_1", "ref_subject_2") + ref_subject_vars = { + name: named_variable(name) for name in subject_names + } + + ref_source_vars = { + name: named_variable(name) + for name in ("ref_source_1", "ref_source_2") + } + for ix, var in enumerate(ref_source_vars.values()): + setattr(var, self.cf_identity, subject_names[ix]) + vars_all = dict( + {"ref_not_subject": named_variable("ref_not_subject")}, + **ref_subject_vars, + **ref_source_vars, + ) + + # Not expecting ref_not_subject. + expected = { + name: CFUGridMeshVariable(name, var) + for name, var in ref_subject_vars.items() + } + result = CFUGridMeshVariable.identify(vars_all) + self.assertDictEqual(expected, result) + + def test_two_part_ref_ignored(self): + # Not expected to handle more than one variable for a mesh + # cf role - invalid UGRID. + subject_name = "ref_subject" + ref_source = named_variable("ref_source") + setattr(ref_source, self.cf_identity, subject_name + " foo") + vars_all = { + subject_name: named_variable(subject_name), + "ref_not_subject": named_variable("ref_not_subject"), + "ref_source": ref_source, + } + + result = CFUGridMeshVariable.identify(vars_all) + self.assertDictEqual({}, result) + + def test_string_type_ignored(self): + subject_name = "ref_subject" + ref_source = named_variable("ref_source") + setattr(ref_source, self.cf_identity, subject_name) + vars_all = { + subject_name: netcdf_ugrid_variable(subject_name, "", np.bytes_), + "ref_not_subject": named_variable("ref_not_subject"), + "ref_source": ref_source, + } + + result = CFUGridMeshVariable.identify(vars_all) + self.assertDictEqual({}, result) + + def test_ignore(self): + subject_names = ("ref_subject_1", "ref_subject_2") + ref_subject_vars = { + name: named_variable(name) for name in subject_names + } + + ref_source_vars = { + name: named_variable(name) + for name in ("ref_source_1", "ref_source_2") + } + for ix, var in enumerate(ref_source_vars.values()): + setattr(var, self.cf_identity, subject_names[ix]) + vars_all = dict( + {"ref_not_subject": named_variable("ref_not_subject")}, + **ref_subject_vars, + **ref_source_vars, + ) + + # ONLY expect the subject variable that hasn't been ignored. + expected_name = subject_names[0] + expected = { + expected_name: CFUGridMeshVariable( + expected_name, ref_subject_vars[expected_name] + ) + } + result = CFUGridMeshVariable.identify( + vars_all, ignore=subject_names[1] + ) + self.assertDictEqual(expected, result) + + def test_target(self): + subject_names = ("ref_subject_1", "ref_subject_2") + ref_subject_vars = { + name: named_variable(name) for name in subject_names + } + + source_names = ("ref_source_1", "ref_source_2") + ref_source_vars = {name: named_variable(name) for name in source_names} + for ix, var in enumerate(ref_source_vars.values()): + setattr(var, self.cf_identity, subject_names[ix]) + vars_all = dict( + {"ref_not_subject": named_variable("ref_not_subject")}, + **ref_subject_vars, + **ref_source_vars, + ) + + # ONLY expect the variable referenced by the named ref_source_var. + expected_name = subject_names[0] + expected = { + expected_name: CFUGridMeshVariable( + expected_name, ref_subject_vars[expected_name] + ) + } + result = CFUGridMeshVariable.identify(vars_all, target=source_names[0]) + self.assertDictEqual(expected, result) + + def test_warn(self): + subject_name = "ref_subject" + ref_source = named_variable("ref_source") + setattr(ref_source, self.cf_identity, subject_name) + vars_all = { + "ref_not_subject": named_variable("ref_not_subject"), + "ref_source": ref_source, + } + + # Missing warning. + with self.assertLogs(logger, level="DEBUG") as log: + result = CFUGridMeshVariable.identify(vars_all, warn=False) + self.assertEqual(0, len(log.output)) + self.assertDictEqual({}, result) + + # Default is warn=True + result = CFUGridMeshVariable.identify(vars_all) + self.assertIn( + f"Missing CF-UGRID mesh variable {subject_name}", log.output[0] + ) + self.assertDictEqual({}, result) + + # String variable warning. + with self.assertLogs(logger, level="DEBUG") as log: + vars_all[subject_name] = netcdf_ugrid_variable( + subject_name, "", np.bytes_ + ) + result = CFUGridMeshVariable.identify(vars_all, warn=False) + self.assertDictEqual({}, result) + + # Default is warn=True + result = CFUGridMeshVariable.identify(vars_all) + self.assertIn("is a CF-netCDF label variable", log.output[0]) + self.assertDictEqual({}, result) diff --git a/lib/iris/tests/unit/experimental/ugrid/test_CFUGridReader.py b/lib/iris/tests/unit/experimental/ugrid/test_CFUGridReader.py new file mode 100644 index 0000000000..237b4cf82b --- /dev/null +++ b/lib/iris/tests/unit/experimental/ugrid/test_CFUGridReader.py @@ -0,0 +1,135 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +""" +Unit tests for the :class:`iris.experimental.ugrid.CFUGridGroup` class. + +todo: fold these tests into cf tests when experimental.ugrid is folded into + standard behaviour. + +""" +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests + +import numpy as np + +from iris.experimental.ugrid import ( + CFUGridGroup, + CFUGridReader, + CFUGridAuxiliaryCoordinateVariable, + CFUGridConnectivityVariable, + CFUGridMeshVariable, +) +from iris.fileformats.cf import CFCoordinateVariable, CFDataVariable +from iris.tests.unit.fileformats.cf.test_CFReader import netcdf_variable + + +from unittest import mock + + +def netcdf_ugrid_variable( + name, + dimensions, + dtype, + coordinates=None, +): + ncvar = netcdf_variable( + name=name, dimensions=dimensions, dtype=dtype, coordinates=coordinates + ) + + # Fill in all the extra UGRID attributes to prevent problems with getattr + # and Mock. Any attribute can be replaced in downstream setUp if present. + ugrid_attrs = ( + CFUGridAuxiliaryCoordinateVariable.cf_identities + + CFUGridConnectivityVariable.cf_identities + + [CFUGridMeshVariable.cf_identity] + ) + for attr in ugrid_attrs: + setattr(ncvar, attr, None) + + return ncvar + + +class Test_build_cf_groups(tests.IrisTest): + @classmethod + def setUpClass(cls): + # Replicating syntax from test_CFReader.Test_build_cf_groups__formula_terms. + cls.mesh = netcdf_ugrid_variable("mesh", "", np.int) + cls.node_x = netcdf_ugrid_variable("node_x", "node", np.float) + cls.node_y = netcdf_ugrid_variable("node_y", "node", np.float) + cls.face_x = netcdf_ugrid_variable("face_x", "face", np.float) + cls.face_y = netcdf_ugrid_variable("face_y", "face", np.float) + cls.face_nodes = netcdf_ugrid_variable( + "face_nodes", "face vertex", np.int + ) + cls.levels = netcdf_ugrid_variable("levels", "levels", np.int) + cls.data = netcdf_ugrid_variable( + "data", "levels face", np.float, coordinates="face_x face_y" + ) + + # Add necessary attributes for mesh recognition. + cls.mesh.cf_role = "mesh_topology" + cls.mesh.node_coordinates = "node_x node_y" + cls.mesh.face_coordinates = "face_x face_y" + cls.mesh.face_node_connectivity = "face_nodes" + cls.face_nodes.cf_role = "face_node_connectivity" + cls.data.mesh = "mesh" + + cls.variables = dict( + mesh=cls.mesh, + node_x=cls.node_x, + node_y=cls.node_y, + face_x=cls.face_x, + face_y=cls.face_y, + face_nodes=cls.face_nodes, + levels=cls.levels, + data=cls.data, + ) + ncattrs = mock.Mock(return_value=[]) + cls.dataset = mock.Mock( + file_format="NetCDF4", variables=cls.variables, ncattrs=ncattrs + ) + + def setUp(self): + # Restrict the CFUGridReader functionality to only performing + # translations and building first level cf-groups for variables. + self.patch("iris.experimental.ugrid.CFUGridReader._reset") + self.patch("netCDF4.Dataset", return_value=self.dataset) + cf_reader = CFUGridReader("dummy") + self.cf_group = cf_reader.cf_group + + def test_inherited(self): + for expected_var, collection in ( + [CFCoordinateVariable("levels", self.levels), "coordinates"], + [CFDataVariable("data", self.data), "data_variables"], + ): + expected = {expected_var.cf_name: expected_var} + self.assertDictEqual(expected, getattr(self.cf_group, collection)) + + def test_connectivities(self): + expected_var = CFUGridConnectivityVariable( + "face_nodes", self.face_nodes + ) + expected = {expected_var.cf_name: expected_var} + self.assertDictEqual(expected, self.cf_group.connectivities) + + def test_mesh(self): + expected_var = CFUGridMeshVariable("mesh", self.mesh) + expected = {expected_var.cf_name: expected_var} + self.assertDictEqual(expected, self.cf_group.meshes) + + def test_ugrid_coords(self): + names = [ + f"{loc}_{ax}" for loc in ("node", "face") for ax in ("x", "y") + ] + expected = { + name: CFUGridAuxiliaryCoordinateVariable(name, getattr(self, name)) + for name in names + } + self.assertDictEqual(expected, self.cf_group.ugrid_coords) + + def test_is_cf_ugrid_group(self): + self.assertIsInstance(self.cf_group, CFUGridGroup) diff --git a/lib/iris/tests/unit/experimental/ugrid/test_Mesh.py b/lib/iris/tests/unit/experimental/ugrid/test_Mesh.py index c487494271..9848809206 100644 --- a/lib/iris/tests/unit/experimental/ugrid/test_Mesh.py +++ b/lib/iris/tests/unit/experimental/ugrid/test_Mesh.py @@ -848,6 +848,37 @@ def test_remove_coords(self): self.mesh.remove_coords(self.EDGE_LON) self.assertEqual(None, self.mesh.edge_coords.edge_x) + def test_to_MeshCoord(self): + location = "node" + axis = "x" + result = self.mesh.to_MeshCoord(location, axis) + self.assertIsInstance(result, ugrid.MeshCoord) + self.assertEqual(location, result.location) + self.assertEqual(axis, result.axis) + + def test_to_MeshCoord_face(self): + location = "face" + axis = "x" + self.assertRaises( + CoordinateNotFoundError, self.mesh.to_MeshCoord, location, axis + ) + + def test_to_MeshCoords(self): + location = "node" + result = self.mesh.to_MeshCoords(location) + self.assertEqual(len(self.mesh.AXES), len(result)) + for ix, axis in enumerate(self.mesh.AXES): + coord = result[ix] + self.assertIsInstance(coord, ugrid.MeshCoord) + self.assertEqual(location, coord.location) + self.assertEqual(axis, coord.axis) + + def test_to_MeshCoords_face(self): + location = "face" + self.assertRaises( + CoordinateNotFoundError, self.mesh.to_MeshCoords, location + ) + class TestOperations2D(TestOperations1D): # Additional/specialised tests for topology_dimension=2. @@ -1004,6 +1035,26 @@ def test_remove_coords(self): self.mesh.remove_coords(include_faces=True) self.assertEqual(None, self.mesh.face_coords.face_x) + def test_to_MeshCoord_face(self): + self.mesh.add_coords(face_x=self.FACE_LON) + location = "face" + axis = "x" + result = self.mesh.to_MeshCoord(location, axis) + self.assertIsInstance(result, ugrid.MeshCoord) + self.assertEqual(location, result.location) + self.assertEqual(axis, result.axis) + + def test_to_MeshCoords_face(self): + self.mesh.add_coords(face_x=self.FACE_LON, face_y=self.FACE_LAT) + location = "face" + result = self.mesh.to_MeshCoords(location) + self.assertEqual(len(self.mesh.AXES), len(result)) + for ix, axis in enumerate(self.mesh.AXES): + coord = result[ix] + self.assertIsInstance(coord, ugrid.MeshCoord) + self.assertEqual(location, coord.location) + self.assertEqual(axis, coord.axis) + class InitValidation(TestMeshCommon): def test_invalid_topology(self): diff --git a/lib/iris/tests/unit/experimental/ugrid/test_ParseUgridOnLoad.py b/lib/iris/tests/unit/experimental/ugrid/test_ParseUgridOnLoad.py new file mode 100644 index 0000000000..d60cdfdd3b --- /dev/null +++ b/lib/iris/tests/unit/experimental/ugrid/test_ParseUgridOnLoad.py @@ -0,0 +1,46 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +""" +Unit tests for the :class:`iris.experimental.ugrid.ParseUgridOnLoad` class. + +todo: remove this module when experimental.ugrid is folded into standard behaviour. + +""" +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests + +from iris.experimental.ugrid import ParseUGridOnLoad, PARSE_UGRID_ON_LOAD + + +class TestClass(tests.IrisTest): + @classmethod + def setUpClass(cls): + cls.cls = ParseUGridOnLoad() + + def test_default(self): + self.assertFalse(self.cls) + + def test_context(self): + self.assertFalse(self.cls) + with self.cls.context(): + self.assertTrue(self.cls) + self.assertFalse(self.cls) + + +class TestConstant(tests.IrisTest): + @classmethod + def setUpClass(cls): + cls.constant = PARSE_UGRID_ON_LOAD + + def test_default(self): + self.assertFalse(self.constant) + + def test_context(self): + self.assertFalse(self.constant) + with self.constant.context(): + self.assertTrue(self.constant) + self.assertFalse(self.constant) diff --git a/lib/iris/tests/unit/fileformats/cf/test_CFGroup.py b/lib/iris/tests/unit/fileformats/cf/test_CFGroup.py new file mode 100644 index 0000000000..6cfb566f34 --- /dev/null +++ b/lib/iris/tests/unit/fileformats/cf/test_CFGroup.py @@ -0,0 +1,51 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +"""Unit tests for the :class:`iris.fileformats.cf.CFGroup` class.""" + +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests + +from iris.fileformats.cf import ( + CFGroup, + CFAuxiliaryCoordinateVariable, + CFCoordinateVariable, + CFDataVariable, +) + +from unittest.mock import MagicMock + + +class Tests(tests.IrisTest): + # TODO: unit tests for existing functionality pre 2021-03-11. + def setUp(self): + self.cf_group = CFGroup() + + def test_non_data_names(self): + data_var = MagicMock(spec=CFDataVariable, cf_name="data_var") + aux_var = MagicMock( + spec=CFAuxiliaryCoordinateVariable, cf_name="aux_var" + ) + coord_var = MagicMock(spec=CFCoordinateVariable, cf_name="coord_var") + coord_var2 = MagicMock(spec=CFCoordinateVariable, cf_name="coord_var2") + duplicate_name_var = MagicMock( + spec=CFCoordinateVariable, cf_name="aux_var" + ) + + for var in ( + data_var, + aux_var, + coord_var, + coord_var2, + duplicate_name_var, + ): + self.cf_group[var.cf_name] = var + + expected_names = [ + var.cf_name for var in (aux_var, coord_var, coord_var2) + ] + expected = set(expected_names) + self.assertEqual(expected, self.cf_group.non_data_variable_names) diff --git a/lib/iris/tests/unit/fileformats/netcdf/test_load_cubes.py b/lib/iris/tests/unit/fileformats/netcdf/test_load_cubes.py new file mode 100644 index 0000000000..5ed908bfd0 --- /dev/null +++ b/lib/iris/tests/unit/fileformats/netcdf/test_load_cubes.py @@ -0,0 +1,319 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the LGPL license. +# See COPYING and COPYING.LESSER in the root of the repository for full +# licensing details. +""" +Unit tests for the :func:`iris.fileformats.netcdf.load_cubes` function. + +todo: migrate the remaining unit-esque tests from iris.tests.test_netcdf, + switching to use netcdf.load_cubes() instead of iris.load()/load_cube(). + +""" + +# Import iris.tests first so that some things can be initialised before +# importing anything else. +import iris.tests as tests + +from cf_units import as_unit +import numpy as np +from pathlib import Path +from shutil import rmtree +from subprocess import check_call +import tempfile + +from iris.coords import AncillaryVariable, CellMeasure +from iris.experimental.ugrid import PARSE_UGRID_ON_LOAD, MeshCoord +from iris.fileformats.netcdf import load_cubes, logger + + +def setUpModule(): + global TMP_DIR + TMP_DIR = Path(tempfile.mkdtemp()) + + +def tearDownModule(): + if TMP_DIR is not None: + rmtree(TMP_DIR) + + +def cdl_to_nc(cdl): + cdl_path = TMP_DIR / "tst.cdl" + nc_path = TMP_DIR / "tst.nc" + # Write CDL string into a temporary CDL file. + with open(cdl_path, "w") as f_out: + f_out.write(cdl) + # Use ncgen to convert this into an actual (temporary) netCDF file. + command = "ncgen -o {} {}".format(nc_path, cdl_path) + check_call(command, shell=True) + return str(nc_path) + + +class Tests(tests.IrisTest): + def test_ancillary_variables(self): + # Note: using a CDL string as a test data reference, rather than a + # binary file. + ref_cdl = """ + netcdf cm_attr { + dimensions: + axv = 3 ; + variables: + int64 qqv(axv) ; + qqv:long_name = "qq" ; + qqv:units = "1" ; + qqv:ancillary_variables = "my_av" ; + int64 axv(axv) ; + axv:units = "1" ; + axv:long_name = "x" ; + double my_av(axv) ; + my_av:units = "1" ; + my_av:long_name = "refs" ; + my_av:custom = "extra-attribute"; + data: + axv = 1, 2, 3; + my_av = 11., 12., 13.; + } + """ + nc_path = cdl_to_nc(ref_cdl) + + # Load with iris.fileformats.netcdf.load_cubes, and check expected content. + cubes = list(load_cubes(nc_path)) + self.assertEqual(len(cubes), 1) + avs = cubes[0].ancillary_variables() + self.assertEqual(len(avs), 1) + expected = AncillaryVariable( + np.ma.array([11.0, 12.0, 13.0]), + long_name="refs", + var_name="my_av", + units="1", + attributes={"custom": "extra-attribute"}, + ) + self.assertEqual(avs[0], expected) + + def test_status_flags(self): + # Note: using a CDL string as a test data reference, rather than a binary file. + ref_cdl = """ + netcdf cm_attr { + dimensions: + axv = 3 ; + variables: + int64 qqv(axv) ; + qqv:long_name = "qq" ; + qqv:units = "1" ; + qqv:ancillary_variables = "my_av" ; + int64 axv(axv) ; + axv:units = "1" ; + axv:long_name = "x" ; + byte my_av(axv) ; + my_av:long_name = "qq status_flag" ; + my_av:flag_values = 1b, 2b ; + my_av:flag_meanings = "a b" ; + data: + axv = 11, 21, 31; + my_av = 1b, 1b, 2b; + } + """ + nc_path = cdl_to_nc(ref_cdl) + + # Load with iris.fileformats.netcdf.load_cubes, and check expected content. + cubes = list(load_cubes(nc_path)) + self.assertEqual(len(cubes), 1) + avs = cubes[0].ancillary_variables() + self.assertEqual(len(avs), 1) + expected = AncillaryVariable( + np.ma.array([1, 1, 2], dtype=np.int8), + long_name="qq status_flag", + var_name="my_av", + units="no_unit", + attributes={ + "flag_values": np.array([1, 2], dtype=np.int8), + "flag_meanings": "a b", + }, + ) + self.assertEqual(avs[0], expected) + + def test_cell_measures(self): + # Note: using a CDL string as a test data reference, rather than a binary file. + ref_cdl = """ + netcdf cm_attr { + dimensions: + axv = 3 ; + ayv = 2 ; + variables: + int64 qqv(ayv, axv) ; + qqv:long_name = "qq" ; + qqv:units = "1" ; + qqv:cell_measures = "area: my_areas" ; + int64 ayv(ayv) ; + ayv:units = "1" ; + ayv:long_name = "y" ; + int64 axv(axv) ; + axv:units = "1" ; + axv:long_name = "x" ; + double my_areas(ayv, axv) ; + my_areas:units = "m2" ; + my_areas:long_name = "standardised cell areas" ; + my_areas:custom = "extra-attribute"; + data: + axv = 11, 12, 13; + ayv = 21, 22; + my_areas = 110., 120., 130., 221., 231., 241.; + } + """ + nc_path = cdl_to_nc(ref_cdl) + + # Load with iris.fileformats.netcdf.load_cubes, and check expected content. + cubes = list(load_cubes(nc_path)) + self.assertEqual(len(cubes), 1) + cms = cubes[0].cell_measures() + self.assertEqual(len(cms), 1) + expected = CellMeasure( + np.ma.array([[110.0, 120.0, 130.0], [221.0, 231.0, 241.0]]), + measure="area", + var_name="my_areas", + long_name="standardised cell areas", + units="m2", + attributes={"custom": "extra-attribute"}, + ) + self.assertEqual(cms[0], expected) + + def test_default_units(self): + # Note: using a CDL string as a test data reference, rather than a binary file. + ref_cdl = """ + netcdf cm_attr { + dimensions: + axv = 3 ; + ayv = 2 ; + variables: + int64 qqv(ayv, axv) ; + qqv:long_name = "qq" ; + qqv:ancillary_variables = "my_av" ; + qqv:cell_measures = "area: my_areas" ; + int64 ayv(ayv) ; + ayv:long_name = "y" ; + int64 axv(axv) ; + axv:units = "1" ; + axv:long_name = "x" ; + double my_av(axv) ; + my_av:long_name = "refs" ; + double my_areas(ayv, axv) ; + my_areas:long_name = "areas" ; + data: + axv = 11, 12, 13; + ayv = 21, 22; + my_areas = 110., 120., 130., 221., 231., 241.; + } + """ + nc_path = cdl_to_nc(ref_cdl) + + # Load with iris.fileformats.netcdf.load_cubes, and check expected content. + cubes = list(load_cubes(nc_path)) + self.assertEqual(len(cubes), 1) + self.assertEqual(cubes[0].units, as_unit("unknown")) + self.assertEqual(cubes[0].coord("y").units, as_unit("unknown")) + self.assertEqual(cubes[0].coord("x").units, as_unit(1)) + self.assertEqual( + cubes[0].ancillary_variable("refs").units, as_unit("unknown") + ) + self.assertEqual( + cubes[0].cell_measure("areas").units, as_unit("unknown") + ) + + +class TestsMesh(tests.IrisTest): + @classmethod + def setUpClass(cls): + cls.ref_cdl = """ + netcdf mesh_test { + dimensions: + node = 3 ; + face = 1 ; + vertex = 3 ; + levels = 2 ; + variables: + int mesh ; + mesh:cf_role = "mesh_topology" ; + mesh:topology_dimension = 2 ; + mesh:node_coordinates = "node_x node_y" ; + mesh:face_coordinates = "face_x face_y" ; + mesh:face_node_connectivity = "face_nodes" ; + float node_x(node) ; + node_x:standard_name = "longitude" ; + float node_y(node) ; + node_y:standard_name = "latitude" ; + float face_x(face) ; + face_x:standard_name = "longitude" ; + float face_y(face) ; + face_y:standard_name = "latitude" ; + int face_nodes(face, vertex) ; + face_nodes:cf_role = "face_node_connectivity" ; + face_nodes:start_index = 0 ; + int levels(levels) ; + float node_data(levels, node) ; + node_data:coordinates = "node_x node_y" ; + node_data:location = "node" ; + node_data:mesh = "mesh" ; + float face_data(levels, face) ; + face_data:coordinates = "face_x face_y" ; + face_data:location = "face" ; + face_data:mesh = "mesh" ; + data: + mesh = 0; + node_x = 0., 2., 1.; + node_y = 0., 0., 1.; + face_x = 0.5; + face_y = 0.5; + face_nodes = 0, 1, 2; + levels = 1, 2; + node_data = 0., 0., 0.; + face_data = 0.; + } + """ + cls.nc_path = cdl_to_nc(cls.ref_cdl) + with PARSE_UGRID_ON_LOAD.context(): + cls.mesh_cubes = list(load_cubes(cls.nc_path)) + + def test_mesh_handled(self): + cubes_no_ugrid = list(load_cubes(self.nc_path)) + self.assertEqual(4, len(cubes_no_ugrid)) + self.assertEqual(2, len(self.mesh_cubes)) + + def test_standard_dims(self): + for cube in self.mesh_cubes: + self.assertIsNotNone(cube.coords("levels")) + + def test_mesh_coord(self): + cube = [ + cube for cube in self.mesh_cubes if cube.var_name == "face_data" + ][0] + face_x = cube.coord("longitude") + face_y = cube.coord("latitude") + + for coord in (face_x, face_y): + self.assertIsInstance(coord, MeshCoord) + self.assertEqual("face", coord.location) + self.assertArrayEqual(np.ma.array([0.5]), coord.points) + + self.assertEqual("x", face_x.axis) + self.assertEqual("y", face_y.axis) + self.assertEqual(face_x.mesh, face_y.mesh) + self.assertArrayEqual(np.ma.array([[0.0, 2.0, 1.0]]), face_x.bounds) + self.assertArrayEqual(np.ma.array([[0.0, 0.0, 1.0]]), face_y.bounds) + + def test_shared_mesh(self): + cube_meshes = [cube.coord("latitude").mesh for cube in self.mesh_cubes] + self.assertEqual(cube_meshes[0], cube_meshes[1]) + + def test_missing_mesh(self): + ref_cdl = self.ref_cdl.replace( + 'face_data:mesh = "mesh"', 'face_data:mesh = "mesh2"' + ) + nc_path = cdl_to_nc(ref_cdl) + + # No error when mesh handling not activated. + _ = list(load_cubes(nc_path)) + + with PARSE_UGRID_ON_LOAD.context(): + with self.assertLogs(logger, level="DEBUG") as log: + _ = list(load_cubes(nc_path)) + self.assertIn("File does not contain mesh", log.output[0])