diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index b90039e96c..cc51ff2d0a 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -44,6 +44,9 @@ This document explains the changes made to Iris for this release so it also converts the values of the attributes ``"actual_range"``, ``"valid_max"``, ``"valid_min"``, and ``"valid_range"``. (:pull:`6416`) +#. `@ukmo-ccbunney`_ fixed loading and merging of masked data in scalar ``AuxCoords``. + (:issue:`3584`, :pull:`6468`) + 💣 Incompatible Changes ======================= diff --git a/lib/iris/_merge.py b/lib/iris/_merge.py index 633ac0f2da..5769a12e4b 100644 --- a/lib/iris/_merge.py +++ b/lib/iris/_merge.py @@ -1406,10 +1406,37 @@ def axis_and_name(name): # TODO: Consider appropriate sort order (ascending, # descending) i.e. use CF positive attribute. cells = sorted(indexes[name]) - points = np.array( - [cell.point for cell in cells], - dtype=metadata[name].points_dtype, - ) + points = [cell.point for cell in cells] + + # If any points are masked then create a masked array type, + # otherwise create a standard ndarray. + if np.ma.masked in points: + dtype = metadata[name].points_dtype + + # Create a pre-filled array with all elements set to `fill_value` for dtype + # This avoids the following problems when trying to do `np.ma.masked_array(points, dtype...)`: + # - Underlying data of masked elements is arbitrary + # - Can't convert a np.ma.masked to an integer type + # - For floating point arrays, numpy raises a warning about "converting masked elements to NaN" + fill_value = np.trunc( + np.ma.default_fill_value(dtype), dtype=dtype + ) # truncation needed to deal with silly default fill values in Numpy + + # create array of fill values; ensures we have consistent data under mask + arr_points = np.ma.repeat(dtype.type(fill_value), len(points)) + + # get mask index and filtered data then store in new array: + mask = np.array([p is np.ma.masked for p in points]) + arr_points.mask = mask + + # Need another list comprehension to avoid numpy warning "converting masked elements to NaN": + arr_points[~mask] = np.array( + [p for p in points if p is not np.ma.masked] + ) + points = arr_points + else: + points = np.array(points, dtype=metadata[name].points_dtype) + if cells[0].bound is not None: bounds = np.array( [cell.bound for cell in cells], @@ -1594,13 +1621,19 @@ def _build_coordinates(self): # the bounds are not monotonic, so try building the coordinate, # and if it fails make the coordinate into an auxiliary coordinate. # This will ultimately make an anonymous dimension. - try: - coord = iris.coords.DimCoord( - template.points, bounds=template.bounds, **template.kwargs - ) - dim_coords_and_dims.append(_CoordAndDims(coord, template.dims)) - except ValueError: + + # If the points contain masked values, if definitely cannot be built + # as a dim coord, so add it to the _aux_templates immediately + if np.ma.is_masked(template.points): self._aux_templates.append(template) + else: + try: + coord = iris.coords.DimCoord( + template.points, bounds=template.bounds, **template.kwargs + ) + dim_coords_and_dims.append(_CoordAndDims(coord, template.dims)) + except ValueError: + self._aux_templates.append(template) # There is the potential that there are still anonymous dimensions. # Get a list of the dimensions which are not anonymous at this stage. @@ -1609,26 +1642,34 @@ def _build_coordinates(self): ] # Build the auxiliary coordinates. + def _build_aux_coord_from_template(template): + # kwarg not applicable to AuxCoord. + template.kwargs.pop("circular", None) + coord = iris.coords.AuxCoord( + template.points, bounds=template.bounds, **template.kwargs + ) + aux_coords_and_dims.append(_CoordAndDims(coord, template.dims)) + for template in self._aux_templates: # Attempt to build a DimCoord and add it to the cube. If this # fails e.g it's non-monontic or multi-dimensional or non-numeric, # then build an AuxCoord. - try: - coord = iris.coords.DimCoord( - template.points, bounds=template.bounds, **template.kwargs - ) - if len(template.dims) == 1 and template.dims[0] not in covered_dims: - dim_coords_and_dims.append(_CoordAndDims(coord, template.dims)) - covered_dims.append(template.dims[0]) - else: - aux_coords_and_dims.append(_CoordAndDims(coord, template.dims)) - except ValueError: - # kwarg not applicable to AuxCoord. - template.kwargs.pop("circular", None) - coord = iris.coords.AuxCoord( - template.points, bounds=template.bounds, **template.kwargs - ) - aux_coords_and_dims.append(_CoordAndDims(coord, template.dims)) + + # Check here whether points are masked? If so then it has to be an AuxCoord + if np.ma.is_masked(template.points): + _build_aux_coord_from_template(template) + else: + try: + coord = iris.coords.DimCoord( + template.points, bounds=template.bounds, **template.kwargs + ) + if len(template.dims) == 1 and template.dims[0] not in covered_dims: + dim_coords_and_dims.append(_CoordAndDims(coord, template.dims)) + covered_dims.append(template.dims[0]) + else: + aux_coords_and_dims.append(_CoordAndDims(coord, template.dims)) + except ValueError: + _build_aux_coord_from_template(template) # Mix in the vector coordinates. for item, dims in zip( diff --git a/lib/iris/coords.py b/lib/iris/coords.py index 9fdd20d6fc..966fe36d55 100644 --- a/lib/iris/coords.py +++ b/lib/iris/coords.py @@ -1237,6 +1237,9 @@ class Cell(namedtuple("Cell", ["point", "bound"])): # Make this class's comparison operators override those of numpy __array_priority__ = 100 + # pre-computed hash for un-hashable `np.ma.masked` value + _MASKED_VALUE_HASH = hash("<<##MASKED_VALUE##>>") + def __new__(cls, point=None, bound=None): """Construct a Cell from point or point-and-bound information.""" if point is None: @@ -1277,13 +1280,17 @@ def __add__(self, mod): def __hash__(self): # See __eq__ for the definition of when two cells are equal. + point = self.point + if np.ma.is_masked(point): + # `np.ma.masked` is unhashable + point = Cell._MASKED_VALUE_HASH if self.bound is None: - return hash(self.point) + return hash(point) bound = self.bound rbound = bound[::-1] if rbound < bound: bound = rbound - return hash((self.point, bound)) + return hash((point, bound)) def __eq__(self, other): """Compare Cell equality depending on the type of the object to be compared.""" @@ -2086,7 +2093,8 @@ def cell(self, index): """ index = iris.util._build_full_slice_given_keys(index, self.ndim) - point = tuple(np.array(self.core_points()[index], ndmin=1).flatten()) + # Use `np.asanyaray` to preserve any masked values: + point = tuple(np.asanyarray(self.core_points()[index]).flatten()) if len(point) != 1: raise IndexError( "The index %s did not uniquely identify a single " @@ -2809,6 +2817,9 @@ def _values(self, points): # Check validity requirements for dimension-coordinate points. self._new_points_requirements(points) # Cast to a numpy array for masked arrays with no mask. + + # NOTE: This is the point where any mask is lost on a coordinate if none of the + # values are actually masked. What if we wanted this to be an AuxCoord with a mask? points = np.array(points) super(DimCoord, self.__class__)._values.fset(self, points) diff --git a/lib/iris/fileformats/netcdf/loader.py b/lib/iris/fileformats/netcdf/loader.py index f07a941350..473a028315 100644 --- a/lib/iris/fileformats/netcdf/loader.py +++ b/lib/iris/fileformats/netcdf/loader.py @@ -253,6 +253,14 @@ def _get_cf_var_data(cf_var, filename): if total_bytes < _LAZYVAR_MIN_BYTES: # Don't make a lazy array, as it will cost more memory AND more time to access. result = cf_var[:] + + # Special handling of masked scalar value; this will be returned as + # an `np.ma.masked` instance which will lose the original dtype. + # Workaround for this it return a 1-element masked array of the + # correct dtype. Note: this is not an issue for masked arrays, + # only masked scalar values. + if result is np.ma.masked: + result = np.ma.masked_all(1, dtype=cf_var.datatype) else: # Get lazy chunked data out of a cf variable. # Creates Dask wrappers around data arrays for any cube components which diff --git a/lib/iris/tests/integration/merge/test_merge.py b/lib/iris/tests/integration/merge/test_merge.py index 7e1acd4ad6..c47231d57d 100644 --- a/lib/iris/tests/integration/merge/test_merge.py +++ b/lib/iris/tests/integration/merge/test_merge.py @@ -4,15 +4,714 @@ # See LICENSE in the root of the repository for full licensing details. """Integration tests for merging cubes.""" -# import iris tests first so that some things can be initialised -# before importing anything else. -import iris.tests as tests # isort:skip +import datetime -from iris.coords import DimCoord +import numpy as np +import pytest + +from iris.coords import AuxCoord, DimCoord from iris.cube import Cube, CubeList +from iris.tests._shared_utils import ( + assert_array_equal, + assert_CML, + get_data_path, + skip_data, +) +import iris.tests.stock + + +class MergeMixin: + """Mix-in class for attributes & utilities common to these test cases.""" + + def test_normal_cubes(self, request): + cubes = iris.load(self._data_path) + assert len(cubes) == self._num_cubes + assert_CML(request, cubes, ["merge", self._prefix + ".cml"]) + + def test_remerge(self): + # After the merge process the coordinates within each cube can be in a + # different order. Until that changes we can't compare the cubes + # directly or with the CML ... so we just make sure the count stays + # the same. + cubes = iris.load(self._data_path) + cubes2 = cubes.merge() + assert len(cubes) == len(cubes2) + + def test_duplication(self): + cubes = iris.load(self._data_path) + pytest.raises(iris.exceptions.DuplicateDataError, (cubes + cubes).merge) + cubes2 = (cubes + cubes).merge(unique=False) + assert len(cubes2) == 2 * len(cubes) + + +@skip_data +class TestSingleCube(MergeMixin): + def setup_method(self): + self._data_path = get_data_path(("PP", "globClim1", "theta.pp")) + self._num_cubes = 1 + self._prefix = "theta" + + +@skip_data +class TestMultiCube(MergeMixin): + def setup_method(self): + self._data_path = get_data_path(("PP", "globClim1", "dec_subset.pp")) + self._num_cubes = 4 + self._prefix = "dec" + + def test_coord_attributes(self): + def custom_coord_callback(cube, field, filename): + cube.coord("time").attributes["monty"] = "python" + cube.coord("time").attributes["brain"] = "hurts" + + # Load slices, decorating a coord with custom attributes + cubes = iris.load_raw(self._data_path, callback=custom_coord_callback) + # Merge + merged = iris.cube.CubeList(cubes).merge() + # Check the custom attributes are in the merged cube + for cube in merged: + assert cube.coord("time").attributes["monty"] == "python" + assert cube.coord("time").attributes["brain"] == "hurts" + + +@skip_data +class TestColpex: + def setup_method(self): + self._data_path = get_data_path(("PP", "COLPEX", "small_colpex_theta_p_alt.pp")) + + def test_colpex(self, request): + cubes = iris.load(self._data_path) + assert len(cubes) == 3 + assert_CML(request, cubes, ("COLPEX", "small_colpex_theta_p_alt.cml")) + + +@skip_data +class TestDataMerge: + def test_extended_proxy_data(self, request): + # Get the empty theta cubes for T+1.5 and T+2 + data_path = get_data_path(("PP", "COLPEX", "theta_and_orog_subset.pp")) + phenom_constraint = iris.Constraint("air_potential_temperature") + datetime_1 = datetime.datetime(2009, 9, 9, 17, 20) + datetime_2 = datetime.datetime(2009, 9, 9, 17, 50) + time_constraint1 = iris.Constraint(time=datetime_1) + time_constraint2 = iris.Constraint(time=datetime_2) + time_constraint_1_and_2 = iris.Constraint( + time=lambda c: c in (datetime_1, datetime_2) + ) + cube1 = iris.load_cube(data_path, phenom_constraint & time_constraint1) + cube2 = iris.load_cube(data_path, phenom_constraint & time_constraint2) + + # Merge the two halves + cubes = iris.cube.CubeList([cube1, cube2]).merge(True) + assert_CML(request, cubes, ("merge", "theta_two_times.cml")) + + # Make sure we get the same result directly from load + cubes = iris.load_cube(data_path, phenom_constraint & time_constraint_1_and_2) + assert_CML(request, cubes, ("merge", "theta_two_times.cml")) + + def test_real_data(self, request): + data_path = get_data_path(("PP", "globClim1", "theta.pp")) + cubes = iris.load_raw(data_path) + # Force the source 2-D cubes to load their data before the merge + for cube in cubes: + _ = cube.data + cubes = cubes.merge() + assert_CML(request, cubes, ["merge", "theta.cml"]) + + +class TestDimensionSplitting: + def _make_cube(self, a, b, c, data): + cube_data = np.empty((4, 5), dtype=np.float32) + cube_data[:] = data + cube = iris.cube.Cube(cube_data) + cube.add_dim_coord( + DimCoord( + np.array([0, 1, 2, 3, 4], dtype=np.int32), + long_name="x", + units="1", + ), + 1, + ) + cube.add_dim_coord( + DimCoord( + np.array([0, 1, 2, 3], dtype=np.int32), + long_name="y", + units="1", + ), + 0, + ) + cube.add_aux_coord( + DimCoord(np.array([a], dtype=np.int32), long_name="a", units="1") + ) + cube.add_aux_coord( + DimCoord(np.array([b], dtype=np.int32), long_name="b", units="1") + ) + cube.add_aux_coord( + DimCoord(np.array([c], dtype=np.int32), long_name="c", units="1") + ) + return cube + + def test_single_split(self, request): + # Test what happens when a cube forces a simple, two-way split. + cubes = [] + cubes.append(self._make_cube(0, 0, 0, 0)) + cubes.append(self._make_cube(0, 1, 1, 1)) + cubes.append(self._make_cube(1, 0, 2, 2)) + cubes.append(self._make_cube(1, 1, 3, 3)) + cubes.append(self._make_cube(2, 0, 4, 4)) + cubes.append(self._make_cube(2, 1, 5, 5)) + cube = iris.cube.CubeList(cubes).merge() + assert_CML(request, cube, ("merge", "single_split.cml")) + + def test_multi_split(self, request): + # Test what happens when a cube forces a three-way split. + cubes = [] + cubes.append(self._make_cube(0, 0, 0, 0)) + cubes.append(self._make_cube(0, 0, 1, 1)) + cubes.append(self._make_cube(0, 1, 0, 2)) + cubes.append(self._make_cube(0, 1, 1, 3)) + cubes.append(self._make_cube(1, 0, 0, 4)) + cubes.append(self._make_cube(1, 0, 1, 5)) + cubes.append(self._make_cube(1, 1, 0, 6)) + cubes.append(self._make_cube(1, 1, 1, 7)) + cubes.append(self._make_cube(2, 0, 0, 8)) + cubes.append(self._make_cube(2, 0, 1, 9)) + cubes.append(self._make_cube(2, 1, 0, 10)) + cubes.append(self._make_cube(2, 1, 1, 11)) + cube = iris.cube.CubeList(cubes).merge() + assert_CML(request, cube, ("merge", "multi_split.cml")) + + +class TestCombination: + def _make_cube(self, a, b, c, d, data=0): + cube_data = np.empty((4, 5), dtype=np.float32) + cube_data[:] = data + cube = iris.cube.Cube(cube_data) + cube.add_dim_coord( + DimCoord( + np.array([0, 1, 2, 3, 4], dtype=np.int32), + long_name="x", + units="1", + ), + 1, + ) + cube.add_dim_coord( + DimCoord( + np.array([0, 1, 2, 3], dtype=np.int32), + long_name="y", + units="1", + ), + 0, + ) + + for name, value in zip(["a", "b", "c", "d"], [a, b, c, d]): + dtype = np.str_ if isinstance(value, str) else np.float32 + cube.add_aux_coord( + AuxCoord(np.array([value], dtype=dtype), long_name=name, units="1") + ) + + return cube + + def test_separable_combination(self, request): + cubes = iris.cube.CubeList() + cubes.append( + self._make_cube("2005", "ECMWF", "HOPE-E, Sys 1, Met 1, ENSEMBLES", 0) + ) + cubes.append( + self._make_cube("2005", "ECMWF", "HOPE-E, Sys 1, Met 1, ENSEMBLES", 1) + ) + cubes.append( + self._make_cube("2005", "ECMWF", "HOPE-E, Sys 1, Met 1, ENSEMBLES", 2) + ) + cubes.append( + self._make_cube( + "2026", "UK Met Office", "HadGEM2, Sys 1, Met 1, ENSEMBLES", 0 + ) + ) + cubes.append( + self._make_cube( + "2026", "UK Met Office", "HadGEM2, Sys 1, Met 1, ENSEMBLES", 1 + ) + ) + cubes.append( + self._make_cube( + "2026", "UK Met Office", "HadGEM2, Sys 1, Met 1, ENSEMBLES", 2 + ) + ) + cubes.append( + self._make_cube("2002", "CERFACS", "GELATO, Sys 0, Met 1, ENSEMBLES", 0) + ) + cubes.append( + self._make_cube("2002", "CERFACS", "GELATO, Sys 0, Met 1, ENSEMBLES", 1) + ) + cubes.append( + self._make_cube("2002", "CERFACS", "GELATO, Sys 0, Met 1, ENSEMBLES", 2) + ) + cubes.append( + self._make_cube("2002", "IFM-GEOMAR", "ECHAM5, Sys 1, Met 10, ENSEMBLES", 0) + ) + cubes.append( + self._make_cube("2002", "IFM-GEOMAR", "ECHAM5, Sys 1, Met 10, ENSEMBLES", 1) + ) + cubes.append( + self._make_cube("2002", "IFM-GEOMAR", "ECHAM5, Sys 1, Met 10, ENSEMBLES", 2) + ) + cubes.append( + self._make_cube( + "2502", "UK Met Office", "HadCM3, Sys 51, Met 10, ENSEMBLES", 0 + ) + ) + cubes.append( + self._make_cube( + "2502", "UK Met Office", "HadCM3, Sys 51, Met 11, ENSEMBLES", 0 + ) + ) + cubes.append( + self._make_cube( + "2502", "UK Met Office", "HadCM3, Sys 51, Met 12, ENSEMBLES", 0 + ) + ) + cubes.append( + self._make_cube( + "2502", "UK Met Office", "HadCM3, Sys 51, Met 13, ENSEMBLES", 0 + ) + ) + cubes.append( + self._make_cube( + "2502", "UK Met Office", "HadCM3, Sys 51, Met 14, ENSEMBLES", 0 + ) + ) + cubes.append( + self._make_cube( + "2502", "UK Met Office", "HadCM3, Sys 51, Met 15, ENSEMBLES", 0 + ) + ) + cubes.append( + self._make_cube( + "2502", "UK Met Office", "HadCM3, Sys 51, Met 16, ENSEMBLES", 0 + ) + ) + cubes.append( + self._make_cube( + "2502", "UK Met Office", "HadCM3, Sys 51, Met 17, ENSEMBLES", 0 + ) + ) + cubes.append( + self._make_cube( + "2502", "UK Met Office", "HadCM3, Sys 51, Met 18, ENSEMBLES", 0 + ) + ) + cube = cubes.merge() + assert_CML( + request, cube, ("merge", "separable_combination.cml"), checksum=False + ) + + +class TestDimSelection: + def _make_cube(self, a, b, data=0, a_dim=False, b_dim=False): + cube_data = np.empty((4, 5), dtype=np.float32) + cube_data[:] = data + cube = iris.cube.Cube(cube_data) + cube.add_dim_coord( + DimCoord( + np.array([0, 1, 2, 3, 4], dtype=np.int32), + long_name="x", + units="1", + ), + 1, + ) + cube.add_dim_coord( + DimCoord( + np.array([0, 1, 2, 3], dtype=np.int32), + long_name="y", + units="1", + ), + 0, + ) + + for name, value, dim in zip(["a", "b"], [a, b], [a_dim, b_dim]): + dtype = np.str_ if isinstance(value, str) else np.float32 + ctype = DimCoord if dim else AuxCoord + coord = ctype(np.array([value], dtype=dtype), long_name=name, units="1") + cube.add_aux_coord(coord) + + return cube + + def test_string_a_with_aux(self, request): + templates = (("a", 0), ("b", 1), ("c", 2), ("d", 3)) + cubes = [self._make_cube(a, b) for a, b in templates] + cube = iris.cube.CubeList(cubes).merge()[0] + assert_CML(request, cube, ("merge", "string_a_with_aux.cml"), checksum=False) + assert isinstance(cube.coord("a"), AuxCoord) + assert isinstance(cube.coord("b"), DimCoord) + assert cube.coord("b") in cube.dim_coords + + def test_string_b_with_aux(self, request): + templates = ((0, "a"), (1, "b"), (2, "c"), (3, "d")) + cubes = [self._make_cube(a, b) for a, b in templates] + cube = iris.cube.CubeList(cubes).merge()[0] + assert_CML(request, cube, ("merge", "string_b_with_aux.cml"), checksum=False) + assert isinstance(cube.coord("a"), DimCoord) + assert cube.coord("a") in cube.dim_coords + assert isinstance(cube.coord("b"), AuxCoord) + + def test_string_a_with_dim(self, request): + templates = (("a", 0), ("b", 1), ("c", 2), ("d", 3)) + cubes = [self._make_cube(a, b, b_dim=True) for a, b in templates] + cube = iris.cube.CubeList(cubes).merge()[0] + assert_CML(request, cube, ("merge", "string_a_with_dim.cml"), checksum=False) + assert isinstance(cube.coord("a"), AuxCoord) + assert isinstance(cube.coord("b"), DimCoord) + assert cube.coord("b") in cube.dim_coords + + def test_string_b_with_dim(self, request): + templates = ((0, "a"), (1, "b"), (2, "c"), (3, "d")) + cubes = [self._make_cube(a, b, a_dim=True) for a, b in templates] + cube = iris.cube.CubeList(cubes).merge()[0] + assert_CML(request, cube, ("merge", "string_b_with_dim.cml"), checksum=False) + assert isinstance(cube.coord("a"), DimCoord) + assert cube.coord("a") in cube.dim_coords + assert isinstance(cube.coord("b"), AuxCoord) + + def test_string_a_b(self, request): + templates = (("a", "0"), ("b", "1"), ("c", "2"), ("d", "3")) + cubes = [self._make_cube(a, b) for a, b in templates] + cube = iris.cube.CubeList(cubes).merge()[0] + assert_CML(request, cube, ("merge", "string_a_b.cml"), checksum=False) + assert isinstance(cube.coord("a"), AuxCoord) + assert isinstance(cube.coord("b"), AuxCoord) + + def test_a_aux_b_aux(self, request): + templates = ((0, 10), (1, 11), (2, 12), (3, 13)) + cubes = [self._make_cube(a, b) for a, b in templates] + cube = iris.cube.CubeList(cubes).merge()[0] + assert_CML(request, cube, ("merge", "a_aux_b_aux.cml"), checksum=False) + assert isinstance(cube.coord("a"), DimCoord) + assert cube.coord("a") in cube.dim_coords + assert isinstance(cube.coord("b"), DimCoord) + assert cube.coord("b") in cube.aux_coords + + def test_a_aux_b_dim(self, request): + templates = ((0, 10), (1, 11), (2, 12), (3, 13)) + cubes = [self._make_cube(a, b, b_dim=True) for a, b in templates] + cube = iris.cube.CubeList(cubes).merge()[0] + assert_CML(request, cube, ("merge", "a_aux_b_dim.cml"), checksum=False) + assert isinstance(cube.coord("a"), DimCoord) + assert cube.coord("a") in cube.aux_coords + assert isinstance(cube.coord("b"), DimCoord) + assert cube.coord("b") in cube.dim_coords + def test_a_dim_b_aux(self, request): + templates = ((0, 10), (1, 11), (2, 12), (3, 13)) + cubes = [self._make_cube(a, b, a_dim=True) for a, b in templates] + cube = iris.cube.CubeList(cubes).merge()[0] + assert_CML(request, cube, ("merge", "a_dim_b_aux.cml"), checksum=False) + assert isinstance(cube.coord("a"), DimCoord) + assert cube.coord("a") in cube.dim_coords + assert isinstance(cube.coord("b"), DimCoord) + assert cube.coord("b") in cube.aux_coords -class TestContiguous(tests.IrisTest): + def test_a_dim_b_dim(self, request): + templates = ((0, 10), (1, 11), (2, 12), (3, 13)) + cubes = [self._make_cube(a, b, a_dim=True, b_dim=True) for a, b in templates] + cube = iris.cube.CubeList(cubes).merge()[0] + assert_CML(request, cube, ("merge", "a_dim_b_dim.cml"), checksum=False) + assert isinstance(cube.coord("a"), DimCoord) + assert cube.coord("a") in cube.dim_coords + assert isinstance(cube.coord("b"), DimCoord) + assert cube.coord("b") in cube.aux_coords + + +class TestTimeTripleMerging: + def _make_cube(self, a, b, c, data=0): + cube_data = np.empty((4, 5), dtype=np.float32) + cube_data[:] = data + cube = iris.cube.Cube(cube_data) + cube.add_dim_coord( + DimCoord( + np.array([0, 1, 2, 3, 4], dtype=np.int32), + long_name="x", + units="1", + ), + 1, + ) + cube.add_dim_coord( + DimCoord( + np.array([0, 1, 2, 3], dtype=np.int32), + long_name="y", + units="1", + ), + 0, + ) + cube.add_aux_coord( + DimCoord( + np.array([a], dtype=np.int32), + standard_name="forecast_period", + units="1", + ) + ) + cube.add_aux_coord( + DimCoord( + np.array([b], dtype=np.int32), + standard_name="forecast_reference_time", + units="1", + ) + ) + cube.add_aux_coord( + DimCoord(np.array([c], dtype=np.int32), standard_name="time", units="1") + ) + return cube + + def _test_triples(self, triples, filename, request): + cubes = [self._make_cube(fp, rt, t) for fp, rt, t in triples] + cube = iris.cube.CubeList(cubes).merge() + assert_CML( + request, cube, ("merge", "time_triple_" + filename + ".cml"), checksum=False + ) + + def test_single_forecast(self, request): + # A single forecast series (i.e. from a single reference time) + # => fp, t: 4; rt: scalar + triples = ( + (0, 10, 10), + (1, 10, 11), + (2, 10, 12), + (3, 10, 13), + ) + self._test_triples(triples, "single_forecast", request) + + def test_successive_forecasts(self, request): + # Three forecast series from successively later reference times + # => rt, t: 3; fp, t: 4 + triples = ( + (0, 10, 10), + (1, 10, 11), + (2, 10, 12), + (3, 10, 13), + (0, 11, 11), + (1, 11, 12), + (2, 11, 13), + (3, 11, 14), + (0, 12, 12), + (1, 12, 13), + (2, 12, 14), + (3, 12, 15), + ) + self._test_triples(triples, "successive_forecasts", request) + + def test_time_vs_ref_time(self, request): + # => fp, t: 4; fp, rt: 3 + triples = ( + (2, 10, 12), + (3, 10, 13), + (4, 10, 14), + (5, 10, 15), + (1, 11, 12), + (2, 11, 13), + (3, 11, 14), + (4, 11, 15), + (0, 12, 12), + (1, 12, 13), + (2, 12, 14), + (3, 12, 15), + ) + self._test_triples(triples, "time_vs_ref_time", request) + + def test_time_vs_forecast(self, request): + # => rt, t: 4, fp, rt: 3 + triples = ( + (0, 10, 10), + (0, 11, 11), + (0, 12, 12), + (0, 13, 13), + (1, 9, 10), + (1, 10, 11), + (1, 11, 12), + (1, 12, 13), + (2, 8, 10), + (2, 9, 11), + (2, 10, 12), + (2, 11, 13), + ) + self._test_triples(triples, "time_vs_forecast", request) + + def test_time_non_dim_coord(self, request): + # => rt: 1 fp, t (bounded): 2 + triples = ( + (5, 0, 2.5), + (10, 0, 5), + ) + cubes = [self._make_cube(fp, rt, t) for fp, rt, t in triples] + for end_time, cube in zip([5, 10], cubes): + cube.coord("time").bounds = [0, end_time] + (cube,) = iris.cube.CubeList(cubes).merge() + assert_CML( + request, + cube, + ("merge", "time_triple_time_non_dim_coord.cml"), + checksum=False, + ) + # make sure that forecast_period is the dimensioned coordinate (as time becomes an AuxCoord) + assert cube.coord(dimensions=0, dim_coords=True).name() == "forecast_period" + + def test_independent(self, request): + # => fp: 2; rt: 2; t: 2 + triples = ( + (0, 10, 10), + (0, 11, 10), + (0, 10, 11), + (0, 11, 11), + (1, 10, 10), + (1, 11, 10), + (1, 10, 11), + (1, 11, 11), + ) + self._test_triples(triples, "independent", request) + + def test_series(self, request): + # => fp, rt, t: 5 (with only t being definitive). + triples = ( + (0, 10, 10), + (0, 11, 11), + (0, 12, 12), + (1, 12, 13), + (2, 12, 14), + ) + self._test_triples(triples, "series", request) + + def test_non_expanding_dimension(self, request): + triples = ( + (0, 10, 0), + (0, 20, 1), + (0, 20, 0), + ) + # => fp: scalar; rt, t: 3 (with no time being definitive) + self._test_triples(triples, "non_expanding", request) + + def test_duplicate_data(self, request): + # test what happens when we have repeated time coordinates (i.e. duplicate data) + cube1 = self._make_cube(0, 10, 0) + cube2 = self._make_cube(1, 20, 1) + cube3 = self._make_cube(1, 20, 1) + + # check that we get a duplicate data error when unique is True + with pytest.raises(iris.exceptions.DuplicateDataError): + iris.cube.CubeList([cube1, cube2, cube3]).merge() + + cubes = iris.cube.CubeList([cube1, cube2, cube3]).merge(unique=False) + assert_CML( + request, cubes, ("merge", "time_triple_duplicate_data.cml"), checksum=False + ) + + def test_simple1(self, request): + cube1 = self._make_cube(0, 10, 0) + cube2 = self._make_cube(1, 20, 1) + cube3 = self._make_cube(2, 20, 0) + cube = iris.cube.CubeList([cube1, cube2, cube3]).merge() + assert_CML(request, cube, ("merge", "time_triple_merging1.cml"), checksum=False) + + def test_simple2(self, request): + cubes = iris.cube.CubeList( + [ + self._make_cube(0, 0, 0), + self._make_cube(1, 0, 1), + self._make_cube(2, 0, 2), + self._make_cube(0, 1, 3), + self._make_cube(1, 1, 4), + self._make_cube(2, 1, 5), + ] + ) + cube = cubes.merge()[0] + assert_CML(request, cube, ("merge", "time_triple_merging2.cml"), checksum=False) + + cube = iris.cube.CubeList(cubes[:-1]).merge()[0] + assert_CML(request, cube, ("merge", "time_triple_merging3.cml"), checksum=False) + + def test_simple3(self, request): + cubes = iris.cube.CubeList( + [ + self._make_cube(0, 0, 0), + self._make_cube(0, 1, 1), + self._make_cube(0, 2, 2), + self._make_cube(1, 0, 3), + self._make_cube(1, 1, 4), + self._make_cube(1, 2, 5), + ] + ) + cube = cubes.merge()[0] + assert_CML(request, cube, ("merge", "time_triple_merging4.cml"), checksum=False) + + cube = iris.cube.CubeList(cubes[:-1]).merge()[0] + assert_CML(request, cube, ("merge", "time_triple_merging5.cml"), checksum=False) + + +class TestCubeMergeTheoretical: + def test_simple_bounds_merge(self, request): + cube1 = iris.tests.stock.simple_2d() + cube2 = iris.tests.stock.simple_2d() + + cube1.add_aux_coord(DimCoord(np.int32(10), long_name="pressure", units="Pa")) + cube2.add_aux_coord(DimCoord(np.int32(11), long_name="pressure", units="Pa")) + + r = iris.cube.CubeList([cube1, cube2]).merge() + assert_CML(request, r, ("cube_merge", "test_simple_bound_merge.cml")) + + def test_simple_multidim_merge(self, request): + cube1 = iris.tests.stock.simple_2d_w_multidim_coords() + cube2 = iris.tests.stock.simple_2d_w_multidim_coords() + + cube1.add_aux_coord(DimCoord(np.int32(10), long_name="pressure", units="Pa")) + cube2.add_aux_coord(DimCoord(np.int32(11), long_name="pressure", units="Pa")) + + r = iris.cube.CubeList([cube1, cube2]).merge()[0] + assert_CML(request, r, ("cube_merge", "multidim_coord_merge.cml")) + + # try transposing the cubes first + cube1.transpose([1, 0]) + cube2.transpose([1, 0]) + r = iris.cube.CubeList([cube1, cube2]).merge()[0] + assert_CML(request, r, ("cube_merge", "multidim_coord_merge_transpose.cml")) + + def test_simple_points_merge(self, request): + cube1 = iris.tests.stock.simple_2d(with_bounds=False) + cube2 = iris.tests.stock.simple_2d(with_bounds=False) + + cube1.add_aux_coord(DimCoord(np.int32(10), long_name="pressure", units="Pa")) + cube2.add_aux_coord(DimCoord(np.int32(11), long_name="pressure", units="Pa")) + + r = iris.cube.CubeList([cube1, cube2]).merge() + assert_CML(request, r, ("cube_merge", "test_simple_merge.cml")) + + # check that the unique merging raises a Duplicate data error + pytest.raises( + iris.exceptions.DuplicateDataError, + iris.cube.CubeList([cube1, cube1]).merge, + unique=True, + ) + + # check that non unique merging returns both cubes + r = iris.cube.CubeList([cube1, cube1]).merge(unique=False) + assert_CML(request, r[0], ("cube_merge", "test_orig_point_cube.cml")) + assert_CML(request, r[1], ("cube_merge", "test_orig_point_cube.cml")) + + # test attribute merging + cube1.attributes["my_attr1"] = "foo" + r = iris.cube.CubeList([cube1, cube2]).merge() + # result should be 2 cubes + assert_CML(request, r, ("cube_merge", "test_simple_attributes1.cml")) + + cube2.attributes["my_attr1"] = "bar" + r = iris.cube.CubeList([cube1, cube2]).merge() + # result should be 2 cubes + assert_CML(request, r, ("cube_merge", "test_simple_attributes2.cml")) + + cube2.attributes["my_attr1"] = "foo" + r = iris.cube.CubeList([cube1, cube2]).merge() + # result should be 1 cube + assert_CML(request, r, ("cube_merge", "test_simple_attributes3.cml")) + + +class TestContiguous: def test_form_contiguous_dimcoord(self): # Test that cube sliced up and remerged in the opposite order maintains # contiguity. @@ -24,10 +723,6 @@ def test_form_contiguous_dimcoord(self): cube2 = cubes.merge_cube() coord2 = cube2.coord("spam") - self.assertTrue(coord2.is_contiguous()) - self.assertArrayEqual(coord2.points, [1, 2, 3]) - self.assertArrayEqual(coord2.bounds, coord1.bounds[::-1, ::-1]) - - -if __name__ == "__main__": - tests.main() + assert coord2.is_contiguous() + assert_array_equal(coord2.points, [1, 2, 3]) + assert_array_equal(coord2.bounds, coord1.bounds[::-1, ::-1]) diff --git a/lib/iris/tests/test_merge.py b/lib/iris/tests/test_merge.py deleted file mode 100644 index 1fc6fd8b10..0000000000 --- a/lib/iris/tests/test_merge.py +++ /dev/null @@ -1,1102 +0,0 @@ -# Copyright Iris contributors -# -# This file is part of Iris and is released under the BSD license. -# See LICENSE in the root of the repository for full licensing details. -"""Test the cube merging mechanism.""" - -# import iris tests first so that some things can be initialised before importing anything else -import iris.tests as tests # isort:skip - -from collections.abc import Iterable -import datetime -import itertools - -import numpy as np -import numpy.ma as ma - -import iris -from iris._lazy_data import as_lazy_data -from iris.coords import AuxCoord, DimCoord -import iris.cube -from iris.cube import CubeAttrsDict -import iris.exceptions -import iris.tests.stock - - -class MergeMixin: - """Mix-in class for attributes & utilities common to these test cases.""" - - def test_normal_cubes(self): - cubes = iris.load(self._data_path) - self.assertEqual(len(cubes), self._num_cubes) - self.assertCML(cubes, ["merge", self._prefix + ".cml"]) - - def test_remerge(self): - # After the merge process the coordinates within each cube can be in a - # different order. Until that changes we can't compare the cubes - # directly or with the CML ... so we just make sure the count stays - # the same. - cubes = iris.load(self._data_path) - cubes2 = cubes.merge() - self.assertEqual(len(cubes), len(cubes2)) - - def test_duplication(self): - cubes = iris.load(self._data_path) - self.assertRaises(iris.exceptions.DuplicateDataError, (cubes + cubes).merge) - cubes2 = (cubes + cubes).merge(unique=False) - self.assertEqual(len(cubes2), 2 * len(cubes)) - - -@tests.skip_data -class TestSingleCube(tests.IrisTest, MergeMixin): - def setUp(self): - self._data_path = tests.get_data_path(("PP", "globClim1", "theta.pp")) - self._num_cubes = 1 - self._prefix = "theta" - - -@tests.skip_data -class TestMultiCube(tests.IrisTest, MergeMixin): - def setUp(self): - self._data_path = tests.get_data_path(("PP", "globClim1", "dec_subset.pp")) - self._num_cubes = 4 - self._prefix = "dec" - - def test_coord_attributes(self): - def custom_coord_callback(cube, field, filename): - cube.coord("time").attributes["monty"] = "python" - cube.coord("time").attributes["brain"] = "hurts" - - # Load slices, decorating a coord with custom attributes - cubes = iris.load_raw(self._data_path, callback=custom_coord_callback) - # Merge - merged = iris.cube.CubeList(cubes).merge() - # Check the custom attributes are in the merged cube - for cube in merged: - assert cube.coord("time").attributes["monty"] == "python" - assert cube.coord("time").attributes["brain"] == "hurts" - - -@tests.skip_data -class TestColpex(tests.IrisTest): - def setUp(self): - self._data_path = tests.get_data_path( - ("PP", "COLPEX", "small_colpex_theta_p_alt.pp") - ) - - def test_colpex(self): - cubes = iris.load(self._data_path) - self.assertEqual(len(cubes), 3) - self.assertCML(cubes, ("COLPEX", "small_colpex_theta_p_alt.cml")) - - -class TestDataMergeCombos(tests.IrisTest): - def _make_data( - self, - data, - dtype=np.dtype("int32"), - fill_value=None, - mask=None, - lazy=False, - N=3, - ): - if isinstance(data, Iterable): - shape = (len(data), N, N) - data = np.array(data).reshape(-1, 1, 1) - else: - shape = (N, N) - if mask is not None: - payload = ma.empty(shape, dtype=dtype, fill_value=fill_value) - payload.data[:] = data - if isinstance(mask, bool): - payload.mask = mask - else: - payload[mask] = ma.masked - else: - payload = np.empty(shape, dtype=dtype) - payload[:] = data - if lazy: - payload = as_lazy_data(payload) - return payload - - def _make_cube( - self, - data, - dtype=np.dtype("int32"), - fill_value=None, - mask=None, - lazy=False, - N=3, - ): - x = np.arange(N) - y = np.arange(N) - payload = self._make_data( - data, dtype=dtype, fill_value=fill_value, mask=mask, lazy=lazy, N=N - ) - cube = iris.cube.Cube(payload) - lat = DimCoord(y, standard_name="latitude", units="degrees") - cube.add_dim_coord(lat, 0) - lon = DimCoord(x, standard_name="longitude", units="degrees") - cube.add_dim_coord(lon, 1) - height = DimCoord(data, standard_name="height", units="m") - cube.add_aux_coord(height) - return cube - - @staticmethod - def _expected_fill_value(fill0="none", fill1="none"): - result = None - if fill0 != "none" or fill1 != "none": - if fill0 == "none": - result = fill1 - elif fill1 == "none": - result = fill0 - elif fill0 == fill1: - result = fill0 - return result - - def _check_fill_value(self, result, fill0="none", fill1="none"): - expected_fill_value = self._expected_fill_value(fill0, fill1) - if expected_fill_value is None: - data = result.data - if ma.isMaskedArray(data): - np_fill_value = ma.masked_array(0, dtype=result.dtype).fill_value - self.assertEqual(data.fill_value, np_fill_value) - else: - data = result.data - if ma.isMaskedArray(data): - self.assertEqual(data.fill_value, expected_fill_value) - - def setUp(self): - self.dtype = np.dtype("int32") - fill_value = 1234 - self.lazy_combos = itertools.product([False, True], [False, True]) - fill_combos = itertools.product([None, fill_value], [fill_value, None]) - single_fill_combos = itertools.product([None, fill_value]) - self.combos = itertools.product(self.lazy_combos, fill_combos) - self.mixed_combos = itertools.product(self.lazy_combos, single_fill_combos) - - def test__ndarray_ndarray(self): - for lazy0, lazy1 in self.lazy_combos: - cubes = iris.cube.CubeList() - cubes.append(self._make_cube(0, dtype=self.dtype, lazy=lazy0)) - cubes.append(self._make_cube(1, dtype=self.dtype, lazy=lazy1)) - result = cubes.merge_cube() - expected = self._make_data([0, 1], dtype=self.dtype) - self.assertArrayEqual(result.data, expected) - self.assertEqual(result.dtype, self.dtype) - self._check_fill_value(result) - - def test__masked_masked(self): - for (lazy0, lazy1), (fill0, fill1) in self.combos: - cubes = iris.cube.CubeList() - mask = ((0,), (0,)) - cubes.append( - self._make_cube( - 0, - mask=mask, - lazy=lazy0, - dtype=self.dtype, - fill_value=fill0, - ) - ) - mask = ((1,), (1,)) - cubes.append( - self._make_cube( - 1, - mask=mask, - lazy=lazy1, - dtype=self.dtype, - fill_value=fill1, - ) - ) - result = cubes.merge_cube() - mask = ((0, 1), (0, 1), (0, 1)) - expected_fill_value = self._expected_fill_value(fill0, fill1) - expected = self._make_data( - [0, 1], - mask=mask, - dtype=self.dtype, - fill_value=expected_fill_value, - ) - self.assertMaskedArrayEqual(result.data, expected) - self.assertEqual(result.dtype, self.dtype) - self._check_fill_value(result, fill0, fill1) - - def test__ndarray_masked(self): - for (lazy0, lazy1), (fill,) in self.mixed_combos: - cubes = iris.cube.CubeList() - cubes.append(self._make_cube(0, lazy=lazy0, dtype=self.dtype)) - mask = [(0, 1), (0, 1)] - cubes.append( - self._make_cube( - 1, mask=mask, lazy=lazy1, dtype=self.dtype, fill_value=fill - ) - ) - result = cubes.merge_cube() - mask = [(1, 1), (0, 1), (0, 1)] - expected_fill_value = self._expected_fill_value(fill) - expected = self._make_data( - [0, 1], - mask=mask, - dtype=self.dtype, - fill_value=expected_fill_value, - ) - self.assertMaskedArrayEqual(result.data, expected) - self.assertEqual(result.dtype, self.dtype) - self._check_fill_value(result, fill1=fill) - - def test__masked_ndarray(self): - for (lazy0, lazy1), (fill,) in self.mixed_combos: - cubes = iris.cube.CubeList() - mask = [(0, 1), (0, 1)] - cubes.append( - self._make_cube( - 0, mask=mask, lazy=lazy0, dtype=self.dtype, fill_value=fill - ) - ) - cubes.append(self._make_cube(1, lazy=lazy1, dtype=self.dtype)) - result = cubes.merge_cube() - mask = [(0, 0), (0, 1), (0, 1)] - expected_fill_value = self._expected_fill_value(fill) - expected = self._make_data( - [0, 1], - mask=mask, - dtype=self.dtype, - fill_value=expected_fill_value, - ) - self.assertMaskedArrayEqual(result.data, expected) - self.assertEqual(result.dtype, self.dtype) - self._check_fill_value(result, fill0=fill) - - def test_maksed_array_preserved(self): - for (lazy0, lazy1), (fill,) in self.mixed_combos: - cubes = iris.cube.CubeList() - mask = False - cubes.append( - self._make_cube( - 0, mask=mask, lazy=lazy0, dtype=self.dtype, fill_value=fill - ) - ) - cubes.append(self._make_cube(1, lazy=lazy1, dtype=self.dtype)) - result = cubes.merge_cube() - mask = False - expected_fill_value = self._expected_fill_value(fill) - expected = self._make_data( - [0, 1], - mask=mask, - dtype=self.dtype, - fill_value=expected_fill_value, - ) - self.assertEqual(type(result.data), ma.MaskedArray) - self.assertMaskedArrayEqual(result.data, expected) - self.assertEqual(result.dtype, self.dtype) - self._check_fill_value(result, fill0=fill) - - def test_fill_value_invariant_to_order__same_non_None(self): - fill_value = 1234 - cubes = [self._make_cube(i, mask=True, fill_value=fill_value) for i in range(3)] - for combo in itertools.permutations(cubes): - result = iris.cube.CubeList(combo).merge_cube() - self.assertEqual(result.data.fill_value, fill_value) - - def test_fill_value_invariant_to_order__all_None(self): - cubes = [self._make_cube(i, mask=True, fill_value=None) for i in range(3)] - for combo in itertools.permutations(cubes): - result = iris.cube.CubeList(combo).merge_cube() - np_fill_value = ma.masked_array(0, dtype=result.dtype).fill_value - self.assertEqual(result.data.fill_value, np_fill_value) - - def test_fill_value_invariant_to_order__different_non_None(self): - cubes = [self._make_cube(0, mask=True, fill_value=1234)] - cubes.append(self._make_cube(1, mask=True, fill_value=2341)) - cubes.append(self._make_cube(2, mask=True, fill_value=3412)) - cubes.append(self._make_cube(3, mask=True, fill_value=4123)) - for combo in itertools.permutations(cubes): - result = iris.cube.CubeList(combo).merge_cube() - np_fill_value = ma.masked_array(0, dtype=result.dtype).fill_value - self.assertEqual(result.data.fill_value, np_fill_value) - - def test_fill_value_invariant_to_order__mixed(self): - cubes = [self._make_cube(0, mask=True, fill_value=None)] - cubes.append(self._make_cube(1, mask=True, fill_value=1234)) - cubes.append(self._make_cube(2, mask=True, fill_value=4321)) - for combo in itertools.permutations(cubes): - result = iris.cube.CubeList(combo).merge_cube() - np_fill_value = ma.masked_array(0, dtype=result.dtype).fill_value - self.assertEqual(result.data.fill_value, np_fill_value) - - -@tests.skip_data -class TestDataMerge(tests.IrisTest): - def test_extended_proxy_data(self): - # Get the empty theta cubes for T+1.5 and T+2 - data_path = tests.get_data_path(("PP", "COLPEX", "theta_and_orog_subset.pp")) - phenom_constraint = iris.Constraint("air_potential_temperature") - datetime_1 = datetime.datetime(2009, 9, 9, 17, 20) - datetime_2 = datetime.datetime(2009, 9, 9, 17, 50) - time_constraint1 = iris.Constraint(time=datetime_1) - time_constraint2 = iris.Constraint(time=datetime_2) - time_constraint_1_and_2 = iris.Constraint( - time=lambda c: c in (datetime_1, datetime_2) - ) - cube1 = iris.load_cube(data_path, phenom_constraint & time_constraint1) - cube2 = iris.load_cube(data_path, phenom_constraint & time_constraint2) - - # Merge the two halves - cubes = iris.cube.CubeList([cube1, cube2]).merge(True) - self.assertCML(cubes, ("merge", "theta_two_times.cml")) - - # Make sure we get the same result directly from load - cubes = iris.load_cube(data_path, phenom_constraint & time_constraint_1_and_2) - self.assertCML(cubes, ("merge", "theta_two_times.cml")) - - def test_real_data(self): - data_path = tests.get_data_path(("PP", "globClim1", "theta.pp")) - cubes = iris.load_raw(data_path) - # Force the source 2-D cubes to load their data before the merge - for cube in cubes: - _ = cube.data - cubes = cubes.merge() - self.assertCML(cubes, ["merge", "theta.cml"]) - - -class TestDimensionSplitting(tests.IrisTest): - def _make_cube(self, a, b, c, data): - cube_data = np.empty((4, 5), dtype=np.float32) - cube_data[:] = data - cube = iris.cube.Cube(cube_data) - cube.add_dim_coord( - DimCoord( - np.array([0, 1, 2, 3, 4], dtype=np.int32), - long_name="x", - units="1", - ), - 1, - ) - cube.add_dim_coord( - DimCoord( - np.array([0, 1, 2, 3], dtype=np.int32), - long_name="y", - units="1", - ), - 0, - ) - cube.add_aux_coord( - DimCoord(np.array([a], dtype=np.int32), long_name="a", units="1") - ) - cube.add_aux_coord( - DimCoord(np.array([b], dtype=np.int32), long_name="b", units="1") - ) - cube.add_aux_coord( - DimCoord(np.array([c], dtype=np.int32), long_name="c", units="1") - ) - return cube - - def test_single_split(self): - # Test what happens when a cube forces a simple, two-way split. - cubes = [] - cubes.append(self._make_cube(0, 0, 0, 0)) - cubes.append(self._make_cube(0, 1, 1, 1)) - cubes.append(self._make_cube(1, 0, 2, 2)) - cubes.append(self._make_cube(1, 1, 3, 3)) - cubes.append(self._make_cube(2, 0, 4, 4)) - cubes.append(self._make_cube(2, 1, 5, 5)) - cube = iris.cube.CubeList(cubes).merge() - self.assertCML(cube, ("merge", "single_split.cml")) - - def test_multi_split(self): - # Test what happens when a cube forces a three-way split. - cubes = [] - cubes.append(self._make_cube(0, 0, 0, 0)) - cubes.append(self._make_cube(0, 0, 1, 1)) - cubes.append(self._make_cube(0, 1, 0, 2)) - cubes.append(self._make_cube(0, 1, 1, 3)) - cubes.append(self._make_cube(1, 0, 0, 4)) - cubes.append(self._make_cube(1, 0, 1, 5)) - cubes.append(self._make_cube(1, 1, 0, 6)) - cubes.append(self._make_cube(1, 1, 1, 7)) - cubes.append(self._make_cube(2, 0, 0, 8)) - cubes.append(self._make_cube(2, 0, 1, 9)) - cubes.append(self._make_cube(2, 1, 0, 10)) - cubes.append(self._make_cube(2, 1, 1, 11)) - cube = iris.cube.CubeList(cubes).merge() - self.assertCML(cube, ("merge", "multi_split.cml")) - - -class TestCombination(tests.IrisTest): - def _make_cube(self, a, b, c, d, data=0): - cube_data = np.empty((4, 5), dtype=np.float32) - cube_data[:] = data - cube = iris.cube.Cube(cube_data) - cube.add_dim_coord( - DimCoord( - np.array([0, 1, 2, 3, 4], dtype=np.int32), - long_name="x", - units="1", - ), - 1, - ) - cube.add_dim_coord( - DimCoord( - np.array([0, 1, 2, 3], dtype=np.int32), - long_name="y", - units="1", - ), - 0, - ) - - for name, value in zip(["a", "b", "c", "d"], [a, b, c, d]): - dtype = np.str_ if isinstance(value, str) else np.float32 - cube.add_aux_coord( - AuxCoord(np.array([value], dtype=dtype), long_name=name, units="1") - ) - - return cube - - def test_separable_combination(self): - cubes = iris.cube.CubeList() - cubes.append( - self._make_cube("2005", "ECMWF", "HOPE-E, Sys 1, Met 1, ENSEMBLES", 0) - ) - cubes.append( - self._make_cube("2005", "ECMWF", "HOPE-E, Sys 1, Met 1, ENSEMBLES", 1) - ) - cubes.append( - self._make_cube("2005", "ECMWF", "HOPE-E, Sys 1, Met 1, ENSEMBLES", 2) - ) - cubes.append( - self._make_cube( - "2026", "UK Met Office", "HadGEM2, Sys 1, Met 1, ENSEMBLES", 0 - ) - ) - cubes.append( - self._make_cube( - "2026", "UK Met Office", "HadGEM2, Sys 1, Met 1, ENSEMBLES", 1 - ) - ) - cubes.append( - self._make_cube( - "2026", "UK Met Office", "HadGEM2, Sys 1, Met 1, ENSEMBLES", 2 - ) - ) - cubes.append( - self._make_cube("2002", "CERFACS", "GELATO, Sys 0, Met 1, ENSEMBLES", 0) - ) - cubes.append( - self._make_cube("2002", "CERFACS", "GELATO, Sys 0, Met 1, ENSEMBLES", 1) - ) - cubes.append( - self._make_cube("2002", "CERFACS", "GELATO, Sys 0, Met 1, ENSEMBLES", 2) - ) - cubes.append( - self._make_cube("2002", "IFM-GEOMAR", "ECHAM5, Sys 1, Met 10, ENSEMBLES", 0) - ) - cubes.append( - self._make_cube("2002", "IFM-GEOMAR", "ECHAM5, Sys 1, Met 10, ENSEMBLES", 1) - ) - cubes.append( - self._make_cube("2002", "IFM-GEOMAR", "ECHAM5, Sys 1, Met 10, ENSEMBLES", 2) - ) - cubes.append( - self._make_cube( - "2502", "UK Met Office", "HadCM3, Sys 51, Met 10, ENSEMBLES", 0 - ) - ) - cubes.append( - self._make_cube( - "2502", "UK Met Office", "HadCM3, Sys 51, Met 11, ENSEMBLES", 0 - ) - ) - cubes.append( - self._make_cube( - "2502", "UK Met Office", "HadCM3, Sys 51, Met 12, ENSEMBLES", 0 - ) - ) - cubes.append( - self._make_cube( - "2502", "UK Met Office", "HadCM3, Sys 51, Met 13, ENSEMBLES", 0 - ) - ) - cubes.append( - self._make_cube( - "2502", "UK Met Office", "HadCM3, Sys 51, Met 14, ENSEMBLES", 0 - ) - ) - cubes.append( - self._make_cube( - "2502", "UK Met Office", "HadCM3, Sys 51, Met 15, ENSEMBLES", 0 - ) - ) - cubes.append( - self._make_cube( - "2502", "UK Met Office", "HadCM3, Sys 51, Met 16, ENSEMBLES", 0 - ) - ) - cubes.append( - self._make_cube( - "2502", "UK Met Office", "HadCM3, Sys 51, Met 17, ENSEMBLES", 0 - ) - ) - cubes.append( - self._make_cube( - "2502", "UK Met Office", "HadCM3, Sys 51, Met 18, ENSEMBLES", 0 - ) - ) - cube = cubes.merge() - self.assertCML(cube, ("merge", "separable_combination.cml"), checksum=False) - - -class TestDimSelection(tests.IrisTest): - def _make_cube(self, a, b, data=0, a_dim=False, b_dim=False): - cube_data = np.empty((4, 5), dtype=np.float32) - cube_data[:] = data - cube = iris.cube.Cube(cube_data) - cube.add_dim_coord( - DimCoord( - np.array([0, 1, 2, 3, 4], dtype=np.int32), - long_name="x", - units="1", - ), - 1, - ) - cube.add_dim_coord( - DimCoord( - np.array([0, 1, 2, 3], dtype=np.int32), - long_name="y", - units="1", - ), - 0, - ) - - for name, value, dim in zip(["a", "b"], [a, b], [a_dim, b_dim]): - dtype = np.str_ if isinstance(value, str) else np.float32 - ctype = DimCoord if dim else AuxCoord - coord = ctype(np.array([value], dtype=dtype), long_name=name, units="1") - cube.add_aux_coord(coord) - - return cube - - def test_string_a_with_aux(self): - templates = (("a", 0), ("b", 1), ("c", 2), ("d", 3)) - cubes = [self._make_cube(a, b) for a, b in templates] - cube = iris.cube.CubeList(cubes).merge()[0] - self.assertCML(cube, ("merge", "string_a_with_aux.cml"), checksum=False) - self.assertIsInstance(cube.coord("a"), AuxCoord) - self.assertIsInstance(cube.coord("b"), DimCoord) - self.assertTrue(cube.coord("b") in cube.dim_coords) - - def test_string_b_with_aux(self): - templates = ((0, "a"), (1, "b"), (2, "c"), (3, "d")) - cubes = [self._make_cube(a, b) for a, b in templates] - cube = iris.cube.CubeList(cubes).merge()[0] - self.assertCML(cube, ("merge", "string_b_with_aux.cml"), checksum=False) - self.assertIsInstance(cube.coord("a"), DimCoord) - self.assertTrue(cube.coord("a") in cube.dim_coords) - self.assertIsInstance(cube.coord("b"), AuxCoord) - - def test_string_a_with_dim(self): - templates = (("a", 0), ("b", 1), ("c", 2), ("d", 3)) - cubes = [self._make_cube(a, b, b_dim=True) for a, b in templates] - cube = iris.cube.CubeList(cubes).merge()[0] - self.assertCML(cube, ("merge", "string_a_with_dim.cml"), checksum=False) - self.assertIsInstance(cube.coord("a"), AuxCoord) - self.assertIsInstance(cube.coord("b"), DimCoord) - self.assertTrue(cube.coord("b") in cube.dim_coords) - - def test_string_b_with_dim(self): - templates = ((0, "a"), (1, "b"), (2, "c"), (3, "d")) - cubes = [self._make_cube(a, b, a_dim=True) for a, b in templates] - cube = iris.cube.CubeList(cubes).merge()[0] - self.assertCML(cube, ("merge", "string_b_with_dim.cml"), checksum=False) - self.assertIsInstance(cube.coord("a"), DimCoord) - self.assertTrue(cube.coord("a") in cube.dim_coords) - self.assertIsInstance(cube.coord("b"), AuxCoord) - - def test_string_a_b(self): - templates = (("a", "0"), ("b", "1"), ("c", "2"), ("d", "3")) - cubes = [self._make_cube(a, b) for a, b in templates] - cube = iris.cube.CubeList(cubes).merge()[0] - self.assertCML(cube, ("merge", "string_a_b.cml"), checksum=False) - self.assertIsInstance(cube.coord("a"), AuxCoord) - self.assertIsInstance(cube.coord("b"), AuxCoord) - - def test_a_aux_b_aux(self): - templates = ((0, 10), (1, 11), (2, 12), (3, 13)) - cubes = [self._make_cube(a, b) for a, b in templates] - cube = iris.cube.CubeList(cubes).merge()[0] - self.assertCML(cube, ("merge", "a_aux_b_aux.cml"), checksum=False) - self.assertIsInstance(cube.coord("a"), DimCoord) - self.assertTrue(cube.coord("a") in cube.dim_coords) - self.assertIsInstance(cube.coord("b"), DimCoord) - self.assertTrue(cube.coord("b") in cube.aux_coords) - - def test_a_aux_b_dim(self): - templates = ((0, 10), (1, 11), (2, 12), (3, 13)) - cubes = [self._make_cube(a, b, b_dim=True) for a, b in templates] - cube = iris.cube.CubeList(cubes).merge()[0] - self.assertCML(cube, ("merge", "a_aux_b_dim.cml"), checksum=False) - self.assertIsInstance(cube.coord("a"), DimCoord) - self.assertTrue(cube.coord("a") in cube.aux_coords) - self.assertIsInstance(cube.coord("b"), DimCoord) - self.assertTrue(cube.coord("b") in cube.dim_coords) - - def test_a_dim_b_aux(self): - templates = ((0, 10), (1, 11), (2, 12), (3, 13)) - cubes = [self._make_cube(a, b, a_dim=True) for a, b in templates] - cube = iris.cube.CubeList(cubes).merge()[0] - self.assertCML(cube, ("merge", "a_dim_b_aux.cml"), checksum=False) - self.assertIsInstance(cube.coord("a"), DimCoord) - self.assertTrue(cube.coord("a") in cube.dim_coords) - self.assertIsInstance(cube.coord("b"), DimCoord) - self.assertTrue(cube.coord("b") in cube.aux_coords) - - def test_a_dim_b_dim(self): - templates = ((0, 10), (1, 11), (2, 12), (3, 13)) - cubes = [self._make_cube(a, b, a_dim=True, b_dim=True) for a, b in templates] - cube = iris.cube.CubeList(cubes).merge()[0] - self.assertCML(cube, ("merge", "a_dim_b_dim.cml"), checksum=False) - self.assertIsInstance(cube.coord("a"), DimCoord) - self.assertTrue(cube.coord("a") in cube.dim_coords) - self.assertIsInstance(cube.coord("b"), DimCoord) - self.assertTrue(cube.coord("b") in cube.aux_coords) - - -class TestTimeTripleMerging(tests.IrisTest): - def _make_cube(self, a, b, c, data=0): - cube_data = np.empty((4, 5), dtype=np.float32) - cube_data[:] = data - cube = iris.cube.Cube(cube_data) - cube.add_dim_coord( - DimCoord( - np.array([0, 1, 2, 3, 4], dtype=np.int32), - long_name="x", - units="1", - ), - 1, - ) - cube.add_dim_coord( - DimCoord( - np.array([0, 1, 2, 3], dtype=np.int32), - long_name="y", - units="1", - ), - 0, - ) - cube.add_aux_coord( - DimCoord( - np.array([a], dtype=np.int32), - standard_name="forecast_period", - units="1", - ) - ) - cube.add_aux_coord( - DimCoord( - np.array([b], dtype=np.int32), - standard_name="forecast_reference_time", - units="1", - ) - ) - cube.add_aux_coord( - DimCoord(np.array([c], dtype=np.int32), standard_name="time", units="1") - ) - return cube - - def _test_triples(self, triples, filename): - cubes = [self._make_cube(fp, rt, t) for fp, rt, t in triples] - cube = iris.cube.CubeList(cubes).merge() - self.assertCML( - cube, ("merge", "time_triple_" + filename + ".cml"), checksum=False - ) - - def test_single_forecast(self): - # A single forecast series (i.e. from a single reference time) - # => fp, t: 4; rt: scalar - triples = ( - (0, 10, 10), - (1, 10, 11), - (2, 10, 12), - (3, 10, 13), - ) - self._test_triples(triples, "single_forecast") - - def test_successive_forecasts(self): - # Three forecast series from successively later reference times - # => rt, t: 3; fp, t: 4 - triples = ( - (0, 10, 10), - (1, 10, 11), - (2, 10, 12), - (3, 10, 13), - (0, 11, 11), - (1, 11, 12), - (2, 11, 13), - (3, 11, 14), - (0, 12, 12), - (1, 12, 13), - (2, 12, 14), - (3, 12, 15), - ) - self._test_triples(triples, "successive_forecasts") - - def test_time_vs_ref_time(self): - # => fp, t: 4; fp, rt: 3 - triples = ( - (2, 10, 12), - (3, 10, 13), - (4, 10, 14), - (5, 10, 15), - (1, 11, 12), - (2, 11, 13), - (3, 11, 14), - (4, 11, 15), - (0, 12, 12), - (1, 12, 13), - (2, 12, 14), - (3, 12, 15), - ) - self._test_triples(triples, "time_vs_ref_time") - - def test_time_vs_forecast(self): - # => rt, t: 4, fp, rt: 3 - triples = ( - (0, 10, 10), - (0, 11, 11), - (0, 12, 12), - (0, 13, 13), - (1, 9, 10), - (1, 10, 11), - (1, 11, 12), - (1, 12, 13), - (2, 8, 10), - (2, 9, 11), - (2, 10, 12), - (2, 11, 13), - ) - self._test_triples(triples, "time_vs_forecast") - - def test_time_non_dim_coord(self): - # => rt: 1 fp, t (bounded): 2 - triples = ( - (5, 0, 2.5), - (10, 0, 5), - ) - cubes = [self._make_cube(fp, rt, t) for fp, rt, t in triples] - for end_time, cube in zip([5, 10], cubes): - cube.coord("time").bounds = [0, end_time] - (cube,) = iris.cube.CubeList(cubes).merge() - self.assertCML( - cube, - ("merge", "time_triple_time_non_dim_coord.cml"), - checksum=False, - ) - # make sure that forecast_period is the dimensioned coordinate (as time becomes an AuxCoord) - self.assertEqual( - cube.coord(dimensions=0, dim_coords=True).name(), "forecast_period" - ) - - def test_independent(self): - # => fp: 2; rt: 2; t: 2 - triples = ( - (0, 10, 10), - (0, 11, 10), - (0, 10, 11), - (0, 11, 11), - (1, 10, 10), - (1, 11, 10), - (1, 10, 11), - (1, 11, 11), - ) - self._test_triples(triples, "independent") - - def test_series(self): - # => fp, rt, t: 5 (with only t being definitive). - triples = ( - (0, 10, 10), - (0, 11, 11), - (0, 12, 12), - (1, 12, 13), - (2, 12, 14), - ) - self._test_triples(triples, "series") - - def test_non_expanding_dimension(self): - triples = ( - (0, 10, 0), - (0, 20, 1), - (0, 20, 0), - ) - # => fp: scalar; rt, t: 3 (with no time being definitive) - self._test_triples(triples, "non_expanding") - - def test_duplicate_data(self): - # test what happens when we have repeated time coordinates (i.e. duplicate data) - cube1 = self._make_cube(0, 10, 0) - cube2 = self._make_cube(1, 20, 1) - cube3 = self._make_cube(1, 20, 1) - - # check that we get a duplicate data error when unique is True - with self.assertRaises(iris.exceptions.DuplicateDataError): - iris.cube.CubeList([cube1, cube2, cube3]).merge() - - cubes = iris.cube.CubeList([cube1, cube2, cube3]).merge(unique=False) - self.assertCML( - cubes, ("merge", "time_triple_duplicate_data.cml"), checksum=False - ) - - def test_simple1(self): - cube1 = self._make_cube(0, 10, 0) - cube2 = self._make_cube(1, 20, 1) - cube3 = self._make_cube(2, 20, 0) - cube = iris.cube.CubeList([cube1, cube2, cube3]).merge() - self.assertCML(cube, ("merge", "time_triple_merging1.cml"), checksum=False) - - def test_simple2(self): - cubes = iris.cube.CubeList( - [ - self._make_cube(0, 0, 0), - self._make_cube(1, 0, 1), - self._make_cube(2, 0, 2), - self._make_cube(0, 1, 3), - self._make_cube(1, 1, 4), - self._make_cube(2, 1, 5), - ] - ) - cube = cubes.merge()[0] - self.assertCML(cube, ("merge", "time_triple_merging2.cml"), checksum=False) - - cube = iris.cube.CubeList(cubes[:-1]).merge()[0] - self.assertCML(cube, ("merge", "time_triple_merging3.cml"), checksum=False) - - def test_simple3(self): - cubes = iris.cube.CubeList( - [ - self._make_cube(0, 0, 0), - self._make_cube(0, 1, 1), - self._make_cube(0, 2, 2), - self._make_cube(1, 0, 3), - self._make_cube(1, 1, 4), - self._make_cube(1, 2, 5), - ] - ) - cube = cubes.merge()[0] - self.assertCML(cube, ("merge", "time_triple_merging4.cml"), checksum=False) - - cube = iris.cube.CubeList(cubes[:-1]).merge()[0] - self.assertCML(cube, ("merge", "time_triple_merging5.cml"), checksum=False) - - -class TestCubeMergeTheoretical(tests.IrisTest): - def test_simple_bounds_merge(self): - cube1 = iris.tests.stock.simple_2d() - cube2 = iris.tests.stock.simple_2d() - - cube1.add_aux_coord(DimCoord(np.int32(10), long_name="pressure", units="Pa")) - cube2.add_aux_coord(DimCoord(np.int32(11), long_name="pressure", units="Pa")) - - r = iris.cube.CubeList([cube1, cube2]).merge() - self.assertCML(r, ("cube_merge", "test_simple_bound_merge.cml")) - - def test_simple_multidim_merge(self): - cube1 = iris.tests.stock.simple_2d_w_multidim_coords() - cube2 = iris.tests.stock.simple_2d_w_multidim_coords() - - cube1.add_aux_coord(DimCoord(np.int32(10), long_name="pressure", units="Pa")) - cube2.add_aux_coord(DimCoord(np.int32(11), long_name="pressure", units="Pa")) - - r = iris.cube.CubeList([cube1, cube2]).merge()[0] - self.assertCML(r, ("cube_merge", "multidim_coord_merge.cml")) - - # try transposing the cubes first - cube1.transpose([1, 0]) - cube2.transpose([1, 0]) - r = iris.cube.CubeList([cube1, cube2]).merge()[0] - self.assertCML(r, ("cube_merge", "multidim_coord_merge_transpose.cml")) - - def test_simple_points_merge(self): - cube1 = iris.tests.stock.simple_2d(with_bounds=False) - cube2 = iris.tests.stock.simple_2d(with_bounds=False) - - cube1.add_aux_coord(DimCoord(np.int32(10), long_name="pressure", units="Pa")) - cube2.add_aux_coord(DimCoord(np.int32(11), long_name="pressure", units="Pa")) - - r = iris.cube.CubeList([cube1, cube2]).merge() - self.assertCML(r, ("cube_merge", "test_simple_merge.cml")) - - # check that the unique merging raises a Duplicate data error - self.assertRaises( - iris.exceptions.DuplicateDataError, - iris.cube.CubeList([cube1, cube1]).merge, - unique=True, - ) - - # check that non unique merging returns both cubes - r = iris.cube.CubeList([cube1, cube1]).merge(unique=False) - self.assertCML(r[0], ("cube_merge", "test_orig_point_cube.cml")) - self.assertCML(r[1], ("cube_merge", "test_orig_point_cube.cml")) - - # test attribute merging - cube1.attributes["my_attr1"] = "foo" - r = iris.cube.CubeList([cube1, cube2]).merge() - # result should be 2 cubes - self.assertCML(r, ("cube_merge", "test_simple_attributes1.cml")) - - cube2.attributes["my_attr1"] = "bar" - r = iris.cube.CubeList([cube1, cube2]).merge() - # result should be 2 cubes - self.assertCML(r, ("cube_merge", "test_simple_attributes2.cml")) - - cube2.attributes["my_attr1"] = "foo" - r = iris.cube.CubeList([cube1, cube2]).merge() - # result should be 1 cube - self.assertCML(r, ("cube_merge", "test_simple_attributes3.cml")) - - -class TestCubeMergeWithAncils(tests.IrisTest): - def _makecube(self, y, cm=False, av=False): - cube = iris.cube.Cube([0, 0]) - cube.add_dim_coord(iris.coords.DimCoord([0, 1], long_name="x"), 0) - cube.add_aux_coord(iris.coords.DimCoord(y, long_name="y")) - if cm: - cube.add_cell_measure(iris.coords.CellMeasure([1, 1], long_name="foo"), 0) - if av: - cube.add_ancillary_variable( - iris.coords.AncillaryVariable([1, 1], long_name="bar"), 0 - ) - return cube - - def test_fail_missing_cell_measure(self): - cube1 = self._makecube(0, cm=True) - cube2 = self._makecube(1) - cubes = iris.cube.CubeList([cube1, cube2]).merge() - self.assertEqual(len(cubes), 2) - - def test_fail_missing_ancillary_variable(self): - cube1 = self._makecube(0, av=True) - cube2 = self._makecube(1) - cubes = iris.cube.CubeList([cube1, cube2]).merge() - self.assertEqual(len(cubes), 2) - - def test_fail_different_cell_measure(self): - cube1 = self._makecube(0, cm=True) - cube2 = self._makecube(1) - cube2.add_cell_measure(iris.coords.CellMeasure([2, 2], long_name="foo"), 0) - cubes = iris.cube.CubeList([cube1, cube2]).merge() - self.assertEqual(len(cubes), 2) - - def test_fail_different_ancillary_variable(self): - cube1 = self._makecube(0, av=True) - cube2 = self._makecube(1) - cube2.add_ancillary_variable( - iris.coords.AncillaryVariable([2, 2], long_name="bar"), 0 - ) - cubes = iris.cube.CubeList([cube1, cube2]).merge() - self.assertEqual(len(cubes), 2) - - def test_merge_with_cell_measure(self): - cube1 = self._makecube(0, cm=True) - cube2 = self._makecube(1, cm=True) - cubes = iris.cube.CubeList([cube1, cube2]).merge() - self.assertEqual(len(cubes), 1) - self.assertEqual(cube1.cell_measures(), cubes[0].cell_measures()) - - def test_merge_with_ancillary_variable(self): - cube1 = self._makecube(0, av=True) - cube2 = self._makecube(1, av=True) - cubes = iris.cube.CubeList([cube1, cube2]).merge() - self.assertEqual(len(cubes), 1) - self.assertEqual(cube1.ancillary_variables(), cubes[0].ancillary_variables()) - - def test_cell_measure_error_msg(self): - msg = "cube.cell_measures differ" - cube1 = self._makecube(0, cm=True) - cube2 = self._makecube(1) - with self.assertRaisesRegex(iris.exceptions.MergeError, msg): - _ = iris.cube.CubeList([cube1, cube2]).merge_cube() - - def test_ancillary_variable_error_msg(self): - msg = "cube.ancillary_variables differ" - cube1 = self._makecube(0, av=True) - cube2 = self._makecube(1) - with self.assertRaisesRegex(iris.exceptions.MergeError, msg): - _ = iris.cube.CubeList([cube1, cube2]).merge_cube() - - -class TestCubeMerge__split_attributes__error_messages(tests.IrisTest): - """Specific tests for the detection and wording of attribute-mismatch errors. - - In particular, the adoption of 'split' attributes with the new - :class:`iris.cube.CubeAttrsDict` introduces some more subtle possible discrepancies - in attributes, where this has also impacted the messaging, so this aims to probe - those cases. - """ - - def _check_merge_error(self, attrs_1, attrs_2, expected_message): - """Check the error from a merge failure caused by a mismatch of attributes. - - Build a pair of cubes with given attributes, merge them + check for a match - to the expected error message. - """ - cube_1 = iris.cube.Cube( - [0], - aux_coords_and_dims=[(AuxCoord([1], long_name="x"), None)], - attributes=attrs_1, - ) - cube_2 = iris.cube.Cube( - [0], - aux_coords_and_dims=[(AuxCoord([2], long_name="x"), None)], - attributes=attrs_2, - ) - with self.assertRaisesRegex(iris.exceptions.MergeError, expected_message): - iris.cube.CubeList([cube_1, cube_2]).merge_cube() - - def test_keys_differ__single(self): - self._check_merge_error( - attrs_1=dict(a=1, b=2), - attrs_2=dict(a=1), - # Note: matching key 'a' does *not* appear in the message - expected_message="cube.attributes keys differ: 'b'", - ) - - def test_keys_differ__multiple(self): - self._check_merge_error( - attrs_1=dict(a=1, b=2), - attrs_2=dict(a=1, c=2), - expected_message="cube.attributes keys differ: 'b', 'c'", - ) - - def test_values_differ__single(self): - self._check_merge_error( - attrs_1=dict(a=1, b=2), # Note: matching key 'a' does not appear - attrs_2=dict(a=1, b=3), - expected_message="cube.attributes values differ for keys: 'b'", - ) - - def test_values_differ__multiple(self): - self._check_merge_error( - attrs_1=dict(a=1, b=2), - attrs_2=dict(a=12, b=22), - expected_message="cube.attributes values differ for keys: 'a', 'b'", - ) - - def test_splitattrs_keys_local_global_mismatch(self): - # Since Cube.attributes is now a "split-attributes" dictionary, it is now - # possible to have "cube1.attributes != cube1.attributes", but also - # "set(cube1.attributes.keys()) == set(cube2.attributes.keys())". - # I.E. it is now necessary to specifically compare ".globals" and ".locals" to - # see *what* differs between two attributes dictionaries. - self._check_merge_error( - attrs_1=CubeAttrsDict(globals=dict(a=1), locals=dict(b=2)), - attrs_2=CubeAttrsDict(locals=dict(a=2)), - expected_message="cube.attributes keys differ: 'a', 'b'", - ) - - def test_splitattrs_keys_local_match_masks_global_mismatch(self): - self._check_merge_error( - attrs_1=CubeAttrsDict(globals=dict(a=1), locals=dict(a=3)), - attrs_2=CubeAttrsDict(globals=dict(a=2), locals=dict(a=3)), - expected_message="cube.attributes values differ for keys: 'a'", - ) - - -if __name__ == "__main__": - tests.main() diff --git a/lib/iris/tests/unit/coords/test_Cell.py b/lib/iris/tests/unit/coords/test_Cell.py index 18d34b7d39..2e4c5944e0 100644 --- a/lib/iris/tests/unit/coords/test_Cell.py +++ b/lib/iris/tests/unit/coords/test_Cell.py @@ -4,31 +4,28 @@ # See LICENSE in the root of the repository for full licensing details. """Unit tests for the :class:`iris.coords.Cell` class.""" -# Import iris.tests first so that some things can be initialised before -# importing anything else. -import iris.tests as tests # isort:skip - import datetime import cftime import numpy as np +import pytest from iris.coords import Cell from iris.time import PartialDateTime -class Test___common_cmp__(tests.IrisTest): +class Test___common_cmp__: def assert_raises_on_comparison(self, cell, other, exception_type, regexp): - with self.assertRaisesRegex(exception_type, regexp): + with pytest.raises(exception_type, match=regexp): cell < other - with self.assertRaisesRegex(exception_type, regexp): + with pytest.raises(exception_type, match=regexp): cell <= other - with self.assertRaisesRegex(exception_type, regexp): + with pytest.raises(exception_type, match=regexp): cell > other - with self.assertRaisesRegex(exception_type, regexp): + with pytest.raises(exception_type, match=regexp): cell >= other - def test_PartialDateTime_bounded_cell(self): + def test_partial_date_time_bounded_cell(self): # Check bounded cell comparisons to a PartialDateTime dt = PartialDateTime(month=6) cell = Cell( @@ -38,10 +35,10 @@ def test_PartialDateTime_bounded_cell(self): datetime.datetime(2011, 1, 1), ], ) - self.assertGreater(dt, cell) - self.assertGreaterEqual(dt, cell) - self.assertLess(cell, dt) - self.assertLessEqual(cell, dt) + assert dt > cell + assert dt >= cell + assert cell < dt + assert cell <= dt def test_cftime_calender_bounded_cell(self): # Check that cell comparisons fail with different calendars @@ -55,46 +52,46 @@ def test_cftime_calender_bounded_cell(self): ) self.assert_raises_on_comparison(cell, dt, TypeError, "different calendars") - def test_PartialDateTime_unbounded_cell(self): + def test_partial_date_time_unbounded_cell(self): # Check that cell comparison works with PartialDateTimes. dt = PartialDateTime(month=6) cell = Cell(cftime.datetime(2010, 3, 1)) - self.assertLess(cell, dt) - self.assertGreater(dt, cell) - self.assertLessEqual(cell, dt) - self.assertGreaterEqual(dt, cell) + assert cell < dt + assert dt > cell + assert cell <= dt + assert dt >= cell def test_datetime_unbounded_cell(self): # Check that cell comparison works with datetimes & cftimes. dt = datetime.datetime(2000, 6, 15) cell = Cell(cftime.datetime(2000, 1, 1)) - self.assertGreater(dt, cell) - self.assertGreaterEqual(dt, cell) - self.assertLess(cell, dt) - self.assertLessEqual(cell, dt) + assert dt > cell + assert dt >= cell + assert cell < dt + assert cell <= dt - def test_0D_numpy_array(self): + def test_0_d_numpy_array(self): # Check that cell comparison works with 0D numpy arrays cell = Cell(1.3) - self.assertGreater(np.array(1.5), cell) - self.assertLess(np.array(1.1), cell) - self.assertGreaterEqual(np.array(1.3), cell) - self.assertLessEqual(np.array(1.3), cell) + assert np.array(1.5) > cell + assert np.array(1.1) < cell + assert np.array(1.3) >= cell + assert np.array(1.3) <= cell def test_len_1_numpy_array(self): # Check that cell comparison works with numpy arrays of len=1 cell = Cell(1.3) - self.assertGreater(np.array([1.5]), cell) - self.assertLess(np.array([1.1]), cell) - self.assertGreaterEqual(np.array([1.3]), cell) - self.assertLessEqual(np.array([1.3]), cell) + assert np.array([1.5]) > cell + assert np.array([1.1]) < cell + assert np.array([1.3]) >= cell + assert np.array([1.3]) <= cell -class Test___eq__(tests.IrisTest): +class Test___eq__: def test_datetimelike(self): # Check that cell equality works with different datetime objects # using the same calendar @@ -103,7 +100,7 @@ def test_datetimelike(self): datetime.datetime(2010, 1, 1), bound=None, ) - self.assertEqual(cell, point) + assert cell == point def test_datetimelike_bounded_cell(self): # Check that cell equality works with bounded cells using different datetime objects @@ -115,7 +112,7 @@ def test_datetimelike_bounded_cell(self): datetime.datetime(2011, 1, 1), ], ) - self.assertEqual(cell, point) + assert cell == point def test_datetimelike_calenders_cell(self): # Check that equality with a cell with a different calendar @@ -128,24 +125,24 @@ def test_datetimelike_calenders_cell(self): datetime.datetime(2011, 1, 1), ], ) - with self.assertRaisesRegex(TypeError, "different calendars"): + with pytest.raises(TypeError, match="different calendars"): cell >= point - def test_PartialDateTime_other(self): + def test_partial_date_time_other(self): cell = Cell(datetime.datetime(2010, 3, 2)) # A few simple cases. - self.assertEqual(cell, PartialDateTime(month=3)) - self.assertNotEqual(cell, PartialDateTime(month=3, hour=12)) - self.assertNotEqual(cell, PartialDateTime(month=4)) + assert cell == PartialDateTime(month=3) + assert cell != PartialDateTime(month=3, hour=12) + assert cell != PartialDateTime(month=4) -class Test_contains_point(tests.IrisTest): +class Test_contains_point: """Test that contains_point works for combinations. Combinations of datetime, cf.datatime, and PartialDateTime objects. """ - def test_datetime_PartialDateTime_point(self): + def test_datetime_partial_date_time_point(self): point = PartialDateTime(month=6) cell = Cell( datetime.datetime(2010, 1, 1), @@ -154,7 +151,7 @@ def test_datetime_PartialDateTime_point(self): datetime.datetime(2011, 1, 1), ], ) - self.assertFalse(cell.contains_point(point)) + assert not cell.contains_point(point) def test_datetime_cftime_standard_point(self): point = cftime.datetime(2010, 6, 15) @@ -165,7 +162,7 @@ def test_datetime_cftime_standard_point(self): datetime.datetime(2011, 1, 1), ], ) - self.assertTrue(cell.contains_point(point)) + assert cell.contains_point(point) def test_datetime_cftime_360day_point(self): point = cftime.datetime(2010, 6, 15, calendar="360_day") @@ -176,10 +173,10 @@ def test_datetime_cftime_360day_point(self): datetime.datetime(2011, 1, 1), ], ) - with self.assertRaisesRegex(TypeError, "different calendars"): + with pytest.raises(TypeError, match="different calendars"): cell.contains_point(point) - def test_cftime_standard_PartialDateTime_point(self): + def test_cftime_standard_partial_date_time_point(self): point = PartialDateTime(month=6) cell = Cell( cftime.datetime(2010, 1, 1), @@ -188,9 +185,9 @@ def test_cftime_standard_PartialDateTime_point(self): cftime.datetime(2011, 1, 1), ], ) - self.assertFalse(cell.contains_point(point)) + assert not cell.contains_point(point) - def test_cftime_360day_PartialDateTime_point(self): + def test_cftime_360day_partial_date_time_point(self): point = PartialDateTime(month=6) cell = Cell( cftime.datetime(2010, 1, 1, calendar="360_day"), @@ -199,7 +196,7 @@ def test_cftime_360day_PartialDateTime_point(self): cftime.datetime(2011, 1, 1, calendar="360_day"), ], ) - self.assertFalse(cell.contains_point(point)) + assert not cell.contains_point(point) def test_cftime_standard_datetime_point(self): point = datetime.datetime(2010, 6, 1) @@ -210,7 +207,7 @@ def test_cftime_standard_datetime_point(self): cftime.datetime(2011, 1, 1), ], ) - self.assertTrue(cell.contains_point(point)) + assert cell.contains_point(point) def test_cftime_360day_datetime_point(self): point = datetime.datetime(2010, 6, 1) @@ -221,7 +218,7 @@ def test_cftime_360day_datetime_point(self): cftime.datetime(2011, 1, 1, calendar="360_day"), ], ) - with self.assertRaisesRegex(TypeError, "different calendars"): + with pytest.raises(TypeError, match="different calendars"): cell.contains_point(point) def test_cftime_360_day_cftime_360day_point(self): @@ -233,10 +230,10 @@ def test_cftime_360_day_cftime_360day_point(self): cftime.datetime(2011, 1, 1, calendar="360_day"), ], ) - self.assertTrue(cell.contains_point(point)) + assert cell.contains_point(point) -class Test_numpy_comparison(tests.IrisTest): +class Test_numpy_comparison: """Unit tests to check that the results of comparisons with numpy types can be used as truth values. """ @@ -253,7 +250,7 @@ def test_cell_lhs(self): bool(cell == n) bool(cell != n) except: # noqa - self.fail("Result of comparison could not be used as a truth value") + pytest.fail("Result of comparison could not be used as a truth value") def test_cell_rhs(self): cell = Cell(point=1.5) @@ -267,8 +264,33 @@ def test_cell_rhs(self): bool(n == cell) bool(n != cell) except: # noqa - self.fail("Result of comparison could not be used as a truth value") - - -if __name__ == "__main__": - tests.main() + pytest.fail("Result of comparison could not be used as a truth value") + + +class Test_hashing: + @pytest.mark.parametrize( + "point", + ( + pytest.param(np.float32(1.0), id="float32"), + pytest.param(np.float64(1.0), id="float64"), + pytest.param(np.int16(1), id="int16"), + pytest.param(np.int32(1), id="int32"), + pytest.param(np.int64(1), id="int64"), + pytest.param(np.uint16(1), id="uint16"), + pytest.param(np.uint32(1), id="uint32"), + pytest.param(np.uint64(1), id="uint64"), + pytest.param(True, id="bool"), + pytest.param(np.ma.masked, id="masked"), + pytest.param(datetime.datetime(2001, 1, 1), id="datetime"), + ), + ) + def test_cell_is_hashable(self, point): + """Test a Cell object is hashable with various point/bound types.""" + # test with no bounds: + cell = Cell(point=point, bound=None) + hash(cell) + + # if a numerical type, then test with bounds based on point: + if isinstance(point, np.number): + cell = Cell(point=input, bound=((point - 1, point + 1))) + hash(cell) diff --git a/lib/iris/tests/unit/merge/test_merge.py b/lib/iris/tests/unit/merge/test_merge.py new file mode 100644 index 0000000000..4174d76a89 --- /dev/null +++ b/lib/iris/tests/unit/merge/test_merge.py @@ -0,0 +1,446 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. +"""Test the cube merging mechanism.""" + +from collections.abc import Iterable +import itertools + +import numpy as np +import numpy.ma as ma +import pytest + +import iris +from iris._lazy_data import as_lazy_data +from iris.coords import AuxCoord, DimCoord +import iris.cube +from iris.cube import CubeAttrsDict +import iris.exceptions +from iris.tests._shared_utils import assert_array_equal, assert_masked_array_equal + + +class TestDataMergeCombos: + def _make_data( + self, + data, + dtype=np.dtype("int32"), + fill_value=None, + mask=None, + lazy=False, + N=3, + ): + if isinstance(data, Iterable): + shape = (len(data), N, N) + data = np.array(data).reshape(-1, 1, 1) + else: + shape = (N, N) + if mask is not None: + payload = ma.empty(shape, dtype=dtype, fill_value=fill_value) + payload.data[:] = data + if isinstance(mask, bool): + payload.mask = mask + else: + payload[mask] = ma.masked + else: + payload = np.empty(shape, dtype=dtype) + payload[:] = data + if lazy: + payload = as_lazy_data(payload) + return payload + + def _make_cube( + self, + data, + dtype=np.dtype("int32"), + fill_value=None, + mask=None, + lazy=False, + N=3, + ): + x = np.arange(N) + y = np.arange(N) + payload = self._make_data( + data, dtype=dtype, fill_value=fill_value, mask=mask, lazy=lazy, N=N + ) + cube = iris.cube.Cube(payload) + lat = DimCoord(y, standard_name="latitude", units="degrees") + cube.add_dim_coord(lat, 0) + lon = DimCoord(x, standard_name="longitude", units="degrees") + cube.add_dim_coord(lon, 1) + height = DimCoord(data, standard_name="height", units="m") + cube.add_aux_coord(height) + return cube + + @staticmethod + def _expected_fill_value(fill0="none", fill1="none"): + result = None + if fill0 != "none" or fill1 != "none": + if fill0 == "none": + result = fill1 + elif fill1 == "none": + result = fill0 + elif fill0 == fill1: + result = fill0 + return result + + def _check_fill_value(self, result, fill0="none", fill1="none"): + expected_fill_value = self._expected_fill_value(fill0, fill1) + if expected_fill_value is None: + data = result.data + if ma.isMaskedArray(data): + np_fill_value = ma.masked_array(0, dtype=result.dtype).fill_value + assert data.fill_value == np_fill_value + else: + data = result.data + if ma.isMaskedArray(data): + assert data.fill_value == expected_fill_value + + def setup_method(self): + self.dtype = np.dtype("int32") + fill_value = 1234 + self.lazy_combos = itertools.product([False, True], [False, True]) + fill_combos = itertools.product([None, fill_value], [fill_value, None]) + single_fill_combos = itertools.product([None, fill_value]) + self.combos = itertools.product(self.lazy_combos, fill_combos) + self.mixed_combos = itertools.product(self.lazy_combos, single_fill_combos) + + def test__ndarray_ndarray(self): + for lazy0, lazy1 in self.lazy_combos: + cubes = iris.cube.CubeList() + cubes.append(self._make_cube(0, dtype=self.dtype, lazy=lazy0)) + cubes.append(self._make_cube(1, dtype=self.dtype, lazy=lazy1)) + result = cubes.merge_cube() + expected = self._make_data([0, 1], dtype=self.dtype) + assert_array_equal(result.data, expected) + assert result.dtype == self.dtype + self._check_fill_value(result) + + def test__masked_masked(self): + for (lazy0, lazy1), (fill0, fill1) in self.combos: + cubes = iris.cube.CubeList() + mask = ((0,), (0,)) + cubes.append( + self._make_cube( + 0, + mask=mask, + lazy=lazy0, + dtype=self.dtype, + fill_value=fill0, + ) + ) + mask = ((1,), (1,)) + cubes.append( + self._make_cube( + 1, + mask=mask, + lazy=lazy1, + dtype=self.dtype, + fill_value=fill1, + ) + ) + result = cubes.merge_cube() + mask = ((0, 1), (0, 1), (0, 1)) + expected_fill_value = self._expected_fill_value(fill0, fill1) + expected = self._make_data( + [0, 1], + mask=mask, + dtype=self.dtype, + fill_value=expected_fill_value, + ) + assert_masked_array_equal(result.data, expected) + assert result.dtype == self.dtype + self._check_fill_value(result, fill0, fill1) + + def test__ndarray_masked(self): + for (lazy0, lazy1), (fill,) in self.mixed_combos: + cubes = iris.cube.CubeList() + cubes.append(self._make_cube(0, lazy=lazy0, dtype=self.dtype)) + mask = [(0, 1), (0, 1)] + cubes.append( + self._make_cube( + 1, mask=mask, lazy=lazy1, dtype=self.dtype, fill_value=fill + ) + ) + result = cubes.merge_cube() + mask = [(1, 1), (0, 1), (0, 1)] + expected_fill_value = self._expected_fill_value(fill) + expected = self._make_data( + [0, 1], + mask=mask, + dtype=self.dtype, + fill_value=expected_fill_value, + ) + assert_masked_array_equal(result.data, expected) + assert result.dtype == self.dtype + self._check_fill_value(result, fill1=fill) + + def test__masked_ndarray(self): + for (lazy0, lazy1), (fill,) in self.mixed_combos: + cubes = iris.cube.CubeList() + mask = [(0, 1), (0, 1)] + cubes.append( + self._make_cube( + 0, mask=mask, lazy=lazy0, dtype=self.dtype, fill_value=fill + ) + ) + cubes.append(self._make_cube(1, lazy=lazy1, dtype=self.dtype)) + result = cubes.merge_cube() + mask = [(0, 0), (0, 1), (0, 1)] + expected_fill_value = self._expected_fill_value(fill) + expected = self._make_data( + [0, 1], + mask=mask, + dtype=self.dtype, + fill_value=expected_fill_value, + ) + assert_masked_array_equal(result.data, expected) + assert result.dtype == self.dtype + self._check_fill_value(result, fill0=fill) + + def test_maksed_array_preserved(self): + for (lazy0, lazy1), (fill,) in self.mixed_combos: + cubes = iris.cube.CubeList() + mask = False + cubes.append( + self._make_cube( + 0, mask=mask, lazy=lazy0, dtype=self.dtype, fill_value=fill + ) + ) + cubes.append(self._make_cube(1, lazy=lazy1, dtype=self.dtype)) + result = cubes.merge_cube() + mask = False + expected_fill_value = self._expected_fill_value(fill) + expected = self._make_data( + [0, 1], + mask=mask, + dtype=self.dtype, + fill_value=expected_fill_value, + ) + assert type(result.data) is ma.MaskedArray + assert_masked_array_equal(result.data, expected) + assert result.dtype == self.dtype + self._check_fill_value(result, fill0=fill) + + def test_fill_value_invariant_to_order__same_non_none(self): + fill_value = 1234 + cubes = [self._make_cube(i, mask=True, fill_value=fill_value) for i in range(3)] + for combo in itertools.permutations(cubes): + result = iris.cube.CubeList(combo).merge_cube() + assert result.data.fill_value == fill_value + + def test_fill_value_invariant_to_order__all_none(self): + cubes = [self._make_cube(i, mask=True, fill_value=None) for i in range(3)] + for combo in itertools.permutations(cubes): + result = iris.cube.CubeList(combo).merge_cube() + np_fill_value = ma.masked_array(0, dtype=result.dtype).fill_value + assert result.data.fill_value == np_fill_value + + def test_fill_value_invariant_to_order__different_non_none(self): + cubes = [self._make_cube(0, mask=True, fill_value=1234)] + cubes.append(self._make_cube(1, mask=True, fill_value=2341)) + cubes.append(self._make_cube(2, mask=True, fill_value=3412)) + cubes.append(self._make_cube(3, mask=True, fill_value=4123)) + for combo in itertools.permutations(cubes): + result = iris.cube.CubeList(combo).merge_cube() + np_fill_value = ma.masked_array(0, dtype=result.dtype).fill_value + assert result.data.fill_value == np_fill_value + + def test_fill_value_invariant_to_order__mixed(self): + cubes = [self._make_cube(0, mask=True, fill_value=None)] + cubes.append(self._make_cube(1, mask=True, fill_value=1234)) + cubes.append(self._make_cube(2, mask=True, fill_value=4321)) + for combo in itertools.permutations(cubes): + result = iris.cube.CubeList(combo).merge_cube() + np_fill_value = ma.masked_array(0, dtype=result.dtype).fill_value + assert result.data.fill_value == np_fill_value + + +class TestCubeMergeWithAncils: + def _makecube(self, y, cm=False, av=False): + cube = iris.cube.Cube([0, 0]) + cube.add_dim_coord(iris.coords.DimCoord([0, 1], long_name="x"), 0) + cube.add_aux_coord(iris.coords.DimCoord(y, long_name="y")) + if cm: + cube.add_cell_measure(iris.coords.CellMeasure([1, 1], long_name="foo"), 0) + if av: + cube.add_ancillary_variable( + iris.coords.AncillaryVariable([1, 1], long_name="bar"), 0 + ) + return cube + + def test_fail_missing_cell_measure(self): + cube1 = self._makecube(0, cm=True) + cube2 = self._makecube(1) + cubes = iris.cube.CubeList([cube1, cube2]).merge() + assert len(cubes) == 2 + + def test_fail_missing_ancillary_variable(self): + cube1 = self._makecube(0, av=True) + cube2 = self._makecube(1) + cubes = iris.cube.CubeList([cube1, cube2]).merge() + assert len(cubes) == 2 + + def test_fail_different_cell_measure(self): + cube1 = self._makecube(0, cm=True) + cube2 = self._makecube(1) + cube2.add_cell_measure(iris.coords.CellMeasure([2, 2], long_name="foo"), 0) + cubes = iris.cube.CubeList([cube1, cube2]).merge() + assert len(cubes) == 2 + + def test_fail_different_ancillary_variable(self): + cube1 = self._makecube(0, av=True) + cube2 = self._makecube(1) + cube2.add_ancillary_variable( + iris.coords.AncillaryVariable([2, 2], long_name="bar"), 0 + ) + cubes = iris.cube.CubeList([cube1, cube2]).merge() + assert len(cubes) == 2 + + def test_merge_with_cell_measure(self): + cube1 = self._makecube(0, cm=True) + cube2 = self._makecube(1, cm=True) + cubes = iris.cube.CubeList([cube1, cube2]).merge() + assert len(cubes) == 1 + assert cube1.cell_measures() == cubes[0].cell_measures() + + def test_merge_with_ancillary_variable(self): + cube1 = self._makecube(0, av=True) + cube2 = self._makecube(1, av=True) + cubes = iris.cube.CubeList([cube1, cube2]).merge() + assert len(cubes) == 1 + assert cube1.ancillary_variables() == cubes[0].ancillary_variables() + + def test_cell_measure_error_msg(self): + msg = "cube.cell_measures differ" + cube1 = self._makecube(0, cm=True) + cube2 = self._makecube(1) + with pytest.raises(iris.exceptions.MergeError, match=msg): + _ = iris.cube.CubeList([cube1, cube2]).merge_cube() + + def test_ancillary_variable_error_msg(self): + msg = "cube.ancillary_variables differ" + cube1 = self._makecube(0, av=True) + cube2 = self._makecube(1) + with pytest.raises(iris.exceptions.MergeError, match=msg): + _ = iris.cube.CubeList([cube1, cube2]).merge_cube() + + +class TestCubeMerge__split_attributes__error_messages: + """Specific tests for the detection and wording of attribute-mismatch errors. + + In particular, the adoption of 'split' attributes with the new + :class:`iris.cube.CubeAttrsDict` introduces some more subtle possible discrepancies + in attributes, where this has also impacted the messaging, so this aims to probe + those cases. + """ + + def _check_merge_error(self, attrs_1, attrs_2, expected_message): + """Check the error from a merge failure caused by a mismatch of attributes. + + Build a pair of cubes with given attributes, merge them + check for a match + to the expected error message. + """ + cube_1 = iris.cube.Cube( + [0], + aux_coords_and_dims=[(AuxCoord([1], long_name="x"), None)], + attributes=attrs_1, + ) + cube_2 = iris.cube.Cube( + [0], + aux_coords_and_dims=[(AuxCoord([2], long_name="x"), None)], + attributes=attrs_2, + ) + with pytest.raises(iris.exceptions.MergeError, match=expected_message): + iris.cube.CubeList([cube_1, cube_2]).merge_cube() + + def test_keys_differ__single(self): + self._check_merge_error( + attrs_1=dict(a=1, b=2), + attrs_2=dict(a=1), + # Note: matching key 'a' does *not* appear in the message + expected_message="cube.attributes keys differ: 'b'", + ) + + def test_keys_differ__multiple(self): + self._check_merge_error( + attrs_1=dict(a=1, b=2), + attrs_2=dict(a=1, c=2), + expected_message="cube.attributes keys differ: 'b', 'c'", + ) + + def test_values_differ__single(self): + self._check_merge_error( + attrs_1=dict(a=1, b=2), # Note: matching key 'a' does not appear + attrs_2=dict(a=1, b=3), + expected_message="cube.attributes values differ for keys: 'b'", + ) + + def test_values_differ__multiple(self): + self._check_merge_error( + attrs_1=dict(a=1, b=2), + attrs_2=dict(a=12, b=22), + expected_message="cube.attributes values differ for keys: 'a', 'b'", + ) + + def test_splitattrs_keys_local_global_mismatch(self): + # Since Cube.attributes is now a "split-attributes" dictionary, it is now + # possible to have "cube1.attributes != cube1.attributes", but also + # "set(cube1.attributes.keys()) == set(cube2.attributes.keys())". + # I.E. it is now necessary to specifically compare ".globals" and ".locals" to + # see *what* differs between two attributes dictionaries. + self._check_merge_error( + attrs_1=CubeAttrsDict(globals=dict(a=1), locals=dict(b=2)), + attrs_2=CubeAttrsDict(locals=dict(a=2)), + expected_message="cube.attributes keys differ: 'a', 'b'", + ) + + def test_splitattrs_keys_local_match_masks_global_mismatch(self): + self._check_merge_error( + attrs_1=CubeAttrsDict(globals=dict(a=1), locals=dict(a=3)), + attrs_2=CubeAttrsDict(globals=dict(a=2), locals=dict(a=3)), + expected_message="cube.attributes values differ for keys: 'a'", + ) + + +@pytest.mark.parametrize( + "dtype", [np.int16, np.int32, np.int64, np.float32, np.float64] +) +class TestCubeMerge_masked_scalar: + """Test for merging of scalar coordinates containing masked data.""" + + def _build_cube(self, scalar_data): + return iris.cube.Cube( + np.arange(5), + standard_name="air_pressure", + aux_coords_and_dims=[ + (AuxCoord(points=scalar_data, standard_name="realization"), None) + ], + ) + + def test_merge_scalar_coords_all_masked(self, dtype): + """Test merging of scalar aux coords all with masked data.""" + n = 5 + cubes = iris.cube.CubeList( + [self._build_cube(np.ma.masked_all(1, dtype=dtype)) for i in range(n)] + ) + merged = cubes.merge_cube() + c = merged.coord("realization") + assert np.ma.isMaskedArray(c.points) + assert np.all(c.points.mask) + assert c.points.dtype.type is dtype + + def test_merge_scalar_coords_some_masked(self, dtype): + """Test merging of scalar aux coords with mix of masked and unmasked data.""" + n = 5 + cubes = iris.cube.CubeList( + [ + self._build_cube(np.ma.masked_array(i, dtype=dtype, mask=i % 2)) + for i in range(n) + ] + ) + merged = cubes.merge_cube() + c = merged.coord("realization") + assert np.ma.isMaskedArray(c.points) + assert all([c.points.mask[i] == i % 2 for i in range(n)]) + assert c.points.dtype.type is dtype