Make array equal work with masked Dask arrays and add tests

docs/src/whatsnew/latest.rst

@@ -65,6 +65,9 @@ This document explains the changes made to Iris for this release
    older NetCDF formats e.g. ``NETCDF4_CLASSIC`` support a maximum precision of
    32-bit. (:issue:`6178`, :pull:`6343`)
 
+#. `@bouweandela`_ fixed handling of masked Dask arrays in
+    :func:`~iris.util.array_equal`.
+
 
 💣 Incompatible Changes
 =======================

lib/iris/coords.py

@@ -589,21 +589,22 @@ def __eq__(self, other):
         if hasattr(other, "metadata"):
             # metadata comparison
             eq = self.metadata == other.metadata
+
+            # Also consider bounds, if we have them.
+            # (N.B. though only Coords can ever actually *have* bounds).
+            if eq and eq is not NotImplemented:
+                eq = self.has_bounds() is other.has_bounds()
+
             # data values comparison
             if eq and eq is not NotImplemented:
                 eq = iris.util.array_equal(
                     self._core_values(), other._core_values(), withnans=True
                 )
-
-            # Also consider bounds, if we have them.
-            # (N.B. though only Coords can ever actually *have* bounds).
             if eq and eq is not NotImplemented:
                 if self.has_bounds() and other.has_bounds():
                     eq = iris.util.array_equal(
                         self.core_bounds(), other.core_bounds(), withnans=True
                     )
-                else:
-                    eq = not self.has_bounds() and not other.has_bounds()
 
         return eq
 

lib/iris/tests/unit/concatenate/test_hashing.py

@@ -9,6 +9,8 @@
 import pytest
 
 from iris import _concatenate
+from iris.tests.unit.util.test_array_equal import TEST_CASES
+from iris.util import array_equal
 
 
 @pytest.mark.parametrize(
@@ -75,6 +77,20 @@ def test_compute_hashes(a, b, eq):
     assert eq == (hashes["a"] == hashes["b"])
 
 
+@pytest.mark.parametrize(
+    "a,b",
+    [
+        (a, b)
+        for (a, b, withnans, eq) in TEST_CASES
+        if isinstance(a, np.ndarray | da.Array) and isinstance(b, np.ndarray | da.Array)
+    ],
+)
+def test_compute_hashes_vs_array_equal(a, b):
+    """Test that hashing give the same answer as `array_equal(withnans=True)`."""
+    hashes = _concatenate._compute_hashes({"a": a, "b": b})
+    assert array_equal(a, b, withnans=True) == (hashes["a"] == hashes["b"])
+
+
 def test_arrayhash_equal_incompatible_chunks_raises():
     hash1 = _concatenate._ArrayHash(1, chunks=((1, 1),))
     hash2 = _concatenate._ArrayHash(1, chunks=((2,),))

lib/iris/tests/unit/util/test_array_equal.py

@@ -4,133 +4,190 @@
 # See LICENSE in the root of the repository for full licensing details.
 """Test function :func:`iris.util.array_equal`."""
 
+import dask.array as da
 import numpy as np
 import numpy.ma as ma
+import pytest
 
 from iris.util import array_equal
 
-
-class Test:
-    def test_0d(self):
-        array_a = np.array(23)
-        array_b = np.array(23)
-        array_c = np.array(7)
-        assert array_equal(array_a, array_b)
-        assert not array_equal(array_a, array_c)
-
-    def test_0d_and_scalar(self):
-        array_a = np.array(23)
-        assert array_equal(array_a, 23)
-        assert not array_equal(array_a, 45)
-
-    def test_1d_and_sequences(self):
-        for sequence_type in (list, tuple):
-            seq_a = sequence_type([1, 2, 3])
-            array_a = np.array(seq_a)
-            assert array_equal(array_a, seq_a)
-            assert not array_equal(array_a, seq_a[:-1])
-            array_a[1] = 45
-            assert not array_equal(array_a, seq_a)
-
-    def test_nd(self):
-        array_a = np.array(np.arange(24).reshape(2, 3, 4))
-        array_b = np.array(np.arange(24).reshape(2, 3, 4))
-        array_c = np.array(np.arange(24).reshape(2, 3, 4))
-        array_c[0, 1, 2] = 100
-        assert array_equal(array_a, array_b)
-        assert not array_equal(array_a, array_c)
-
-    def test_masked_is_not_ignored(self):
-        array_a = ma.masked_array([1, 2, 3], mask=[1, 0, 1])
-        array_b = ma.masked_array([2, 2, 2], mask=[1, 0, 1])
-        assert array_equal(array_a, array_b)
-
-    def test_masked_is_different(self):
-        array_a = ma.masked_array([1, 2, 3], mask=[1, 0, 1])
-        array_b = ma.masked_array([1, 2, 3], mask=[0, 0, 1])
-        assert not array_equal(array_a, array_b)
-
-    def test_masked_isnt_unmasked(self):
-        array_a = np.array([1, 2, 2])
-        array_b = ma.masked_array([1, 2, 2], mask=[0, 0, 1])
-        assert not array_equal(array_a, array_b)
-
-    def test_masked_unmasked_equivelance(self):
-        array_a = np.array([1, 2, 2])
-        array_b = ma.masked_array([1, 2, 2])
-        array_c = ma.masked_array([1, 2, 2], mask=[0, 0, 0])
-        assert array_equal(array_a, array_b)
-        assert array_equal(array_a, array_c)
-
-    def test_fully_masked_arrays(self):
-        array_a = ma.masked_array(np.arange(24).reshape(2, 3, 4), mask=True)
-        array_b = ma.masked_array(np.arange(24).reshape(2, 3, 4), mask=True)
-        assert array_equal(array_a, array_b)
-
-    def test_fully_masked_0d_arrays(self):
-        array_a = ma.masked_array(3, mask=True)
-        array_b = ma.masked_array(3, mask=True)
-        assert array_equal(array_a, array_b)
-
-    def test_fully_masked_string_arrays(self):
-        array_a = ma.masked_array(["a", "b", "c"], mask=True)
-        array_b = ma.masked_array(["a", "b", "c"], mask=[1, 1, 1])
-        assert array_equal(array_a, array_b)
-
-    def test_partially_masked_string_arrays(self):
-        array_a = ma.masked_array(["a", "b", "c"], mask=[1, 0, 1])
-        array_b = ma.masked_array(["a", "b", "c"], mask=[1, 0, 1])
-        assert array_equal(array_a, array_b)
-
-    def test_string_arrays_equal(self):
-        array_a = np.array(["abc", "def", "efg"])
-        array_b = np.array(["abc", "def", "efg"])
-        assert array_equal(array_a, array_b)
-
-    def test_string_arrays_different_contents(self):
-        array_a = np.array(["abc", "def", "efg"])
-        array_b = np.array(["abc", "de", "efg"])
-        assert not array_equal(array_a, array_b)
-
-    def test_string_arrays_subset(self):
-        array_a = np.array(["abc", "def", "efg"])
-        array_b = np.array(["abc", "def"])
-        assert not array_equal(array_a, array_b)
-        assert not array_equal(array_b, array_a)
-
-    def test_string_arrays_unequal_dimensionality(self):
-        array_a = np.array("abc")
-        array_b = np.array(["abc"])
-        array_c = np.array([["abc"]])
-        assert not array_equal(array_a, array_b)
-        assert not array_equal(array_b, array_a)
-        assert not array_equal(array_a, array_c)
-        assert not array_equal(array_b, array_c)
-
-    def test_string_arrays_0d_and_scalar(self):
-        array_a = np.array("foobar")
-        assert array_equal(array_a, "foobar")
-        assert not array_equal(array_a, "foo")
-        assert not array_equal(array_a, "foobar.")
-
-    def test_nan_equality_nan_ne_nan(self):
-        array_a = np.array([1.0, np.nan, 2.0, np.nan, 3.0])
-        array_b = array_a.copy()
-        assert not array_equal(array_a, array_a)
-        assert not array_equal(array_a, array_b)
-
-    def test_nan_equality_nan_naneq_nan(self):
-        array_a = np.array([1.0, np.nan, 2.0, np.nan, 3.0])
-        array_b = np.array([1.0, np.nan, 2.0, np.nan, 3.0])
-        assert array_equal(array_a, array_a, withnans=True)
-        assert array_equal(array_a, array_b, withnans=True)
-
-    def test_nan_equality_nan_nanne_a(self):
-        array_a = np.array([1.0, np.nan, 2.0, np.nan, 3.0])
-        array_b = np.array([1.0, np.nan, 2.0, 0.0, 3.0])
-        assert not array_equal(array_a, array_b, withnans=True)
-
-    def test_nan_equality_a_nanne_b(self):
-        array_a = np.array([1.0, np.nan, 2.0, np.nan, 3.0])
-        array_b = np.array([1.0, np.nan, 2.0, np.nan, 4.0])
-        assert not array_equal(array_a, array_b, withnans=True)
+ARRAY1 = np.array(np.arange(24).reshape(2, 3, 4))
+ARRAY1[0, 1, 2] = 100
+
+ARRAY2 = np.array([1.0, np.nan, 2.0, np.nan, 3.0])
+
+TEST_CASES = [
+    # test empty
+    (np.array([]), np.array([]), False, True),
+    (np.array([]), np.array([], dtype=np.float64), True, True),
+    # test 0d
+    (np.array(23), np.array(23), False, True),
+    (np.array(23), np.array(7), False, False),
+    # test 0d and scalar
+    (np.array(23), 23, False, True),
+    (np.array(23), 45, False, False),
+    # test 1d and sequences
+    (np.array([1, 2, 3]), [1, 2, 3], False, True),
+    (np.array([1, 2, 3]), [1, 2], False, False),
+    (np.array([1, 45, 3]), [1, 2, 3], False, False),
+    (np.array([1, 2, 3]), (1, 2, 3), False, True),
+    (np.array([1, 2, 3]), (1, 2), False, False),
+    (np.array([1, 45, 3]), (1, 2, 3), False, False),
+    # test 3d
+    (
+        np.array(np.arange(24).reshape(2, 3, 4)),
+        np.array(np.arange(24).reshape(2, 3, 4)),
+        False,
+        True,
+    ),
+    (
+        np.array(np.arange(24).reshape(2, 3, 4)),
+        ARRAY1,
+        False,
+        False,
+    ),
+    # test masked is not ignored
+    (
+        ma.masked_array([1, 2, 3], mask=[1, 0, 1]),
+        ma.masked_array([2, 2, 2], mask=[1, 0, 1]),
+        False,
+        True,
+    ),
+    # test masked is different
+    (
+        ma.masked_array([1, 2, 3], mask=[1, 0, 1]),
+        ma.masked_array([1, 2, 3], mask=[0, 0, 1]),
+        False,
+        False,
+    ),
+    # test masked isn't unmasked
+    (
+        np.array([1, 2, 2]),
+        ma.masked_array([1, 2, 2], mask=[0, 0, 1]),
+        False,
+        False,
+    ),
+    (
+        ma.masked_array([1, 2, 2], mask=[0, 0, 1]),
+        ma.masked_array([1, 2, 2]),
+        False,
+        False,
+    ),
+    (
+        np.array([1, 2]),
+        ma.masked_array([1, 3], mask=[0, 1]),
+        False,
+        False,
+    ),
+    # test masked/unmasked_equivalence
+    (
+        np.array([1, 2, 2]),
+        ma.masked_array([1, 2, 2]),
+        False,
+        True,
+    ),
+    (
+        np.array([1, 2, 2]),
+        ma.masked_array([1, 2, 2], mask=[0, 0, 0]),
+        False,
+        True,
+    ),
+    # test fully masked arrays
+    (
+        ma.masked_array(np.arange(24).reshape(2, 3, 4), mask=True),
+        ma.masked_array(np.arange(24).reshape(2, 3, 4), mask=True),
+        False,
+        True,
+    ),
+    # test fully masked 0d arrays
+    (
+        ma.masked_array(3, mask=True),
+        ma.masked_array(3, mask=True),
+        False,
+        True,
+    ),
+    # test fully masked string arrays
+    (
+        ma.masked_array(["a", "b", "c"], mask=True),
+        ma.masked_array(["a", "b", "c"], mask=[1, 1, 1]),
+        False,
+        True,
+    ),
+    # test partially masked string arrays
+    (
+        ma.masked_array(["a", "b", "c"], mask=[1, 0, 1]),
+        ma.masked_array(["a", "b", "c"], mask=[1, 0, 1]),
+        False,
+        True,
+    ),
+    # test string arrays equal
+    (
+        np.array(["abc", "def", "efg"]),
+        np.array(["abc", "def", "efg"]),
+        False,
+        True,
+    ),
+    # test string arrays different contents
+    (
+        np.array(["abc", "def", "efg"]),
+        np.array(["abc", "de", "efg"]),
+        False,
+        False,
+    ),
+    # test string arrays subset
+    (
+        np.array(["abc", "def", "efg"]),
+        np.array(["abc", "def"]),
+        False,
+        False,
+    ),
+    (
+        np.array(["abc", "def"]),
+        np.array(["abc", "def", "efg"]),
+        False,
+        False,
+    ),
+    # test string arrays unequal dimensionality
+    (np.array("abc"), np.array(["abc"]), False, False),
+    (np.array(["abc"]), np.array("abc"), False, False),
+    (np.array("abc"), np.array([["abc"]]), False, False),
+    (np.array(["abc"]), np.array([["abc"]]), False, False),
+    # test string arrays 0d and scalar
+    (np.array("foobar"), "foobar", False, True),
+    (np.array("foobar"), "foo", False, False),
+    (np.array("foobar"), "foobar.", False, False),
+    # test nan equality nan ne nan
+    (ARRAY2, ARRAY2, False, False),
+    (ARRAY2, ARRAY2.copy(), False, False),
+    # test nan equality nan naneq nan
+    (ARRAY2, ARRAY2, True, True),
+    (ARRAY2, ARRAY2.copy(), True, True),
+    # test nan equality nan nanne a
+    (
+        np.array([1.0, np.nan, 2.0, np.nan, 3.0]),
+        np.array([1.0, np.nan, 2.0, 0.0, 3.0]),
+        True,
+        False,
+    ),
+    # test nan equality a nanne b
+    (
+        np.array([1.0, np.nan, 2.0, np.nan, 3.0]),
+        np.array([1.0, np.nan, 2.0, np.nan, 4.0]),
+        True,
+        False,
+    ),
+]
+
+
+@pytest.mark.parametrize("lazy", [False, True])
+@pytest.mark.parametrize("array_a,array_b,withnans,eq", TEST_CASES)
+def test_array_equal(array_a, array_b, withnans, eq, lazy):
+    if lazy:
+        identical = array_a is array_b
+        if isinstance(array_a, np.ndarray):
+            array_a = da.asarray(array_a, chunks=2)
+        if isinstance(array_b, np.ndarray):
+            array_b = da.asarray(array_b, chunks=1)
+        if identical:
+            array_b = array_a
+    assert eq == array_equal(array_a, array_b, withnans=withnans)