Upgrade utilities unit tests

improver/utilities/common_input_handle.py

@@ -27,18 +27,15 @@ def as_cubelist(*cubes: Union[Cube, CubeList]):
 
     Args:
         cubes:
-            Input data provided in the form of one or more cubes or cubelists (or mixture thereof).
+            Input data provided in the form of one or more cubes or cubelists
+            (or mixture thereof).
             Any iterable is supported.
 
     Returns:
         CubeList:
             A CubeList containing all the cubes provided as input.
     """
     cubes = CubeList(flatten(cubes))
-    # Remove CubeList verification for iris >=3.3.0
-    for cube in cubes:
-        if not hasattr(cube, "add_aux_coord"):
-            raise TypeError("A non iris Cube object has been provided.")
     if len(cubes) == 0:
         raise ValueError("One or more cubes should be provided.")
     return cubes

improver/utilities/cube_extraction.py

@@ -472,8 +472,8 @@ def _one_way_fill(
         for c in range(first, last, step):
             c_slice = [slice(None)] * data.ndim
             c_slice[coordinate_axis] = slice(c, c + 1)
-            data[c_slice] = np.ma.where(
-                data.mask[c_slice], data[last_slice] + local_increment, data[c_slice]
+            data[*c_slice] = np.ma.where(
+                data.mask[*c_slice], data[*last_slice] + local_increment, data[*c_slice]
             )
             last_slice = c_slice
 

improver/utilities/generalized_additive_models.py

@@ -120,6 +120,14 @@ def process(self, predictors: np.ndarray, targets: np.ndarray):
         Returns:
             A fitted pyGAM GAM model.
         """
+        # Monkey patch for pyGAM due to handling of sparse arrays in some versions of
+        # scipy.
+        import scipy.sparse
+
+        def to_array(self):
+            return self.toarray()
+
+        scipy.sparse.spmatrix.A = property(to_array)
         # Import from pygam here to minimize dependencies
         from pygam import GAM
 

improver/utilities/gradient_between_vertical_levels.py

@@ -92,7 +92,7 @@ def gradient_over_vertical_levels(
                 if geopotential_height:
                     height_ASL = geopotential_height.extract(
                         iris.Constraint(pressure=cube.coord("pressure").points)
-                    )
+                    ).data
                     coord_used.append("pressure")
                 else:
                     raise ValueError(
@@ -101,7 +101,7 @@ def gradient_over_vertical_levels(
                     )
             else:
                 if orography:
-                    height_ASL = orography + cube_height
+                    height_ASL = orography.data + cube_height
                     coord_used.append("height")
                 elif not (orography or geopotential_height):
                     height_ASL = cube_height
@@ -114,7 +114,7 @@ def gradient_over_vertical_levels(
             cube_heights.append(height_ASL)
 
         height_diff = cube_heights[0] - cube_heights[1]
-        height_diff.data = np.ma.masked_where(height_diff.data == 0, height_diff.data)
+        height_diff = np.ma.masked_where(height_diff == 0, height_diff)
 
         if "height" in coord_used and "pressure" in coord_used:
             try:

improver/utilities/indexing_operations.py

@@ -76,10 +76,7 @@ def choose(index_array: ndarray, array_set: ndarray) -> ndarray:
         raise IndexError(msg)
 
     result = np.array(
-        [
-            array_set[index_array[i]][i[1:]]
-            for i in np.lib.index_tricks.ndindex(index_array.shape)
-        ]
+        [array_set[index_array[i]][i[1:]] for i in np.ndindex(index_array.shape)]
     ).reshape(index_array.shape)
 
     return result

improver/utilities/temporal.py

@@ -104,10 +104,15 @@ def iris_time_to_datetime(
     """
     coord = time_coord.copy()
     coord.convert_units("seconds since 1970-01-01 00:00:00")
+
     if point_or_bound == "point":
-        datetime_list = [value.point for value in coord.cells()]
+        datetime_list = [value.point._to_real_datetime() for value in coord.cells()]
     elif point_or_bound == "bound":
-        datetime_list = [value.bound for value in coord.cells()]
+        datetime_list = [
+            [c.bound[0]._to_real_datetime(), c.bound[1]._to_real_datetime()]
+            for c in coord.cells()
+        ]
+
     return datetime_list
 
 

improver/utilities/temporal_interpolation.py

@@ -804,7 +804,7 @@ def allocate_data(self, cube: Cube, period: int) -> Cube:
         # Split the whole period duration into allocations for each fidelity
         # period.
         intervals = period // self.fidelity
-        interval_data = cube.data / intervals
+        interval_data = (cube.data / intervals).astype(cube.data.dtype)
 
         daynightplugin = DayNightMask()
         start_time, _ = cube.coord("time").bounds.flatten()
@@ -934,7 +934,7 @@ def process(self, cube: Cube) -> Cube:
         # Ensure that the cube is already self-consistent and does not include
         # any durations that exceed the period described. This is mostly to
         # handle grib packing errors for ECMWF data.
-        cube.data = np.clip(cube.data, 0, period)
+        cube.data = np.clip(cube.data, 0, period, dtype=cube.data.dtype)
 
         fidelity_period_cube = self.allocate_data(cube, period)
         factor = self.renormalisation_factor(cube, fidelity_period_cube)

improver_tests/utilities/common_input_handle/test_as_cubelist.py

@@ -71,10 +71,3 @@ def test_empty_list_provided():
     msg = "One or more cubes should be provided."
     with pytest.raises(ValueError, match=msg):
         as_cubelist([])
-
-
-def test_non_cube_cubelist_provided():
-    """Test when a CubeList containing a non cube is provided."""
-    msg = "A non iris Cube object has been provided."
-    with pytest.raises(TypeError, match=msg):
-        as_cubelist(CubeList(["not_a_cube"]))

improver_tests/utilities/temporal/test_temporal.py

@@ -90,9 +90,9 @@ class Test_cycletime_to_number(IrisTest):
     def test_basic(self):
         """Test that a number is returned of the expected value."""
         cycletime = "20171122T0000Z"
-        dt = 419808.0
+        dt = 419808
         result = cycletime_to_number(cycletime)
-        self.assertIsInstance(result, float)
+        self.assertIsInstance(result, np.int64)
         self.assertAlmostEqual(result, dt)
 
     def test_cycletime_format_defined(self):

improver_tests/utilities/test_GAMFit.py

@@ -112,6 +112,14 @@ def test_process():
     The "wage" dataset used in this test consists of the features Year, Age, and
     Education (as a category) with the target being a value for the expected wage.
     """
+    # Monkey patch for pyGAM due to handling of sparse arrays in some versions of
+    # scipy.
+    import scipy.sparse
+
+    def to_array(self):
+        return self.toarray()
+
+    scipy.sparse.spmatrix.A = property(to_array)
     # Skip test if pyGAM not available.
     pytest.importorskip("pygam")
     from pygam import GAM, f, s

improver_tests/utilities/test_GAMPredict.py

@@ -46,6 +46,14 @@ def test_process(X_new, expected):
     The "wage" dataset used in this test consists of the features Year, Age, and
     Education (as a category) with the target being a value for the expected wage.
     """
+    # Monkey patch for pyGAM due to handling of sparse arrays in some versions of
+    # scipy.
+    import scipy.sparse
+
+    def to_array(self):
+        return self.toarray()
+
+    scipy.sparse.spmatrix.A = property(to_array)
     # Skip test if pyGAM not available.
     pytest.importorskip("pygam")
     from pygam import GAM, f, s

improver_tests/utilities/test_GradientBetweenVerticalLevels.py

@@ -87,7 +87,7 @@ def test_height_and_pressure(
         expected_bounds = {"pressure": [[85000, 100000]]}
         expected_coord = {"pressure": 100000}
 
-    cubes.append([temperature_at_850hPa, temperature_at_screen_level])
+    cubes.extend([temperature_at_850hPa, temperature_at_screen_level])
 
     expected = [[0.03, 0.01], [-0.01, -0.03]]
     result = GradientBetweenVerticalLevels()(iris.cube.CubeList(cubes))

improver_tests/utilities/test_TemporalInterpolation.py

@@ -424,10 +424,10 @@ def test_solar_interpolation(solar_expected, realizations):
     assert result.coord("time").points == 1509523200
     assert result.coord("forecast_period").points[0] == 3600 * 4
     if result.ndim == 2:
-        np.testing.assert_almost_equal(result.data, solar_expected)
+        np.testing.assert_almost_equal(result.data, solar_expected, decimal=6)
     else:
         for dslice in result.data:
-            np.testing.assert_almost_equal(dslice, solar_expected)
+            np.testing.assert_almost_equal(dslice, solar_expected, decimal=6)
 
 
 @pytest.mark.parametrize("realizations", (None, [0, 1, 2]))

improver_tests/utilities/test_load.py

@@ -209,6 +209,10 @@ def test_no_lazy_load(self):
 
     def test_lazy_load(self):
         """Test that the loading works correctly with lazy loading."""
+        # Override default value within iris to ensure that the file is loaded lazily.
+        from iris.fileformats.netcdf import loader
+
+        loader._LAZYVAR_MIN_BYTES = 0
         result = load_cube(self.filepath)
         self.assertTrue(result.has_lazy_data())
 
@@ -405,8 +409,11 @@ def test_no_lazy_load(self):
             )
 
     def test_lazy_load(self):
-        """Test that the cubelist returned upon loading does contain
-        lazy data."""
+        """Test that the cubelist returned upon loading does contain lazy data."""
+        # Override default value within iris to ensure that the file is loaded lazily.
+        from iris.fileformats.netcdf import loader
+
+        loader._LAZYVAR_MIN_BYTES = 0
         result = load_cubelist([self.filepath, self.filepath])
         self.assertArrayEqual([True, True], [_.has_lazy_data() for _ in result])
 

improver_tests/utilities/test_mathematical_operations.py

@@ -226,7 +226,7 @@ def test_basic(self):
             result.coord("height").points, np.array([10.0, 5.0])
         )
         self.assertArrayAlmostEqual(
-            result.coord("height").bounds, np.array([[10.0, 20.0], [5.0, 10.0]])
+            result.coord("height").bounds, np.array([[20.0, 10.0], [10.0, 5.0]])
         )
 
     def test_positive_values_in_data(self):
@@ -429,7 +429,7 @@ def test_basic(self):
             result.coord("height").points, np.array([10.0, 5.0])
         )
         self.assertArrayAlmostEqual(
-            result.coord("height").bounds, np.array([[10.0, 20.0], [5.0, 10.0]])
+            result.coord("height").bounds, np.array([[20.0, 10.0], [10.0, 5.0]])
         )
 
     def test_metadata(self):

improver_tests/utilities/test_neighbourhood_tools.py

@@ -85,7 +85,7 @@ def test_padding_neighbourhood_size_2(array_size_5):
         + [[i, 0, j, 0] for i in range(5) for j in [0, 1]]
         + [[i, 5, j, 1] for i in range(5) for j in [0, 1]]
     )
-    outer_part = padded[list(zip(*border_index))]
+    outer_part = padded[tuple(zip(*border_index))]
     np.testing.assert_array_equal(outer_part, np.zeros(40, dtype=np.int32))
 
 
@@ -98,7 +98,7 @@ def test_padding_non_zero(array_size_5):
         + [[i, 0, j, 0] for i in range(5) for j in [0, 1]]
         + [[i, 5, j, 1] for i in range(5) for j in [0, 1]]
     )
-    outer_part = padded[list(zip(*border_index))]
+    outer_part = padded[tuple(zip(*border_index))]
     np.testing.assert_array_equal(outer_part, np.ones(40, dtype=np.int32))