diff --git a/lib/iris/_lazy_data.py b/lib/iris/_lazy_data.py index 985e79af13..c89d086e80 100644 --- a/lib/iris/_lazy_data.py +++ b/lib/iris/_lazy_data.py @@ -72,7 +72,7 @@ def is_lazy_data(data): _MAX_CHUNK_SIZE = 8 * 1024 * 1024 * 2 -def as_lazy_data(data, chunks=_MAX_CHUNK_SIZE): +def as_lazy_data(data, chunks=_MAX_CHUNK_SIZE, asarray=False): """ Convert the input array `data` to a dask array. @@ -89,12 +89,15 @@ def as_lazy_data(data, chunks=_MAX_CHUNK_SIZE): For more information see http://dask.pydata.org/en/latest/array-creation.html#chunks. + * asarray: + If True, then chunks will be converted to instances of `ndarray`. + Set to False (default) to pass passed chunks through unchanged. + Returns: The input array converted to a dask array. """ if not is_lazy_data(data): - asarray = not ma.isMaskedArray(data) data = da.from_array(data, chunks=chunks, asarray=asarray) return data diff --git a/lib/iris/fileformats/pp.py b/lib/iris/fileformats/pp.py index 89c374ffc7..157a83b4c5 100644 --- a/lib/iris/fileformats/pp.py +++ b/lib/iris/fileformats/pp.py @@ -861,8 +861,7 @@ def _lbpack_getter(self): @property def dtype(self): - return np.dtype('f8') if self.src_dtype.kind == 'i' else \ - self.src_dtype.newbyteorder('=') + return self.src_dtype.newbyteorder('=') @property def fill_value(self): @@ -1040,11 +1039,9 @@ def _data_bytes_to_shaped_array(data_bytes, lbpack, boundary_packing, # Reform in row-column order data.shape = data_shape - # Convert mdi to NaN. + # Mask the array if mdi in data: - if data.dtype.kind == 'i': - data = data.astype(np.dtype('f8')) - data[data == mdi] = np.nan + data = ma.masked_values(data, mdi, copy=False) return data @@ -1382,11 +1379,6 @@ def save(self, file_handle): # Get the actual data content. data = self.data if ma.is_masked(data): - # Fill missing data points with the MDI value from the header. - if data.dtype.kind in 'biu': - # Integer or Boolean data : No masking is supported. - msg = 'Non-floating masked data cannot be saved to PP.' - raise ValueError(msg) data = data.filled(fill_value=self.bmdi) # Make sure the data is big-endian diff --git a/lib/iris/tests/results/cube_to_pp/append_single.txt b/lib/iris/tests/results/cube_to_pp/append_single.txt index 6aee209281..642d40cb2a 100644 --- a/lib/iris/tests/results/cube_to_pp/append_single.txt +++ b/lib/iris/tests/results/cube_to_pp/append_single.txt @@ -49,7 +49,7 @@ bdy: -2.5 bzx: -3.75 bdx: 3.75 - bmdi: 9999.0 + bmdi: -1e+30 bmks: 1.0 data: [[ 254.64399719 254.64399719 254.64399719 ..., 254.64399719 254.64399719 254.64399719] @@ -115,7 +115,7 @@ bdy: -2.5 bzx: -3.75 bdx: 3.75 - bmdi: 9999.0 + bmdi: -1e+30 bmks: 1.0 data: [[ 254.64399719 254.64399719 254.64399719 ..., 254.64399719 254.64399719 254.64399719] diff --git a/lib/iris/tests/results/cube_to_pp/replace_single.txt b/lib/iris/tests/results/cube_to_pp/replace_single.txt index afff9c6f41..c5fdb48515 100644 --- a/lib/iris/tests/results/cube_to_pp/replace_single.txt +++ b/lib/iris/tests/results/cube_to_pp/replace_single.txt @@ -49,7 +49,7 @@ bdy: -2.5 bzx: -3.75 bdx: 3.75 - bmdi: 9999.0 + bmdi: -1e+30 bmks: 1.0 data: [[ 254.64399719 254.64399719 254.64399719 ..., 254.64399719 254.64399719 254.64399719] diff --git a/lib/iris/tests/results/cube_to_pp/simple.txt b/lib/iris/tests/results/cube_to_pp/simple.txt index afff9c6f41..c5fdb48515 100644 --- a/lib/iris/tests/results/cube_to_pp/simple.txt +++ b/lib/iris/tests/results/cube_to_pp/simple.txt @@ -49,7 +49,7 @@ bdy: -2.5 bzx: -3.75 bdx: 3.75 - bmdi: 9999.0 + bmdi: -1e+30 bmks: 1.0 data: [[ 254.64399719 254.64399719 254.64399719 ..., 254.64399719 254.64399719 254.64399719] diff --git a/lib/iris/tests/results/cube_to_pp/user_rules.txt b/lib/iris/tests/results/cube_to_pp/user_rules.txt index e6d241aa86..9cac1047e0 100644 --- a/lib/iris/tests/results/cube_to_pp/user_rules.txt +++ b/lib/iris/tests/results/cube_to_pp/user_rules.txt @@ -49,7 +49,7 @@ bdy: -2.5 bzx: -3.75 bdx: 3.75 - bmdi: 9999.0 + bmdi: -1e+30 bmks: 1.0 data: [[ 254.64399719 254.64399719 254.64399719 ..., 254.64399719 254.64399719 254.64399719] diff --git a/lib/iris/tests/test_cdm.py b/lib/iris/tests/test_cdm.py index 1c4a8cefb8..5944acf184 100644 --- a/lib/iris/tests/test_cdm.py +++ b/lib/iris/tests/test_cdm.py @@ -1181,7 +1181,7 @@ def test_save_and_merge(self): # extract the 2d field that has SOME missing values masked_slice = cube[0] - masked_slice.fill_value = fill_value + masked_slice.data.fill_value = fill_value # test saving masked data reference_txt_path = tests.get_result_path(('cdm', 'masked_save_pp.txt')) @@ -1199,7 +1199,7 @@ def test_save_and_merge(self): self.assertEqual(len(merged_cubes), 1, "expected a single merged cube") merged_cube = merged_cubes[0] self.assertEqual(merged_cube.dtype, dtype) - self.assertEqual(merged_cube.fill_value, fill_value) + self.assertEqual(merged_cube.data.fill_value, fill_value) @tests.skip_data diff --git a/lib/iris/tests/test_merge.py b/lib/iris/tests/test_merge.py index 328f902ce2..6a58aa84ad 100644 --- a/lib/iris/tests/test_merge.py +++ b/lib/iris/tests/test_merge.py @@ -136,7 +136,7 @@ def _make_cube(self, data, dtype=np.dtype('int32'), fill_value=None, y = np.arange(N) payload = self._make_data(data, dtype=dtype, fill_value=fill_value, mask=mask, lazy=lazy, N=N) - cube = iris.cube.Cube(payload, dtype=dtype, fill_value=fill_value) + cube = iris.cube.Cube(payload) lat = DimCoord(y, standard_name='latitude', units='degrees') cube.add_dim_coord(lat, 0) lon = DimCoord(x, standard_name='longitude', units='degrees') @@ -146,23 +146,26 @@ def _make_cube(self, data, dtype=np.dtype('int32'), fill_value=None, return cube @staticmethod - def _expected_fill_value(fill0, fill1): + def _expected_fill_value(fill0='none', fill1='none'): result = None - if fill0 == fill1: - result = fill0 + if fill0 != 'none' or fill1 != 'none': + if fill0 == 'none': + result = fill1 + elif fill1 == 'none': + result = fill0 + elif fill0 == fill1: + result = fill0 return result - def _check_fill_value(self, result, fill0, fill1): + def _check_fill_value(self, result, fill0='none', fill1='none'): expected_fill_value = self._expected_fill_value(fill0, fill1) if expected_fill_value is None: - self.assertIsNone(result.fill_value) data = result.data if ma.isMaskedArray(data): np_fill_value = ma.masked_array(0, dtype=result.dtype).fill_value self.assertEqual(data.fill_value, np_fill_value) else: - self.assertEqual(result.fill_value, expected_fill_value) data = result.data if ma.isMaskedArray(data): self.assertEqual(data.fill_value, expected_fill_value) @@ -170,24 +173,25 @@ def _check_fill_value(self, result, fill0, fill1): def setUp(self): self.dtype = np.dtype('int32') fill_value = 1234 - lazy_combos = itertools.product([False, True], - [False, True]) + self.lazy_combos = itertools.product([False, True], + [False, True]) fill_combos = itertools.product([None, fill_value], [fill_value, None]) - self.combos = itertools.product(lazy_combos, fill_combos) + single_fill_combos = itertools.product([None, fill_value]) + self.combos = itertools.product(self.lazy_combos, fill_combos) + self.mixed_combos = itertools.product(self.lazy_combos, + single_fill_combos) def test__ndarray_ndarray(self): - for (lazy0, lazy1), (fill0, fill1) in self.combos: + for (lazy0, lazy1) in self.lazy_combos: cubes = iris.cube.CubeList() - cubes.append(self._make_cube(0, dtype=self.dtype, lazy=lazy0, - fill_value=fill0)) - cubes.append(self._make_cube(1, dtype=self.dtype, lazy=lazy1, - fill_value=fill1)) + cubes.append(self._make_cube(0, dtype=self.dtype, lazy=lazy0)) + cubes.append(self._make_cube(1, dtype=self.dtype, lazy=lazy1)) result = cubes.merge_cube() expected = self._make_data([0, 1], dtype=self.dtype) self.assertArrayEqual(result.data, expected) self.assertEqual(result.dtype, self.dtype) - self._check_fill_value(result, fill0, fill1) + self._check_fill_value(result) def test__masked_masked(self): for (lazy0, lazy1), (fill0, fill1) in self.combos: @@ -210,40 +214,38 @@ def test__masked_masked(self): self._check_fill_value(result, fill0, fill1) def test__ndarray_masked(self): - for (lazy0, lazy1), (fill0, fill1) in self.combos: + for (lazy0, lazy1), (fill,) in self.mixed_combos: cubes = iris.cube.CubeList() - cubes.append(self._make_cube(0, lazy=lazy0, dtype=self.dtype, - fill_value=fill0)) + cubes.append(self._make_cube(0, lazy=lazy0, dtype=self.dtype)) mask = [(0, 1), (0, 1)] cubes.append(self._make_cube(1, mask=mask, lazy=lazy1, dtype=self.dtype, - fill_value=fill1)) + fill_value=fill)) result = cubes.merge_cube() mask = [(1, 1), (0, 1), (0, 1)] - expected_fill_value = self._expected_fill_value(fill0, fill1) + expected_fill_value = self._expected_fill_value(fill) expected = self._make_data([0, 1], mask=mask, dtype=self.dtype, fill_value=expected_fill_value) self.assertMaskedArrayEqual(result.data, expected) self.assertEqual(result.dtype, self.dtype) - self._check_fill_value(result, fill0, fill1) + self._check_fill_value(result, fill1=fill1) def test__masked_ndarray(self): - for (lazy0, lazy1), (fill0, fill1) in self.combos: + for (lazy0, lazy1), (fill,) in self.mixed_combos: cubes = iris.cube.CubeList() mask = [(0, 1), (0, 1)] cubes.append(self._make_cube(0, mask=mask, lazy=lazy0, dtype=self.dtype, - fill_value=fill0)) - cubes.append(self._make_cube(1, lazy=lazy1, dtype=self.dtype, - fill_value=fill1)) + fill_value=fill)) + cubes.append(self._make_cube(1, lazy=lazy1, dtype=self.dtype)) result = cubes.merge_cube() mask = [(0, 0), (0, 1), (0, 1)] - expected_fill_value = self._expected_fill_value(fill0, fill1) + expected_fill_value = self._expected_fill_value(fill) expected = self._make_data([0, 1], mask=mask, dtype=self.dtype, fill_value=expected_fill_value) self.assertMaskedArrayEqual(result.data, expected) self.assertEqual(result.dtype, self.dtype) - self._check_fill_value(result, fill0, fill1) + self._check_fill_value(result, fill0=fill) def test_fill_value_invariant_to_order__same_non_None(self): fill_value = 1234 @@ -251,7 +253,6 @@ def test_fill_value_invariant_to_order__same_non_None(self): fill_value=fill_value) for i in range(3)] for combo in itertools.permutations(cubes): result = iris.cube.CubeList(combo).merge_cube() - self.assertEqual(result.fill_value, fill_value) self.assertEqual(result.data.fill_value, fill_value) def test_fill_value_invariant_to_order__all_None(self): @@ -259,7 +260,6 @@ def test_fill_value_invariant_to_order__all_None(self): fill_value=None) for i in range(3)] for combo in itertools.permutations(cubes): result = iris.cube.CubeList(combo).merge_cube() - self.assertIsNone(result.fill_value) np_fill_value = ma.masked_array(0, dtype=result.dtype).fill_value self.assertEqual(result.data.fill_value, np_fill_value) @@ -270,7 +270,6 @@ def test_fill_value_invariant_to_order__different_non_None(self): cubes.append(self._make_cube(3, mask=True, fill_value=4123)) for combo in itertools.permutations(cubes): result = iris.cube.CubeList(combo).merge_cube() - self.assertIsNone(result.fill_value) np_fill_value = ma.masked_array(0, dtype=result.dtype).fill_value self.assertEqual(result.data.fill_value, np_fill_value) @@ -280,7 +279,6 @@ def test_fill_value_invariant_to_order__mixed(self): cubes.append(self._make_cube(2, mask=True, fill_value=4321)) for combo in itertools.permutations(cubes): result = iris.cube.CubeList(combo).merge_cube() - self.assertIsNone(result.fill_value) np_fill_value = ma.masked_array(0, dtype=result.dtype).fill_value self.assertEqual(result.data.fill_value, np_fill_value)