Skip to content

PP + merge test fixes #2728

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lbdreyer merged 2 commits into from
Aug 4, 2017
Merged

PP + merge test fixes #2728

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
22995f0
PP + merge test fixes
bjlittle Aug 4, 2017
5137503
Review actions.
bjlittle Aug 4, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions lib/iris/_lazy_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ def is_lazy_data(data):
_MAX_CHUNK_SIZE = 8 * 1024 * 1024 * 2


def as_lazy_data(data, chunks=_MAX_CHUNK_SIZE):
def as_lazy_data(data, chunks=_MAX_CHUNK_SIZE, asarray=False):
"""
Convert the input array `data` to a dask array.

Expand All @@ -89,12 +89,15 @@ def as_lazy_data(data, chunks=_MAX_CHUNK_SIZE):
For more information see
http://dask.pydata.org/en/latest/array-creation.html#chunks.

* asarray:
If True, then chunks will be converted to instances of `ndarray`.
Set to False (default) to pass passed chunks through unchanged.

Returns:
The input array converted to a dask array.

"""
if not is_lazy_data(data):
asarray = not ma.isMaskedArray(data)
data = da.from_array(data, chunks=chunks, asarray=asarray)
return data

Expand Down
14 changes: 3 additions & 11 deletions lib/iris/fileformats/pp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -861,8 +861,7 @@ def _lbpack_getter(self):

@property
def dtype(self):
return np.dtype('f8') if self.src_dtype.kind == 'i' else \
self.src_dtype.newbyteorder('=')
return self.src_dtype.newbyteorder('=')

@property
def fill_value(self):
Expand Down Expand Up @@ -1040,11 +1039,9 @@ def _data_bytes_to_shaped_array(data_bytes, lbpack, boundary_packing,
# Reform in row-column order
data.shape = data_shape

# Convert mdi to NaN.
# Mask the array
if mdi in data:
if data.dtype.kind == 'i':
data = data.astype(np.dtype('f8'))
data[data == mdi] = np.nan
data = ma.masked_values(data, mdi, copy=False)

return data

Expand Down Expand Up @@ -1382,11 +1379,6 @@ def save(self, file_handle):
# Get the actual data content.
data = self.data
if ma.is_masked(data):
# Fill missing data points with the MDI value from the header.
if data.dtype.kind in 'biu':
# Integer or Boolean data : No masking is supported.
msg = 'Non-floating masked data cannot be saved to PP.'
raise ValueError(msg)
Copy link
Member

@lbdreyer lbdreyer Aug 4, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This was added in #2564 which has now been removed anyway

data = data.filled(fill_value=self.bmdi)

# Make sure the data is big-endian
Expand Down
4 changes: 2 additions & 2 deletions lib/iris/tests/results/cube_to_pp/append_single.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
bdy: -2.5
bzx: -3.75
bdx: 3.75
bmdi: 9999.0
bmdi: -1e+30
bmks: 1.0
data: [[ 254.64399719 254.64399719 254.64399719 ..., 254.64399719
254.64399719 254.64399719]
Expand Down Expand Up @@ -115,7 +115,7 @@
bdy: -2.5
bzx: -3.75
bdx: 3.75
bmdi: 9999.0
bmdi: -1e+30
bmks: 1.0
data: [[ 254.64399719 254.64399719 254.64399719 ..., 254.64399719
254.64399719 254.64399719]
Expand Down
2 changes: 1 addition & 1 deletion lib/iris/tests/results/cube_to_pp/replace_single.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
bdy: -2.5
bzx: -3.75
bdx: 3.75
bmdi: 9999.0
bmdi: -1e+30
bmks: 1.0
data: [[ 254.64399719 254.64399719 254.64399719 ..., 254.64399719
254.64399719 254.64399719]
Expand Down
2 changes: 1 addition & 1 deletion lib/iris/tests/results/cube_to_pp/simple.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
bdy: -2.5
bzx: -3.75
bdx: 3.75
bmdi: 9999.0
bmdi: -1e+30
bmks: 1.0
data: [[ 254.64399719 254.64399719 254.64399719 ..., 254.64399719
254.64399719 254.64399719]
Expand Down
2 changes: 1 addition & 1 deletion lib/iris/tests/results/cube_to_pp/user_rules.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
bdy: -2.5
bzx: -3.75
bdx: 3.75
bmdi: 9999.0
bmdi: -1e+30
bmks: 1.0
data: [[ 254.64399719 254.64399719 254.64399719 ..., 254.64399719
254.64399719 254.64399719]
Expand Down
4 changes: 2 additions & 2 deletions lib/iris/tests/test_cdm.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1181,7 +1181,7 @@ def test_save_and_merge(self):

# extract the 2d field that has SOME missing values
masked_slice = cube[0]
masked_slice.fill_value = fill_value
masked_slice.data.fill_value = fill_value

# test saving masked data
reference_txt_path = tests.get_result_path(('cdm', 'masked_save_pp.txt'))
Expand All @@ -1199,7 +1199,7 @@ def test_save_and_merge(self):
self.assertEqual(len(merged_cubes), 1, "expected a single merged cube")
merged_cube = merged_cubes[0]
self.assertEqual(merged_cube.dtype, dtype)
self.assertEqual(merged_cube.fill_value, fill_value)
self.assertEqual(merged_cube.data.fill_value, fill_value)


@tests.skip_data
Expand Down
62 changes: 30 additions & 32 deletions lib/iris/tests/test_merge.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ def _make_cube(self, data, dtype=np.dtype('int32'), fill_value=None,
y = np.arange(N)
payload = self._make_data(data, dtype=dtype, fill_value=fill_value,
mask=mask, lazy=lazy, N=N)
cube = iris.cube.Cube(payload, dtype=dtype, fill_value=fill_value)
cube = iris.cube.Cube(payload)
lat = DimCoord(y, standard_name='latitude', units='degrees')
cube.add_dim_coord(lat, 0)
lon = DimCoord(x, standard_name='longitude', units='degrees')
Expand All @@ -146,48 +146,52 @@ def _make_cube(self, data, dtype=np.dtype('int32'), fill_value=None,
return cube

@staticmethod
def _expected_fill_value(fill0, fill1):
def _expected_fill_value(fill0='none', fill1='none'):
result = None
if fill0 == fill1:
result = fill0
if fill0 != 'none' or fill1 != 'none':
if fill0 == 'none':
result = fill1
elif fill1 == 'none':
result = fill0
elif fill0 == fill1:
result = fill0
return result

def _check_fill_value(self, result, fill0, fill1):
def _check_fill_value(self, result, fill0='none', fill1='none'):
expected_fill_value = self._expected_fill_value(fill0, fill1)
if expected_fill_value is None:
self.assertIsNone(result.fill_value)
data = result.data
if ma.isMaskedArray(data):
np_fill_value = ma.masked_array(0,
dtype=result.dtype).fill_value
self.assertEqual(data.fill_value, np_fill_value)
else:
self.assertEqual(result.fill_value, expected_fill_value)
data = result.data
if ma.isMaskedArray(data):
self.assertEqual(data.fill_value, expected_fill_value)

def setUp(self):
self.dtype = np.dtype('int32')
fill_value = 1234
lazy_combos = itertools.product([False, True],
[False, True])
self.lazy_combos = itertools.product([False, True],
[False, True])
fill_combos = itertools.product([None, fill_value],
[fill_value, None])
self.combos = itertools.product(lazy_combos, fill_combos)
single_fill_combos = itertools.product([None, fill_value])
self.combos = itertools.product(self.lazy_combos, fill_combos)
self.mixed_combos = itertools.product(self.lazy_combos,
single_fill_combos)

def test__ndarray_ndarray(self):
for (lazy0, lazy1), (fill0, fill1) in self.combos:
for (lazy0, lazy1) in self.lazy_combos:
cubes = iris.cube.CubeList()
cubes.append(self._make_cube(0, dtype=self.dtype, lazy=lazy0,
fill_value=fill0))
cubes.append(self._make_cube(1, dtype=self.dtype, lazy=lazy1,
fill_value=fill1))
cubes.append(self._make_cube(0, dtype=self.dtype, lazy=lazy0))
cubes.append(self._make_cube(1, dtype=self.dtype, lazy=lazy1))
result = cubes.merge_cube()
expected = self._make_data([0, 1], dtype=self.dtype)
self.assertArrayEqual(result.data, expected)
self.assertEqual(result.dtype, self.dtype)
self._check_fill_value(result, fill0, fill1)
self._check_fill_value(result)
Copy link
Member

@lbdreyer lbdreyer Aug 4, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

For tests including ndarray (e.g. test__ndarray_ndarray, test__masked_ndarray, test__ndarray_masked) it seems unnecessary to test the fill_combos since we already know they won't have fill_values.

I would do something like:

def test__ndarray_ndarray(self):
    for (lazy0, laz1) in self.lazy_combos:
        cubes = iris.cube.CubeList()
        cubes.append(self._make_cube(0, dtype=self.dtype, lazy=lazy0)
        cubes.append(self._make_cube(1, dtype=self.dtype, lazy=lazy1)
        result = cubes.merge_cube()
        expected = self._make_data([0, 1], dtype=self.dtype)
        self.assertArrayEqual(result.data, expected)
        self.assertEqual(result.dtype, self.dtype)

Copy link
Member

@lbdreyer lbdreyer Aug 4, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unless you want to check that after a merge, the resultant data isn't a masked array for some reason


def test__masked_masked(self):
for (lazy0, lazy1), (fill0, fill1) in self.combos:
Expand All @@ -210,56 +214,52 @@ def test__masked_masked(self):
self._check_fill_value(result, fill0, fill1)

def test__ndarray_masked(self):
for (lazy0, lazy1), (fill0, fill1) in self.combos:
for (lazy0, lazy1), (fill,) in self.mixed_combos:
cubes = iris.cube.CubeList()
cubes.append(self._make_cube(0, lazy=lazy0, dtype=self.dtype,
fill_value=fill0))
cubes.append(self._make_cube(0, lazy=lazy0, dtype=self.dtype))
mask = [(0, 1), (0, 1)]
cubes.append(self._make_cube(1, mask=mask, lazy=lazy1,
dtype=self.dtype,
fill_value=fill1))
fill_value=fill))
result = cubes.merge_cube()
mask = [(1, 1), (0, 1), (0, 1)]
expected_fill_value = self._expected_fill_value(fill0, fill1)
expected_fill_value = self._expected_fill_value(fill)
expected = self._make_data([0, 1], mask=mask, dtype=self.dtype,
fill_value=expected_fill_value)
self.assertMaskedArrayEqual(result.data, expected)
self.assertEqual(result.dtype, self.dtype)
self._check_fill_value(result, fill0, fill1)
self._check_fill_value(result, fill1=fill1)

def test__masked_ndarray(self):
for (lazy0, lazy1), (fill0, fill1) in self.combos:
for (lazy0, lazy1), (fill,) in self.mixed_combos:
cubes = iris.cube.CubeList()
mask = [(0, 1), (0, 1)]
cubes.append(self._make_cube(0, mask=mask, lazy=lazy0,
dtype=self.dtype,
fill_value=fill0))
cubes.append(self._make_cube(1, lazy=lazy1, dtype=self.dtype,
fill_value=fill1))
fill_value=fill))
cubes.append(self._make_cube(1, lazy=lazy1, dtype=self.dtype))
result = cubes.merge_cube()
mask = [(0, 0), (0, 1), (0, 1)]
expected_fill_value = self._expected_fill_value(fill0, fill1)
expected_fill_value = self._expected_fill_value(fill)
expected = self._make_data([0, 1], mask=mask, dtype=self.dtype,
fill_value=expected_fill_value)
self.assertMaskedArrayEqual(result.data, expected)
self.assertEqual(result.dtype, self.dtype)
self._check_fill_value(result, fill0, fill1)
self._check_fill_value(result, fill0=fill)

def test_fill_value_invariant_to_order__same_non_None(self):
fill_value = 1234
cubes = [self._make_cube(i, mask=True,
fill_value=fill_value) for i in range(3)]
for combo in itertools.permutations(cubes):
result = iris.cube.CubeList(combo).merge_cube()
self.assertEqual(result.fill_value, fill_value)
self.assertEqual(result.data.fill_value, fill_value)

def test_fill_value_invariant_to_order__all_None(self):
cubes = [self._make_cube(i, mask=True,
fill_value=None) for i in range(3)]
for combo in itertools.permutations(cubes):
result = iris.cube.CubeList(combo).merge_cube()
self.assertIsNone(result.fill_value)
np_fill_value = ma.masked_array(0, dtype=result.dtype).fill_value
self.assertEqual(result.data.fill_value, np_fill_value)

Expand All @@ -270,7 +270,6 @@ def test_fill_value_invariant_to_order__different_non_None(self):
cubes.append(self._make_cube(3, mask=True, fill_value=4123))
for combo in itertools.permutations(cubes):
result = iris.cube.CubeList(combo).merge_cube()
self.assertIsNone(result.fill_value)
np_fill_value = ma.masked_array(0, dtype=result.dtype).fill_value
self.assertEqual(result.data.fill_value, np_fill_value)

Expand All @@ -280,7 +279,6 @@ def test_fill_value_invariant_to_order__mixed(self):
cubes.append(self._make_cube(2, mask=True, fill_value=4321))
for combo in itertools.permutations(cubes):
result = iris.cube.CubeList(combo).merge_cube()
self.assertIsNone(result.fill_value)
np_fill_value = ma.masked_array(0, dtype=result.dtype).fill_value
self.assertEqual(result.data.fill_value, np_fill_value)

Expand Down