Skip to content

Fix concatenate tests. #18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bjlittle merged 1 commit into from
Mar 15, 2017
Merged

Fix concatenate tests. #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion lib/iris/cube.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1777,7 +1777,7 @@ def data(self, value):
self._dask_array = None

# Cancel any 'realisation' datatype conversion, and fill value.
self._dtype = None
self.dtype = None
self.fill_value = None

def has_lazy_data(self):
Expand Down
129 changes: 70 additions & 59 deletions lib/iris/tests/test_concatenate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,8 @@


def _make_cube(x, y, data, aux=None, offset=0, scalar=None,
dtype=np.dtype('float32'), fill_value=None):
dtype=np.dtype('float32'), fill_value=None,
mask=None):
"""
A convenience test function that creates a custom 2D cube.

Expand Down Expand Up @@ -67,6 +68,10 @@ def _make_cube(x, y, data, aux=None, offset=0, scalar=None,
* scalar:
Create a 'height' scalar coordinate with the given value.

* mask:
Create cube masked data with the specified mask. Boolean or
array indicies of points to be masked.

Returns:
The newly created 2D :class:`iris.cube.Cube`.

Expand All @@ -75,9 +80,18 @@ def _make_cube(x, y, data, aux=None, offset=0, scalar=None,
y_range = np.arange(*y, dtype=dtype)
x_size = len(x_range)
y_size = len(y_range)
shape = (y_size, x_size)

cube_data = np.empty((y_size, x_size), dtype=dtype)
cube_data[:] = data
if mask is not None:
cube_data = ma.empty(shape, dtype=dtype)
cube_data.data[:] = data
if isinstance(mask, bool):
cube_data.mask = mask
else:
cube_data[mask] = ma.masked
else:
cube_data = np.empty(shape, dtype=dtype)
cube_data[:] = data
cube = iris.cube.Cube(cube_data, fill_value=fill_value)
coord = DimCoord(y_range, long_name='y')
coord.guess_bounds()
Expand Down Expand Up @@ -217,15 +231,18 @@ def concatenate(cubes, order=None):
result = cubelist.concatenate()

for cube in result:
# Setting the cube.data clears the cube.dtype and cube.fill_value.
# We want to maintain the fill_value for testing purposes, even
# though we have lost the lazy data in order to pin down the array
# data order to overcome testing on different architectures.
Copy link

@lbdreyer lbdreyer Mar 15, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we be putting this comment everywhere we have done

fill_value = cube.fill_value
cube.data = ...
cube.fill_value = fill_value

Copy link
Owner Author

@bjlittle bjlittle Mar 15, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've tried to leave some clue's (for my future self) but to be honest if we have a step change in the "persisting fill_value across a data setter operation" then we can always find the usage of fill_value in the code base.

I just added a bit more of context for concatenate because it's testing is a bit weird and, at the time, I thought an exception to the norm.

fill_value = cube.fill_value
if ma.isMaskedArray(cube.data):
# cube.data = ma.copy(cube.data, order=order)
data = np.array(cube.data.data, copy=True, order=order)
mask = np.array(cube.data.mask, copy=True, order=order)
fill_value = cube.data.fill_value
cube.data = ma.array(data, mask=mask, fill_value=fill_value)
cube.data = ma.array(data, mask=mask)
else:
# cube.data = np.copy(cube.data, order=order)
cube.data = np.array(cube.data, copy=True, order=order)
cube.fill_value = fill_value

return result

Expand Down Expand Up @@ -321,12 +338,10 @@ def test_order(self):
def test_masked_fill_value(self):
cubes = []
y = (0, 2)
cube = _make_cube((0, 2), y, 1)
cube.data = ma.asarray(cube.data)
cube = _make_cube((0, 2), y, 1, mask=True)
cube.fill_value = 10
cubes.append(cube)
cube = _make_cube((2, 4), y, 1)
cube.data = ma.asarray(cube.data)
cube = _make_cube((2, 4), y, 1, mask=True)
cube.fill_value = 20
cubes.append(cube)
result = concatenate(cubes)
Expand All @@ -337,8 +352,7 @@ class Test2D(tests.IrisTest):
def test_masked_and_unmasked(self):
cubes = []
y = (0, 2)
cube = _make_cube((0, 2), y, 1)
cube.data = ma.asarray(cube.data)
cube = _make_cube((0, 2), y, 1, mask=True)
cubes.append(cube)
cubes.append(_make_cube((2, 4), y, 2))
result = concatenate(cubes)
Expand All @@ -347,42 +361,39 @@ def test_masked_and_unmasked(self):
def test_masked_and_unmasked_int16(self):
cubes = []
y = (0, 2)
cube = _make_cube((0, 2), y, 1, dtype=np.dtype('int16'))
cube.data = ma.asarray(cube.data)
dtype = np.dtype('int16')
cube = _make_cube((0, 2), y, 1, dtype=dtype, mask=True)
cubes.append(cube)
cubes.append(_make_cube((2, 4), y, 2, dtype=np.dtype('int16')))
cubes.append(_make_cube((2, 4), y, 2, dtype=dtype))
result = concatenate(cubes)
self.assertEqual(len(result), 1)

def test_unmasked_and_masked(self):
cubes = []
y = (0, 2)
cubes.append(_make_cube((0, 2), y, 1))
cube = _make_cube((2, 4), y, 2)
cube.data = ma.asarray(cube.data)
cube = _make_cube((2, 4), y, 2, mask=True)
cubes.append(cube)
result = concatenate(cubes)
self.assertEqual(len(result), 1)

def test_unmasked_and_masked_int16(self):
cubes = []
y = (0, 2)
cubes.append(_make_cube((0, 2), y, 1, dtype=np.dtype('int16')))
cube = _make_cube((2, 4), y, 2, dtype=np.dtype('int16'))
cube.data = ma.asarray(cube.data)
dtype = np.dtype('int16')
cubes.append(_make_cube((0, 2), y, 1, dtype=dtype))
cube = _make_cube((2, 4), y, 2, dtype=dtype, mask=True)
cubes.append(cube)
result = concatenate(cubes)
self.assertEqual(len(result), 1)

def test_masked_fill_value(self):
cubes = []
y = (0, 2)
cube = _make_cube((0, 2), y, 1)
cube.data = ma.asarray(cube.data)
cube = _make_cube((0, 2), y, 1, mask=True)
cube.data.fill_value = 10
cubes.append(cube)
cube = _make_cube((2, 4), y, 1)
cube.data = ma.asarray(cube.data)
cube = _make_cube((2, 4), y, 1, mask=True)
cube.data.fill_value = 20
cubes.append(cube)
result = concatenate(cubes)
Expand All @@ -391,13 +402,11 @@ def test_masked_fill_value(self):
def test_concat_masked_2x2d(self):
cubes = []
y = (0, 2)
cube = _make_cube((0, 2), y, 1)
cube.data = ma.asarray(cube.data)
cube.data[(0, 1), (0, 1)] = ma.masked
mask = [(0, 1), (0, 1)]
cube = _make_cube((0, 2), y, 1, mask=mask)
cubes.append(cube)
cube = _make_cube((2, 4), y, 2)
cube.data = ma.asarray(cube.data)
cube.data[(0, 1), (1, 0)] = ma.masked
mask = [(0, 1), (1, 0)]
cube = _make_cube((2, 4), y, 2, mask=mask)
cubes.append(cube)
result = concatenate(cubes)
self.assertCML(result, ('concatenate', 'concat_masked_2x2d.cml'))
Expand All @@ -410,13 +419,11 @@ def test_concat_masked_2x2d(self):
def test_concat_masked_2y2d(self):
cubes = []
x = (0, 2)
cube = _make_cube(x, (0, 2), 1)
cube.data = np.ma.asarray(cube.data)
cube.data[(0, 1), (0, 1)] = ma.masked
mask = [(0, 1), (0, 1)]
cube = _make_cube(x, (0, 2), 1, mask=mask)
cubes.append(cube)
cube = _make_cube(x, (2, 4), 2)
cube.data = ma.asarray(cube.data)
cube.data[(0, 1), (1, 0)] = ma.masked
mask = [(0, 1), (1, 0)]
cube = _make_cube(x, (2, 4), 2, mask=mask)
cubes.append(cube)
result = concatenate(cubes)
self.assertCML(result, ('concatenate', 'concat_masked_2y2d.cml'))
Expand All @@ -431,13 +438,15 @@ def test_concat_masked_2y2d(self):
def test_concat_masked_2y2d_int16(self):
cubes = []
x = (0, 2)
cube = _make_cube(x, (0, 2), 1, dtype=np.int16, fill_value=-37)
cube.data = np.ma.asarray(cube.data)
cube.data[(0, 1), (0, 1)] = ma.masked
dtype = np.dtype('int16')
fill_value = -37
mask = [(0, 1), (0, 1)]
cube = _make_cube(x, (0, 2), 1, dtype=dtype, fill_value=fill_value,
mask=mask)
cubes.append(cube)
cube = _make_cube(x, (2, 4), 2, dtype=np.int16, fill_value=-37)
cube.data = ma.asarray(cube.data)
cube.data[(0, 1), (1, 0)] = ma.masked
mask = [(0, 1), (1, 0)]
cube = _make_cube(x, (2, 4), 2, dtype=dtype, fill_value=fill_value,
mask=mask)
cubes.append(cube)
result = concatenate(cubes)
self.assertCML(result, ('concatenate', 'concat_masked_2y2d_int16.cml'))
Expand All @@ -452,16 +461,17 @@ def test_concat_masked_2y2d_int16(self):
def test_concat_masked_2y2d_int16_with_concrete_and_lazy(self):
cubes = []
x = (0, 2)
cube = _make_cube(x, (0, 2), 1, dtype=np.int16, fill_value=-37)
cube.data = np.ma.asarray(cube.data)
cube.data[(0, 1), (0, 1)] = ma.masked
dtype = np.dtype('int16')
fill_value = -37
mask = [(0, 1), (0, 1)]
cube = _make_cube(x, (0, 2), 1, dtype=dtype, fill_value=fill_value,
mask=mask)
cubes.append(cube)
cube = _make_cube(x, (2, 4), 2, dtype=np.int16)
cube.data = ma.asarray(cube.data)
cube.data[(0, 1), (1, 0)] = ma.masked
mask = [(0, 1), (1, 0)]
cube = _make_cube(x, (2, 4), 2, dtype=dtype, mask=mask)
cube.data = cube.lazy_data()
cube.dtype = np.dtype('int16')
cube.fill_value = -37
cube.dtype = dtype
cube.fill_value = fill_value
cubes.append(cube)
result = concatenate(cubes)
self.assertCML(result, ('concatenate', 'concat_masked_2y2d_int16.cml'))
Expand All @@ -476,16 +486,17 @@ def test_concat_masked_2y2d_int16_with_concrete_and_lazy(self):
def test_concat_masked_2y2d_int16_with_lazy_and_concrete(self):
cubes = []
x = (0, 2)
cube = _make_cube(x, (2, 4), 2, dtype=np.int16)
cube.data = ma.asarray(cube.data)
cube.data[(0, 1), (1, 0)] = ma.masked
dtype = np.dtype('int16')
fill_value = -37
mask = [(0, 1), (1, 0)]
cube = _make_cube(x, (2, 4), 2, dtype=dtype, mask=mask)
cube.data = cube.lazy_data()
cube.dtype = np.dtype('int16')
cube.fill_value = -37
cube.dtype = dtype
cube.fill_value = fill_value
cubes.append(cube)
cube = _make_cube(x, (0, 2), 1, dtype=np.int16, fill_value=-37)
cube.data = np.ma.asarray(cube.data)
cube.data[(0, 1), (0, 1)] = ma.masked
mask = [(0, 1), (0, 1)]
cube = _make_cube(x, (0, 2), 1, dtype=dtype, fill_value=fill_value,
mask=mask)
cubes.append(cube)
result = concatenate(cubes)
self.assertCML(result, ('concatenate', 'concat_masked_2y2d_int16.cml'))
Expand Down