Skip to content

Fix PP bmdi handling #2564

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
DPeterK merged 9 commits into from
May 18, 2017
Merged

Fix PP bmdi handling #2564

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
212739f
Various un-skips.
pp-mo May 15, 2017
15d05d9
Checked new result for PPfields repr.
pp-mo May 15, 2017
8642922
Fix mock field data in TestVertical.
pp-mo May 15, 2017
941829f
Fix usage of BMDI in pp load and save.
pp-mo May 17, 2017
01ae38f
Fix test problem.
pp-mo May 17, 2017
23114a8
Tidy out old comments.
pp-mo May 17, 2017
ef4d607
Use new field.realised_dtype where appropriate.
pp-mo May 18, 2017
c2b8f10
More test fixes.
pp-mo May 18, 2017
64aaec2
Review changes, mostly cosmetic.
pp-mo May 18, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 13 additions & 9 deletions lib/iris/fileformats/pp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1284,10 +1284,8 @@ def data(self):
of the pp file

"""
# Cache the real data on first use
if self.realised_dtype.kind == 'i' and self.bmdi == -1e30:
self.bmdi = -9999
if is_lazy_data(self._data):
# Replace with real data on the first access.
self._data = as_concrete_data(self._data,
nans_replacement=ma.masked,
result_dtype=self.realised_dtype)
Expand Down Expand Up @@ -1395,12 +1393,18 @@ def save(self, file_handle):

"""

# Before we can actually write to file, we need to calculate the header
# elements. First things first, make sure the data is big-endian
# Get the actual data content.
data = self.data
if ma.isMaskedArray(data):
data = data.filled(fill_value=self.bmdi)

if ma.is_masked(data):
# Fill missing data points with the MDI value from the header.
if data.dtype.kind in 'biu':
# Integer or Boolean data : No masking is supported.
msg = 'Non-floating masked data cannot be saved to PP.'
raise ValueError(msg)
fill_value = self.bmdi
data = data.filled(fill_value=fill_value)

# Make sure the data is big-endian
if data.dtype.newbyteorder('>') != data.dtype:
# take a copy of the data when byteswapping
data = data.byteswap(False)
Expand All @@ -1412,7 +1416,7 @@ def save(self, file_handle):
b = np.empty(shape=NUM_FLOAT_HEADERS,
dtype=np.dtype(">f%d" % PP_WORD_DEPTH))

# Populate the arrays from the PPField
# Fill in the header elements from the PPField
for name, pos in self.HEADER_DEFN:
try:
header_elem = getattr(self, name)
Expand Down
12 changes: 9 additions & 3 deletions lib/iris/fileformats/rules.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -897,13 +897,19 @@ def _make_cube(field, converter):
# Convert the field to a Cube.
metadata = converter(field)

cube = iris.cube.Cube(field.core_data(),
cube_data = field.core_data()
cube_dtype = field.realised_dtype
if cube_dtype.kind in 'biu':
# Don't adopt BMDI as a fill value for integer data.
cube_fill_value = None
else:
cube_fill_value = field.bmdi
cube = iris.cube.Cube(cube_data,
attributes=metadata.attributes,
cell_methods=metadata.cell_methods,
dim_coords_and_dims=metadata.dim_coords_and_dims,
aux_coords_and_dims=metadata.aux_coords_and_dims,
fill_value=field.bmdi, dtype=field.realised_dtype)

fill_value=cube_fill_value, dtype=cube_dtype)

# Temporary code to deal with invalid standard names in the
# translation table.
Expand Down
46 changes: 28 additions & 18 deletions lib/iris/tests/integration/test_pp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,18 +47,19 @@ def _test_coord(self, cube, point, bounds=None, **kwargs):
if bounds is not None:
self.assertArrayEqual(coords[0].bounds, [bounds])

# hits a segfault, very odd
@tests.skip_biggus
def test_soil_level_round_trip(self):
# Use pp.load_cubes() to convert a fake PPField into a Cube.
# NB. Use MagicMock so that SplittableInt header items, such as
# LBCODE, support len().
soil_level = 1234
mock_data = np.zeros(1)
mock_core_data = mock.MagicMock(return_value=mock_data)
field = mock.MagicMock(lbvc=6, lblev=soil_level,
stash=iris.fileformats.pp.STASH(1, 0, 9),
lbuser=[0] * 7, lbrsvd=[0] * 4,
brsvd=[0] * 4, brlev=0,
_data=np.zeros(1))
core_data=mock_core_data,
realised_dtype=mock_data.dtype)
load = mock.Mock(return_value=iter([field]))
with mock.patch('iris.fileformats.pp.load', new=load) as load:
cube = next(iris.fileformats.pp.load_cubes('DUMMY'))
Expand All @@ -82,19 +83,20 @@ def test_soil_level_round_trip(self):
self.assertEqual(field.brsvd[0], 0)
self.assertEqual(field.brlev, 0)

# hits a segfault, very odd
@tests.skip_biggus
def test_soil_depth_round_trip(self):
# Use pp.load_cubes() to convert a fake PPField into a Cube.
# NB. Use MagicMock so that SplittableInt header items, such as
# LBCODE, support len().
lower, point, upper = 1.2, 3.4, 5.6
brsvd = [lower, 0, 0, 0]
mock_data = np.zeros(1)
mock_core_data = mock.MagicMock(return_value=mock_data)
field = mock.MagicMock(lbvc=6, blev=point,
stash=iris.fileformats.pp.STASH(1, 0, 9),
lbuser=[0] * 7, lbrsvd=[0] * 4,
brsvd=brsvd, brlev=upper,
_data=np.zeros(1))
core_data=mock_core_data,
realised_dtype=mock_data.dtype)
load = mock.Mock(return_value=iter([field]))
with mock.patch('iris.fileformats.pp.load', new=load) as load:
cube = next(iris.fileformats.pp.load_cubes('DUMMY'))
Expand All @@ -118,18 +120,19 @@ def test_soil_depth_round_trip(self):
self.assertEqual(field.brsvd[0], lower)
self.assertEqual(field.brlev, upper)

# hits a segfault, very odd
@tests.skip_biggus
def test_potential_temperature_level_round_trip(self):
# Check save+load for data on 'potential temperature' levels.

# Use pp.load_cubes() to convert a fake PPField into a Cube.
# NB. Use MagicMock so that SplittableInt header items, such as
# LBCODE, support len().
potm_value = 22.5
mock_data = np.zeros(1)
mock_core_data = mock.MagicMock(return_value=mock_data)
field = mock.MagicMock(lbvc=19, blev=potm_value,
lbuser=[0] * 7, lbrsvd=[0] * 4,
_data=np.zeros(1))
core_data=mock_core_data,
realised_dtype=mock_data.dtype)
load = mock.Mock(return_value=iter([field]))
with mock.patch('iris.fileformats.pp.load', new=load):
cube = next(iris.fileformats.pp.load_cubes('DUMMY'))
Expand All @@ -154,12 +157,15 @@ def test_hybrid_pressure_round_trip(self):
# LBCODE, support len().
def field_with_data(scale=1):
x, y = 40, 30
return_value = np.arange(1200).reshape(y, x) * scale
core_data = mock.MagicMock(return_value=return_value)
field = mock.MagicMock(core_data=core_data, lbcode=[1],
mock_data = np.arange(1200).reshape(y, x) * scale
mock_core_data = mock.MagicMock(return_value=mock_data)
field = mock.MagicMock(core_data=mock_core_data,
realised_dtype=mock_data.dtype,
lbcode=[1],
lbnpt=x, lbrow=y, bzx=350, bdx=1.5,
bzy=40, bdy=1.5, lbuser=[0] * 7,
lbrsvd=[0] * 4)

field._x_coord_name = lambda: 'longitude'
field._y_coord_name = lambda: 'latitude'
field.coord_system = lambda: None
Expand Down Expand Up @@ -235,9 +241,11 @@ def field_with_data(scale=1):
def test_hybrid_pressure_with_duplicate_references(self):
def field_with_data(scale=1):
x, y = 40, 30
return_value = np.arange(1200).reshape(y, x) * scale
core_data = mock.MagicMock(return_value=return_value)
field = mock.MagicMock(core_data=core_data, lbcode=[1],
mock_data = np.arange(1200).reshape(y, x) * scale
mock_core_data = mock.MagicMock(return_value=mock_data)
field = mock.MagicMock(core_data=mock_core_data,
realised_dtype=mock_data.dtype,
lbcode=[1],
lbnpt=x, lbrow=y, bzx=350, bdx=1.5,
bzy=40, bdy=1.5, lbuser=[0] * 7,
lbrsvd=[0] * 4)
Expand Down Expand Up @@ -350,9 +358,11 @@ def test_hybrid_height_round_trip_no_reference(self):
# LBCODE, support len().
def field_with_data(scale=1):
x, y = 40, 30
return_value = np.arange(1200).reshape(y, x) * scale
core_data = mock.MagicMock(return_value=return_value)
field = mock.MagicMock(core_data=core_data, lbcode=[1],
mock_data = np.arange(1200).reshape(y, x) * scale
mock_core_data = mock.MagicMock(return_value=mock_data)
field = mock.MagicMock(core_data=mock_core_data,
realised_dtype=mock_data.dtype,
lbcode=[1],
lbnpt=x, lbrow=y, bzx=350, bdx=1.5,
bzy=40, bdy=1.5, lbuser=[0] * 7,
lbrsvd=[0] * 4)
Expand Down
65 changes: 22 additions & 43 deletions lib/iris/tests/results/PP/rle_unpacked.pp.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,22 +51,15 @@
bdx: 1.0
bmdi: -1.07374e+09
bmks: 1.0
data.data: [[ 0. 0. 0. ..., 0. 0. 0. ]
[ 0. 0. 0. ..., 0. 0. 0. ]
[ 0. 0. 0. ..., 0. 0. 0. ]
data: [[-- -- -- ..., -- -- --]
[-- -- -- ..., -- -- --]
[-- -- -- ..., -- -- --]
...,
[ 233.78448486 233.66023254 233.53538513 ..., 234.14790344
234.02868652 233.90744019]
[ 232.06645203 231.93363953 231.80679321 ..., 232.4972229 232.34857178
232.20487976]
[ 0. 0. 0. ..., 0. 0. 0. ]]
data.mask: [[ True True True ..., True True True]
[ True True True ..., True True True]
[ True True True ..., True True True]
...,
[False False False ..., False False False]
[False False False ..., False False False]
[ True True True ..., True True True]]
[233.78448486328125 233.6602325439453 233.53538513183594 ...,
234.1479034423828 234.0286865234375 233.90744018554688]
[232.0664520263672 231.9336395263672 231.80679321289062 ...,
232.49722290039062 232.34857177734375 232.2048797607422]
[-- -- -- ..., -- -- --]]
y: [-90. -89. -88. -87. -86. -85. -84.
-83. -82. -81. -80. -79. -78. -77.
-76. -75. -74. -73. -72. -71. -70.
Expand Down Expand Up @@ -268,22 +261,15 @@
bdx: 1.0
bmdi: -1.07374e+09
bmks: 1.0
data.data: [[ 0. 0. 0. ..., 0. 0. 0. ]
[ 0. 0. 0. ..., 0. 0. 0. ]
[ 0. 0. 0. ..., 0. 0. 0. ]
...,
[ 0.00864433 0.00852169 0.00838812 ..., 0.00917711 0.00897506
0.0088051 ]
[ 0.00563598 0.00565888 0.00567179 ..., 0.00559308 0.00561324
0.00562014]
[ 0. 0. 0. ..., 0. 0. 0. ]]
data.mask: [[ True True True ..., True True True]
[ True True True ..., True True True]
[ True True True ..., True True True]
data: [[-- -- -- ..., -- -- --]
[-- -- -- ..., -- -- --]
[-- -- -- ..., -- -- --]
...,
[False False False ..., False False False]
[False False False ..., False False False]
[ True True True ..., True True True]]
[0.00864433217793703 0.008521686308085918 0.008388116024434566 ...,
0.009177112020552158 0.008975056000053883 0.008805099874734879]
[0.005635980516672134 0.005658876616507769 0.005671785678714514 ...,
0.00559308473020792 0.00561324181035161 0.005620136857032776]
[-- -- -- ..., -- -- --]]
y: [-90. -89. -88. -87. -86. -85. -84.
-83. -82. -81. -80. -79. -78. -77.
-76. -75. -74. -73. -72. -71. -70.
Expand Down Expand Up @@ -485,20 +471,13 @@
bdx: 1.0
bmdi: -1.07374e+09
bmks: 1.0
data.data: [[ 0. 0. 0. ..., 0. 0. 0.]
[ 0. 0. 0. ..., 0. 0. 0.]
[ 0. 0. 0. ..., 0. 0. 0.]
...,
[ 0. 0. 0. ..., 0. 0. 0.]
[ 0. 0. 0. ..., 0. 0. 0.]
[ 0. 0. 0. ..., 0. 0. 0.]]
data.mask: [[ True True True ..., True True True]
[ True True True ..., True True True]
[ True True True ..., True True True]
data: [[-- -- -- ..., -- -- --]
[-- -- -- ..., -- -- --]
[-- -- -- ..., -- -- --]
...,
[False False False ..., False False False]
[False False False ..., False False False]
[ True True True ..., True True True]]
[0.0 0.0 0.0 ..., 0.0 0.0 0.0]
[0.0 0.0 0.0 ..., 0.0 0.0 0.0]
[-- -- -- ..., -- -- --]]
y: [-90. -89. -88. -87. -86. -85. -84.
-83. -82. -81. -80. -79. -78. -77.
-76. -75. -74. -73. -72. -71. -70.
Expand Down
2 changes: 1 addition & 1 deletion lib/iris/tests/results/cube_to_pp/no_forecast_period.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
bdy: 1.0
bzx: -2.0
bdx: 1.0
bmdi: -9999.0
bmdi: -1e+30
bmks: 1.0
data: [[ 0 1 2 3]
[ 4 5 6 7]
Expand Down
2 changes: 1 addition & 1 deletion lib/iris/tests/results/cube_to_pp/no_forecast_time.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@
bdy: 1.0
bzx: -2.0
bdx: 1.0
bmdi: -9999.0
bmdi: -1e+30
bmks: 1.0
data: [[ 0 1 2 3]
[ 4 5 6 7]
Expand Down
2 changes: 1 addition & 1 deletion lib/iris/tests/results/usecases/pp_to_cf_conversion/from_netcdf/integer.b_0.cml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<?xml version="1.0" ?>
<cubes xmlns="urn:x-iris:cubeml-0.2">
<cube core-dtype="int32" dtype="int32" standard_name="land_binary_mask" units="1" var_name="land_binary_mask">
<cube core-dtype="float64" dtype="int32" standard_name="land_binary_mask" units="1" var_name="land_binary_mask">
<attributes>
<attribute name="Conventions" value="CF-1.5"/>
<attribute name="source" value="Data from Met Office Unified Model"/>
Expand Down
2 changes: 1 addition & 1 deletion lib/iris/tests/results/usecases/pp_to_cf_conversion/from_pp/integer.b.cml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
<?xml version="1.0" ?>
<cubes xmlns="urn:x-iris:cubeml-0.2">
<cube core-dtype="int32" dtype="int32" fill_value="-1e+30" standard_name="land_binary_mask" units="1">
<cube core-dtype="float64" dtype="int32" standard_name="land_binary_mask" units="1">
<attributes>
<attribute name="source" value="Data from Met Office Unified Model"/>
</attributes>
Expand Down
6 changes: 0 additions & 6 deletions lib/iris/tests/test_cube_to_pp.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -230,10 +230,6 @@ def geog_cs(self):


class TestPPSaveRules(tests.IrisTest, pp.PPTest):
# Skip this test, there appears to be a long standing bug in PP saving
# for int32, which is made worse by assigning the 'default' bmdi of
# 1e30 into int arrays
@tests.skip_biggus
Copy link
Member

@DPeterK DPeterK May 18, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nice to see these all being fixed! 💯

def test_default_coord_system(self):
GeogCS = iris.coord_systems.GeogCS
cube = iris.tests.stock.lat_lon_cube()
Expand Down Expand Up @@ -266,8 +262,6 @@ def lbproc_from_pp(self, filename):
field = next(pp_file)
return field.lbproc

# see related comment #236
@tests.skip_biggus
def test_pp_save_rules(self):
# Test single process flags
for _, process_desc in iris.fileformats.pp.LBPROC_PAIRS[1:]:
Expand Down
2 changes: 0 additions & 2 deletions lib/iris/tests/test_pp_cf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,8 +80,6 @@ def callback_aaxzc_n10r13xy_b_pp(cube, field, filename):
cube.add_aux_coord(height_coord)


# XXX: Issue with integer.b.pp and invalid bmdi fill_value for int32 dtype
@tests.skip_biggus
@tests.skip_data
class TestAll(tests.IrisTest, pp.PPTest):
_ref_dir = ('usecases', 'pp_to_cf_conversion')
Expand Down
1 change: 0 additions & 1 deletion lib/iris/tests/test_pp_module.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,6 @@ def test_wgdos_mo_pack(self):
for orig_field, saved_field in zip(orig_fields, saved_fields):
assert_array_equal(orig_field.data, saved_field.data)

@tests.skip_biggus
def test_rle(self):
r = pp.load(tests.get_data_path(('PP', 'ocean_rle', 'ocean_rle.pp')))

Expand Down
5 changes: 4 additions & 1 deletion lib/iris/tests/unit/fileformats/rules/test__make_cube.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,10 @@ def test_invalid_units(self):
dim_coords_and_dims, aux_coords_and_dims)
converter = mock.Mock(return_value=metadata)

field = mock.Mock(core_data=lambda: np.arange(3.), bmdi=9999.)
test_data = np.arange(3.)
field = mock.Mock(core_data=lambda: test_data,
bmdi=9999.,
realised_dtype=test_data.dtype)
with warnings.catch_warnings(record=True) as warn:
warnings.simplefilter("always")
cube, factories, references = _make_cube(field, converter)
Expand Down
Loading