diff --git a/lib/iris/etc/pp_save_rules.txt b/lib/iris/etc/pp_save_rules.txt index ee78cbfd01..b13bbb6ecf 100644 --- a/lib/iris/etc/pp_save_rules.txt +++ b/lib/iris/etc/pp_save_rules.txt @@ -759,18 +759,6 @@ THEN pp.brsvd[1] = aux_factory(cm, iris.aux_factory.HybridPressureFactory).dependencies['delta'].bounds[0, 1] -#MDI -IF - ma.isMaskedArray(cm.data) -THEN - pp.bmdi = cm.data.fill_value - -IF - not isinstance(cm.data, ma.core.MaskedArray) -THEN - pp.bmdi = -1e30 - - # CFname mega rule IF (cm.standard_name, cm.long_name, str(cm.units)) in iris.fileformats.um_cf_map.CF_TO_LBFC diff --git a/lib/iris/fileformats/netcdf.py b/lib/iris/fileformats/netcdf.py index 76d44c90d6..b95f971fed 100644 --- a/lib/iris/fileformats/netcdf.py +++ b/lib/iris/fileformats/netcdf.py @@ -1989,16 +1989,19 @@ def store(data, cf_var, fill_value): if dtype.itemsize == 1 and fill_value is None: if is_masked: - warnings.warn("Cube '{}' contains masked byte data and will " - "be interpreted as unmasked. To save as masked " - "data please explicitly provide a fill value." - .format(cube.name())) + msg = ("Cube '{}' contains byte data with masked points, but " + "no fill_value keyword was given. As saved, these " + "points will read back as valid values. To save as " + "masked byte data, please explicitly specify the " + "'fill_value' keyword.") + warnings.warn(msg.format(cube.name())) elif contains_fill_value: - warnings.warn("Cube '{}' contains data points equal to the fill " - "value {}. The points will be interpreted as being " - "masked. Please provide a fill_value argument not " - "equal to any data point.".format(cube.name(), - fill_value)) + msg = ("Cube '{}' contains unmasked data points equal to the " + "fill-value, {}. As saved, these points will read back " + "as missing data. To save these as normal values, please " + "specify a 'fill_value' keyword not equal to any valid " + "data points.") + warnings.warn(msg.format(cube.name(), fill_value)) if cube.standard_name: _setncattr(cf_var, 'standard_name', cube.standard_name) diff --git a/lib/iris/fileformats/pp.py b/lib/iris/fileformats/pp.py index 157a83b4c5..a046f709c2 100644 --- a/lib/iris/fileformats/pp.py +++ b/lib/iris/fileformats/pp.py @@ -1378,8 +1378,18 @@ def save(self, file_handle): # Get the actual data content. data = self.data - if ma.is_masked(data): - data = data.filled(fill_value=self.bmdi) + mdi = self.bmdi + if np.any(data == mdi): + msg = ('PPField data contains unmasked points equal to the fill ' + "value, {}. As saved, these points will read back as " + "missing data. To save these as normal values, please " + "set the field BMDI not equal to any valid data points.") + warnings.warn(msg.format(mdi)) + if isinstance(data, ma.MaskedArray): + if ma.is_masked(data): + data = data.filled(fill_value=mdi) + else: + data = data.data # Make sure the data is big-endian if data.dtype.newbyteorder('>') != data.dtype: @@ -2414,6 +2424,9 @@ def save_pairs_from_cube(cube, field_coords=None, target=None): pp_field.lbcode = 1 # Grid code. pp_field.bmks = 1.0 # Some scaley thing. pp_field.lbproc = 0 + # Set the missing-data value to the standard default value. + # The save code uses this header word to fill masked data points. + pp_field.bmdi = -1e30 # From UM doc F3: "Set to -99 if LBEGIN not known" pp_field.lbuser[1] = -99 diff --git a/lib/iris/tests/__init__.py b/lib/iris/tests/__init__.py index 5c6b717852..aa982cd105 100644 --- a/lib/iris/tests/__init__.py +++ b/lib/iris/tests/__init__.py @@ -49,6 +49,7 @@ import math import os import os.path +import re import shutil import subprocess import sys @@ -527,6 +528,28 @@ def assertRaisesRegexp(self, *args, **kwargs): return six.assertRaisesRegex(super(IrisTest_nometa, self), *args, **kwargs) + @contextlib.contextmanager + def assertGivesWarning(self, expected_regexp='', expect_warning=True): + # Check that a warning is raised matching a given expression, or that + # no warning matching the given expression is raised. + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + yield + messages = [str(warning.message) for warning in w] + expr = re.compile(expected_regexp) + matches = [message for message in messages if expr.search(message)] + warning_raised = any(matches) + if expect_warning: + if not warning_raised: + msg = "Warning matching '{}' not raised." + msg = msg.format(expected_regexp) + self.assertEqual(expect_warning, warning_raised, msg) + else: + if warning_raised: + msg = "Unexpected warning(s) raised, matching '{}' : {!r}." + msg = msg.format(expected_regexp, matches) + self.assertEqual(expect_warning, warning_raised, msg) + def _assertMaskedArray(self, assertion, a, b, strict, **kwargs): # Define helper function to extract unmasked values as a 1d # array. diff --git a/lib/iris/tests/results/cdm/masked_save_pp.txt b/lib/iris/tests/results/cdm/masked_save_pp.txt index 6b6350613b..fe6a1741ab 100644 --- a/lib/iris/tests/results/cdm/masked_save_pp.txt +++ b/lib/iris/tests/results/cdm/masked_save_pp.txt @@ -49,7 +49,7 @@ bdy: 1.0 bzx: -1.0 bdx: 1.0 - bmdi: 123456.0 + bmdi: -1e+30 bmks: 1.0 data: [[-- -- -- -- -- 0.6458941102027893 0.4375872015953064 0.891772985458374 0.9636627435684204 0.3834415078163147 0.7917250394821167 diff --git a/lib/iris/tests/test_cdm.py b/lib/iris/tests/test_cdm.py index 5944acf184..358b2c5cff 100644 --- a/lib/iris/tests/test_cdm.py +++ b/lib/iris/tests/test_cdm.py @@ -1199,7 +1199,8 @@ def test_save_and_merge(self): self.assertEqual(len(merged_cubes), 1, "expected a single merged cube") merged_cube = merged_cubes[0] self.assertEqual(merged_cube.dtype, dtype) - self.assertEqual(merged_cube.data.fill_value, fill_value) + # Check that the original masked-array fill-value is *ignored*. + self.assertArrayAllClose(merged_cube.data.fill_value, -1e30) @tests.skip_data diff --git a/lib/iris/tests/unit/fileformats/netcdf/test_Saver.py b/lib/iris/tests/unit/fileformats/netcdf/test_Saver.py index 3ab339b21e..fcf70c1b3c 100644 --- a/lib/iris/tests/unit/fileformats/netcdf/test_Saver.py +++ b/lib/iris/tests/unit/fileformats/netcdf/test_Saver.py @@ -25,7 +25,6 @@ import iris.tests as tests from contextlib import contextmanager -import warnings import netCDF4 as nc import numpy as np @@ -358,20 +357,6 @@ def _netCDF_var(self, cube, **kwargs): if var.standard_name == standard_name] yield var - @contextmanager - def _warning_check(self, message_text='', expect_warning=True): - # Check that a warning is raised containing a given string, or that - # no warning containing a given string is raised. - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - yield - matches = (message_text in str(warning.message) for warning in w) - warning_raised = any(matches) - msg = "Warning containing text '{}' not raised." if expect_warning \ - else "Warning containing text '{}' unexpectedly raised." - self.assertEqual(expect_warning, warning_raised, - msg.format(message_text)) - def test_fill_value(self): # Test that a passed fill value is saved as a _FillValue attribute. cube = self._make_cube('>f4') @@ -425,8 +410,8 @@ def test_contains_fill_value_passed(self): # Test that a warning is raised if the data contains the fill value. cube = self._make_cube('>f4') fill_value = 1 - with self._warning_check( - 'contains data points equal to the fill value'): + with self.assertGivesWarning( + 'contains unmasked data points equal to the fill-value'): with self._netCDF_var(cube, fill_value=fill_value): pass @@ -435,8 +420,8 @@ def test_contains_fill_value_byte(self): # when it is of a byte type. cube = self._make_cube('>i1') fill_value = 1 - with self._warning_check( - 'contains data points equal to the fill value'): + with self.assertGivesWarning( + 'contains unmasked data points equal to the fill-value'): with self._netCDF_var(cube, fill_value=fill_value): pass @@ -445,8 +430,8 @@ def test_contains_default_fill_value(self): # value if no fill_value argument is supplied. cube = self._make_cube('>f4') cube.data[0, 0] = nc.default_fillvals['f4'] - with self._warning_check( - 'contains data points equal to the fill value'): + with self.assertGivesWarning( + 'contains unmasked data points equal to the fill-value'): with self._netCDF_var(cube): pass @@ -455,8 +440,7 @@ def test_contains_default_fill_value_byte(self): # value if no fill_value argument is supplied when the data is of a # byte type. cube = self._make_cube('>i1') - with self._warning_check( - 'contains data points equal to the fill value', False): + with self.assertGivesWarning(r'\(fill\|mask\)', expect_warning=False): with self._netCDF_var(cube): pass @@ -465,8 +449,7 @@ def test_contains_masked_fill_value(self): # a masked point. fill_value = 1 cube = self._make_cube('>f4', masked_value=fill_value) - with self._warning_check( - 'contains data points equal to the fill value', False): + with self.assertGivesWarning(r'\(fill\|mask\)', expect_warning=False): with self._netCDF_var(cube, fill_value=fill_value): pass @@ -474,7 +457,7 @@ def test_masked_byte_default_fill_value(self): # Test that a warning is raised when saving masked byte data with no # fill value supplied. cube = self._make_cube('>i1', masked_value=1) - with self._warning_check('contains masked byte data', True): + with self.assertGivesWarning(r'\(fill\|mask\)', expect_warning=False): with self._netCDF_var(cube): pass @@ -483,7 +466,7 @@ def test_masked_byte_fill_value_passed(self): # fill value supplied if the the data does not contain the fill_value. fill_value = 100 cube = self._make_cube('>i1', masked_value=2) - with self._warning_check('contains masked byte data', False): + with self.assertGivesWarning(r'\(fill\|mask\)', expect_warning=False): with self._netCDF_var(cube, fill_value=fill_value): pass diff --git a/lib/iris/tests/unit/fileformats/pp/test_PPField.py b/lib/iris/tests/unit/fileformats/pp/test_PPField.py index 3838058560..774f7feda0 100644 --- a/lib/iris/tests/unit/fileformats/pp/test_PPField.py +++ b/lib/iris/tests/unit/fileformats/pp/test_PPField.py @@ -23,6 +23,9 @@ # importing anything else. import iris.tests as tests +from contextlib import contextmanager +import warnings + import numpy as np import iris.fileformats.pp as pp @@ -37,18 +40,22 @@ # items when written to disk and get consistent results. -DUMMY_HEADER = [('dummy1', (0, 13)), +DUMMY_HEADER = [('dummy1', (0, 11)), ('lbtim', (12,)), + ('dummy2', (13,)), ('lblrec', (14,)), - ('dummy2', (15, 18)), + ('dummy3', (15, 16)), ('lbrow', (17,)), + ('dummy4', (18,)), ('lbext', (19,)), ('lbpack', (20,)), - ('dummy3', (21, 37)), + ('dummy5', (21, 37)), ('lbuser', (38, 39, 40, 41, 42, 43, 44,)), ('brsvd', (45, 46, 47, 48)), ('bdatum', (49,)), - ('dummy4', (45, 63)), + ('dummy6', (50, 61)), + ('bmdi', (62, )), + ('dummy7', (63,)), ] @@ -57,13 +64,32 @@ class TestPPField(PPField): HEADER_DEFN = DUMMY_HEADER HEADER_DICT = dict(DUMMY_HEADER) + def _ready_for_save(self): + self.dummy1 = 0 + self.dummy2 = 0 + self.dummy3 = 0 + self.dummy4 = 0 + self.dummy5 = 0 + self.dummy6 = 0 + self.dummy7 = 0 + self.lbtim = 0 + self.lblrec = 0 + self.lbrow = 0 + self.lbext = 0 + self.lbpack = 0 + self.lbuser = 0 + self.brsvd = 0 + self.bdatum = 0 + self.bmdi = -1e30 + return self + @property def t1(self): - return netcdftime.datetime(2013, 10, 14, 10, 4) + return None @property def t2(self): - return netcdftime.datetime(2013, 10, 14, 10, 5) + return None class Test_save(tests.IrisTest): @@ -71,19 +97,7 @@ def test_float64(self): # Tests down-casting of >f8 data to >f4. def field_checksum(data): - field = TestPPField() - field.dummy1 = 0 - field.dummy2 = 0 - field.dummy3 = 0 - field.dummy4 = 0 - field.lbtim = 0 - field.lblrec = 0 - field.lbrow = 0 - field.lbext = 0 - field.lbpack = 0 - field.lbuser = 0 - field.brsvd = 0 - field.bdatum = 0 + field = TestPPField()._ready_for_save() field.data = data with self.temp_filename('.pp') as temp_filename: with open(temp_filename, 'wb') as pp_file: @@ -93,14 +107,49 @@ def field_checksum(data): data_64 = np.linspace(0, 1, num=10, endpoint=False).reshape(2, 5) checksum_32 = field_checksum(data_64.astype('>f4')) - with mock.patch('warnings.warn') as warn: + msg = 'Downcasting array precision from float64 to float32 for save.' + with self.assertGivesWarning(msg): checksum_64 = field_checksum(data_64.astype('>f8')) - self.assertEqual(checksum_32, checksum_64) - warn.assert_called_once_with( - 'Downcasting array precision from float64 to float32 for save.' - 'If float64 precision is required then please save in a ' - 'different format') + + def test_masked_mdi_value_warning(self): + # Check that an unmasked MDI value raises a warning. + field = TestPPField()._ready_for_save() + field.bmdi = -123.4 + # Make float32 data, as float64 default produces an extra warning. + field.data = np.ma.masked_array([1., field.bmdi, 3.], dtype=np.float32) + msg = 'PPField data contains unmasked points' + with self.assertGivesWarning(msg): + with self.temp_filename('.pp') as temp_filename: + with open(temp_filename, 'wb') as pp_file: + field.save(pp_file) + + def test_unmasked_mdi_value_warning(self): + # Check that MDI in *unmasked* data raises a warning. + field = TestPPField()._ready_for_save() + field.bmdi = -123.4 + # Make float32 data, as float64 default produces an extra warning. + field.data = np.array([1., field.bmdi, 3.], dtype=np.float32) + msg = 'PPField data contains unmasked points' + with self.assertGivesWarning(msg): + with self.temp_filename('.pp') as temp_filename: + with open(temp_filename, 'wb') as pp_file: + field.save(pp_file) + + def test_mdi_masked_value_nowarning(self): + # Check that a *masked* MDI value does not raise a warning. + field = TestPPField()._ready_for_save() + field.bmdi = -123.4 + # Make float32 data, as float64 default produces an extra warning. + field.data = np.ma.masked_array([1., 2., 3.], mask=[0, 1, 0], + dtype=np.float32) + # Set underlying data value at masked point to BMDI value. + field.data.data[1] = field.bmdi + self.assertArrayAllClose(field.data.data[1], field.bmdi) + with self.assertGivesWarning(r'\(mask\|fill\)', expect_warning=False): + with self.temp_filename('.pp') as temp_filename: + with open(temp_filename, 'wb') as pp_file: + field.save(pp_file) class Test_calendar(tests.IrisTest): diff --git a/lib/iris/tests/unit/fileformats/pp/test_as_pairs.py b/lib/iris/tests/unit/fileformats/pp/test_save_pairs_from_cube.py similarity index 74% rename from lib/iris/tests/unit/fileformats/pp/test_as_pairs.py rename to lib/iris/tests/unit/fileformats/pp/test_save_pairs_from_cube.py index 90a0792c89..8a2990e883 100644 --- a/lib/iris/tests/unit/fileformats/pp/test_as_pairs.py +++ b/lib/iris/tests/unit/fileformats/pp/test_save_pairs_from_cube.py @@ -14,7 +14,7 @@ # # You should have received a copy of the GNU Lesser General Public License # along with Iris. If not, see . -"""Unit tests for the `iris.fileformats.pp.as_pairs` function.""" +"""Unit tests for the `iris.fileformats.pp.save_pairs_from_cube` function.""" from __future__ import (absolute_import, division, print_function) from six.moves import (filter, input, map, range, zip) # noqa @@ -23,41 +23,44 @@ # importing anything else. import iris.tests as tests -from iris.coords import DimCoord -from iris.fileformats._ff_cross_references import STASH_TRANS -import iris.fileformats.pp as pp -from iris.tests import mock import iris.tests.stock as stock +from iris.fileformats.pp import save_pairs_from_cube -class TestAsFields(tests.IrisTest): + +class TestSaveFields(tests.IrisTest): def setUp(self): self.cube = stock.realistic_3d() def test_cube_only(self): - slices_and_fields = pp.as_pairs(self.cube) + slices_and_fields = save_pairs_from_cube(self.cube) for aslice, field in slices_and_fields: self.assertEqual(aslice.shape, (9, 11)) self.assertEqual(field.lbcode, 101) def test_field_coords(self): - slices_and_fields = pp.as_pairs(self.cube, - field_coords=['grid_longitude', - 'grid_latitude']) + slices_and_fields = save_pairs_from_cube( + self.cube, + field_coords=['grid_longitude', + 'grid_latitude']) for aslice, field in slices_and_fields: self.assertEqual(aslice.shape, (11, 9)) self.assertEqual(field.lbcode, 101) - @tests.skip_dask_mask def test_lazy_data(self): cube = self.cube.copy() # "Rebase" the cube onto a lazy version of its data. cube.data = cube.lazy_data() # Check that lazy data is preserved in save-pairs generation. - slices_and_fields = pp.as_pairs(cube) + slices_and_fields = save_pairs_from_cube(cube) for aslice, _ in slices_and_fields: self.assertTrue(aslice.has_lazy_data()) + def test_default_bmdi(self): + slices_and_fields = save_pairs_from_cube(self.cube) + _, field = next(slices_and_fields) + self.assertEqual(field.bmdi, -1e30) + if __name__ == "__main__": tests.main()