Fix MDI and preserve lazy data in PP saves.

lib/iris/etc/pp_save_rules.txt

@@ -759,18 +759,6 @@ THEN
     pp.brsvd[1] = aux_factory(cm, iris.aux_factory.HybridPressureFactory).dependencies['delta'].bounds[0, 1]
 
 
-#MDI
-IF
-    ma.isMaskedArray(cm.data)
-THEN
-    pp.bmdi = cm.data.fill_value
-
-IF
-    not isinstance(cm.data, ma.core.MaskedArray)
-THEN
-    pp.bmdi = -1e30
-
-
 # CFname mega rule
 IF
     (cm.standard_name, cm.long_name, str(cm.units)) in iris.fileformats.um_cf_map.CF_TO_LBFC

lib/iris/fileformats/netcdf.py

@@ -1989,16 +1989,19 @@ def store(data, cf_var, fill_value):
 
         if dtype.itemsize == 1 and fill_value is None:
             if is_masked:
-                warnings.warn("Cube '{}' contains masked byte data and will "
-                              "be interpreted as unmasked. To save as masked "
-                              "data please explicitly provide a fill value."
-                              .format(cube.name()))
+                msg = ("Cube '{}' contains byte data with masked points, but "
+                       "no fill_value keyword was given. As saved, these "
+                       "points will read back as valid values. To save as "
+                       "masked byte data, please explicitly specify the "
+                       "'fill_value' keyword.")
+                warnings.warn(msg.format(cube.name()))
         elif contains_fill_value:
-            warnings.warn("Cube '{}' contains data points equal to the fill "
-                          "value {}. The points will be interpreted as being "
-                          "masked. Please provide a fill_value argument not "
-                          "equal to any data point.".format(cube.name(),
-                                                            fill_value))
+            msg = ("Cube '{}' contains unmasked data points equal to the "
+                   "fill-value, {}. As saved, these points will read back "
+                   "as missing data. To save these as normal values, please "
+                   "specify a 'fill_value' keyword not equal to any valid "
+                   "data points.")
+            warnings.warn(msg.format(cube.name(), fill_value))
 
         if cube.standard_name:
             _setncattr(cf_var, 'standard_name', cube.standard_name)

lib/iris/fileformats/pp.py

@@ -1378,8 +1378,18 @@ def save(self, file_handle):
 
         # Get the actual data content.
         data = self.data
-        if ma.is_masked(data):
-            data = data.filled(fill_value=self.bmdi)
+        mdi = self.bmdi
+        if np.any(data == mdi):
+            msg = ('PPField data contains unmasked points equal to the fill '
+                   "value, {}. As saved, these points will read back as "
+                   "missing data. To save these as normal values, please "
+                   "set the field BMDI not equal to any valid data points.")
+            warnings.warn(msg.format(mdi))
+        if isinstance(data, ma.MaskedArray):
+            if ma.is_masked(data):
+                data = data.filled(fill_value=mdi)
+            else:
+                data = data.data
 
         # Make sure the data is big-endian
         if data.dtype.newbyteorder('>') != data.dtype:
@@ -2414,6 +2424,9 @@ def save_pairs_from_cube(cube, field_coords=None, target=None):
         pp_field.lbcode = 1     # Grid code.
         pp_field.bmks = 1.0     # Some scaley thing.
         pp_field.lbproc = 0
+        # Set the missing-data value to the standard default value.
+        # The save code uses this header word to fill masked data points.
+        pp_field.bmdi = -1e30
         # From UM doc F3: "Set to -99 if LBEGIN not known"
         pp_field.lbuser[1] = -99
 

lib/iris/tests/__init__.py

@@ -49,6 +49,7 @@
 import math
 import os
 import os.path
+import re
 import shutil
 import subprocess
 import sys
@@ -527,6 +528,28 @@ def assertRaisesRegexp(self, *args, **kwargs):
         return six.assertRaisesRegex(super(IrisTest_nometa, self),
                                      *args, **kwargs)
 
+    @contextlib.contextmanager
+    def assertGivesWarning(self, expected_regexp='', expect_warning=True):
+        # Check that a warning is raised matching a given expression, or that
+        # no warning matching the given expression is raised.
+        with warnings.catch_warnings(record=True) as w:
+            warnings.simplefilter('always')
+            yield
+        messages = [str(warning.message) for warning in w]
+        expr = re.compile(expected_regexp)
+        matches = [message for message in messages if expr.search(message)]
+        warning_raised = any(matches)
+        if expect_warning:
+            if not warning_raised:
+                msg = "Warning matching '{}' not raised."
+                msg = msg.format(expected_regexp)
+                self.assertEqual(expect_warning, warning_raised, msg)
+        else:
+            if warning_raised:
+                msg = "Unexpected warning(s) raised, matching '{}' : {!r}."
+                msg = msg.format(expected_regexp, matches)
+                self.assertEqual(expect_warning, warning_raised, msg)
+
     def _assertMaskedArray(self, assertion, a, b, strict, **kwargs):
         # Define helper function to extract unmasked values as a 1d
         # array.

lib/iris/tests/results/cdm/masked_save_pp.txt

@@ -49,7 +49,7 @@
    bdy: 1.0
    bzx: -1.0
    bdx: 1.0
-   bmdi: 123456.0
+   bmdi: -1e+30
    bmks: 1.0
    data: [[-- -- -- -- -- 0.6458941102027893 0.4375872015953064 0.891772985458374
   0.9636627435684204 0.3834415078163147 0.7917250394821167

lib/iris/tests/test_cdm.py

@@ -1199,7 +1199,8 @@ def test_save_and_merge(self):
             self.assertEqual(len(merged_cubes), 1, "expected a single merged cube")
             merged_cube = merged_cubes[0]
             self.assertEqual(merged_cube.dtype, dtype)
-            self.assertEqual(merged_cube.data.fill_value, fill_value)
+            # Check that the original masked-array fill-value is *ignored*.
+            self.assertArrayAllClose(merged_cube.data.fill_value, -1e30)
 
 
 @tests.skip_data

lib/iris/tests/unit/fileformats/netcdf/test_Saver.py

@@ -25,7 +25,6 @@
 import iris.tests as tests
 
 from contextlib import contextmanager
-import warnings
 
 import netCDF4 as nc
 import numpy as np
@@ -358,20 +357,6 @@ def _netCDF_var(self, cube, **kwargs):
                     if var.standard_name == standard_name]
             yield var
 
-    @contextmanager
-    def _warning_check(self, message_text='', expect_warning=True):
-        # Check that a warning is raised containing a given string, or that
-        # no warning containing a given string is raised.
-        with warnings.catch_warnings(record=True) as w:
-            warnings.simplefilter('always')
-            yield
-        matches = (message_text in str(warning.message) for warning in w)
-        warning_raised = any(matches)
-        msg = "Warning containing text '{}' not raised." if expect_warning \
-            else "Warning containing text '{}' unexpectedly raised."
-        self.assertEqual(expect_warning, warning_raised,
-                         msg.format(message_text))
-
     def test_fill_value(self):
         # Test that a passed fill value is saved as a _FillValue attribute.
         cube = self._make_cube('>f4')
@@ -425,8 +410,8 @@ def test_contains_fill_value_passed(self):
         # Test that a warning is raised if the data contains the fill value.
         cube = self._make_cube('>f4')
         fill_value = 1
-        with self._warning_check(
-                'contains data points equal to the fill value'):
+        with self.assertGivesWarning(
+                'contains unmasked data points equal to the fill-value'):
             with self._netCDF_var(cube, fill_value=fill_value):
                 pass
 
@@ -435,8 +420,8 @@ def test_contains_fill_value_byte(self):
         # when it is of a byte type.
         cube = self._make_cube('>i1')
         fill_value = 1
-        with self._warning_check(
-                'contains data points equal to the fill value'):
+        with self.assertGivesWarning(
+                'contains unmasked data points equal to the fill-value'):
             with self._netCDF_var(cube, fill_value=fill_value):
                 pass
 
@@ -445,8 +430,8 @@ def test_contains_default_fill_value(self):
         # value if no fill_value argument is supplied.
         cube = self._make_cube('>f4')
         cube.data[0, 0] = nc.default_fillvals['f4']
-        with self._warning_check(
-                'contains data points equal to the fill value'):
+        with self.assertGivesWarning(
+                'contains unmasked data points equal to the fill-value'):
             with self._netCDF_var(cube):
                 pass
 
@@ -455,8 +440,7 @@ def test_contains_default_fill_value_byte(self):
         # value if no fill_value argument is supplied when the data is of a
         # byte type.
         cube = self._make_cube('>i1')
-        with self._warning_check(
-                'contains data points equal to the fill value', False):
+        with self.assertGivesWarning(r'\(fill\|mask\)', expect_warning=False):
             with self._netCDF_var(cube):
                 pass
 
@@ -465,16 +449,15 @@ def test_contains_masked_fill_value(self):
         # a masked point.
         fill_value = 1
         cube = self._make_cube('>f4', masked_value=fill_value)
-        with self._warning_check(
-                'contains data points equal to the fill value', False):
+        with self.assertGivesWarning(r'\(fill\|mask\)', expect_warning=False):
             with self._netCDF_var(cube, fill_value=fill_value):
                 pass
 
     def test_masked_byte_default_fill_value(self):
         # Test that a warning is raised when saving masked byte data with no
         # fill value supplied.
         cube = self._make_cube('>i1', masked_value=1)
-        with self._warning_check('contains masked byte data', True):
+        with self.assertGivesWarning(r'\(fill\|mask\)', expect_warning=False):
             with self._netCDF_var(cube):
                 pass
 
@@ -483,7 +466,7 @@ def test_masked_byte_fill_value_passed(self):
         # fill value supplied if the the data does not contain the fill_value.
         fill_value = 100
         cube = self._make_cube('>i1', masked_value=2)
-        with self._warning_check('contains masked byte data', False):
+        with self.assertGivesWarning(r'\(fill\|mask\)', expect_warning=False):
             with self._netCDF_var(cube, fill_value=fill_value):
                 pass
 

lib/iris/tests/unit/fileformats/pp/test_PPField.py

@@ -23,6 +23,9 @@
 # importing anything else.
 import iris.tests as tests
 
+from contextlib import contextmanager
+import warnings
+
 import numpy as np
 
 import iris.fileformats.pp as pp
@@ -37,18 +40,22 @@
 # items when written to disk and get consistent results.
 
 
-DUMMY_HEADER = [('dummy1', (0, 13)),
+DUMMY_HEADER = [('dummy1', (0, 11)),
                 ('lbtim', (12,)),
+                ('dummy2', (13,)),
                 ('lblrec', (14,)),
-                ('dummy2', (15, 18)),
+                ('dummy3', (15, 16)),
                 ('lbrow', (17,)),
+                ('dummy4', (18,)),
                 ('lbext',  (19,)),
                 ('lbpack', (20,)),
-                ('dummy3', (21, 37)),
+                ('dummy5', (21, 37)),
                 ('lbuser', (38, 39, 40, 41, 42, 43, 44,)),
                 ('brsvd', (45, 46, 47, 48)),
                 ('bdatum', (49,)),
-                ('dummy4', (45, 63)),
+                ('dummy6', (50, 61)),
+                ('bmdi',   (62, )),
+                ('dummy7', (63,)),
                 ]
 
 
@@ -57,33 +64,40 @@ class TestPPField(PPField):
     HEADER_DEFN = DUMMY_HEADER
     HEADER_DICT = dict(DUMMY_HEADER)
 
+    def _ready_for_save(self):
+        self.dummy1 = 0
+        self.dummy2 = 0
+        self.dummy3 = 0
+        self.dummy4 = 0
+        self.dummy5 = 0
+        self.dummy6 = 0
+        self.dummy7 = 0
+        self.lbtim = 0
+        self.lblrec = 0
+        self.lbrow = 0
+        self.lbext = 0
+        self.lbpack = 0
+        self.lbuser = 0
+        self.brsvd = 0
+        self.bdatum = 0
+        self.bmdi = -1e30
+        return self
+
     @property
     def t1(self):
-        return netcdftime.datetime(2013, 10, 14, 10, 4)
+        return None
 
     @property
     def t2(self):
-        return netcdftime.datetime(2013, 10, 14, 10, 5)
+        return None
 
 
 class Test_save(tests.IrisTest):
     def test_float64(self):
         # Tests down-casting of >f8 data to >f4.
 
         def field_checksum(data):
-            field = TestPPField()
-            field.dummy1 = 0
-            field.dummy2 = 0
-            field.dummy3 = 0
-            field.dummy4 = 0
-            field.lbtim = 0
-            field.lblrec = 0
-            field.lbrow = 0
-            field.lbext = 0
-            field.lbpack = 0
-            field.lbuser = 0
-            field.brsvd = 0
-            field.bdatum = 0
+            field = TestPPField()._ready_for_save()
             field.data = data
             with self.temp_filename('.pp') as temp_filename:
                 with open(temp_filename, 'wb') as pp_file:
@@ -93,14 +107,49 @@ def field_checksum(data):
 
         data_64 = np.linspace(0, 1, num=10, endpoint=False).reshape(2, 5)
         checksum_32 = field_checksum(data_64.astype('>f4'))
-        with mock.patch('warnings.warn') as warn:
+        msg = 'Downcasting array precision from float64 to float32 for save.'
+        with self.assertGivesWarning(msg):
             checksum_64 = field_checksum(data_64.astype('>f8'))
-
         self.assertEqual(checksum_32, checksum_64)
-        warn.assert_called_once_with(
-            'Downcasting array precision from float64 to float32 for save.'
-            'If float64 precision is required then please save in a '
-            'different format')
+
+    def test_masked_mdi_value_warning(self):
+        # Check that an unmasked MDI value raises a warning.
+        field = TestPPField()._ready_for_save()
+        field.bmdi = -123.4
+        # Make float32 data, as float64 default produces an extra warning.
+        field.data = np.ma.masked_array([1., field.bmdi, 3.], dtype=np.float32)
+        msg = 'PPField data contains unmasked points'
+        with self.assertGivesWarning(msg):
+            with self.temp_filename('.pp') as temp_filename:
+                with open(temp_filename, 'wb') as pp_file:
+                    field.save(pp_file)
+
+    def test_unmasked_mdi_value_warning(self):
+        # Check that MDI in *unmasked* data raises a warning.
+        field = TestPPField()._ready_for_save()
+        field.bmdi = -123.4
+        # Make float32 data, as float64 default produces an extra warning.
+        field.data = np.array([1., field.bmdi, 3.], dtype=np.float32)
+        msg = 'PPField data contains unmasked points'
+        with self.assertGivesWarning(msg):
+            with self.temp_filename('.pp') as temp_filename:
+                with open(temp_filename, 'wb') as pp_file:
+                    field.save(pp_file)
+
+    def test_mdi_masked_value_nowarning(self):
+        # Check that a *masked* MDI value does not raise a warning.
+        field = TestPPField()._ready_for_save()
+        field.bmdi = -123.4
+        # Make float32 data, as float64 default produces an extra warning.
+        field.data = np.ma.masked_array([1., 2., 3.], mask=[0, 1, 0],
+                                        dtype=np.float32)
+        # Set underlying data value at masked point to BMDI value.
+        field.data.data[1] = field.bmdi
+        self.assertArrayAllClose(field.data.data[1], field.bmdi)
+        with self.assertGivesWarning(r'\(mask\|fill\)', expect_warning=False):
+            with self.temp_filename('.pp') as temp_filename:
+                with open(temp_filename, 'wb') as pp_file:
+                    field.save(pp_file)
 
 
 class Test_calendar(tests.IrisTest):