diff --git a/lib/iris/etc/pp_save_rules.txt b/lib/iris/etc/pp_save_rules.txt index 8971fe6625..f8b97d4af5 100644 --- a/lib/iris/etc/pp_save_rules.txt +++ b/lib/iris/etc/pp_save_rules.txt @@ -761,12 +761,12 @@ THEN #MDI IF - isinstance(cm.data, ma.core.MaskedArray) + cm.fill_value is not None THEN - pp.bmdi = cm.data.fill_value + pp.bmdi = cm.fill_value IF - not isinstance(cm.data, ma.core.MaskedArray) + cm.fill_value == None THEN pp.bmdi = -1e30 diff --git a/lib/iris/fileformats/pp.py b/lib/iris/fileformats/pp.py index ea921dca20..83ec3022fc 100644 --- a/lib/iris/fileformats/pp.py +++ b/lib/iris/fileformats/pp.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2010 - 2016, Met Office +# (C) British Crown Copyright 2010 - 2017, Met Office # # This file is part of Iris. # @@ -2322,6 +2322,18 @@ def as_pairs(cube, field_coords=None, target=None): target=target) +def _data_fill_value(cube): + # Function to deduce a fill_value for a cube's data. + # This should eventually be superceded by a cube 'fill_value' property. + if cube.has_lazy_data(): + fill_value = cube.lazy_data().fill_value + elif isinstance(cube.data, ma.MaskedArray): + fill_value = cube.data.fill_value + else: + fill_value = None + return fill_value + + def save_pairs_from_cube(cube, field_coords=None, target=None): """ Use the PP saving rules (and any user rules) to convert a cube or @@ -2404,6 +2416,11 @@ def save_pairs_from_cube(cube, field_coords=None, target=None): # Save each named or latlon slice2D in the cube for slice2D in cube.slices(field_coords): + # Attach an extra cube "fill_value" property, allowing the save rules + # to deduce MDI more easily without realising the data. + # NOTE: it is done this way because this property may exist in future. + slice2D.fill_value = _data_fill_value(slice2D) + # Start with a blank PPField pp_field = PPField3() @@ -2426,8 +2443,12 @@ def save_pairs_from_cube(cube, field_coords=None, target=None): # From UM doc F3: "Set to -99 if LBEGIN not known" pp_field.lbuser[1] = -99 - # Set the data - pp_field.data = slice2D.data + # Set the data, keeping it lazy where possible. + if slice2D.has_lazy_data(): + slice_core_data = slice2D.lazy_data() + else: + slice_core_data = slice2D.data + pp_field._data = slice_core_data # Run the PP save rules on the slice2D, to fill the PPField, # recording the rules that were used diff --git a/lib/iris/tests/integration/test_pp.py b/lib/iris/tests/integration/test_pp.py index 27c9d777d7..3c83202f50 100644 --- a/lib/iris/tests/integration/test_pp.py +++ b/lib/iris/tests/integration/test_pp.py @@ -38,6 +38,16 @@ import iris.util +def run_save_rules(cube, field): + # Add an extra "fill_value" property, as required by PP save rules, and + # now normally supplied by pp.save_pairs_from_cube. + cube.fill_value = iris.fileformats.pp._data_fill_value(cube) + # Ensure save rules are ready. + iris.fileformats.pp._ensure_save_rules_loaded() + # Run save rules to populate the field from the cube + iris.fileformats.pp._save_rules.verify(cube, field) + + class TestVertical(tests.IrisTest): def _test_coord(self, cube, point, bounds=None, **kwargs): coords = cube.coords(**kwargs) @@ -69,8 +79,7 @@ def test_soil_level_round_trip(self): field.lbvc = 0 field.brsvd = [None] * 4 field.brlev = None - iris.fileformats.pp._ensure_save_rules_loaded() - iris.fileformats.pp._save_rules.verify(cube, field) + run_save_rules(cube, field) # Check the vertical coordinate is as originally specified. self.assertEqual(field.lbvc, 6) @@ -103,8 +112,7 @@ def test_soil_depth_round_trip(self): field.lbvc = 0 field.brlev = None field.brsvd = [None] * 4 - iris.fileformats.pp._ensure_save_rules_loaded() - iris.fileformats.pp._save_rules.verify(cube, field) + run_save_rules(cube, field) # Check the vertical coordinate is as originally specified. self.assertEqual(field.lbvc, 6) @@ -132,8 +140,7 @@ def test_potential_temperature_level_round_trip(self): field = iris.fileformats.pp.PPField3() field.lbfc = 0 field.lbvc = 0 - iris.fileformats.pp._ensure_save_rules_loaded() - iris.fileformats.pp._save_rules.verify(cube, field) + run_save_rules(cube, field) # Check the vertical coordinate is as originally specified. self.assertEqual(field.lbvc, 19) @@ -197,15 +204,14 @@ def field_with_data(scale=1): pressure_field.lbvc = 0 pressure_field.brsvd = [None, None] pressure_field.lbuser = [None] * 7 - iris.fileformats.pp._ensure_save_rules_loaded() - iris.fileformats.pp._save_rules.verify(pressure_cube, pressure_field) + run_save_rules(pressure_cube, pressure_field) data_field = iris.fileformats.pp.PPField3() data_field.lbfc = 0 data_field.lbvc = 0 data_field.brsvd = [None, None] data_field.lbuser = [None] * 7 - iris.fileformats.pp._save_rules.verify(data_cube, data_field) + run_save_rules(data_cube, data_field) # The reference surface field should have STASH=409 self.assertArrayEqual(pressure_field.lbuser, @@ -285,8 +291,7 @@ def test_hybrid_height_with_non_standard_coords(self): field.lbvc = 0 field.brsvd = [None, None] field.lbuser = [None] * 7 - iris.fileformats.pp._ensure_save_rules_loaded() - iris.fileformats.pp._save_rules.verify(cube, field) + run_save_rules(cube, field) self.assertEqual(field.blev, delta) self.assertEqual(field.brlev, delta_lower) @@ -322,8 +327,7 @@ def test_hybrid_pressure_with_non_standard_coords(self): field.lbvc = 0 field.brsvd = [None, None] field.lbuser = [None] * 7 - iris.fileformats.pp._ensure_save_rules_loaded() - iris.fileformats.pp._save_rules.verify(cube, field) + run_save_rules(cube, field) self.assertEqual(field.bhlev, delta) self.assertEqual(field.bhrlev, delta_lower) @@ -384,8 +388,7 @@ def field_with_data(scale=1): data_field.lbvc = 0 data_field.brsvd = [None, None] data_field.lbuser = [None] * 7 - iris.fileformats.pp._ensure_save_rules_loaded() - iris.fileformats.pp._save_rules.verify(data_cube, data_field) + run_save_rules(data_cube, data_field) # Check the data field has the vertical coordinate as originally # specified. @@ -421,8 +424,7 @@ def convert_cube_to_field(self, cube): field.lbfc = 0 field.lbvc = 0 field.lbtim = 0 - iris.fileformats.pp._ensure_save_rules_loaded() - iris.fileformats.pp._save_rules.verify(cube, field) + run_save_rules(cube, field) return field def test_time_mean_from_forecast_period(self): @@ -608,8 +610,7 @@ def create_cube(self, longitude_coord='longitude'): def convert_cube_to_field(self, cube): field = iris.fileformats.pp.PPField3() field.lbvc = 0 - iris.fileformats.pp._ensure_save_rules_loaded() - iris.fileformats.pp._save_rules.verify(cube, field) + run_save_rules(cube, field) return field def test_time_mean_only(self): diff --git a/lib/iris/tests/results/cube_to_pp/append_single.txt b/lib/iris/tests/results/cube_to_pp/append_single.txt index 642d40cb2a..6aee209281 100644 --- a/lib/iris/tests/results/cube_to_pp/append_single.txt +++ b/lib/iris/tests/results/cube_to_pp/append_single.txt @@ -49,7 +49,7 @@ bdy: -2.5 bzx: -3.75 bdx: 3.75 - bmdi: -1e+30 + bmdi: 9999.0 bmks: 1.0 data: [[ 254.64399719 254.64399719 254.64399719 ..., 254.64399719 254.64399719 254.64399719] @@ -115,7 +115,7 @@ bdy: -2.5 bzx: -3.75 bdx: 3.75 - bmdi: -1e+30 + bmdi: 9999.0 bmks: 1.0 data: [[ 254.64399719 254.64399719 254.64399719 ..., 254.64399719 254.64399719 254.64399719] diff --git a/lib/iris/tests/results/cube_to_pp/replace_single.txt b/lib/iris/tests/results/cube_to_pp/replace_single.txt index c5fdb48515..afff9c6f41 100644 --- a/lib/iris/tests/results/cube_to_pp/replace_single.txt +++ b/lib/iris/tests/results/cube_to_pp/replace_single.txt @@ -49,7 +49,7 @@ bdy: -2.5 bzx: -3.75 bdx: 3.75 - bmdi: -1e+30 + bmdi: 9999.0 bmks: 1.0 data: [[ 254.64399719 254.64399719 254.64399719 ..., 254.64399719 254.64399719 254.64399719] diff --git a/lib/iris/tests/results/cube_to_pp/simple.txt b/lib/iris/tests/results/cube_to_pp/simple.txt index c5fdb48515..afff9c6f41 100644 --- a/lib/iris/tests/results/cube_to_pp/simple.txt +++ b/lib/iris/tests/results/cube_to_pp/simple.txt @@ -49,7 +49,7 @@ bdy: -2.5 bzx: -3.75 bdx: 3.75 - bmdi: -1e+30 + bmdi: 9999.0 bmks: 1.0 data: [[ 254.64399719 254.64399719 254.64399719 ..., 254.64399719 254.64399719 254.64399719] diff --git a/lib/iris/tests/results/cube_to_pp/user_rules.txt b/lib/iris/tests/results/cube_to_pp/user_rules.txt index 9cac1047e0..e6d241aa86 100644 --- a/lib/iris/tests/results/cube_to_pp/user_rules.txt +++ b/lib/iris/tests/results/cube_to_pp/user_rules.txt @@ -49,7 +49,7 @@ bdy: -2.5 bzx: -3.75 bdx: 3.75 - bmdi: -1e+30 + bmdi: 9999.0 bmks: 1.0 data: [[ 254.64399719 254.64399719 254.64399719 ..., 254.64399719 254.64399719 254.64399719] diff --git a/lib/iris/tests/test_cube_to_pp.py b/lib/iris/tests/test_cube_to_pp.py index dab7298570..7041d4216a 100644 --- a/lib/iris/tests/test_cube_to_pp.py +++ b/lib/iris/tests/test_cube_to_pp.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2010 - 2016, Met Office +# (C) British Crown Copyright 2010 - 2017, Met Office # # This file is part of Iris. # @@ -28,12 +28,10 @@ import cf_units import numpy as np -import iris import iris.coords import iris.coord_systems -import iris.fileformats.pp as ff_pp +import iris.fileformats.pp from iris.fileformats.pp import PPField3 -import iris.io from iris.tests import mock import iris.tests.pp as pp import iris.util @@ -207,12 +205,14 @@ def test_365_calendar_export(self): new_unit = cf_units.Unit('hours since 1970-01-01 00:00:00', calendar=cf_units.CALENDAR_365_DAY) cube.coord('time').units = new_unit + # Add an extra "fill_value" property, as used by the save rules. + cube.fill_value = None pp_field = mock.MagicMock(spec=PPField3) iris.fileformats.pp._ensure_save_rules_loaded() iris.fileformats.pp._save_rules.verify(cube, pp_field) self.assertEqual(pp_field.lbtim.ic, 4) - + class FakePPEnvironment(object): ''' fake a minimal PP environment for use in cross-section coords, as in PP save rules ''' y = [1, 2, 3, 4] @@ -340,7 +340,7 @@ def fields_from_cube(cubes): fh.seek(0) # load in the saved pp fields and check the appropriate metadata - for field in ff_pp.load(tmp_file.name): + for field in iris.fileformats.pp.load(tmp_file.name): yield field diff --git a/lib/iris/tests/unit/fileformats/pp/test_as_pairs.py b/lib/iris/tests/unit/fileformats/pp/test_as_pairs.py index 6f21c2f8b0..391338d043 100644 --- a/lib/iris/tests/unit/fileformats/pp/test_as_pairs.py +++ b/lib/iris/tests/unit/fileformats/pp/test_as_pairs.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2015, Met Office +# (C) British Crown Copyright 2015 - 2017, Met Office # # This file is part of Iris. # @@ -48,6 +48,15 @@ def test_field_coords(self): self.assertEqual(aslice.shape, (11, 9)) self.assertEqual(field.lbcode, 101) + def test_lazy_data(self): + cube = self.cube.copy() + # "Rebase" the cube onto a lazy version of its data. + cube.lazy_data(cube.lazy_data()) + # Check that lazy data is preserved in save-pairs generation. + slices_and_fields = pp.as_pairs(cube) + for aslice, _ in slices_and_fields: + self.assertTrue(aslice.has_lazy_data()) + if __name__ == "__main__": tests.main()