From ca5ed87601eeda09a4b400d142dee9d282d8bb12 Mon Sep 17 00:00:00 2001
From: lbdreyer <laura.dreyer@metoffice.gov.uk>
Date: Thu, 20 Feb 2020 12:24:41 +0000
Subject: [PATCH 1/2] Port tests from iris.tests.integration.test_grib2

---
 .../load_convert/test_data_section.py         |  73 +++++++++
 .../test_product_definition_section.py        |  74 +++++++++
 .../test_grid_definition_section.py           | 147 ++++++++++++++++++
 .../test_product_definition_section.py        |  81 ++++++++++
 .../grid_complex_spatial_differencing.cml     |  40 +++++
 ...d_complex_spatial_differencing.data.0.json |   1 +
 .../data_section/TestGDT30/lambert.cml        |  37 +++++
 .../TestGDT30/lambert.data.0.json             |   1 +
 .../data_section/TestGDT40/reduced.cml        |  46 ++++++
 .../TestGDT40/reduced.data.0.json             |   1 +
 .../data_section/TestGDT40/regular.cml        |  35 +++++
 .../TestGDT40/regular.data.0.json             |   1 +
 .../data_section/TestImport/gdt1.cml          |  36 +++++
 .../data_section/TestImport/gdt1.data.0.json  |   1 +
 .../TestImport/gdt90_with_bitmap.cml          |  40 +++++
 .../TestImport/gdt90_with_bitmap.data.0.json  |   1 +
 .../tests/unit/test_load_pairs_from_fields.py |  52 +++++++
 17 files changed, 667 insertions(+)
 create mode 100644 iris_grib/tests/integration/load_convert/test_data_section.py
 create mode 100644 iris_grib/tests/integration/load_convert/test_product_definition_section.py
 create mode 100644 iris_grib/tests/integration/round_trip/test_grid_definition_section.py
 create mode 100644 iris_grib/tests/integration/round_trip/test_product_definition_section.py
 create mode 100644 iris_grib/tests/results/integration/load_convert/data_section/TestDRT3/grid_complex_spatial_differencing.cml
 create mode 100644 iris_grib/tests/results/integration/load_convert/data_section/TestDRT3/grid_complex_spatial_differencing.data.0.json
 create mode 100644 iris_grib/tests/results/integration/load_convert/data_section/TestGDT30/lambert.cml
 create mode 100644 iris_grib/tests/results/integration/load_convert/data_section/TestGDT30/lambert.data.0.json
 create mode 100644 iris_grib/tests/results/integration/load_convert/data_section/TestGDT40/reduced.cml
 create mode 100644 iris_grib/tests/results/integration/load_convert/data_section/TestGDT40/reduced.data.0.json
 create mode 100644 iris_grib/tests/results/integration/load_convert/data_section/TestGDT40/regular.cml
 create mode 100644 iris_grib/tests/results/integration/load_convert/data_section/TestGDT40/regular.data.0.json
 create mode 100644 iris_grib/tests/results/integration/load_convert/data_section/TestImport/gdt1.cml
 create mode 100644 iris_grib/tests/results/integration/load_convert/data_section/TestImport/gdt1.data.0.json
 create mode 100644 iris_grib/tests/results/integration/load_convert/data_section/TestImport/gdt90_with_bitmap.cml
 create mode 100644 iris_grib/tests/results/integration/load_convert/data_section/TestImport/gdt90_with_bitmap.data.0.json
 create mode 100644 iris_grib/tests/unit/test_load_pairs_from_fields.py

diff --git a/iris_grib/tests/integration/load_convert/test_data_section.py b/iris_grib/tests/integration/load_convert/test_data_section.py
new file mode 100644
index 00000000..9aa8583e
--- /dev/null
+++ b/iris_grib/tests/integration/load_convert/test_data_section.py
@@ -0,0 +1,73 @@
+# Copyright iris-grib contributors
+#
+# This file is part of iris-grib and is released under the LGPL license.
+# See COPYING and COPYING.LESSER in the root of the repository for full
+# licensing details.
+"""
+Integration tests to confirm data is loaded correctly.
+
+"""
+
+# import iris_grib.tests first so that some things can be initialised
+# before importing anything else.
+import iris_grib.tests as tests
+
+import numpy.ma as ma
+
+from iris import load_cube
+
+
+class TestImport(tests.IrisGribTest):
+    def test_gdt1(self):
+        path = tests.get_data_path(
+            ("GRIB", "rotated_nae_t", "sensible_pole.grib2")
+        )
+        cube = load_cube(path)
+        self.assertCMLApproxData(cube)
+
+    def test_gdt90_with_bitmap(self):
+        path = tests.get_data_path(("GRIB", "umukv", "ukv_chan9.grib2"))
+        cube = load_cube(path)
+        # Pay particular attention to the orientation.
+        self.assertIsNot(cube.data[0, 0], ma.masked)
+        self.assertIs(cube.data[-1, 0], ma.masked)
+        self.assertIs(cube.data[0, -1], ma.masked)
+        self.assertIs(cube.data[-1, -1], ma.masked)
+        x = cube.coord("projection_x_coordinate").points
+        y = cube.coord("projection_y_coordinate").points
+        self.assertGreater(x[0], x[-1])  # Decreasing X coordinate
+        self.assertLess(y[0], y[-1])  # Increasing Y coordinate
+        # Check everything else.
+        self.assertCMLApproxData(cube)
+
+
+class TestGDT30(tests.IrisGribTest):
+    def test_lambert(self):
+        path = tests.get_data_path(("GRIB", "lambert", "lambert.grib2"))
+        cube = load_cube(path)
+        self.assertCMLApproxData(cube)
+
+
+class TestGDT40(tests.IrisGribTest):
+    def test_regular(self):
+        path = tests.get_data_path(("GRIB", "gaussian", "regular_gg.grib2"))
+        cube = load_cube(path)
+        self.assertCMLApproxData(cube)
+
+    def test_reduced(self):
+        path = tests.get_data_path(("GRIB", "reduced", "reduced_gg.grib2"))
+        cube = load_cube(path)
+        self.assertCMLApproxData(cube)
+
+
+class TestDRT3(tests.IrisGribTest):
+    def test_grid_complex_spatial_differencing(self):
+        path = tests.get_data_path(
+            ("GRIB", "missing_values", "missing_values.grib2")
+        )
+        cube = load_cube(path)
+        self.assertCMLApproxData(cube)
+
+
+if __name__ == '__main__':
+    tests.main()
diff --git a/iris_grib/tests/integration/load_convert/test_product_definition_section.py b/iris_grib/tests/integration/load_convert/test_product_definition_section.py
new file mode 100644
index 00000000..741a4d63
--- /dev/null
+++ b/iris_grib/tests/integration/load_convert/test_product_definition_section.py
@@ -0,0 +1,74 @@
+# Copyright iris-grib contributors
+#
+# This file is part of iris-grib and is released under the LGPL license.
+# See COPYING and COPYING.LESSER in the root of the repository for full
+# licensing details.
+"""
+Integration tests for loading various production definitions.
+
+"""
+
+# Import iris_grib.tests first so that some things can be initialised before
+# importing anything else.
+import iris_grib.tests as tests
+
+
+from iris import load_cube
+
+
+class TestPDT8(tests.IrisGribTest):
+    def setUp(self):
+        # Load from the test file.
+        file_path = tests.get_data_path(
+            ("GRIB", "time_processed", "time_bound.grib2")
+        )
+        self.cube = load_cube(file_path)
+
+    def test_coords(self):
+        # Check the result has main coordinates as expected.
+        for name, shape, is_bounded in [
+            ("forecast_reference_time", (1,), False),
+            ("time", (1,), True),
+            ("forecast_period", (1,), True),
+            ("pressure", (1,), False),
+            ("latitude", (73,), False),
+            ("longitude", (96,), False),
+        ]:
+            coords = self.cube.coords(name)
+            self.assertEqual(
+                len(coords),
+                1,
+                "expected one {!r} coord, found {}".format(name, len(coords)),
+            )
+            (coord,) = coords
+            self.assertEqual(
+                coord.shape,
+                shape,
+                "coord {!r} shape is {} instead of {!r}.".format(
+                    name, coord.shape, shape
+                ),
+            )
+            self.assertEqual(
+                coord.has_bounds(),
+                is_bounded,
+                "coord {!r} has_bounds={}, expected {}.".format(
+                    name, coord.has_bounds(), is_bounded
+                ),
+            )
+
+    def test_cell_method(self):
+        # Check the result has the expected cell method.
+        cell_methods = self.cube.cell_methods
+        self.assertEqual(
+            len(cell_methods),
+            1,
+            "result has {} cell methods, expected one.".format(
+                len(cell_methods)
+            ),
+        )
+        (cell_method,) = cell_methods
+        self.assertEqual(cell_method.coord_names, ("time",))
+
+
+if __name__ == "__main__":
+    tests.main()
diff --git a/iris_grib/tests/integration/round_trip/test_grid_definition_section.py b/iris_grib/tests/integration/round_trip/test_grid_definition_section.py
new file mode 100644
index 00000000..4a1040b1
--- /dev/null
+++ b/iris_grib/tests/integration/round_trip/test_grid_definition_section.py
@@ -0,0 +1,147 @@
+# Copyright iris-grib contributors
+#
+# This file is part of iris-grib and is released under the LGPL license.
+# See COPYING and COPYING.LESSER in the root of the repository for full
+# licensing details.
+"""
+Integration test for round-trip loading and saving of various grids.
+
+"""
+
+# import iris_grib.tests first so that some things can be initialised
+# before importing anything else.
+import iris_grib.tests as tests
+
+from iris import load_cube, save
+from iris.coord_systems import RotatedGeogCS
+from iris.fileformats.pp import EARTH_RADIUS as UM_DEFAULT_EARTH_RADIUS
+from iris.util import is_regular
+
+from iris.tests import TestGribMessage
+
+from iris_grib.grib_phenom_translation import GRIBCode
+
+
+class TestGDT5(TestGribMessage):
+    def test_save_load(self):
+        # Load sample UKV data (variable-resolution rotated grid).
+        path = tests.get_data_path(("PP", "ukV1", "ukVpmslont.pp"))
+        cube = load_cube(path)
+
+        # Extract a single 2D field, for simplicity.
+        self.assertEqual(cube.ndim, 3)
+        self.assertEqual(cube.coord_dims("time"), (0,))
+        cube = cube[0]
+
+        # FOR NOW: **also** fix the data so that it is square, i.e. nx=ny.
+        # This is needed because of a bug in the gribapi.
+        # See : https://software.ecmwf.int/issues/browse/SUP-1096
+        ny, nx = cube.shape
+        nn = min(nx, ny)
+        cube = cube[:nn, :nn]
+
+        # Check that it has a rotated-pole variable-spaced grid, as expected.
+        x_coord = cube.coord(axis="x")
+        self.assertIsInstance(x_coord.coord_system, RotatedGeogCS)
+        self.assertFalse(is_regular(x_coord))
+
+        # Write to temporary file, check that key contents are in the file,
+        # then load back in.
+        with self.temp_filename("ukv_sample.grib2") as temp_file_path:
+            save(cube, temp_file_path)
+
+            # Check that various aspects of the saved file are as expected.
+            expect_values = (
+                (0, "editionNumber", 2),
+                (3, "gridDefinitionTemplateNumber", 5),
+                (3, "Ni", cube.shape[-1]),
+                (3, "Nj", cube.shape[-2]),
+                (3, "shapeOfTheEarth", 1),
+                (
+                    3,
+                    "scaledValueOfRadiusOfSphericalEarth",
+                    int(UM_DEFAULT_EARTH_RADIUS),
+                ),
+                (3, "resolutionAndComponentFlags", 0),
+                (3, "latitudeOfSouthernPole", -37500000),
+                (3, "longitudeOfSouthernPole", 357500000),
+                (3, "angleOfRotation", 0),
+            )
+            self.assertGribMessageContents(temp_file_path, expect_values)
+
+            # Load the Grib file back into a new cube.
+            cube_loaded_from_saved = load_cube(temp_file_path)
+            # Also load data, before the temporary file gets deleted.
+            cube_loaded_from_saved.data
+
+        # The re-loaded result will not match the original in every respect:
+        #  * cube attributes are discarded
+        #  * horizontal coordinates are rounded to an integer representation
+        #  * bounds on horizontal coords are lost
+        # Thus the following "equivalence tests" are rather piecemeal..
+
+        # Check those re-loaded properties which should match the original.
+        for test_cube in (cube, cube_loaded_from_saved):
+            self.assertEqual(
+                test_cube.standard_name, "air_pressure_at_sea_level"
+            )
+            self.assertEqual(test_cube.units, "Pa")
+            self.assertEqual(test_cube.shape, (744, 744))
+            self.assertEqual(test_cube.cell_methods, ())
+
+        # Check only the GRIB_PARAM attribute exists on the re-loaded cube.
+        # Note: this does *not* match the original, but is as expected.
+        self.assertEqual(
+            cube_loaded_from_saved.attributes,
+            {"GRIB_PARAM": GRIBCode("GRIB2:d000c003n001")},
+        )
+
+        # Now remaining to check: coordinates + data...
+
+        # Check they have all the same coordinates.
+        co_names = [coord.name() for coord in cube.coords()]
+        co_names_reload = [
+            coord.name() for coord in cube_loaded_from_saved.coords()
+        ]
+        self.assertEqual(sorted(co_names_reload), sorted(co_names))
+
+        # Check all the coordinates.
+        for coord_name in co_names:
+            try:
+                co_orig = cube.coord(coord_name)
+                co_load = cube_loaded_from_saved.coord(coord_name)
+
+                # Check shape.
+                self.assertEqual(
+                    co_load.shape,
+                    co_orig.shape,
+                    'Shape of re-loaded "{}" coord is {} '
+                    "instead of {}".format(
+                        coord_name, co_load.shape, co_orig.shape
+                    ),
+                )
+
+                # Check coordinate points equal, within a tolerance.
+                self.assertArrayAllClose(
+                    co_load.points, co_orig.points, rtol=1.0e-6
+                )
+
+                # Check all coords are unbounded.
+                # (NOTE: this is not so for the original X and Y coordinates,
+                # but Grib does not store those bounds).
+                self.assertIsNone(co_load.bounds)
+
+            except AssertionError as err:
+                self.assertTrue(
+                    False,
+                    'Failed on coordinate "{}" : {}'.format(
+                        coord_name, str(err)
+                    ),
+                )
+
+        # Check that main data array also matches.
+        self.assertArrayAllClose(cube.data, cube_loaded_from_saved.data)
+
+
+if __name__ == "__main__":
+    tests.main()
diff --git a/iris_grib/tests/integration/round_trip/test_product_definition_section.py b/iris_grib/tests/integration/round_trip/test_product_definition_section.py
new file mode 100644
index 00000000..b0fed1cc
--- /dev/null
+++ b/iris_grib/tests/integration/round_trip/test_product_definition_section.py
@@ -0,0 +1,81 @@
+# Copyright iris-grib contributors
+#
+# This file is part of iris-grib and is released under the LGPL license.
+# See COPYING and COPYING.LESSER in the root of the repository for full
+# licensing details.
+"""
+Integration tests for round-trip loading and saving various product
+definitions.
+
+"""
+
+# Import iris_grib.tests first so that some things can be initialised before
+# importing anything else.
+import iris_grib.tests as tests
+
+from cf_units import Unit
+from iris import load_cube, save
+import iris.coords
+import iris.coord_systems
+
+from iris.tests import TestGribMessage
+import iris.tests.stock as stock
+
+from iris_grib.grib_phenom_translation import GRIBCode
+
+
+class TestPDT11(TestGribMessage):
+    def test_perturbation(self):
+        path = tests.get_data_path(
+            ("NetCDF", "global", "xyt", "SMALL_hires_wind_u_for_ipcc4.nc")
+        )
+        cube = load_cube(path)
+        # trim to 1 time and regular lats
+        cube = cube[0, 12:144, :]
+        crs = iris.coord_systems.GeogCS(6371229)
+        cube.coord("latitude").coord_system = crs
+        cube.coord("longitude").coord_system = crs
+        # add a realization coordinate
+        cube.add_aux_coord(
+            iris.coords.DimCoord(
+                points=1, standard_name="realization", units="1"
+            )
+        )
+        with self.temp_filename("testPDT11.GRIB2") as temp_file_path:
+            iris.save(cube, temp_file_path)
+
+            # Check that various aspects of the saved file are as expected.
+            expect_values = (
+                (0, "editionNumber", 2),
+                (3, "gridDefinitionTemplateNumber", 0),
+                (4, "productDefinitionTemplateNumber", 11),
+                (4, "perturbationNumber", 1),
+                (4, "typeOfStatisticalProcessing", 0),
+                (4, "numberOfForecastsInEnsemble", 255),
+            )
+            self.assertGribMessageContents(temp_file_path, expect_values)
+
+
+class TestPDT40(tests.IrisTest):
+    def test_save_load(self):
+        cube = stock.lat_lon_cube()
+        cube.rename("atmosphere_mole_content_of_ozone")
+        cube.units = Unit("Dobson")
+        tcoord = iris.coords.DimCoord(
+            23, "time", units=Unit("days since epoch", calendar="standard")
+        )
+        fpcoord = iris.coords.DimCoord(24, "forecast_period",
+                                       units=Unit("hours"))
+        cube.add_aux_coord(tcoord)
+        cube.add_aux_coord(fpcoord)
+        cube.attributes["WMO_constituent_type"] = 0
+        cube.attributes["GRIB_PARAM"] = GRIBCode("GRIB2:d000c014n000")
+
+        with self.temp_filename("test_grib_pdt40.grib2") as temp_file_path:
+            save(cube, temp_file_path)
+            loaded = load_cube(temp_file_path)
+            self.assertEqual(loaded.attributes, cube.attributes)
+
+
+if __name__ == "__main__":
+    tests.main()
diff --git a/iris_grib/tests/results/integration/load_convert/data_section/TestDRT3/grid_complex_spatial_differencing.cml b/iris_grib/tests/results/integration/load_convert/data_section/TestDRT3/grid_complex_spatial_differencing.cml
new file mode 100644
index 00000000..2cfe06f8
--- /dev/null
+++ b/iris_grib/tests/results/integration/load_convert/data_section/TestDRT3/grid_complex_spatial_differencing.cml
@@ -0,0 +1,40 @@
+<?xml version="1.0" ?>
+<cubes xmlns="urn:x-iris:cubeml-0.2">
+  <cube dtype="float64" standard_name="x_wind" units="m s-1">
+    <attributes>
+      <attribute name="GRIB_PARAM" value="GRIB2:d000c002n002"/>
+    </attributes>
+    <coords>
+      <coord>
+        <dimCoord id="1d45e087" points="[0]" shape="(1,)" standard_name="forecast_period" units="Unit('hours')" value_type="int64"/>
+      </coord>
+      <coord>
+        <dimCoord id="9c8bdf81" points="[380304.0]" shape="(1,)" standard_name="forecast_reference_time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
+      </coord>
+      <coord datadims="[0]">
+        <dimCoord id="77a50eb5" points="[90.0, 87.5, 85.0, 82.5, 80.0, 77.5, 75.0, 72.5,
+		70.0, 67.5, 65.0, 62.5, 60.0, 57.5, 55.0, 52.5,
+		50.0, 47.5, 45.0, 42.5, 40.0, 37.5, 35.0, 32.5,
+		30.0, 27.5, 25.0, 22.5, 20.0, 17.5, 15.0, 12.5,
+		10.0, 7.5, 5.0, 2.5, 0.0, -2.5, -5.0, -7.5,
+		-10.0, -12.5, -15.0, -17.5, -20.0, -22.5, -25.0,
+		-27.5, -30.0, -32.5, -35.0, -37.5, -40.0, -42.5,
+		-45.0, -47.5, -50.0, -52.5, -55.0, -57.5, -60.0,
+		-62.5, -65.0, -67.5, -70.0, -72.5, -75.0, -77.5,
+		-80.0, -82.5, -85.0, -87.5, -90.0]" shape="(73,)" standard_name="latitude" units="Unit('degrees')" value_type="float64">
+          <geogCS earth_radius="6371229.0"/>
+        </dimCoord>
+      </coord>
+      <coord datadims="[1]">
+        <dimCoord circular="True" id="f913a8b3" points="[0.0, 2.5, 5.0, ..., 352.5, 355.0, 357.5]" shape="(144,)" standard_name="longitude" units="Unit('degrees')" value_type="float64">
+          <geogCS earth_radius="6371229.0"/>
+        </dimCoord>
+      </coord>
+      <coord>
+        <dimCoord id="cb784457" points="[380304.0]" shape="(1,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
+      </coord>
+    </coords>
+    <cellMethods/>
+    <data dtype="float64" shape="(73, 144)" state="loaded"/>
+  </cube>
+</cubes>
diff --git a/iris_grib/tests/results/integration/load_convert/data_section/TestDRT3/grid_complex_spatial_differencing.data.0.json b/iris_grib/tests/results/integration/load_convert/data_section/TestDRT3/grid_complex_spatial_differencing.data.0.json
new file mode 100644
index 00000000..f42d355d
--- /dev/null
+++ b/iris_grib/tests/results/integration/load_convert/data_section/TestDRT3/grid_complex_spatial_differencing.data.0.json
@@ -0,0 +1 @@
+{"std": 7.798695691713748, "min": -34.43, "max": 33.009999999999998, "shape": [73, 144], "masked": true, "mean": 2.3147813807531383}
\ No newline at end of file
diff --git a/iris_grib/tests/results/integration/load_convert/data_section/TestGDT30/lambert.cml b/iris_grib/tests/results/integration/load_convert/data_section/TestGDT30/lambert.cml
new file mode 100644
index 00000000..215a0de8
--- /dev/null
+++ b/iris_grib/tests/results/integration/load_convert/data_section/TestGDT30/lambert.cml
@@ -0,0 +1,37 @@
+<?xml version="1.0" ?>
+<cubes xmlns="urn:x-iris:cubeml-0.2">
+  <cube dtype="float64" standard_name="air_temperature" units="K">
+    <attributes>
+      <attribute name="GRIB_PARAM" value="GRIB2:d000c000n000"/>
+    </attributes>
+    <coords>
+      <coord>
+        <dimCoord id="1d45e087" points="[2]" shape="(1,)" standard_name="forecast_period" units="Unit('hours')" value_type="int64"/>
+      </coord>
+      <coord>
+        <dimCoord id="9c8bdf81" points="[379980.0]" shape="(1,)" standard_name="forecast_reference_time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
+      </coord>
+      <coord>
+        <dimCoord id="ecc4ad5a" long_name="height" points="[2.0]" shape="(1,)" units="Unit('m')" value_type="float64"/>
+      </coord>
+      <coord datadims="[1]">
+        <dimCoord id="0f43c07d" points="[-2396487.0124, -2392487.0124, -2388487.0124,
+		..., 2387512.9876, 2391512.9876, 2395512.9876]" shape="(1199,)" standard_name="projection_x_coordinate" units="Unit('m')" value_type="float64">
+          <lambertConformal central_lat="60.0" central_lon="262.0" ellipsoid="GeogCS(6371229.0)" false_easting="0" false_northing="0" secant_latitudes="(60.0, 30.0)"/>
+        </dimCoord>
+      </coord>
+      <coord datadims="[0]">
+        <dimCoord id="1e5b5e41" points="[-3870311.24926, -3866311.24926, -3862311.24926,
+		..., -686311.249256, -682311.249256,
+		-678311.249256]" shape="(799,)" standard_name="projection_y_coordinate" units="Unit('m')" value_type="float64">
+          <lambertConformal central_lat="60.0" central_lon="262.0" ellipsoid="GeogCS(6371229.0)" false_easting="0" false_northing="0" secant_latitudes="(60.0, 30.0)"/>
+        </dimCoord>
+      </coord>
+      <coord>
+        <dimCoord id="cb784457" points="[379982.0]" shape="(1,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
+      </coord>
+    </coords>
+    <cellMethods/>
+    <data dtype="float64" shape="(799, 1199)" state="loaded"/>
+  </cube>
+</cubes>
diff --git a/iris_grib/tests/results/integration/load_convert/data_section/TestGDT30/lambert.data.0.json b/iris_grib/tests/results/integration/load_convert/data_section/TestGDT30/lambert.data.0.json
new file mode 100644
index 00000000..99d58c5e
--- /dev/null
+++ b/iris_grib/tests/results/integration/load_convert/data_section/TestGDT30/lambert.data.0.json
@@ -0,0 +1 @@
+{"std": 5.3916288115779398, "min": 265.550048828125, "max": 300.862548828125, "shape": [799, 1199], "masked": false, "mean": 287.6306666328037}
\ No newline at end of file
diff --git a/iris_grib/tests/results/integration/load_convert/data_section/TestGDT40/reduced.cml b/iris_grib/tests/results/integration/load_convert/data_section/TestGDT40/reduced.cml
new file mode 100644
index 00000000..3a963b32
--- /dev/null
+++ b/iris_grib/tests/results/integration/load_convert/data_section/TestGDT40/reduced.cml
@@ -0,0 +1,46 @@
+<?xml version="1.0" ?>
+<cubes xmlns="urn:x-iris:cubeml-0.2">
+  <cube dtype="float64" standard_name="geopotential" units="m2 s-2">
+    <attributes>
+      <attribute name="GRIB_PARAM" value="GRIB2:d000c003n004"/>
+      <attribute name="centre" value="European Centre for Medium Range Weather Forecasts"/>
+    </attributes>
+    <coords>
+      <coord>
+        <dimCoord id="1d45e087" points="[0]" shape="(1,)" standard_name="forecast_period" units="Unit('hours')" value_type="int64"/>
+      </coord>
+      <coord>
+        <dimCoord id="9c8bdf81" points="[377688.0]" shape="(1,)" standard_name="forecast_reference_time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
+      </coord>
+      <coord datadims="[0]">
+        <auxCoord id="77a50eb5" points="[88.572168514, 88.572168514, 88.572168514, ...,
+		-88.572168514, -88.572168514, -88.572168514]" shape="(13280,)" standard_name="latitude" units="Unit('degrees')" value_type="float64">
+          <geogCS earth_radius="6371229.0"/>
+        </auxCoord>
+      </coord>
+      <coord>
+        <dimCoord id="c9091dc6" long_name="level_pressure" points="[2.00004005432]" shape="(1,)" units="Unit('Pa')" value_type="float64"/>
+      </coord>
+      <coord datadims="[0]">
+        <auxCoord id="f913a8b3" points="[0.0, 18.0, 36.0, ..., 306.0, 324.0, 342.0]" shape="(13280,)" standard_name="longitude" units="Unit('degrees')" value_type="float64">
+          <geogCS earth_radius="6371229.0"/>
+        </auxCoord>
+      </coord>
+      <coord>
+        <dimCoord id="38fd0305" points="[1]" shape="(1,)" standard_name="model_level_number" units="Unit('1')" value_type="int64">
+          <attributes>
+            <attribute name="positive" value="up"/>
+          </attributes>
+        </dimCoord>
+      </coord>
+      <coord>
+        <dimCoord id="a5c170db" long_name="sigma" points="[0.0]" shape="(1,)" units="Unit('1')" value_type="float64"/>
+      </coord>
+      <coord>
+        <dimCoord id="cb784457" points="[377688.0]" shape="(1,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
+      </coord>
+    </coords>
+    <cellMethods/>
+    <data dtype="float64" shape="(13280,)" state="loaded"/>
+  </cube>
+</cubes>
diff --git a/iris_grib/tests/results/integration/load_convert/data_section/TestGDT40/reduced.data.0.json b/iris_grib/tests/results/integration/load_convert/data_section/TestGDT40/reduced.data.0.json
new file mode 100644
index 00000000..4cd44b53
--- /dev/null
+++ b/iris_grib/tests/results/integration/load_convert/data_section/TestGDT40/reduced.data.0.json
@@ -0,0 +1 @@
+{"std": 6295.5250434859099, "min": -6419.0146484375, "max": 55403.9853515625, "shape": [13280], "masked": false, "mean": 2446.3044780685241}
\ No newline at end of file
diff --git a/iris_grib/tests/results/integration/load_convert/data_section/TestGDT40/regular.cml b/iris_grib/tests/results/integration/load_convert/data_section/TestGDT40/regular.cml
new file mode 100644
index 00000000..e5eea0fc
--- /dev/null
+++ b/iris_grib/tests/results/integration/load_convert/data_section/TestGDT40/regular.cml
@@ -0,0 +1,35 @@
+<?xml version="1.0" ?>
+<cubes xmlns="urn:x-iris:cubeml-0.2">
+  <cube dtype="float64" standard_name="geopotential_height" units="m">
+    <attributes>
+      <attribute name="GRIB_PARAM" value="GRIB2:d000c003n005"/>
+    </attributes>
+    <coords>
+      <coord>
+        <dimCoord id="1d45e087" points="[69]" shape="(1,)" standard_name="forecast_period" units="Unit('hours')" value_type="int64"/>
+      </coord>
+      <coord>
+        <dimCoord id="9c8bdf81" points="[382320.0]" shape="(1,)" standard_name="forecast_reference_time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
+      </coord>
+      <coord datadims="[0]">
+        <dimCoord id="77a50eb5" points="[88.572168514, 86.7225309547, 84.861970292, ...,
+		-84.861970292, -86.7225309547, -88.572168514]" shape="(96,)" standard_name="latitude" units="Unit('degrees')" value_type="float64">
+          <geogCS earth_radius="6371229.0"/>
+        </dimCoord>
+      </coord>
+      <coord datadims="[1]">
+        <dimCoord circular="True" id="f913a8b3" points="[0.0, 1.875, 3.75, ..., 354.375, 356.25, 358.125]" shape="(192,)" standard_name="longitude" units="Unit('degrees')" value_type="float64">
+          <geogCS earth_radius="6371229.0"/>
+        </dimCoord>
+      </coord>
+      <coord>
+        <dimCoord id="302bf438" long_name="pressure" points="[52500.0]" shape="(1,)" units="Unit('Pa')" value_type="float64"/>
+      </coord>
+      <coord>
+        <dimCoord id="cb784457" points="[382389.0]" shape="(1,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
+      </coord>
+    </coords>
+    <cellMethods/>
+    <data dtype="float64" shape="(96, 192)" state="loaded"/>
+  </cube>
+</cubes>
diff --git a/iris_grib/tests/results/integration/load_convert/data_section/TestGDT40/regular.data.0.json b/iris_grib/tests/results/integration/load_convert/data_section/TestGDT40/regular.data.0.json
new file mode 100644
index 00000000..9595f90e
--- /dev/null
+++ b/iris_grib/tests/results/integration/load_convert/data_section/TestGDT40/regular.data.0.json
@@ -0,0 +1 @@
+{"std": 290.49181302067751, "min": 4388.16162109375, "max": 5576.53662109375, "shape": [96, 192], "masked": false, "mean": 5210.564598931207}
\ No newline at end of file
diff --git a/iris_grib/tests/results/integration/load_convert/data_section/TestImport/gdt1.cml b/iris_grib/tests/results/integration/load_convert/data_section/TestImport/gdt1.cml
new file mode 100644
index 00000000..d304d8a8
--- /dev/null
+++ b/iris_grib/tests/results/integration/load_convert/data_section/TestImport/gdt1.cml
@@ -0,0 +1,36 @@
+<?xml version="1.0" ?>
+<cubes xmlns="urn:x-iris:cubeml-0.2">
+  <cube dtype="float64" standard_name="air_temperature" units="K">
+    <attributes>
+      <attribute name="GRIB_PARAM" value="GRIB2:d000c000n000"/>
+    </attributes>
+    <coords>
+      <coord>
+        <dimCoord id="1d45e087" points="[0]" shape="(1,)" standard_name="forecast_period" units="Unit('hours')" value_type="int64"/>
+      </coord>
+      <coord>
+        <dimCoord id="9c8bdf81" points="[344904.0]" shape="(1,)" standard_name="forecast_reference_time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
+      </coord>
+      <coord datadims="[0]">
+        <dimCoord id="f971e3af" points="[-20.069997, -19.959999, -19.850001, ...,
+		19.199289, 19.309287, 19.419285]" shape="(360,)" standard_name="grid_latitude" units="Unit('degrees')" value_type="float64">
+          <rotatedGeogCS ellipsoid="GeogCS(6367470.0)" grid_north_pole_latitude="37.5" grid_north_pole_longitude="152.4669799999999"/>
+        </dimCoord>
+      </coord>
+      <coord datadims="[1]">
+        <dimCoord id="e5e4f895" points="[326.219965, 326.329959, 326.439953, ...,
+		391.886383, 391.996377, 392.106371]" shape="(600,)" standard_name="grid_longitude" units="Unit('degrees')" value_type="float64">
+          <rotatedGeogCS ellipsoid="GeogCS(6367470.0)" grid_north_pole_latitude="37.5" grid_north_pole_longitude="152.4669799999999"/>
+        </dimCoord>
+      </coord>
+      <coord>
+        <dimCoord id="302bf438" long_name="pressure" points="[100000.0]" shape="(1,)" units="Unit('Pa')" value_type="float64"/>
+      </coord>
+      <coord>
+        <dimCoord id="cb784457" points="[344904.0]" shape="(1,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
+      </coord>
+    </coords>
+    <cellMethods/>
+    <data dtype="float64" shape="(360, 600)" state="loaded"/>
+  </cube>
+</cubes>
diff --git a/iris_grib/tests/results/integration/load_convert/data_section/TestImport/gdt1.data.0.json b/iris_grib/tests/results/integration/load_convert/data_section/TestImport/gdt1.data.0.json
new file mode 100644
index 00000000..2b416f6a
--- /dev/null
+++ b/iris_grib/tests/results/integration/load_convert/data_section/TestImport/gdt1.data.0.json
@@ -0,0 +1 @@
+{"std": 6.1629553729365423, "min": 266.625, "max": 302.25, "shape": [360, 600], "masked": false, "mean": 284.43164236111113}
\ No newline at end of file
diff --git a/iris_grib/tests/results/integration/load_convert/data_section/TestImport/gdt90_with_bitmap.cml b/iris_grib/tests/results/integration/load_convert/data_section/TestImport/gdt90_with_bitmap.cml
new file mode 100644
index 00000000..3118f868
--- /dev/null
+++ b/iris_grib/tests/results/integration/load_convert/data_section/TestImport/gdt90_with_bitmap.cml
@@ -0,0 +1,40 @@
+<?xml version="1.0" ?>
+<cubes xmlns="urn:x-iris:cubeml-0.2">
+  <cube dtype="float64" units="unknown">
+    <attributes>
+      <attribute name="GRIB_PARAM" value="GRIB2:d003c000n002"/>
+    </attributes>
+    <coords>
+      <coord>
+        <dimCoord id="87abc70d" long_name="instrument_type" points="[207]" shape="(1,)" units="Unit('1')" value_type="int64"/>
+      </coord>
+      <coord datadims="[1]">
+        <dimCoord id="31b8c928" points="[369081.561454, 366080.898353, 363080.235252,
+		..., -792175.058731, -795175.721833,
+		-798176.384934]" shape="(390,)" standard_name="projection_x_coordinate" units="Unit('m')" value_type="float64">
+          <verticalPerspective ellipsoid="GeogCS(semi_major_axis=6378168.800000001, semi_minor_axis=6356584.0)" false_easting="0.0" false_northing="0.0" latitude_of_projection_origin="0.0" longitude_of_projection_origin="0.0" perspective_point_height="35785825.85377121"/>
+        </dimCoord>
+      </coord>
+      <coord datadims="[0]">
+        <dimCoord id="1f763ccc" points="[4392884.59202, 4395885.19625, 4398885.80048,
+		..., 5065019.93943, 5068020.54366, 5071021.14789]" shape="(227,)" standard_name="projection_y_coordinate" units="Unit('m')" value_type="float64">
+          <verticalPerspective ellipsoid="GeogCS(semi_major_axis=6378168.800000001, semi_minor_axis=6356584.0)" false_easting="0.0" false_northing="0.0" latitude_of_projection_origin="0.0" longitude_of_projection_origin="0.0" perspective_point_height="35785825.85377121"/>
+        </dimCoord>
+      </coord>
+      <coord>
+        <dimCoord id="527908e7" long_name="satellite_number" points="[56]" shape="(1,)" units="Unit('1')" value_type="int64"/>
+      </coord>
+      <coord>
+        <dimCoord id="7de00380" long_name="satellite_series" points="[333]" shape="(1,)" units="Unit('1')" value_type="int64"/>
+      </coord>
+      <coord>
+        <dimCoord id="f7861546" points="[92.0]" shape="(1,)" standard_name="sensor_band_central_radiation_wavenumber" units="Unit('m-1')" value_type="float64"/>
+      </coord>
+      <coord>
+        <dimCoord id="cb784457" points="[356661.0]" shape="(1,)" standard_name="time" units="Unit('hours since 1970-01-01 00:00:00', calendar='gregorian')" value_type="float64"/>
+      </coord>
+    </coords>
+    <cellMethods/>
+    <data dtype="float64" shape="(227, 390)" state="loaded"/>
+  </cube>
+</cubes>
diff --git a/iris_grib/tests/results/integration/load_convert/data_section/TestImport/gdt90_with_bitmap.data.0.json b/iris_grib/tests/results/integration/load_convert/data_section/TestImport/gdt90_with_bitmap.data.0.json
new file mode 100644
index 00000000..08c3cb82
--- /dev/null
+++ b/iris_grib/tests/results/integration/load_convert/data_section/TestImport/gdt90_with_bitmap.data.0.json
@@ -0,0 +1 @@
+{"std": 13.780125280995835, "min": 208.90541992187502, "max": 287.25541992187499, "shape": [227, 390], "masked": true, "mean": 266.3984425053925}
\ No newline at end of file
diff --git a/iris_grib/tests/unit/test_load_pairs_from_fields.py b/iris_grib/tests/unit/test_load_pairs_from_fields.py
new file mode 100644
index 00000000..9f429154
--- /dev/null
+++ b/iris_grib/tests/unit/test_load_pairs_from_fields.py
@@ -0,0 +1,52 @@
+# Copyright iris-grib contributors
+#
+# This file is part of iris-grib and is released under the LGPL license.
+# See COPYING and COPYING.LESSER in the root of the repository for full
+# licensing details.
+"""Unit tests for the `iris_grib.load_pairs_from_fields` function."""
+
+import iris_grib.tests as tests
+
+from iris_grib import load_pairs_from_fields
+from iris_grib.message import GribMessage
+
+
+class TestAsCubes(tests.IrisTest):
+    def setUp(self):
+        # Load from the test file.
+        self.file_path = tests.get_data_path(
+            ("GRIB", "time_processed", "time_bound.grib2")
+        )
+
+    def test_year_filter(self):
+        msgs = GribMessage.messages_from_filename(self.file_path)
+        chosen_messages = []
+        for gmsg in msgs:
+            if gmsg.sections[1]["year"] == 1998:
+                chosen_messages.append(gmsg)
+        cubes_msgs = list(load_pairs_from_fields(chosen_messages))
+        self.assertEqual(len(cubes_msgs), 1)
+
+    def test_year_filter_none(self):
+        msgs = GribMessage.messages_from_filename(self.file_path)
+        chosen_messages = []
+        for gmsg in msgs:
+            if gmsg.sections[1]["year"] == 1958:
+                chosen_messages.append(gmsg)
+        cubes_msgs = list(load_pairs_from_fields(chosen_messages))
+        self.assertEqual(len(cubes_msgs), 0)
+
+    def test_as_pairs(self):
+        messages = GribMessage.messages_from_filename(self.file_path)
+        cubes = []
+        cube_msg_pairs = load_pairs_from_fields(messages)
+        for cube, gmsg in cube_msg_pairs:
+            if gmsg.sections[1]["year"] == 1998:
+                cube.attributes["the year is"] = gmsg.sections[1]["year"]
+                cubes.append(cube)
+        self.assertEqual(len(cubes), 1)
+        self.assertEqual(cubes[0].attributes["the year is"], 1998)
+
+
+if __name__ == "__main__":
+    tests.main()

From 9f14bdeef36b31dda3f84fc6fd6b3c8eb5f4f626 Mon Sep 17 00:00:00 2001
From: lbdreyer <laura.dreyer@metoffice.gov.uk>
Date: Thu, 20 Feb 2020 18:11:38 +0000
Subject: [PATCH 2/2] Remove squaring off of cube's data

---
 .../round_trip/test_grid_definition_section.py           | 9 +--------
 1 file changed, 1 insertion(+), 8 deletions(-)

diff --git a/iris_grib/tests/integration/round_trip/test_grid_definition_section.py b/iris_grib/tests/integration/round_trip/test_grid_definition_section.py
index 4a1040b1..a7bcc122 100644
--- a/iris_grib/tests/integration/round_trip/test_grid_definition_section.py
+++ b/iris_grib/tests/integration/round_trip/test_grid_definition_section.py
@@ -33,13 +33,6 @@ def test_save_load(self):
         self.assertEqual(cube.coord_dims("time"), (0,))
         cube = cube[0]
 
-        # FOR NOW: **also** fix the data so that it is square, i.e. nx=ny.
-        # This is needed because of a bug in the gribapi.
-        # See : https://software.ecmwf.int/issues/browse/SUP-1096
-        ny, nx = cube.shape
-        nn = min(nx, ny)
-        cube = cube[:nn, :nn]
-
         # Check that it has a rotated-pole variable-spaced grid, as expected.
         x_coord = cube.coord(axis="x")
         self.assertIsInstance(x_coord.coord_system, RotatedGeogCS)
@@ -86,7 +79,7 @@ def test_save_load(self):
                 test_cube.standard_name, "air_pressure_at_sea_level"
             )
             self.assertEqual(test_cube.units, "Pa")
-            self.assertEqual(test_cube.shape, (744, 744))
+            self.assertEqual(test_cube.shape, (928, 744))
             self.assertEqual(test_cube.cell_methods, ())
 
         # Check only the GRIB_PARAM attribute exists on the re-loaded cube.