Eccodesmissing

.travis.yml

@@ -59,7 +59,7 @@ install:
       conda install --quiet --file minimal-conda-requirements.txt;
     else
       if [[ "$TRAVIS_PYTHON_VERSION" == 3* ]]; then
-        sed -e '/ecmwf_grib/d' -e '/esmpy/d' -e 's/#.\+$//' conda-requirements.txt | xargs conda install --quiet;
+        sed -e '/esmpy/d' -e 's/#.\+$//' conda-requirements.txt | xargs conda install --quiet;
       else
         conda install --quiet --file conda-requirements.txt;
       fi

README.md

@@ -4,6 +4,7 @@ Iris
 [![Join the chat at https://gitter.im/SciTools/iris](https://badges.gitter.im/SciTools/iris.svg)](https://gitter.im/SciTools/iris?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 [![Build Status](https://api.travis-ci.org/repositories/SciTools/iris.svg?branch=master)](https://travis-ci.org/SciTools/iris/branches)
 [![DOI](https://zenodo.org/badge/doi/10.5281/zenodo.51860.svg)](https://dx.doi.org/10.5281/zenodo.51860)
+[![Documentation for master branch ](https://img.shields.io/badge/docs-master-blue.svg)](https://scitools-docs.github.io/iris/master/index.html)
 
 (C) British Crown Copyright 2010 - 2017, Met Office
 

conda-requirements.txt

@@ -25,7 +25,7 @@ imagehash
 requests
 
 # Optional iris dependencies
-python-ecmwf_grib
+python-eccodes
 esmpy>=7.0
 gdal
 libmo_unpack

lib/iris/fileformats/grib/_load_convert.py

@@ -158,7 +158,9 @@ def _unscale(v, f):
 
     if isinstance(value, Iterable) or isinstance(factor, Iterable):
         def _masker(item):
-            result = ma.masked_equal(item, _MDI)
+            result = item
+            for _MDI in _MDIs:
+                result = ma.masked_equal(result, _MDI)
             if ma.count_masked(result):
                 # Circumvent downstream NumPy "RuntimeWarning"
                 # of "overflow encountered in power" in _unscale
@@ -172,24 +174,24 @@ def _masker(item):
             result = result.data
     else:
         result = ma.masked
-        if value != _MDI and factor != _MDI:
+        if value not in _MDIs and factor not in _MDIs:
             result = _unscale(value, factor)
     return result
 
 
 # Regulations 92.1.4 and 92.1.5.
-_MDI = 2 ** 32 - 1
+_MDIs = [2 ** 32 - 1, 2 ** 32 - 1]
 # Note:
 #   1. Integer "on-disk" values (aka. coded keys) in GRIB messages:
 #       - Are 8-, 16-, or 32-bit.
 #       - Are either signed or unsigned, with signed values stored as
 #         sign-and-magnitude (*not* twos-complement).
 #       - Use all bits set to indicate a missing value (MDI).
-#   2. Irrespective of the on-disk form, the ECMWF GRIB API *always*:
+#   2. Irrespective of the on-disk form, ECCodes: the ECMWF GRIB API *always*:
 #       - Returns values as 64-bit signed integers, either as native
 #         Python 'int' or numpy 'int64'.
-#       - Returns missing values as 2**32 - 1, but not all keys are
-#         defined as supporting missing values.
+#       - Returns missing values as 2**32 - 1 or 2**31 - 1, but not all keys
+#         are defined as supporting missing values.
 #   NB. For keys which support missing values, the MDI value is reliably
 #   distinct from the valid range of either signed or unsigned 8-, 16-,
 #   or 32-bit values. For example:
@@ -634,10 +636,10 @@ def grid_definition_template_4_and_5(section, metadata, y_name, x_name, cs):
     basicAngleOfTheInitialProductDomain = section[key]
     subdivisionsOfBasicAngle = section['subdivisionsOfBasicAngle']
 
-    if basicAngleOfTheInitialProductDomain in [0, _MDI]:
+    if basicAngleOfTheInitialProductDomain in [0] + _MDIs:
         basicAngleOfTheInitialProductDomain = 1.
 
-    if subdivisionsOfBasicAngle in [0, _MDI]:
+    if subdivisionsOfBasicAngle in [0] + _MDIs:
         subdivisionsOfBasicAngle = 1. / _GRID_ACCURACY_IN_DEGREES
 
     resolution = np.float64(basicAngleOfTheInitialProductDomain)
@@ -1078,7 +1080,7 @@ def grid_definition_template_90(section, metadata):
         :class:`collections.OrderedDict` of metadata.
 
     """
-    if section['Nr'] == _MDI:
+    if section['Nr'] in _MDIs:
         raise TranslationError('Unsupported orthographic grid.')
     elif section['Nr'] == 0:
         raise TranslationError('Unsupported zero height for space-view.')
@@ -1403,7 +1405,7 @@ def vertical_coords(section, metadata):
 
         if fixed_surface is None:
             if typeOfFirstFixedSurface != _TYPE_OF_FIXED_SURFACE_MISSING:
-                if scaledValueOfFirstFixedSurface == _MDI:
+                if scaledValueOfFirstFixedSurface in _MDIs:
                     if options.warn_on_unsupported:
                         msg = 'Unable to translate type of first fixed ' \
                             'surface with missing scaled value.'
@@ -1427,7 +1429,7 @@ def vertical_coords(section, metadata):
                 key = 'scaledValueOfSecondFixedSurface'
                 scaledValueOfSecondFixedSurface = section[key]
 
-                if scaledValueOfSecondFixedSurface == _MDI:
+                if scaledValueOfSecondFixedSurface in _MDIs:
                     msg = 'Product definition section 4 has missing ' \
                         'scaled value of second fixed surface'
                     raise TranslationError(msg)
@@ -1686,8 +1688,8 @@ def data_cutoff(hoursAfterDataCutoff, minutesAfterDataCutoff):
         Message section 4, octet 17.
 
     """
-    if (hoursAfterDataCutoff != _MDI or
-            minutesAfterDataCutoff != _MDI):
+    if (hoursAfterDataCutoff not in _MDIs or
+            minutesAfterDataCutoff not in _MDIs):
         if options.warn_on_unsupported:
             warnings.warn('Unable to translate "hours and/or minutes '
                           'after data cutoff".')
@@ -1937,12 +1939,12 @@ def product_definition_template_9(section, metadata, frt_coord):
     if probability_typecode == 1:
         # Type is "above upper level".
         threshold_value = section['scaledValueOfUpperLimit']
-        if threshold_value == _MDI:
+        if threshold_value in _MDIs:
             msg = 'Product definition section 4 has missing ' \
                 'scaled value of upper limit'
             raise TranslationError(msg)
         threshold_scaling = section['scaleFactorOfUpperLimit']
-        if threshold_scaling == _MDI:
+        if threshold_scaling in _MDIs:
             msg = 'Product definition section 4 has missing ' \
                 'scale factor of upper limit'
             raise TranslationError(msg)

lib/iris/fileformats/grib/_save_rules.py

@@ -257,8 +257,18 @@ def latlon_first_last(x_coord, y_coord, grib):
 def dx_dy(x_coord, y_coord, grib):
     x_step = regular_step(x_coord)
     y_step = regular_step(y_coord)
-    gribapi.grib_set(grib, "DxInDegrees", float(abs(x_step)))
-    gribapi.grib_set(grib, "DyInDegrees", float(abs(y_step)))
+    if x_coord.units == 'degrees':
+        gribapi.grib_set(grib, "iDirectionIncrement",
+                         round(1e6 * float(abs(x_step))))
+    else:
+        raise ValueError('X coordinate must be in degrees, not {}'
+                         '.'.format(x_coord.units))
+    if y_coord.units == 'degrees':
+        gribapi.grib_set(grib, "jDirectionIncrement",
+                         round(1e6 * float(abs(y_step))))
+    else:
+        raise ValueError('Y coordinate must be in degrees, not {}'
+                         '.'.format(y_coord.units))
 
 
 def scanning_mode_flags(x_coord, y_coord, grib):

lib/iris/tests/integration/format_interop/test_name_grib.py

@@ -1,4 +1,4 @@
-# (C) British Crown Copyright 2013 - 2016, Met Office
+# (C) British Crown Copyright 2013 - 2017, Met Office
 #
 # This file is part of Iris.
 #
@@ -72,21 +72,18 @@ def check_common(self, name_cube, grib_cube):
     def test_name2_field(self):
         filepath = tests.get_data_path(('NAME', 'NAMEII_field.txt'))
         name_cubes = iris.load(filepath)
-        # Check gribapi version, because we currently have a known load/save
-        # problem with gribapi 1v14 (at least).
-        gribapi_ver = gribapi.grib_get_api_version()
-        gribapi_fully_supported_version = \
-            (StrictVersion(gribapi.grib_get_api_version()) <
-             StrictVersion('1.13'))
+
+        # we currently have a known load/save
+        # problem with gribapi version and zero only data, where min == max.
+
         for i, name_cube in enumerate(name_cubes):
-            if not gribapi_fully_supported_version:
-                data = name_cube.data
-                if np.min(data) == np.max(data):
-                    msg = ('NAMEII cube #{}, "{}" has empty data : '
-                           'SKIPPING test for this cube, as save/load will '
-                           'not currently work with gribabi > 1v12.')
-                    warnings.warn(msg.format(i, name_cube.name()))
-                    continue
+            data = name_cube.data
+            if np.min(data) == np.max(data):
+                msg = ('NAMEII cube #{}, "{}" has empty data : '
+                       'SKIPPING test for this cube, as save/load will '
+                       'not currently work with gribabi > 1v12.')
+                warnings.warn(msg.format(i, name_cube.name()))
+                continue
 
             with self.temp_filename('.grib2') as temp_filename:
                 iris.save(name_cube, temp_filename)

lib/iris/tests/test_grib_save.py

@@ -1,4 +1,4 @@
-# (C) British Crown Copyright 2010 - 2016, Met Office
+# (C) British Crown Copyright 2010 - 2017, Met Office
 #
 # This file is part of Iris.
 #
@@ -34,7 +34,7 @@
 
 if tests.GRIB_AVAILABLE:
     import gribapi
-
+    from iris.fileformats.grib._load_convert import _MDIs
 
 @tests.skip_data
 @tests.skip_grib
@@ -50,10 +50,10 @@ def test_latlon_forecast_plev(self):
             iris.save(cubes, temp_file_path)
             expect_diffs = {'totalLength': (4837, 4832),
                             'productionStatusOfProcessedData': (0, 255),
-                            'scaleFactorOfRadiusOfSphericalEarth': (4294967295,
+                            'scaleFactorOfRadiusOfSphericalEarth': (_MDIs[0],
                                                                     0),
                             'shapeOfTheEarth': (0, 1),
-                            'scaledValueOfRadiusOfSphericalEarth': (4294967295,
+                            'scaledValueOfRadiusOfSphericalEarth': (_MDIs[0],
                                                                     6367470),
                             'typeOfGeneratingProcess': (0, 255),
                             'generatingProcessIdentifier': (128, 255),
@@ -70,10 +70,10 @@ def test_rotated_latlon(self):
             iris.save(cubes, temp_file_path)
             expect_diffs = {'totalLength': (648196, 648191),
                             'productionStatusOfProcessedData': (0, 255),
-                            'scaleFactorOfRadiusOfSphericalEarth': (4294967295,
+                            'scaleFactorOfRadiusOfSphericalEarth': (_MDIs[0],
                                                                     0),
                             'shapeOfTheEarth': (0, 1),
-                            'scaledValueOfRadiusOfSphericalEarth': (4294967295,
+                            'scaledValueOfRadiusOfSphericalEarth': (_MDIs[0],
                                                                     6367470),
                             'longitudeOfLastGridPoint': (392109982, 32106370),
                             'latitudeOfLastGridPoint': (19419996, 19419285),
@@ -91,10 +91,10 @@ def test_time_mean(self):
         cubes = iris.load(source_grib)
         expect_diffs = {'totalLength': (21232, 21227),
                         'productionStatusOfProcessedData': (0, 255),
-                        'scaleFactorOfRadiusOfSphericalEarth': (4294967295,
+                        'scaleFactorOfRadiusOfSphericalEarth': (_MDIs[0],
                                                                 0),
                         'shapeOfTheEarth': (0, 1),
-                        'scaledValueOfRadiusOfSphericalEarth': (4294967295,
+                        'scaledValueOfRadiusOfSphericalEarth': (_MDIs[0],
                                                                 6367470),
                         'longitudeOfLastGridPoint': (356249908, 356249809),
                         'latitudeOfLastGridPoint': (-89999938, -89999944),

lib/iris/tests/unit/fileformats/grib/load_convert/test_data_cutoff.py

@@ -1,4 +1,4 @@
-# (C) British Crown Copyright 2014 - 2015, Met Office
+# (C) British Crown Copyright 2014 - 2017, Met Office
 #
 # This file is part of Iris.
 #
@@ -26,10 +26,12 @@
 # before importing anything else.
 import iris.tests as tests
 
-from iris.fileformats.grib._load_convert import _MDI as MDI
+from iris.fileformats.grib._load_convert import _MDIs
 from iris.fileformats.grib._load_convert import data_cutoff
 from iris.tests import mock
 
+MDI = _MDIs[1]
+
 
 class TestDataCutoff(tests.IrisTest):
     def _check(self, hours, minutes, request_warning, expect_warning=False):

lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_4_and_5.py

@@ -34,11 +34,11 @@
 
 from iris.coords import DimCoord
 from iris.fileformats.grib._load_convert import \
-    _MDI as MDI, \
+    _MDIs, \
     grid_definition_template_4_and_5
 from iris.tests import mock
 
-
+MDI = _MDIs[1]
 RESOLUTION = 1e6
 
 

lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_40.py

@@ -29,9 +29,11 @@
 
 import iris.coords
 from iris.fileformats.grib._load_convert import \
-    _MDI, product_definition_template_40
+    _MDIs, product_definition_template_40
 from iris.tests.unit.fileformats.grib.load_convert import empty_metadata
 
+_MDI = _MDIs[1]
+
 
 class Test(tests.IrisTest):
     def setUp(self):

lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_9.py

@@ -1,4 +1,4 @@
-# (C) British Crown Copyright 2014 - 2015, Met Office
+# (C) British Crown Copyright 2014 - 2017, Met Office
 #
 # This file is part of Iris.
 #
@@ -29,9 +29,11 @@
 
 from iris.exceptions import TranslationError
 from iris.fileformats.grib._load_convert import product_definition_template_9
-from iris.fileformats.grib._load_convert import Probability, _MDI
+from iris.fileformats.grib._load_convert import Probability, _MDIs
 from iris.tests import mock
 
+_MDI = _MDIs[1]
+
 
 class Test(tests.IrisTest):
     def setUp(self):

lib/iris/tests/unit/fileformats/grib/load_convert/test_unscale.py

@@ -29,9 +29,10 @@
 import numpy as np
 import numpy.ma as ma
 
-from iris.fileformats.grib._load_convert import _MDI as MDI, unscale
+from iris.fileformats.grib._load_convert import _MDIs, unscale
 
 # Reference GRIB2 Regulation 92.1.12.
+MDI = _MDIs[1]
 
 
 class Test(tests.IrisTest):

lib/iris/tests/unit/fileformats/grib/load_convert/test_vertical_coords.py

@@ -31,11 +31,13 @@
 from iris.coords import DimCoord
 from iris.exceptions import TranslationError
 from iris.fileformats.grib._load_convert import \
-    _MDI as MISSING_LEVEL, \
+    _MDIs, \
     _TYPE_OF_FIXED_SURFACE_MISSING as MISSING_SURFACE, \
     vertical_coords
 from iris.tests import mock
 
+MISSING_LEVEL = _MDIs[1]
+
 
 class Test(tests.IrisTest):
     def setUp(self):

lib/iris/tests/unit/fileformats/grib/message/test_GribMessage.py

@@ -54,7 +54,12 @@ def test_release_file(self):
         filename = tests.get_data_path(('GRIB', '3_layer_viz',
                                         '3_layer.grib2'))
         my_file = open(filename)
-        self.patch('__builtin__.open', mock.Mock(return_value=my_file))
+        if six.PY2:
+            self.patch('__builtin__.open', mock.Mock(return_value=my_file))
+        else:
+            import builtins
+            self.patch('builtins.open', mock.Mock(return_value=my_file))
+
         messages = list(GribMessage.messages_from_filename(filename))
         self.assertFalse(my_file.closed)
         del messages

lib/iris/tests/unit/fileformats/grib/save_rules/test_grid_definition_template_0.py

@@ -76,8 +76,8 @@ def test__grid_points(self):
         self._check_key("longitudeOfLastGridPoint", 7000000)
         self._check_key("latitudeOfFirstGridPoint", 4000000)
         self._check_key("latitudeOfLastGridPoint", 9000000)
-        self._check_key("DxInDegrees", 2.0)
-        self._check_key("DyInDegrees", 5.0)
+        self._check_key("iDirectionIncrement", 2000000)
+        self._check_key("jDirectionIncrement", 5000000)
 
     def test__scanmode(self):
         grid_definition_template_0(self.test_cube, self.mock_grib)

lib/iris/tests/unit/fileformats/grib/save_rules/test_grid_definition_template_1.py

@@ -91,8 +91,8 @@ def test__grid_points(self):
         self._check_key("longitudeOfLastGridPoint", 7000000)
         self._check_key("latitudeOfFirstGridPoint", 4000000)
         self._check_key("latitudeOfLastGridPoint", 9000000)
-        self._check_key("DxInDegrees", 2.0)
-        self._check_key("DyInDegrees", 5.0)
+        self._check_key("iDirectionIncrement", 2000000)
+        self._check_key("jDirectionIncrement", 5000000)
 
     def test__scanmode(self):
         grid_definition_template_1(self.test_cube, self.mock_grib)