From 077b6eea44ef20bd19f803b3397303bba276b37a Mon Sep 17 00:00:00 2001 From: lbdreyer Date: Thu, 13 Jul 2017 11:46:36 +0100 Subject: [PATCH 1/6] Add docs badge to README (#2684) --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 23d1222f37..c764b8fedd 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,7 @@ Iris [![Join the chat at https://gitter.im/SciTools/iris](https://badges.gitter.im/SciTools/iris.svg)](https://gitter.im/SciTools/iris?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![Build Status](https://api.travis-ci.org/repositories/SciTools/iris.svg?branch=master)](https://travis-ci.org/SciTools/iris/branches) [![DOI](https://zenodo.org/badge/doi/10.5281/zenodo.51860.svg)](https://dx.doi.org/10.5281/zenodo.51860) +[![Documentation for master branch ](https://img.shields.io/badge/docs-master-blue.svg)](https://scitools-docs.github.io/iris/master/index.html) (C) British Crown Copyright 2010 - 2017, Met Office From 2bf11c79f506c1be526ef42a2d0cd464fe71568b Mon Sep 17 00:00:00 2001 From: markh Date: Thu, 15 Jun 2017 08:27:46 +0000 Subject: [PATCH 2/6] adopt ecCodes --- .travis.yml | 2 +- conda-requirements.txt | 2 +- lib/iris/fileformats/grib/_load_convert.py | 32 ++++++++++--------- lib/iris/tests/test_grib_save.py | 14 ++++---- .../grib/load_convert/test_data_cutoff.py | 6 ++-- .../test_grid_definition_template_4_and_5.py | 4 +-- .../test_product_definition_template_40.py | 4 ++- .../test_product_definition_template_9.py | 6 ++-- .../grib/load_convert/test_unscale.py | 3 +- .../grib/load_convert/test_vertical_coords.py | 4 ++- .../grib/message/test_GribMessage.py | 7 +++- 11 files changed, 50 insertions(+), 34 deletions(-) diff --git a/.travis.yml b/.travis.yml index bf618a27af..7f579c6f5d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,7 +59,7 @@ install: conda install --quiet --file minimal-conda-requirements.txt; else if [[ "$TRAVIS_PYTHON_VERSION" == 3* ]]; then - sed -e '/ecmwf_grib/d' -e '/esmpy/d' -e 's/#.\+$//' conda-requirements.txt | xargs conda install --quiet; + sed -e '/esmpy/d' -e 's/#.\+$//' conda-requirements.txt | xargs conda install --quiet; else conda install --quiet --file conda-requirements.txt; fi diff --git a/conda-requirements.txt b/conda-requirements.txt index e0206a539d..7884dbe7c7 100644 --- a/conda-requirements.txt +++ b/conda-requirements.txt @@ -25,7 +25,7 @@ imagehash requests # Optional iris dependencies -python-ecmwf_grib +python-eccodes esmpy>=7.0 gdal libmo_unpack diff --git a/lib/iris/fileformats/grib/_load_convert.py b/lib/iris/fileformats/grib/_load_convert.py index 9ce3df9137..ab1217c8da 100644 --- a/lib/iris/fileformats/grib/_load_convert.py +++ b/lib/iris/fileformats/grib/_load_convert.py @@ -158,7 +158,9 @@ def _unscale(v, f): if isinstance(value, Iterable) or isinstance(factor, Iterable): def _masker(item): - result = ma.masked_equal(item, _MDI) + result = item + for _MDI in _MDIs: + result = ma.masked_equal(result, _MDI) if ma.count_masked(result): # Circumvent downstream NumPy "RuntimeWarning" # of "overflow encountered in power" in _unscale @@ -172,24 +174,24 @@ def _masker(item): result = result.data else: result = ma.masked - if value != _MDI and factor != _MDI: + if value not in _MDIs and factor not in _MDIs: result = _unscale(value, factor) return result # Regulations 92.1.4 and 92.1.5. -_MDI = 2 ** 32 - 1 +_MDIs = [2 ** 31 - 1, 2 ** 32 - 1] # Note: # 1. Integer "on-disk" values (aka. coded keys) in GRIB messages: # - Are 8-, 16-, or 32-bit. # - Are either signed or unsigned, with signed values stored as # sign-and-magnitude (*not* twos-complement). # - Use all bits set to indicate a missing value (MDI). -# 2. Irrespective of the on-disk form, the ECMWF GRIB API *always*: +# 2. Irrespective of the on-disk form, ECCodes: the ECMWF GRIB API *always*: # - Returns values as 64-bit signed integers, either as native # Python 'int' or numpy 'int64'. -# - Returns missing values as 2**32 - 1, but not all keys are -# defined as supporting missing values. +# - Returns missing values as 2**32 - 1 or 2**31 - 1, but not all keys +# are defined as supporting missing values. # NB. For keys which support missing values, the MDI value is reliably # distinct from the valid range of either signed or unsigned 8-, 16-, # or 32-bit values. For example: @@ -634,10 +636,10 @@ def grid_definition_template_4_and_5(section, metadata, y_name, x_name, cs): basicAngleOfTheInitialProductDomain = section[key] subdivisionsOfBasicAngle = section['subdivisionsOfBasicAngle'] - if basicAngleOfTheInitialProductDomain in [0, _MDI]: + if basicAngleOfTheInitialProductDomain in [0] + _MDIs: basicAngleOfTheInitialProductDomain = 1. - if subdivisionsOfBasicAngle in [0, _MDI]: + if subdivisionsOfBasicAngle in [0] + _MDIs: subdivisionsOfBasicAngle = 1. / _GRID_ACCURACY_IN_DEGREES resolution = np.float64(basicAngleOfTheInitialProductDomain) @@ -1078,7 +1080,7 @@ def grid_definition_template_90(section, metadata): :class:`collections.OrderedDict` of metadata. """ - if section['Nr'] == _MDI: + if section['Nr'] in _MDIs: raise TranslationError('Unsupported orthographic grid.') elif section['Nr'] == 0: raise TranslationError('Unsupported zero height for space-view.') @@ -1403,7 +1405,7 @@ def vertical_coords(section, metadata): if fixed_surface is None: if typeOfFirstFixedSurface != _TYPE_OF_FIXED_SURFACE_MISSING: - if scaledValueOfFirstFixedSurface == _MDI: + if scaledValueOfFirstFixedSurface in _MDIs: if options.warn_on_unsupported: msg = 'Unable to translate type of first fixed ' \ 'surface with missing scaled value.' @@ -1427,7 +1429,7 @@ def vertical_coords(section, metadata): key = 'scaledValueOfSecondFixedSurface' scaledValueOfSecondFixedSurface = section[key] - if scaledValueOfSecondFixedSurface == _MDI: + if scaledValueOfSecondFixedSurface in _MDIs: msg = 'Product definition section 4 has missing ' \ 'scaled value of second fixed surface' raise TranslationError(msg) @@ -1686,8 +1688,8 @@ def data_cutoff(hoursAfterDataCutoff, minutesAfterDataCutoff): Message section 4, octet 17. """ - if (hoursAfterDataCutoff != _MDI or - minutesAfterDataCutoff != _MDI): + if (hoursAfterDataCutoff not in _MDIs or + minutesAfterDataCutoff not in _MDIs): if options.warn_on_unsupported: warnings.warn('Unable to translate "hours and/or minutes ' 'after data cutoff".') @@ -1937,12 +1939,12 @@ def product_definition_template_9(section, metadata, frt_coord): if probability_typecode == 1: # Type is "above upper level". threshold_value = section['scaledValueOfUpperLimit'] - if threshold_value == _MDI: + if threshold_value in _MDIs: msg = 'Product definition section 4 has missing ' \ 'scaled value of upper limit' raise TranslationError(msg) threshold_scaling = section['scaleFactorOfUpperLimit'] - if threshold_scaling == _MDI: + if threshold_scaling in _MDIs: msg = 'Product definition section 4 has missing ' \ 'scale factor of upper limit' raise TranslationError(msg) diff --git a/lib/iris/tests/test_grib_save.py b/lib/iris/tests/test_grib_save.py index 53c14a169c..d5013e1e79 100644 --- a/lib/iris/tests/test_grib_save.py +++ b/lib/iris/tests/test_grib_save.py @@ -34,7 +34,7 @@ if tests.GRIB_AVAILABLE: import gribapi - + from iris.fileformats.grib._load_convert import _MDIs @tests.skip_data @tests.skip_grib @@ -50,10 +50,10 @@ def test_latlon_forecast_plev(self): iris.save(cubes, temp_file_path) expect_diffs = {'totalLength': (4837, 4832), 'productionStatusOfProcessedData': (0, 255), - 'scaleFactorOfRadiusOfSphericalEarth': (4294967295, + 'scaleFactorOfRadiusOfSphericalEarth': (_MDIs[0], 0), 'shapeOfTheEarth': (0, 1), - 'scaledValueOfRadiusOfSphericalEarth': (4294967295, + 'scaledValueOfRadiusOfSphericalEarth': (_MDIs[0], 6367470), 'typeOfGeneratingProcess': (0, 255), 'generatingProcessIdentifier': (128, 255), @@ -70,10 +70,10 @@ def test_rotated_latlon(self): iris.save(cubes, temp_file_path) expect_diffs = {'totalLength': (648196, 648191), 'productionStatusOfProcessedData': (0, 255), - 'scaleFactorOfRadiusOfSphericalEarth': (4294967295, + 'scaleFactorOfRadiusOfSphericalEarth': (_MDIs[0], 0), 'shapeOfTheEarth': (0, 1), - 'scaledValueOfRadiusOfSphericalEarth': (4294967295, + 'scaledValueOfRadiusOfSphericalEarth': (_MDIs[0], 6367470), 'longitudeOfLastGridPoint': (392109982, 32106370), 'latitudeOfLastGridPoint': (19419996, 19419285), @@ -91,10 +91,10 @@ def test_time_mean(self): cubes = iris.load(source_grib) expect_diffs = {'totalLength': (21232, 21227), 'productionStatusOfProcessedData': (0, 255), - 'scaleFactorOfRadiusOfSphericalEarth': (4294967295, + 'scaleFactorOfRadiusOfSphericalEarth': (_MDIs[0], 0), 'shapeOfTheEarth': (0, 1), - 'scaledValueOfRadiusOfSphericalEarth': (4294967295, + 'scaledValueOfRadiusOfSphericalEarth': (_MDIs[0], 6367470), 'longitudeOfLastGridPoint': (356249908, 356249809), 'latitudeOfLastGridPoint': (-89999938, -89999944), diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_data_cutoff.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_data_cutoff.py index 729d2b9b30..a27efc4208 100644 --- a/lib/iris/tests/unit/fileformats/grib/load_convert/test_data_cutoff.py +++ b/lib/iris/tests/unit/fileformats/grib/load_convert/test_data_cutoff.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2014 - 2015, Met Office +# (C) British Crown Copyright 2014 - 2017, Met Office # # This file is part of Iris. # @@ -26,10 +26,12 @@ # before importing anything else. import iris.tests as tests -from iris.fileformats.grib._load_convert import _MDI as MDI +from iris.fileformats.grib._load_convert import _MDIs from iris.fileformats.grib._load_convert import data_cutoff from iris.tests import mock +MDI = _MDIs[1] + class TestDataCutoff(tests.IrisTest): def _check(self, hours, minutes, request_warning, expect_warning=False): diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_4_and_5.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_4_and_5.py index 534b41474e..59d85d03a0 100644 --- a/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_4_and_5.py +++ b/lib/iris/tests/unit/fileformats/grib/load_convert/test_grid_definition_template_4_and_5.py @@ -34,11 +34,11 @@ from iris.coords import DimCoord from iris.fileformats.grib._load_convert import \ - _MDI as MDI, \ + _MDIs, \ grid_definition_template_4_and_5 from iris.tests import mock - +MDI = _MDIs[1] RESOLUTION = 1e6 diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_40.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_40.py index 7ccecdd76b..90625317e9 100644 --- a/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_40.py +++ b/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_40.py @@ -29,9 +29,11 @@ import iris.coords from iris.fileformats.grib._load_convert import \ - _MDI, product_definition_template_40 + _MDIs, product_definition_template_40 from iris.tests.unit.fileformats.grib.load_convert import empty_metadata +_MDI = _MDIs[1] + class Test(tests.IrisTest): def setUp(self): diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_9.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_9.py index e1aede2dbf..e8068f17ff 100644 --- a/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_9.py +++ b/lib/iris/tests/unit/fileformats/grib/load_convert/test_product_definition_template_9.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2014 - 2015, Met Office +# (C) British Crown Copyright 2014 - 2017, Met Office # # This file is part of Iris. # @@ -29,9 +29,11 @@ from iris.exceptions import TranslationError from iris.fileformats.grib._load_convert import product_definition_template_9 -from iris.fileformats.grib._load_convert import Probability, _MDI +from iris.fileformats.grib._load_convert import Probability, _MDIs from iris.tests import mock +_MDI = _MDIs[1] + class Test(tests.IrisTest): def setUp(self): diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_unscale.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_unscale.py index 1f891177cc..37ffb56165 100644 --- a/lib/iris/tests/unit/fileformats/grib/load_convert/test_unscale.py +++ b/lib/iris/tests/unit/fileformats/grib/load_convert/test_unscale.py @@ -29,9 +29,10 @@ import numpy as np import numpy.ma as ma -from iris.fileformats.grib._load_convert import _MDI as MDI, unscale +from iris.fileformats.grib._load_convert import _MDIs, unscale # Reference GRIB2 Regulation 92.1.12. +MDI = _MDIs[1] class Test(tests.IrisTest): diff --git a/lib/iris/tests/unit/fileformats/grib/load_convert/test_vertical_coords.py b/lib/iris/tests/unit/fileformats/grib/load_convert/test_vertical_coords.py index daa508891a..ae4c3d8814 100644 --- a/lib/iris/tests/unit/fileformats/grib/load_convert/test_vertical_coords.py +++ b/lib/iris/tests/unit/fileformats/grib/load_convert/test_vertical_coords.py @@ -31,11 +31,13 @@ from iris.coords import DimCoord from iris.exceptions import TranslationError from iris.fileformats.grib._load_convert import \ - _MDI as MISSING_LEVEL, \ + _MDIs, \ _TYPE_OF_FIXED_SURFACE_MISSING as MISSING_SURFACE, \ vertical_coords from iris.tests import mock +MISSING_LEVEL = _MDIs[1] + class Test(tests.IrisTest): def setUp(self): diff --git a/lib/iris/tests/unit/fileformats/grib/message/test_GribMessage.py b/lib/iris/tests/unit/fileformats/grib/message/test_GribMessage.py index 3bbd9c8e04..a5693b8032 100644 --- a/lib/iris/tests/unit/fileformats/grib/message/test_GribMessage.py +++ b/lib/iris/tests/unit/fileformats/grib/message/test_GribMessage.py @@ -54,7 +54,12 @@ def test_release_file(self): filename = tests.get_data_path(('GRIB', '3_layer_viz', '3_layer.grib2')) my_file = open(filename) - self.patch('__builtin__.open', mock.Mock(return_value=my_file)) + if six.PY2: + self.patch('__builtin__.open', mock.Mock(return_value=my_file)) + else: + import builtins + self.patch('builtins.open', mock.Mock(return_value=my_file)) + messages = list(GribMessage.messages_from_filename(filename)) self.assertFalse(my_file.closed) del messages From 87f4e79a1ebf01c48ea604def0fda0ac83d732bf Mon Sep 17 00:00:00 2001 From: markh Date: Thu, 15 Jun 2017 14:40:46 +0000 Subject: [PATCH 3/6] eccodes py2 py3 behaviour: DxInDegrees --- lib/iris/fileformats/grib/_save_rules.py | 14 ++++++++++++-- lib/iris/tests/test_grib_save.py | 2 +- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/lib/iris/fileformats/grib/_save_rules.py b/lib/iris/fileformats/grib/_save_rules.py index 7a5a8fce49..9a3e11fce9 100644 --- a/lib/iris/fileformats/grib/_save_rules.py +++ b/lib/iris/fileformats/grib/_save_rules.py @@ -257,8 +257,18 @@ def latlon_first_last(x_coord, y_coord, grib): def dx_dy(x_coord, y_coord, grib): x_step = regular_step(x_coord) y_step = regular_step(y_coord) - gribapi.grib_set(grib, "DxInDegrees", float(abs(x_step))) - gribapi.grib_set(grib, "DyInDegrees", float(abs(y_step))) + if x_coord.units == 'degrees': + gribapi.grib_set(grib, "iDirectionIncrement", + round(1e6 * float(abs(x_step)))) + else: + raise ValueError('X coordinate must be in degrees, not {}' + '.'.format(x_coord.units)) + if y_coord.units == 'degrees': + gribapi.grib_set(grib, "jDirectionIncrement", + round(1e6 * float(abs(y_step)))) + else: + raise ValueError('Y coordinate must be in degrees, not {}' + '.'.format(y_coord.units)) def scanning_mode_flags(x_coord, y_coord, grib): diff --git a/lib/iris/tests/test_grib_save.py b/lib/iris/tests/test_grib_save.py index d5013e1e79..f80734eee8 100644 --- a/lib/iris/tests/test_grib_save.py +++ b/lib/iris/tests/test_grib_save.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2010 - 2016, Met Office +# (C) British Crown Copyright 2010 - 2017, Met Office # # This file is part of Iris. # From a5642faa957909b55814561f6afbafde6bd960d1 Mon Sep 17 00:00:00 2001 From: markh Date: Thu, 15 Jun 2017 15:06:34 +0000 Subject: [PATCH 4/6] unit test i/jDirectionIncrement --- .../grib/save_rules/test_grid_definition_template_0.py | 4 ++-- .../grib/save_rules/test_grid_definition_template_1.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/iris/tests/unit/fileformats/grib/save_rules/test_grid_definition_template_0.py b/lib/iris/tests/unit/fileformats/grib/save_rules/test_grid_definition_template_0.py index a4f581b78e..d5ee9f771e 100644 --- a/lib/iris/tests/unit/fileformats/grib/save_rules/test_grid_definition_template_0.py +++ b/lib/iris/tests/unit/fileformats/grib/save_rules/test_grid_definition_template_0.py @@ -76,8 +76,8 @@ def test__grid_points(self): self._check_key("longitudeOfLastGridPoint", 7000000) self._check_key("latitudeOfFirstGridPoint", 4000000) self._check_key("latitudeOfLastGridPoint", 9000000) - self._check_key("DxInDegrees", 2.0) - self._check_key("DyInDegrees", 5.0) + self._check_key("iDirectionIncrement", 2000000) + self._check_key("jDirectionIncrement", 5000000) def test__scanmode(self): grid_definition_template_0(self.test_cube, self.mock_grib) diff --git a/lib/iris/tests/unit/fileformats/grib/save_rules/test_grid_definition_template_1.py b/lib/iris/tests/unit/fileformats/grib/save_rules/test_grid_definition_template_1.py index f8cdf181d4..1e7ce01a32 100644 --- a/lib/iris/tests/unit/fileformats/grib/save_rules/test_grid_definition_template_1.py +++ b/lib/iris/tests/unit/fileformats/grib/save_rules/test_grid_definition_template_1.py @@ -91,8 +91,8 @@ def test__grid_points(self): self._check_key("longitudeOfLastGridPoint", 7000000) self._check_key("latitudeOfFirstGridPoint", 4000000) self._check_key("latitudeOfLastGridPoint", 9000000) - self._check_key("DxInDegrees", 2.0) - self._check_key("DyInDegrees", 5.0) + self._check_key("iDirectionIncrement", 2000000) + self._check_key("jDirectionIncrement", 5000000) def test__scanmode(self): grid_definition_template_1(self.test_cube, self.mock_grib) From 1fbf6b4ba618fd28a84b435a991c93a1af81f068 Mon Sep 17 00:00:00 2001 From: markh Date: Thu, 15 Jun 2017 15:56:33 +0000 Subject: [PATCH 5/6] do not protect failing test with version checks --- .../format_interop/test_name_grib.py | 27 +++++++++---------- 1 file changed, 12 insertions(+), 15 deletions(-) diff --git a/lib/iris/tests/integration/format_interop/test_name_grib.py b/lib/iris/tests/integration/format_interop/test_name_grib.py index 192f0082b0..afb147f48c 100644 --- a/lib/iris/tests/integration/format_interop/test_name_grib.py +++ b/lib/iris/tests/integration/format_interop/test_name_grib.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2013 - 2016, Met Office +# (C) British Crown Copyright 2013 - 2017, Met Office # # This file is part of Iris. # @@ -72,21 +72,18 @@ def check_common(self, name_cube, grib_cube): def test_name2_field(self): filepath = tests.get_data_path(('NAME', 'NAMEII_field.txt')) name_cubes = iris.load(filepath) - # Check gribapi version, because we currently have a known load/save - # problem with gribapi 1v14 (at least). - gribapi_ver = gribapi.grib_get_api_version() - gribapi_fully_supported_version = \ - (StrictVersion(gribapi.grib_get_api_version()) < - StrictVersion('1.13')) + + # we currently have a known load/save + # problem with gribapi version and zero only data, where min == max. + for i, name_cube in enumerate(name_cubes): - if not gribapi_fully_supported_version: - data = name_cube.data - if np.min(data) == np.max(data): - msg = ('NAMEII cube #{}, "{}" has empty data : ' - 'SKIPPING test for this cube, as save/load will ' - 'not currently work with gribabi > 1v12.') - warnings.warn(msg.format(i, name_cube.name())) - continue + data = name_cube.data + if np.min(data) == np.max(data): + msg = ('NAMEII cube #{}, "{}" has empty data : ' + 'SKIPPING test for this cube, as save/load will ' + 'not currently work with gribabi > 1v12.') + warnings.warn(msg.format(i, name_cube.name())) + continue with self.temp_filename('.grib2') as temp_filename: iris.save(name_cube, temp_filename) From ba65435668c1a5c3d75ed94ad795ec10f035d3b5 Mon Sep 17 00:00:00 2001 From: markh Date: Fri, 14 Jul 2017 09:43:03 +0000 Subject: [PATCH 6/6] break mdi for testing --- lib/iris/fileformats/grib/_load_convert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/iris/fileformats/grib/_load_convert.py b/lib/iris/fileformats/grib/_load_convert.py index ab1217c8da..6ec7230ad2 100644 --- a/lib/iris/fileformats/grib/_load_convert.py +++ b/lib/iris/fileformats/grib/_load_convert.py @@ -180,7 +180,7 @@ def _masker(item): # Regulations 92.1.4 and 92.1.5. -_MDIs = [2 ** 31 - 1, 2 ** 32 - 1] +_MDIs = [2 ** 32 - 1, 2 ** 32 - 1] # Note: # 1. Integer "on-disk" values (aka. coded keys) in GRIB messages: # - Are 8-, 16-, or 32-bit.