diff --git a/.cirrus.yml b/.cirrus.yml index 0e64f2ae..186fc951 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -96,13 +96,23 @@ linux_task_template: &LINUX_TASK_TEMPLATE tests_task: matrix: - env: - PY_VER: 3.6 - env: - PY_VER: 3.7 - # env: - # PY_VER: 3.8 - name: "${CIRRUS_OS}: py${PY_VER} tests" + - name: "${CIRRUS_OS} tests: iris=repo-main python=3.7" + env: + PY_VER: 3.7 + IRIS_SOURCE: "source" + - name: "${CIRRUS_OS} tests: iris=repo-main python=3.8" + env: + PY_VER: 3.8 + IRIS_SOURCE: "source" + - name: "${CIRRUS_OS} tests: iris=conda-release python=3.7" + env: + PY_VER: 3.7 + IRIS_SOURCE: "conda-forge" + - name: "${CIRRUS_OS} tests: iris=conda-release python=3.8" + env: + PY_VER: 3.8 + IRIS_SOURCE: "conda-forge" + << : *LINUX_TASK_TEMPLATE tests_script: - export CONDA_OVERRIDE_LINUX="$(uname -r | cut -d'+' -f1)" diff --git a/iris_grib/__init__.py b/iris_grib/__init__.py index ae718c6e..03f42cbc 100644 --- a/iris_grib/__init__.py +++ b/iris_grib/__init__.py @@ -25,7 +25,6 @@ from iris._lazy_data import as_lazy_data import iris.coord_systems as coord_systems from iris.exceptions import TranslationError, NotYetImplementedError -from iris.util import _array_slice_ifempty from . import grib_phenom_translation as gptx from . import _save_rules @@ -98,17 +97,13 @@ def ndim(self): return len(self.shape) def __getitem__(self, keys): - # Avoid fetching file data just to return an 'empty' result. - # Needed because of how dask.array.from_array behaves since Dask v2.0. - result = _array_slice_ifempty(keys, self.shape, self.dtype) - if result is None: - with open(self.path, 'rb') as grib_fh: - grib_fh.seek(self.offset) - grib_message = gribapi.grib_new_from_file(grib_fh) - data = _message_values(grib_message, self.shape) - gribapi.grib_release(grib_message) - - result = data.__getitem__(keys) + with open(self.path, 'rb') as grib_fh: + grib_fh.seek(self.offset) + grib_message = gribapi.grib_new_from_file(grib_fh) + data = _message_values(grib_message, self.shape) + gribapi.grib_release(grib_message) + + result = data.__getitem__(keys) return result diff --git a/iris_grib/message.py b/iris_grib/message.py index 503d4cd6..21ea52a7 100644 --- a/iris_grib/message.py +++ b/iris_grib/message.py @@ -12,7 +12,6 @@ import re import gribapi -from iris_grib import _array_slice_ifempty import numpy as np import numpy.ma as ma @@ -233,32 +232,28 @@ def __getitem__(self, keys): # NB. Currently assumes that the validity of this interpretation # is checked before this proxy is created. - # Avoid fetching file data just to return an 'empty' result. - # Needed because of how dask.array.from_array behaves since Dask v2.0. - result = _array_slice_ifempty(keys, self.shape, self.dtype) - if result is None: - message = self.recreate_raw() - sections = message.sections - bitmap_section = sections[6] - bitmap = self._bitmap(bitmap_section) - data = sections[7]['codedValues'] - - if bitmap is not None: - # Note that bitmap and data are both 1D arrays at this point. - if np.count_nonzero(bitmap) == data.shape[0]: - # Only the non-masked values are included in codedValues. - _data = np.empty(shape=bitmap.shape) - _data[bitmap.astype(bool)] = data - # `ma.masked_array` masks where input = 1, the opposite of - # the behaviour specified by the GRIB spec. - data = ma.masked_array(_data, mask=np.logical_not(bitmap), - fill_value=np.nan) - else: - msg = 'Shapes of data and bitmap do not match.' - raise TranslationError(msg) - - data = data.reshape(self.shape) - result = data.__getitem__(keys) + message = self.recreate_raw() + sections = message.sections + bitmap_section = sections[6] + bitmap = self._bitmap(bitmap_section) + data = sections[7]['codedValues'] + + if bitmap is not None: + # Note that bitmap and data are both 1D arrays at this point. + if np.count_nonzero(bitmap) == data.shape[0]: + # Only the non-masked values are included in codedValues. + _data = np.empty(shape=bitmap.shape) + _data[bitmap.astype(bool)] = data + # `ma.masked_array` masks where input = 1, the opposite of + # the behaviour specified by the GRIB spec. + data = ma.masked_array(_data, mask=np.logical_not(bitmap), + fill_value=np.nan) + else: + msg = 'Shapes of data and bitmap do not match.' + raise TranslationError(msg) + + data = data.reshape(self.shape) + result = data.__getitem__(keys) return result diff --git a/iris_grib/tests/unit/message/test__DataProxy.py b/iris_grib/tests/unit/message/test__DataProxy.py index d7494d5e..b828c97c 100644 --- a/iris_grib/tests/unit/message/test__DataProxy.py +++ b/iris_grib/tests/unit/message/test__DataProxy.py @@ -43,31 +43,5 @@ def test_bitmap__invalid_indicator(self): data_proxy._bitmap(section_6) -class Test_emptyfetch(tests.IrisGribTest): - # See : - # iris.tests.unit.fileformats.pp.test_PPDataProxy.Test__getitem__slicing - # In this case, test *only* the no-data-read effect, not the method which - # is part of Iris. - def test_empty_slice(self): - # Check behaviour of the getitem call with an 'empty' slicing. - # This is necessary because, since Dask 2.0, the "from_array" function - # takes a zero-length slice of its array argument, to capture array - # metadata, and in those cases we want to avoid file access. - test_dtype = np.dtype(np.float32) - mock_datafetch = mock.MagicMock() - proxy = _DataProxy(shape=(3, 4), - dtype=np.dtype(np.float32), - recreate_raw=mock_datafetch) - - # Test the special no-data indexing operation. - result = proxy[0:0, 0:0] - - # Check the behaviour and results were as expected. - self.assertEqual(mock_datafetch.call_count, 0) - self.assertIsInstance(result, np.ndarray) - self.assertEqual(result.dtype, test_dtype) - self.assertEqual(result.shape, (0, 0)) - - if __name__ == '__main__': tests.main() diff --git a/noxfile.py b/noxfile.py index e9a029fe..fa801a49 100644 --- a/noxfile.py +++ b/noxfile.py @@ -25,7 +25,7 @@ PACKAGE = str("iris_grib") #: Cirrus-CI environment variable hook. -PY_VER = os.environ.get("PY_VER", ["3.6", "3.7"]) +PY_VER = os.environ.get("PY_VER", ["3.7", "3.8"]) IRIS_SOURCE = os.environ.get("IRIS_SOURCE", ['source', 'conda-forge']) #: Default cartopy cache directory. @@ -284,14 +284,14 @@ def tests(session, iris): iris_dir = f"{session.create_tmp()}/iris" if os.path.exists(iris_dir): - # cached. update by pulling from origin/master + # cached. update by pulling from origin/main session.run( "git", "-C", iris_dir, "pull", "origin", - "master", + "main", external=True # use git from host environment ) else: @@ -330,8 +330,6 @@ def tests(session, iris): "-m", "iris_grib.tests.runner", "--default-tests", - "--unit-tests", - "--integration-tests", ) @@ -385,4 +383,4 @@ def linkcheck(session): "make", "linkcheck", external=True, - ) \ No newline at end of file + )