From 702e8b5d8c75d5186f2145f4536bf798fad0ef83 Mon Sep 17 00:00:00 2001 From: Bill Little Date: Thu, 11 Jul 2019 10:00:56 +0100 Subject: [PATCH 1/3] unpin dask --- requirements/core.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/core.txt b/requirements/core.txt index 19e261613e..e39777edb7 100644 --- a/requirements/core.txt +++ b/requirements/core.txt @@ -7,7 +7,7 @@ cartopy #conda: proj4<5 cf-units>=2 cftime -dask[array]<2 #conda: dask<2 +dask[array] #conda: dask matplotlib>=2,<3 netcdf4 numpy>=1.14 From 9ae944c0ea9a28f9421b6685ac37ea213bd02313 Mon Sep 17 00:00:00 2001 From: Bill Little Date: Thu, 11 Jul 2019 11:18:25 +0100 Subject: [PATCH 2/3] fix tests --- lib/iris/tests/unit/fileformats/pp/test__create_field_data.py | 2 +- lib/iris/tests/unit/lazy_data/test_as_concrete_data.py | 3 ++- lib/iris/tests/unit/lazy_data/test_co_realise_cubes.py | 3 ++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/iris/tests/unit/fileformats/pp/test__create_field_data.py b/lib/iris/tests/unit/fileformats/pp/test__create_field_data.py index d9816c64b9..94abb974cd 100644 --- a/lib/iris/tests/unit/fileformats/pp/test__create_field_data.py +++ b/lib/iris/tests/unit/fileformats/pp/test__create_field_data.py @@ -64,7 +64,7 @@ def test_deferred_bytes(self): field = mock.Mock(core_data=core_data) data_shape = (100, 120) proxy = mock.Mock(dtype=np.dtype('f4'), shape=data_shape, - spec=pp.PPDataProxy) + spec=pp.PPDataProxy, ndim=len(data_shape)) # We can't directly inspect the concrete data source underlying # the dask array, so instead we patch the proxy creation and check it's # being created and invoked correctly. diff --git a/lib/iris/tests/unit/lazy_data/test_as_concrete_data.py b/lib/iris/tests/unit/lazy_data/test_as_concrete_data.py index 3d64e038b5..d9a055e4b5 100644 --- a/lib/iris/tests/unit/lazy_data/test_as_concrete_data.py +++ b/lib/iris/tests/unit/lazy_data/test_as_concrete_data.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2017, Met Office +# (C) British Crown Copyright 2017 - 2019, Met Office # # This file is part of Iris. # @@ -34,6 +34,7 @@ class MyProxy(object): def __init__(self, a): self.shape = a.shape self.dtype = a.dtype + self.ndim = a.ndim self.a = a def __getitem__(self, keys): diff --git a/lib/iris/tests/unit/lazy_data/test_co_realise_cubes.py b/lib/iris/tests/unit/lazy_data/test_co_realise_cubes.py index 03782cda85..b633adc4df 100644 --- a/lib/iris/tests/unit/lazy_data/test_co_realise_cubes.py +++ b/lib/iris/tests/unit/lazy_data/test_co_realise_cubes.py @@ -1,4 +1,4 @@ -# (C) British Crown Copyright 2018, Met Office +# (C) British Crown Copyright 2018 - 2019, Met Office # # This file is part of Iris. # @@ -36,6 +36,7 @@ class ArrayAccessCounter(object): def __init__(self, array): self.dtype = array.dtype self.shape = array.shape + self.ndim = array.ndim self._array = array self.access_count = 0 From 66877cce4594f7371baa6377e2b7a6e29be98dbe Mon Sep 17 00:00:00 2001 From: Bill Little Date: Thu, 11 Jul 2019 11:55:46 +0100 Subject: [PATCH 3/3] fix test_combined_access --- .../unit/lazy_data/test_co_realise_cubes.py | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/iris/tests/unit/lazy_data/test_co_realise_cubes.py b/lib/iris/tests/unit/lazy_data/test_co_realise_cubes.py index b633adc4df..f1b69e4d69 100644 --- a/lib/iris/tests/unit/lazy_data/test_co_realise_cubes.py +++ b/lib/iris/tests/unit/lazy_data/test_co_realise_cubes.py @@ -72,15 +72,29 @@ def test_multi(self): self.assertTrue(cube_inner.has_lazy_data()) def test_combined_access(self): + import dask + from distutils.version import StrictVersion as Version + wrapped_array = ArrayAccessCounter(np.arange(3.)) lazy_array = as_lazy_data(wrapped_array) derived_a = lazy_array + 1 derived_b = lazy_array + 2 + derived_c = lazy_array + 3 cube_a = Cube(derived_a) cube_b = Cube(derived_b) - co_realise_cubes(cube_a, cube_b) - # Though used twice, the source data should only get fetched once. - self.assertEqual(wrapped_array.access_count, 1) + cube_c = Cube(derived_c) + co_realise_cubes(cube_a, cube_b, cube_c) + # Though used more than once, the source data should only get fetched + # once by dask. + if Version(dask.__version__) < Version('2.0.0'): + self.assertEqual(wrapped_array.access_count, 1) + else: + # dask 2+, now performs an initial data access with no data payload + # to ascertain the metadata associated with the dask.array, thus + # accounting for one additional access to the data from the + # perspective of this particular unit test. + # See dask.array.utils.meta_from_array + self.assertEqual(wrapped_array.access_count, 2) if __name__ == '__main__':