From b033fb7f5d2cbd85ab2849bd531f72243cff1871 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 3 Apr 2023 17:32:51 +0100 Subject: [PATCH 1/6] Small netCDF variable data is real. --- lib/iris/fileformats/netcdf/loader.py | 65 +++++++++++++------ .../netcdf/loader/test__get_cf_var_data.py | 32 +++++++-- 2 files changed, 73 insertions(+), 24 deletions(-) diff --git a/lib/iris/fileformats/netcdf/loader.py b/lib/iris/fileformats/netcdf/loader.py index 8fcab61d17..113f40b3c9 100644 --- a/lib/iris/fileformats/netcdf/loader.py +++ b/lib/iris/fileformats/netcdf/loader.py @@ -173,26 +173,53 @@ def _get_actual_dtype(cf_var): return dummy_data.dtype +# An arbitrary variable array size, below which we will fetch real data from a variable +# rather than making a lazy array for deferred access. +# Set by experiment at roughly the point where it begins to save us memory, but actually +# mostly done for speed improvement. See https://github.com/SciTools/iris/pull/5069 +_LAZYVAR_MIN_BYTES = 5000 + + def _get_cf_var_data(cf_var, filename): - # Get lazy chunked data out of a cf variable. - dtype = _get_actual_dtype(cf_var) - - # Create cube with deferred data, but no metadata - fill_value = getattr( - cf_var.cf_data, - "_FillValue", - _thread_safe_nc.default_fillvals[cf_var.dtype.str[1:]], - ) - proxy = NetCDFDataProxy( - cf_var.shape, dtype, filename, cf_var.cf_name, fill_value - ) - # Get the chunking specified for the variable : this is either a shape, or - # maybe the string "contiguous". - chunks = cf_var.cf_data.chunking() - # In the "contiguous" case, pass chunks=None to 'as_lazy_data'. - if chunks == "contiguous": - chunks = None - return as_lazy_data(proxy, chunks=chunks) + """ + Get an array representing the data of a CF variable. + + This is typically a lazy array based around a NetCDFDataProxy, but if the variable + is "sufficiently small", we instead fetch the data as a real (numpy) array. + The latter is especially valuable for scalar coordinates, which are otherwise + unnecessarily slow + wasteful of memory. + + """ + total_bytes = cf_var.size * cf_var.dtype.itemsize + if total_bytes < _LAZYVAR_MIN_BYTES: + # Don't make a lazy array, as it will cost more memory AND more time to access. + # Instead fetch the data immediately, as a real array, and return that. + result = cf_var[:] + + else: + # Get lazy chunked data out of a cf variable. + dtype = _get_actual_dtype(cf_var) + + # Make a data-proxy that mimics array access and can fetch from the file. + fill_value = getattr( + cf_var.cf_data, + "_FillValue", + _thread_safe_nc.default_fillvals[cf_var.dtype.str[1:]], + ) + proxy = NetCDFDataProxy( + cf_var.shape, dtype, filename, cf_var.cf_name, fill_value + ) + # Get the chunking specified for the variable : this is either a shape, or + # maybe the string "contiguous". + chunks = cf_var.cf_data.chunking() + # In the "contiguous" case, pass chunks=None to 'as_lazy_data'. + if chunks == "contiguous": + chunks = None + + # Return a dask array providing deferred access. + result = as_lazy_data(proxy, chunks=chunks) + + return result class _OrderedAddableList(list): diff --git a/lib/iris/tests/unit/fileformats/netcdf/loader/test__get_cf_var_data.py b/lib/iris/tests/unit/fileformats/netcdf/loader/test__get_cf_var_data.py index 054c8e2db1..7912648786 100644 --- a/lib/iris/tests/unit/fileformats/netcdf/loader/test__get_cf_var_data.py +++ b/lib/iris/tests/unit/fileformats/netcdf/loader/test__get_cf_var_data.py @@ -4,7 +4,6 @@ # See COPYING and COPYING.LESSER in the root of the repository for full # licensing details. """Unit tests for the `iris.fileformats.netcdf._get_cf_var_data` function.""" - # Import iris.tests first so that some things can be initialised before # importing anything else. import iris.tests as tests # isort:skip @@ -25,16 +24,24 @@ def setUp(self): self.shape = (300000, 240, 200) self.expected_chunks = _optimum_chunksize(self.shape, self.shape) - def _make(self, chunksizes): - cf_data = mock.Mock(_FillValue=None) + def _make(self, chunksizes=None, shape=None, dtype="i4"): + cf_data = mock.MagicMock( + _FillValue=None, + __getitem__="", + ) cf_data.chunking = mock.MagicMock(return_value=chunksizes) + if shape is None: + shape = self.shape + dtype = np.dtype(dtype) cf_var = mock.MagicMock( spec=iris.fileformats.cf.CFVariable, - dtype=np.dtype("i4"), + dtype=dtype, cf_data=cf_data, cf_name="DUMMY_VAR", - shape=self.shape, + shape=shape, + size=np.prod(shape), ) + cf_var.__getitem__.return_value = mock.sentinel.real_data_accessed return cf_var def test_cf_data_type(self): @@ -68,6 +75,21 @@ def test_cf_data_contiguous(self): lazy_data_chunks = [c[0] for c in lazy_data.chunks] self.assertArrayEqual(lazy_data_chunks, self.expected_chunks) + def test_type__1kf8_is_lazy(self): + cf_var = self._make(shape=(1000,), dtype="f8") + var_data = _get_cf_var_data(cf_var, self.filename) + self.assertIsInstance(var_data, dask_array) + + def test_arraytype__1ki2_is_real(self): + cf_var = self._make(shape=(1000,), dtype="i2") + var_data = _get_cf_var_data(cf_var, self.filename) + self.assertIs(var_data, mock.sentinel.real_data_accessed) + + def test_arraytype__100f8_is_real(self): + cf_var = self._make(shape=(100,), dtype="f8") + var_data = _get_cf_var_data(cf_var, self.filename) + self.assertIs(var_data, mock.sentinel.real_data_accessed) + if __name__ == "__main__": tests.main() From 5c40db594a1cfb367b245a62d4d487adb6cf81a0 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 3 Apr 2023 18:37:48 +0100 Subject: [PATCH 2/6] Various test fixes. --- .../tests/integration/netcdf/test_general.py | 7 +++- lib/iris/tests/integration/test_cube.py | 10 ++++- .../netcdf/int64_auxiliary_coord_netcdf3.cml | 15 ------- .../netcdf/int64_dimension_coord_netcdf3.cml | 15 ------- .../tests/results/netcdf/save_load_traj.cml | 41 ------------------- .../netcdf/uint32_auxiliary_coord_netcdf3.cml | 15 ------- .../netcdf/uint32_dimension_coord_netcdf3.cml | 15 ------- .../unit/aux_factory/test_AuxCoordFactory.py | 8 +++- .../helpers/test_build_ancil_var.py | 1 + .../test_build_auxiliary_coordinate.py | 34 +++++++++++---- .../helpers/test_build_cell_measure.py | 1 + .../netcdf/loader/test__load_cube.py | 2 + 12 files changed, 52 insertions(+), 112 deletions(-) delete mode 100644 lib/iris/tests/results/netcdf/int64_auxiliary_coord_netcdf3.cml delete mode 100644 lib/iris/tests/results/netcdf/int64_dimension_coord_netcdf3.cml delete mode 100644 lib/iris/tests/results/netcdf/save_load_traj.cml delete mode 100644 lib/iris/tests/results/netcdf/uint32_auxiliary_coord_netcdf3.cml delete mode 100644 lib/iris/tests/results/netcdf/uint32_dimension_coord_netcdf3.cml diff --git a/lib/iris/tests/integration/netcdf/test_general.py b/lib/iris/tests/integration/netcdf/test_general.py index 63b977674d..339a38fd1f 100644 --- a/lib/iris/tests/integration/netcdf/test_general.py +++ b/lib/iris/tests/integration/netcdf/test_general.py @@ -13,6 +13,7 @@ import os.path import shutil import tempfile +from unittest import mock import warnings import numpy as np @@ -34,7 +35,11 @@ def test_lazy_preserved_save(self): fpath = tests.get_data_path( ("NetCDF", "label_and_climate", "small_FC_167_mon_19601101.nc") ) - acube = iris.load_cube(fpath, "air_temperature") + # While loading, "turn off" loading small variables as real data. + with mock.patch( + "iris.fileformats.netcdf.loader._LAZYVAR_MIN_BYTES", 0 + ): + acube = iris.load_cube(fpath, "air_temperature") self.assertTrue(acube.has_lazy_data()) # Also check a coord with lazy points + bounds. self.assertTrue(acube.coord("forecast_period").has_lazy_points()) diff --git a/lib/iris/tests/integration/test_cube.py b/lib/iris/tests/integration/test_cube.py index 996362f594..ad6666d28e 100644 --- a/lib/iris/tests/integration/test_cube.py +++ b/lib/iris/tests/integration/test_cube.py @@ -9,6 +9,8 @@ # importing anything else. import iris.tests as tests # isort:skip +from unittest import mock + import numpy as np import iris @@ -23,7 +25,13 @@ def test_agg_by_aux_coord(self): problem_test_file = tests.get_data_path( ("NetCDF", "testing", "small_theta_colpex.nc") ) - cube = iris.load_cube(problem_test_file, "air_potential_temperature") + # While loading, "turn off" loading small variables as real data. + with mock.patch( + "iris.fileformats.netcdf.loader._LAZYVAR_MIN_BYTES", 0 + ): + cube = iris.load_cube( + problem_test_file, "air_potential_temperature" + ) # Test aggregating by aux coord, notably the `forecast_period` aux # coord on `cube`, whose `_points` attribute is a lazy array. diff --git a/lib/iris/tests/results/netcdf/int64_auxiliary_coord_netcdf3.cml b/lib/iris/tests/results/netcdf/int64_auxiliary_coord_netcdf3.cml deleted file mode 100644 index e48cf41d2a..0000000000 --- a/lib/iris/tests/results/netcdf/int64_auxiliary_coord_netcdf3.cml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/lib/iris/tests/results/netcdf/int64_dimension_coord_netcdf3.cml b/lib/iris/tests/results/netcdf/int64_dimension_coord_netcdf3.cml deleted file mode 100644 index 78fec459e9..0000000000 --- a/lib/iris/tests/results/netcdf/int64_dimension_coord_netcdf3.cml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/lib/iris/tests/results/netcdf/save_load_traj.cml b/lib/iris/tests/results/netcdf/save_load_traj.cml deleted file mode 100644 index 9b225d127f..0000000000 --- a/lib/iris/tests/results/netcdf/save_load_traj.cml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/lib/iris/tests/results/netcdf/uint32_auxiliary_coord_netcdf3.cml b/lib/iris/tests/results/netcdf/uint32_auxiliary_coord_netcdf3.cml deleted file mode 100644 index e48cf41d2a..0000000000 --- a/lib/iris/tests/results/netcdf/uint32_auxiliary_coord_netcdf3.cml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/lib/iris/tests/results/netcdf/uint32_dimension_coord_netcdf3.cml b/lib/iris/tests/results/netcdf/uint32_dimension_coord_netcdf3.cml deleted file mode 100644 index 78fec459e9..0000000000 --- a/lib/iris/tests/results/netcdf/uint32_dimension_coord_netcdf3.cml +++ /dev/null @@ -1,15 +0,0 @@ - - - - - - - - - - - - - - - diff --git a/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py b/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py index 3375f63bf2..f8bd54093f 100644 --- a/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py +++ b/lib/iris/tests/unit/aux_factory/test_AuxCoordFactory.py @@ -12,6 +12,8 @@ # importing anything else. import iris.tests as tests # isort:skip +from unittest import mock + import numpy as np import iris @@ -143,7 +145,11 @@ def setUp(self): path = tests.get_data_path( ["NetCDF", "testing", "small_theta_colpex.nc"] ) - self.cube = iris.load_cube(path, "air_potential_temperature") + # While loading, "turn off" loading small variables as real data. + with mock.patch( + "iris.fileformats.netcdf.loader._LAZYVAR_MIN_BYTES", 0 + ): + self.cube = iris.load_cube(path, "air_potential_temperature") def _check_lazy(self): coords = self.cube.aux_coords + self.cube.derived_coords diff --git a/lib/iris/tests/unit/fileformats/nc_load_rules/helpers/test_build_ancil_var.py b/lib/iris/tests/unit/fileformats/nc_load_rules/helpers/test_build_ancil_var.py index b057a41a3e..36ef025bfa 100644 --- a/lib/iris/tests/unit/fileformats/nc_load_rules/helpers/test_build_ancil_var.py +++ b/lib/iris/tests/unit/fileformats/nc_load_rules/helpers/test_build_ancil_var.py @@ -40,6 +40,7 @@ def mock_cf_av_var(monkeypatch): long_name="wibble", units="m2", shape=data.shape, + size=np.prod(data.shape), dtype=data.dtype, __getitem__=lambda self, key: data[key], ) diff --git a/lib/iris/tests/unit/fileformats/nc_load_rules/helpers/test_build_auxiliary_coordinate.py b/lib/iris/tests/unit/fileformats/nc_load_rules/helpers/test_build_auxiliary_coordinate.py index 13622b72e2..26f0c6e945 100644 --- a/lib/iris/tests/unit/fileformats/nc_load_rules/helpers/test_build_auxiliary_coordinate.py +++ b/lib/iris/tests/unit/fileformats/nc_load_rules/helpers/test_build_auxiliary_coordinate.py @@ -8,11 +8,11 @@ build_auxilliary_coordinate`. """ - # import iris tests first so that some things can be initialised before # importing anything else import iris.tests as tests # isort:skip +import contextlib from unittest import mock import numpy as np @@ -48,6 +48,7 @@ def setUp(self): long_name="wibble", units="m", shape=points.shape, + size=np.prod(points.shape), dtype=points.dtype, __getitem__=lambda self, key: points[key], ) @@ -111,6 +112,7 @@ def _make_cf_bounds_var(self, dimension_names): cf_name="wibble_bnds", cf_data=cf_data, shape=bounds.shape, + size=np.prod(bounds.shape), dtype=bounds.dtype, __getitem__=lambda self, key: bounds[key], ) @@ -165,6 +167,7 @@ def setUp(self): long_name="wibble", units="m", shape=points.shape, + size=np.prod(points.shape), dtype=points.dtype, __getitem__=lambda self, key: points[key], ) @@ -176,21 +179,34 @@ def setUp(self): cube_parts=dict(coordinates=[]), ) + # self.deferred_load_patch = mock.patch( + # "iris.fileformats.netcdf.NetCDFDataProxy.__getitem__", + # new=patched__getitem__, + # ) + + @contextlib.contextmanager + def deferred_load_patch(self): def patched__getitem__(proxy_self, keys): if proxy_self.variable_name == self.cf_coord_var.cf_name: return self.cf_coord_var[keys] raise RuntimeError() - self.deferred_load_patch = mock.patch( - "iris.fileformats.netcdf.NetCDFDataProxy.__getitem__", - new=patched__getitem__, - ) + # Fix for deferred load, and prevent loading small variables as real arrays. + with ( + mock.patch( + "iris.fileformats.netcdf.NetCDFDataProxy.__getitem__", + new=patched__getitem__, + ), + # While loading, "turn off" loading small variables as real data. + mock.patch("iris.fileformats.netcdf.loader._LAZYVAR_MIN_BYTES", 0), + ): + yield def test_scale_factor_add_offset_int(self): self.cf_coord_var.scale_factor = 3 self.cf_coord_var.add_offset = 5 - with self.deferred_load_patch: + with self.deferred_load_patch(): build_auxiliary_coordinate(self.engine, self.cf_coord_var) coord, _ = self.engine.cube_parts["coordinates"][0] @@ -199,7 +215,7 @@ def test_scale_factor_add_offset_int(self): def test_scale_factor_float(self): self.cf_coord_var.scale_factor = 3.0 - with self.deferred_load_patch: + with self.deferred_load_patch(): build_auxiliary_coordinate(self.engine, self.cf_coord_var) coord, _ = self.engine.cube_parts["coordinates"][0] @@ -208,7 +224,7 @@ def test_scale_factor_float(self): def test_add_offset_float(self): self.cf_coord_var.add_offset = 5.0 - with self.deferred_load_patch: + with self.deferred_load_patch(): build_auxiliary_coordinate(self.engine, self.cf_coord_var) coord, _ = self.engine.cube_parts["coordinates"][0] @@ -239,6 +255,7 @@ def setUp(self): units="days since 1970-01-01", calendar=None, shape=points.shape, + size=np.prod(points.shape), dtype=points.dtype, __getitem__=lambda self, key: points[key], ) @@ -251,6 +268,7 @@ def setUp(self): cf_name="wibble_bnds", cf_data=mock.MagicMock(chunking=mock.Mock(return_value=None)), shape=bounds.shape, + size=np.prod(bounds.shape), dtype=bounds.dtype, __getitem__=lambda self, key: bounds[key], ) diff --git a/lib/iris/tests/unit/fileformats/nc_load_rules/helpers/test_build_cell_measure.py b/lib/iris/tests/unit/fileformats/nc_load_rules/helpers/test_build_cell_measure.py index efbb0649c9..3fe64056bb 100644 --- a/lib/iris/tests/unit/fileformats/nc_load_rules/helpers/test_build_cell_measure.py +++ b/lib/iris/tests/unit/fileformats/nc_load_rules/helpers/test_build_cell_measure.py @@ -40,6 +40,7 @@ def mock_cf_cm_var(monkeypatch): long_name="wibble", units="m2", shape=data.shape, + size=np.prod(data.shape), dtype=data.dtype, __getitem__=lambda self, key: data[key], cf_measure="area", diff --git a/lib/iris/tests/unit/fileformats/netcdf/loader/test__load_cube.py b/lib/iris/tests/unit/fileformats/netcdf/loader/test__load_cube.py index 6e28a2f8e4..b67c546aa0 100644 --- a/lib/iris/tests/unit/fileformats/netcdf/loader/test__load_cube.py +++ b/lib/iris/tests/unit/fileformats/netcdf/loader/test__load_cube.py @@ -60,6 +60,7 @@ def _make(self, names, attrs): cf_name="DUMMY_VAR", cf_group=coords, shape=shape, + size=np.prod(shape), ) return cf, cf_var @@ -139,6 +140,7 @@ def _make(self, attrs): cf_group=mock.Mock(), cf_attrs_unused=cf_attrs_unused, shape=shape, + size=np.prod(shape), ) return cf_var From 7e8ef44a30d3ea52e82cf202c948e4af62197338 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 3 Apr 2023 18:49:56 +0100 Subject: [PATCH 3/6] More test fixing. --- .../netcdf/int64_auxiliary_coord_netcdf3.cml | 15 +++++++ .../netcdf/int64_dimension_coord_netcdf3.cml | 15 +++++++ .../tests/results/netcdf/save_load_traj.cml | 41 +++++++++++++++++++ .../netcdf/uint32_auxiliary_coord_netcdf3.cml | 15 +++++++ .../netcdf/uint32_dimension_coord_netcdf3.cml | 15 +++++++ .../test_build_auxiliary_coordinate.py | 23 ++++------- 6 files changed, 110 insertions(+), 14 deletions(-) create mode 100644 lib/iris/tests/results/netcdf/int64_auxiliary_coord_netcdf3.cml create mode 100644 lib/iris/tests/results/netcdf/int64_dimension_coord_netcdf3.cml create mode 100644 lib/iris/tests/results/netcdf/save_load_traj.cml create mode 100644 lib/iris/tests/results/netcdf/uint32_auxiliary_coord_netcdf3.cml create mode 100644 lib/iris/tests/results/netcdf/uint32_dimension_coord_netcdf3.cml diff --git a/lib/iris/tests/results/netcdf/int64_auxiliary_coord_netcdf3.cml b/lib/iris/tests/results/netcdf/int64_auxiliary_coord_netcdf3.cml new file mode 100644 index 0000000000..616b338a25 --- /dev/null +++ b/lib/iris/tests/results/netcdf/int64_auxiliary_coord_netcdf3.cml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/netcdf/int64_dimension_coord_netcdf3.cml b/lib/iris/tests/results/netcdf/int64_dimension_coord_netcdf3.cml new file mode 100644 index 0000000000..2a604d90fc --- /dev/null +++ b/lib/iris/tests/results/netcdf/int64_dimension_coord_netcdf3.cml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/netcdf/save_load_traj.cml b/lib/iris/tests/results/netcdf/save_load_traj.cml new file mode 100644 index 0000000000..c14ff4798f --- /dev/null +++ b/lib/iris/tests/results/netcdf/save_load_traj.cml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/netcdf/uint32_auxiliary_coord_netcdf3.cml b/lib/iris/tests/results/netcdf/uint32_auxiliary_coord_netcdf3.cml new file mode 100644 index 0000000000..616b338a25 --- /dev/null +++ b/lib/iris/tests/results/netcdf/uint32_auxiliary_coord_netcdf3.cml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/results/netcdf/uint32_dimension_coord_netcdf3.cml b/lib/iris/tests/results/netcdf/uint32_dimension_coord_netcdf3.cml new file mode 100644 index 0000000000..2a604d90fc --- /dev/null +++ b/lib/iris/tests/results/netcdf/uint32_dimension_coord_netcdf3.cml @@ -0,0 +1,15 @@ + + + + + + + + + + + + + + + diff --git a/lib/iris/tests/unit/fileformats/nc_load_rules/helpers/test_build_auxiliary_coordinate.py b/lib/iris/tests/unit/fileformats/nc_load_rules/helpers/test_build_auxiliary_coordinate.py index 26f0c6e945..a0e105b795 100644 --- a/lib/iris/tests/unit/fileformats/nc_load_rules/helpers/test_build_auxiliary_coordinate.py +++ b/lib/iris/tests/unit/fileformats/nc_load_rules/helpers/test_build_auxiliary_coordinate.py @@ -179,11 +179,6 @@ def setUp(self): cube_parts=dict(coordinates=[]), ) - # self.deferred_load_patch = mock.patch( - # "iris.fileformats.netcdf.NetCDFDataProxy.__getitem__", - # new=patched__getitem__, - # ) - @contextlib.contextmanager def deferred_load_patch(self): def patched__getitem__(proxy_self, keys): @@ -191,16 +186,16 @@ def patched__getitem__(proxy_self, keys): return self.cf_coord_var[keys] raise RuntimeError() - # Fix for deferred load, and prevent loading small variables as real arrays. - with ( - mock.patch( - "iris.fileformats.netcdf.NetCDFDataProxy.__getitem__", - new=patched__getitem__, - ), - # While loading, "turn off" loading small variables as real data. - mock.patch("iris.fileformats.netcdf.loader._LAZYVAR_MIN_BYTES", 0), + # Fix for deferred load, *AND* avoid loading small variable data in real arrays. + with mock.patch( + "iris.fileformats.netcdf.NetCDFDataProxy.__getitem__", + new=patched__getitem__, ): - yield + # While loading, "turn off" loading small variables as real data. + with mock.patch( + "iris.fileformats.netcdf.loader._LAZYVAR_MIN_BYTES", 0 + ): + yield def test_scale_factor_add_offset_int(self): self.cf_coord_var.scale_factor = 3 From 8a18243f6dc90773cd8b97eaf51add90ad42c732 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Tue, 4 Apr 2023 10:18:59 +0100 Subject: [PATCH 4/6] Fix printout in Mesh documentation. --- docs/src/further_topics/ugrid/operations.rst | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/src/further_topics/ugrid/operations.rst b/docs/src/further_topics/ugrid/operations.rst index a4e0e593d7..e179942b1e 100644 --- a/docs/src/further_topics/ugrid/operations.rst +++ b/docs/src/further_topics/ugrid/operations.rst @@ -430,20 +430,20 @@ creating any associated :class:`~iris.cube.Cube`\s: node node_dimension: 'Mesh2d_node' node coordinates - shape(5,)> - shape(5,)> + + edge edge_dimension: 'Mesh2d_edge' - edge_node_connectivity: shape(6, 2)> + edge_node_connectivity: edge coordinates - shape(6,)> - shape(6,)> + + face face_dimension: 'Mesh2d_face' - face_node_connectivity: shape(2, 4)> + face_node_connectivity: face coordinates - shape(2,)> - shape(2,)> + + long_name: 'my_mesh' var_name: 'my_mesh' From 79d16d1d5986cef9988210ae312ba95150698b6e Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Tue, 4 Apr 2023 10:57:34 +0100 Subject: [PATCH 5/6] Whatsnew + doctests fix. --- docs/src/userguide/real_and_lazy_data.rst | 15 +++++++-------- docs/src/whatsnew/latest.rst | 4 +++- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/src/userguide/real_and_lazy_data.rst b/docs/src/userguide/real_and_lazy_data.rst index 9d66a2f086..38f06aab99 100644 --- a/docs/src/userguide/real_and_lazy_data.rst +++ b/docs/src/userguide/real_and_lazy_data.rst @@ -188,17 +188,17 @@ coordinates' lazy points and bounds: .. doctest:: - >>> cube = iris.load_cube(iris.sample_data_path('hybrid_height.nc'), 'air_potential_temperature') + >>> cube = iris.load_cube(iris.sample_data_path('orca2_votemper.nc'),'votemper') - >>> dim_coord = cube.coord('model_level_number') + >>> dim_coord = cube.coord('depth') >>> print(dim_coord.has_lazy_points()) False >>> print(dim_coord.has_bounds()) - False + True >>> print(dim_coord.has_lazy_bounds()) False - >>> aux_coord = cube.coord('sigma') + >>> aux_coord = cube.coord('longitude') >>> print(aux_coord.has_lazy_points()) True >>> print(aux_coord.has_bounds()) @@ -213,7 +213,9 @@ coordinates' lazy points and bounds: >>> print(aux_coord.has_lazy_bounds()) True - >>> derived_coord = cube.coord('altitude') + # Fetch a derived coordinate, from a different file: These can also have lazy data. + >>> cube2 = iris.load_cube(iris.sample_data_path('hybrid_height.nc'), 'air_potential_temperature') + >>> derived_coord = cube2.coord('altitude') >>> print(derived_coord.has_lazy_points()) True >>> print(derived_coord.has_bounds()) @@ -221,9 +223,6 @@ coordinates' lazy points and bounds: >>> print(derived_coord.has_lazy_bounds()) True -.. note:: - Printing a lazy :class:`~iris.coords.AuxCoord` will realise its points and bounds arrays! - Dask Processing Options ----------------------- diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index 51adfb9d8d..b8aac0fbc9 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -78,7 +78,9 @@ This document explains the changes made to Iris for this release 🚀 Performance Enhancements =========================== -#. N/A +#. `@pp-mo`_ changed the netCDF loader to fetch data immediately from small netCDF + variables, instead of creating a dask array. + This saves both time and memory. (:pull:`5229`) 🔥 Deprecations From 3dabecae5f10874fc7b460a2b4e290f0155e0a36 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Tue, 4 Apr 2023 11:14:13 +0100 Subject: [PATCH 6/6] Tweak whatsnew. --- docs/src/whatsnew/latest.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index b8aac0fbc9..d13d24a558 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -79,8 +79,9 @@ This document explains the changes made to Iris for this release =========================== #. `@pp-mo`_ changed the netCDF loader to fetch data immediately from small netCDF - variables, instead of creating a dask array. - This saves both time and memory. (:pull:`5229`) + variables, instead of creating a dask array: This saves both time and memory. + Note that some cubes, coordinates etc loaded from netCDF will now have real data + where previously it was lazy. (:pull:`5229`) 🔥 Deprecations