From f16409fd378ca56a2bae125274af8430001deb6e Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Thu, 20 Feb 2025 10:58:45 +0000 Subject: [PATCH 01/27] Remove unused 'unique' option from load collections and combine methods. --- lib/iris/_combine.py | 10 ++++------ lib/iris/loading.py | 22 +++++----------------- lib/iris/tests/unit/test_combine_cubes.py | 2 +- 3 files changed, 10 insertions(+), 24 deletions(-) diff --git a/lib/iris/_combine.py b/lib/iris/_combine.py index 7b01dfc87e..b401f82638 100644 --- a/lib/iris/_combine.py +++ b/lib/iris/_combine.py @@ -246,7 +246,7 @@ def context(self, settings=None, **kwargs): self.set(saved_settings) -def _combine_cubes(cubes, options, merge_require_unique): +def _combine_cubes(cubes, options): """Combine cubes as for load, according to "loading policy" options. Applies :meth:`~iris.cube.CubeList.merge`/:meth:`~iris.cube.CubeList.concatenate` @@ -258,8 +258,6 @@ def _combine_cubes(cubes, options, merge_require_unique): A list of cubes to combine. options : dict Settings, as described for :class:`iris.CombineOptions`. - merge_require_unique : bool - Value for the 'unique' keyword in any merge operations. Returns ------- @@ -290,7 +288,7 @@ def _combine_cubes(cubes, options, merge_require_unique): cubes = cubes.concatenate() if "m" in sequence: # merge if requested - cubes = cubes.merge(unique=merge_require_unique) + cubes = cubes.merge() if sequence[-1] == "c": # concat if it comes last cubes = cubes.concatenate() @@ -302,7 +300,7 @@ def _combine_cubes(cubes, options, merge_require_unique): return cubes -def _combine_load_cubes(cubes, merge_require_unique=False): +def _combine_load_cubes(cubes): # A special version to call _combine_cubes while also implementing the # _MULTIREF_DETECTION behaviour from iris import LOAD_POLICY @@ -318,4 +316,4 @@ def _combine_load_cubes(cubes, merge_require_unique=False): if _MULTIREF_DETECTION.found_multiple_refs: options["merge_concat_sequence"] += "c" - return _combine_cubes(cubes, options, merge_require_unique=merge_require_unique) + return _combine_cubes(cubes, options) diff --git a/lib/iris/loading.py b/lib/iris/loading.py index f68cb7d9e5..39eb438447 100644 --- a/lib/iris/loading.py +++ b/lib/iris/loading.py @@ -58,23 +58,17 @@ def add(self, cube): if sub_cube is not None: self.cubes.append(sub_cube) - def combined(self, unique=False): + def combined(self): """Return a new :class:`_CubeFilter` by combining the list of cubes. Combines the list of cubes with :func:`~iris._combine_load_cubes`. - Parameters - ---------- - unique : bool, default=False - If True, raises `iris.exceptions.DuplicateDataError` if - duplicate cubes are detected. - """ from iris._combine import _combine_load_cubes return _CubeFilter( self.constraint, - _combine_load_cubes(self.cubes, merge_require_unique=unique), + _combine_load_cubes(self.cubes), ) @@ -110,19 +104,13 @@ def cubes(self): result.extend(pair.cubes) return result - def combined(self, unique=False): + def combined(self): """Return a new :class:`_CubeFilterCollection` by combining all the cube lists of this collection. Combines each list of cubes using :func:`~iris._combine_load_cubes`. - Parameters - ---------- - unique : bool, default=False - If True, raises `iris.exceptions.DuplicateDataError` if - duplicate cubes are detected. - """ - return _CubeFilterCollection([pair.combined(unique) for pair in self.pairs]) + return _CubeFilterCollection([pair.combined() for pair in self.pairs]) def _load_collection(uris, constraints=None, callback=None): @@ -203,7 +191,7 @@ def load_cube(uris, constraint=None, callback=None): if len(constraints) != 1: raise ValueError("only a single constraint is allowed") - cubes = _load_collection(uris, constraints, callback).combined(unique=False).cubes() + cubes = _load_collection(uris, constraints, callback).combined().cubes() try: # NOTE: this call currently retained to preserve the legacy exceptions diff --git a/lib/iris/tests/unit/test_combine_cubes.py b/lib/iris/tests/unit/test_combine_cubes.py index a60831ed4c..107b7d4a4f 100644 --- a/lib/iris/tests/unit/test_combine_cubes.py +++ b/lib/iris/tests/unit/test_combine_cubes.py @@ -28,7 +28,7 @@ def options(request): def combine_cubes(cubes, settings_name="default", **kwargs): options = LoadPolicy.SETTINGS[settings_name] options.update(kwargs) - return _combine_cubes(cubes, options, merge_require_unique=False) + return _combine_cubes(cubes, options) class Test: From 2ff233c4ba8f30b7fa8ab295517a8c609f80de29 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Thu, 20 Feb 2025 15:04:25 +0000 Subject: [PATCH 02/27] Reinstate 'unique=False' in combine merges to fix load(). --- lib/iris/_combine.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/iris/_combine.py b/lib/iris/_combine.py index b401f82638..b39599002f 100644 --- a/lib/iris/_combine.py +++ b/lib/iris/_combine.py @@ -288,7 +288,9 @@ def _combine_cubes(cubes, options): cubes = cubes.concatenate() if "m" in sequence: # merge if requested - cubes = cubes.merge() + # NOTE: this needs "unique=False" to make "iris.load()" work correctly. + # TODO: make configurable via options. + cubes = cubes.merge(unique=False) if sequence[-1] == "c": # concat if it comes last cubes = cubes.concatenate() From 713ed1b39e9c3ee103c6fada79fc2ed28d2a3ff0 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 24 Feb 2025 12:17:37 +0000 Subject: [PATCH 03/27] Fix typing for combine submodule. --- lib/iris/_combine.py | 53 ++++++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 21 deletions(-) diff --git a/lib/iris/_combine.py b/lib/iris/_combine.py index b39599002f..ff92930fc1 100644 --- a/lib/iris/_combine.py +++ b/lib/iris/_combine.py @@ -13,7 +13,11 @@ import contextlib import threading -from typing import Mapping +import typing +from typing import List + +if typing.TYPE_CHECKING: + from iris.cube import Cube, CubeList class CombineOptions(threading.local): @@ -97,6 +101,7 @@ class CombineOptions(threading.local): """ # Useful constants + #: Valid option names OPTION_KEYS = ( "support_multiple_references", "merge_concat_sequence", @@ -107,6 +112,7 @@ class CombineOptions(threading.local): "merge_concat_sequence": ("", "m", "c", "mc", "cm"), "repeat_until_unchanged": (False, True), } + #: Settings content SETTINGS = { "legacy": dict( support_multiple_references=False, @@ -129,6 +135,8 @@ class CombineOptions(threading.local): repeat_until_unchanged=True, ), } + #: Valid settings names + SETTINGS_NAMES = list(SETTINGS.keys()) def __init__(self, options: str | dict | None = None, **kwargs): """Create loading strategy control object.""" @@ -168,28 +176,31 @@ def set(self, options: str | dict | None = None, **kwargs): """ if options is None: - options = {} + options_dict = {} elif isinstance(options, str) and options in self.SETTINGS: - options = self.SETTINGS[options] - elif not isinstance(options, Mapping): + options_dict = self.SETTINGS[options] + elif isinstance(options, dict): + options_dict = options + else: msg = ( - f"Invalid arg options={options!r} : " - f"must be a dict, or one of {tuple(self.SETTINGS.keys())}" + f"'options' arg has unexpected type {type(options)!r}, " + "expected (None | str | dict)." ) - raise TypeError(msg) + raise ValueError(msg) # Override any options with keywords - options.update(**kwargs) - bad_keys = [key for key in options if key not in self.OPTION_KEYS] + options_dict = options_dict.copy() # do not modify source (!) + options_dict.update(**kwargs) + bad_keys = [key for key in options_dict if key not in self.OPTION_KEYS] if bad_keys: msg = f"Unknown options {bad_keys} : valid options are {self.OPTION_KEYS}." raise ValueError(msg) # Implement all options by changing own content. - for key, value in options.items(): + for key, value in options_dict.items(): setattr(self, key, value) - def settings(self): + def settings(self) -> dict: """Return an options dict containing the current settings.""" return {key: getattr(self, key) for key in self.OPTION_KEYS} @@ -200,7 +211,7 @@ def __repr__(self): return msg @contextlib.contextmanager - def context(self, settings=None, **kwargs): + def context(self, settings: str | dict | None = None, **kwargs): """Return a context manager applying given options. Parameters @@ -246,7 +257,7 @@ def context(self, settings=None, **kwargs): self.set(saved_settings) -def _combine_cubes(cubes, options): +def _combine_cubes(cubes: List[Cube], options: dict) -> CubeList: """Combine cubes as for load, according to "loading policy" options. Applies :meth:`~iris.cube.CubeList.merge`/:meth:`~iris.cube.CubeList.concatenate` @@ -257,7 +268,7 @@ def _combine_cubes(cubes, options): cubes : list of :class:`~iris.cube.Cube` A list of cubes to combine. options : dict - Settings, as described for :class:`iris.CombineOptions`. + Dictionary of settings options, as described for :class:`iris.CombineOptions`. Returns ------- @@ -277,7 +288,7 @@ def _combine_cubes(cubes, options): from iris.cube import CubeList if not isinstance(cubes, CubeList): - cubes = CubeList(cubes) + cubelist = CubeList(cubes) while True: n_original_cubes = len(cubes) @@ -285,24 +296,24 @@ def _combine_cubes(cubes, options): if sequence[0] == "c": # concat if it comes first - cubes = cubes.concatenate() + cubelist = cubelist.concatenate() if "m" in sequence: # merge if requested # NOTE: this needs "unique=False" to make "iris.load()" work correctly. # TODO: make configurable via options. - cubes = cubes.merge(unique=False) + cubelist = cubelist.merge(unique=False) if sequence[-1] == "c": # concat if it comes last - cubes = cubes.concatenate() + cubelist = cubelist.concatenate() # Repeat if requested, *and* this step reduced the number of cubes - if not options["repeat_until_unchanged"] or len(cubes) >= n_original_cubes: + if not options["repeat_until_unchanged"] or len(cubelist) >= n_original_cubes: break - return cubes + return cubelist -def _combine_load_cubes(cubes): +def _combine_load_cubes(cubes: List[Cube]) -> CubeList: # A special version to call _combine_cubes while also implementing the # _MULTIREF_DETECTION behaviour from iris import LOAD_POLICY From 797f698e9e3f28c0655f0ca1d59701a0cc66ca4e Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 24 Feb 2025 12:51:21 +0000 Subject: [PATCH 04/27] More typing and other small improvements to combine code. --- lib/iris/_combine.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/iris/_combine.py b/lib/iris/_combine.py index ff92930fc1..bba25fe58c 100644 --- a/lib/iris/_combine.py +++ b/lib/iris/_combine.py @@ -287,12 +287,14 @@ def _combine_cubes(cubes: List[Cube], options: dict) -> CubeList: """ from iris.cube import CubeList - if not isinstance(cubes, CubeList): + if isinstance(cubes, CubeList): + cubelist = cubes + else: cubelist = CubeList(cubes) + sequence = options["merge_concat_sequence"] while True: - n_original_cubes = len(cubes) - sequence = options["merge_concat_sequence"] + n_original_cubes = len(cubelist) if sequence[0] == "c": # concat if it comes first From 97e3bd1c0ac2198a215210878cf58c89023458dc Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 24 Feb 2025 15:10:25 +0000 Subject: [PATCH 05/27] Move loading-specific parts from CombineOptions to LoadPolicy. --- lib/iris/_combine.py | 125 +++++++++++-------------------------------- lib/iris/loading.py | 90 +++++++++++++++++++++++++++++-- 2 files changed, 117 insertions(+), 98 deletions(-) diff --git a/lib/iris/_combine.py b/lib/iris/_combine.py index bba25fe58c..2cfc5fe80b 100644 --- a/lib/iris/_combine.py +++ b/lib/iris/_combine.py @@ -11,57 +11,44 @@ publicly available. """ -import contextlib +from __future__ import annotations + import threading -import typing -from typing import List +from typing import TYPE_CHECKING, List -if typing.TYPE_CHECKING: +if TYPE_CHECKING: from iris.cube import Cube, CubeList class CombineOptions(threading.local): """A container for cube combination options. - Controls for generalised merge/concatenate options. - - Also controls the detection and handling of cases where a hybrid coordinate - uses multiple reference fields during loading : for example, a UM file which - contains a series of fields describing time-varying orography. - - Options can be set directly, or via :meth:`~iris.LoadPolicy.set`, or changed for - the scope of a code block with :meth:`~iris.LoadPolicy.context`. - - .. note :: + Controls for generalised merge/concatenate options. These are used as controls for + both the :func:`iris.util.combine_cubes` utility method and the core Iris loading + functions : see also :data:`iris.loading.LoadPolicy`. - The default behaviour will "fix" loading for cases like the time-varying - orography case described above. However, this is not strictly - backwards-compatible. If this causes problems, you can force identical loading - behaviour to earlier Iris versions with ``LOAD_POLICY.set("legacy")`` or - equivalent. - - .. testsetup:: - - from iris import LOAD_POLICY + It specifies a number of possible operations which may be applied to a list of + cubes, in a definite sequence, all of which tend to combine cubes into a smaller + number of larger or higher-dimensional cubes. Notes ----- The individual configurable options are : - * ``support_multiple_references`` = True / False - When enabled, the presence of multiple aux-factory reference cubes, which merge - to define a extra dimension, will add that dimension to the loaded cubes. - This is essential for correct support of time-dependent hybrid coordinates (i.e. - aux factories) when loading from fields-based data (e.g. PP or GRIB). - For example (notably) time-dependent orography in UM data on hybrid-heights. + * ``merge_concat_sequence`` = "m" / "c" / "cm" / "mc" + Specifies whether to apply :meth:`~iris.cube.CubeList.merge`, or + :meth:`~iris.cube.CubeList.concatenate` operations, or both, in either order. - In addition, when such multiple references are detected, an extra concatenate - step is added to the 'merge_concat_sequence' (see below), if none is already - configured there. + * ``merge_uses_unique`` = True / False + When True, any merge operation will error if its result contains multiple + identical cubes. Otherwise (unique=False), that is a permitted result. - * ``merge_concat_sequence`` = "m" / "c" / "cm" / "mc" - Specifies whether to merge, or concatenate, or both in either order. - This is the "combine" operation which is applied to loaded data. + .. Note:: + + By default, in a normal :meth:`~iris.cube.CubeList.merge` operation on a + :class:`~iris.cube.CubeList`, ``unique`` defaults to ``True``. + For loading operations, however, the default is ``unique=False``, as this + produces the intended behaviour when loading with multiple constraints. * ``repeat_until_unchanged`` = True / False When enabled, the configured "combine" operation will be repeated until the @@ -102,35 +89,29 @@ class CombineOptions(threading.local): # Useful constants #: Valid option names - OPTION_KEYS = ( - "support_multiple_references", + OPTION_KEYS = [ "merge_concat_sequence", "repeat_until_unchanged", - ) + ] # this is a list, so we can update it in an inheriting class _OPTIONS_ALLOWED_VALUES = { - "support_multiple_references": (False, True), "merge_concat_sequence": ("", "m", "c", "mc", "cm"), "repeat_until_unchanged": (False, True), } #: Settings content SETTINGS = { "legacy": dict( - support_multiple_references=False, merge_concat_sequence="m", repeat_until_unchanged=False, ), "default": dict( - support_multiple_references=True, merge_concat_sequence="m", repeat_until_unchanged=False, ), "recommended": dict( - support_multiple_references=True, merge_concat_sequence="mc", repeat_until_unchanged=False, ), "comprehensive": dict( - support_multiple_references=True, merge_concat_sequence="mc", repeat_until_unchanged=True, ), @@ -163,8 +144,8 @@ def set(self, options: str | dict | None = None, **kwargs): Parameters ---------- * options : str or dict, optional - A dictionary of options values, or the name of one of the - :data:`~iris.LoadPolicy.SETTINGS` standard option sets, + A dictionary of options values, or one of the + :data:`~iris.LoadPolicy.SETTINGS_NAMES` standard settings names, e.g. "legacy" or "comprehensive". * kwargs : dict Individual option settings, from :data:`~iris.LoadPolicy.OPTION_KEYS`. @@ -183,13 +164,13 @@ def set(self, options: str | dict | None = None, **kwargs): options_dict = options else: msg = ( - f"'options' arg has unexpected type {type(options)!r}, " - "expected (None | str | dict)." + f"arg `options` has unexpected type {type(options)!r}, " + f"expected one of (None | str | dcit)." ) - raise ValueError(msg) + raise TypeError(msg) # Override any options with keywords - options_dict = options_dict.copy() # do not modify source (!) + options_dict = options_dict.copy() # don't modify original options_dict.update(**kwargs) bad_keys = [key for key in options_dict if key not in self.OPTION_KEYS] if bad_keys: @@ -210,52 +191,6 @@ def __repr__(self): msg += ")" return msg - @contextlib.contextmanager - def context(self, settings: str | dict | None = None, **kwargs): - """Return a context manager applying given options. - - Parameters - ---------- - settings : str or dict - Options dictionary or name, as for :meth:`~LoadPolicy.set`. - kwargs : dict - Option values, as for :meth:`~LoadPolicy.set`. - - Examples - -------- - .. testsetup:: - - import iris - from iris import LOAD_POLICY, sample_data_path - - >>> # Show how a CombineOptions acts in the context of a load operation - >>> # (N.B. the LOAD_POLICY actually *is* a CombineOptions type object) - >>> path = sample_data_path("time_varying_hybrid_height", "*.pp") - >>> # "legacy" load behaviour allows merge but not concatenate - >>> with LOAD_POLICY.context("legacy"): - ... cubes = iris.load(path, "x_wind") - >>> print(cubes) - 0: x_wind / (m s-1) (time: 2; model_level_number: 5; latitude: 144; longitude: 192) - 1: x_wind / (m s-1) (time: 12; model_level_number: 5; latitude: 144; longitude: 192) - 2: x_wind / (m s-1) (model_level_number: 5; latitude: 144; longitude: 192) - >>> - >>> # "recommended" behaviour enables concatenation - >>> with LOAD_POLICY.context("recommended"): - ... cubes = iris.load(path, "x_wind") - >>> print(cubes) - 0: x_wind / (m s-1) (model_level_number: 5; time: 15; latitude: 144; longitude: 192) - """ - # Save the current state - saved_settings = self.settings() - - # Apply the new options and execute the context - try: - self.set(settings, **kwargs) - yield - finally: - # Re-establish the former state - self.set(saved_settings) - def _combine_cubes(cubes: List[Cube], options: dict) -> CubeList: """Combine cubes as for load, according to "loading policy" options. diff --git a/lib/iris/loading.py b/lib/iris/loading.py index 39eb438447..eaa86c1a7b 100644 --- a/lib/iris/loading.py +++ b/lib/iris/loading.py @@ -4,6 +4,7 @@ # See LICENSE in the root of the repository for full licensing details. """Iris general file loading mechanism.""" +import contextlib import itertools from typing import Iterable @@ -284,7 +285,32 @@ def load_raw(uris, constraints=None, callback=None): class LoadPolicy(CombineOptions): """A control object for Iris loading options. - Incorporates all the settings of a :class:`~iris.CombineOptions`. + Incorporates all the settings of a :class:`~iris.CombineOptions`, and adds an + additional ``support_multiple_references`` option. + + Also adds :meth:`~iris.LoadPolicy.context`, allowing loading behaviours to be + modified for the duration of a code block. + + In addition to controlling the "combine" operations during loading, LoadPolicy also + provides a ``support_multiple_references`` option to manage the detection and + handling of cases where a hybrid coordinate has multiple reference fields : for + example, a UM file which contains a series of fields describing a time-varying + orography. + + The ``support_multiple_references`` option takes the value ``True`` or ``False`` to + enable or disable this. The default is ``True``. + + .. note :: + + The default behaviour will "fix" loading for cases like the time-varying + orography case described above. However, this is not strictly + backwards-compatible. If this causes problems, you can force identical loading + behaviour to earlier Iris versions with ``LOAD_POLICY.set("legacy")`` or + equivalent. + + .. testsetup:: + + from iris import LOAD_POLICY Examples -------- @@ -302,10 +328,68 @@ class LoadPolicy(CombineOptions): LoadPolicy(support_multiple_references=True, merge_concat_sequence='mc', repeat_until_unchanged=True) >>> print(LOAD_POLICY) LoadPolicy(support_multiple_references=True, merge_concat_sequence='cm', repeat_until_unchanged=False) - """ - pass + # Option keys are as for CombineOptions, plus the multiple-refs control + OPTION_KEYS = CombineOptions.OPTION_KEYS + ["support_multiple_references"] + + # Allowed values are as for CombineOptions, plus boolean values for multiple-refs + _OPTIONS_ALLOWED_VALUES = dict( + list(CombineOptions._OPTIONS_ALLOWED_VALUES.items()) + + [("support_multiple_references", (True, False))] + ) + + # Settings dicts are as for CombineOptions, but all have multiple load refs enabled + SETTINGS = { + key: dict(list(settings.items()) + [("support_multiple_references", True)]) + for key, settings in CombineOptions.SETTINGS.items() + } + + @contextlib.contextmanager + def context(self, settings: str | dict, **kwargs): + """Return a context manager applying given options changes during a scope. + + Parameters + ---------- + settings : str or dict + A settings name or options dictionary, as for :meth:`~LoadPolicy.set`. + kwargs : dict + Option values, as for :meth:`~LoadPolicy.set`. + + Examples + -------- + .. testsetup:: + + import iris + from iris import LOAD_POLICY, sample_data_path + + >>> # Show how a CombineOptions acts in the context of a load operation + >>> path = sample_data_path("time_varying_hybrid_height", "*.pp") + >>> + >>> # Show that "legacy" load behaviour allows merge but not concatenate + >>> with LOAD_POLICY.context("legacy"): + ... cubes = iris.load(path, "x_wind") + >>> print(cubes) + 0: x_wind / (m s-1) (time: 2; model_level_number: 5; latitude: 144; longitude: 192) + 1: x_wind / (m s-1) (time: 12; model_level_number: 5; latitude: 144; longitude: 192) + 2: x_wind / (m s-1) (model_level_number: 5; latitude: 144; longitude: 192) + >>> + >>> # Show how "recommended" behaviour enables concatenation also + >>> with LOAD_POLICY.context("recommended"): + ... cubes = iris.load(path, "x_wind") + >>> print(cubes) + 0: x_wind / (m s-1) (model_level_number: 5; time: 15; latitude: 144; longitude: 192) + """ + # Save the current state + saved_settings = self.settings() + + # Apply the new options and execute the context + try: + self.set(settings, **kwargs) + yield + finally: + # Re-establish the former state + self.set(saved_settings) #: A control object containing the current file loading strategy options. From 7358b2532b1e0191f265dcaaa3b2413dac87bb59 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 24 Feb 2025 15:33:52 +0000 Subject: [PATCH 06/27] Fix LoadPolicy error handling + test. --- lib/iris/_combine.py | 21 ++++++++++++++------- lib/iris/tests/unit/test_LoadPolicy.py | 13 ++++++++----- 2 files changed, 22 insertions(+), 12 deletions(-) diff --git a/lib/iris/_combine.py b/lib/iris/_combine.py index 2cfc5fe80b..30dbd9d0d3 100644 --- a/lib/iris/_combine.py +++ b/lib/iris/_combine.py @@ -158,16 +158,23 @@ def set(self, options: str | dict | None = None, **kwargs): """ if options is None: options_dict = {} - elif isinstance(options, str) and options in self.SETTINGS: - options_dict = self.SETTINGS[options] + elif isinstance(options, str): + if options in self.SETTINGS: + options_dict = self.SETTINGS[options] + else: + msg = ( + f"arg 'options'={options!r}, which is not a valid settings name, " + f"expected one of {self.SETTINGS_NAMES}." + ) + raise ValueError(msg) elif isinstance(options, dict): options_dict = options else: - msg = ( - f"arg `options` has unexpected type {type(options)!r}, " - f"expected one of (None | str | dcit)." - ) - raise TypeError(msg) + msg = ( # type: ignore[unreachable] + f"arg 'options' has unexpected type {type(options)!r}, " + f"expected one of (None | str | dict)." + ) # type: ignore[unreachable] + raise TypeError(msg) # type: ignore[unreachable] # Override any options with keywords options_dict = options_dict.copy() # don't modify original diff --git a/lib/iris/tests/unit/test_LoadPolicy.py b/lib/iris/tests/unit/test_LoadPolicy.py index 8772b089c1..7bc7130cf2 100644 --- a/lib/iris/tests/unit/test_LoadPolicy.py +++ b/lib/iris/tests/unit/test_LoadPolicy.py @@ -36,7 +36,7 @@ def test_settings(self): options = LoadPolicy() settings = options.settings() assert isinstance(settings, dict) - assert tuple(settings.keys()) == LoadPolicy.OPTION_KEYS + assert list(settings.keys()) == LoadPolicy.OPTION_KEYS for key in LoadPolicy.OPTION_KEYS: assert settings[key] == getattr(options, key) @@ -74,13 +74,16 @@ def test_arg_bad_dict(self): def test_arg_bad_string(self): options = LoadPolicy() - expected = "Invalid arg options='unknown' : must be a dict, or one of" - with pytest.raises(TypeError, match=expected): - options.set("unknown") + expected = ( + r"arg 'options'='oddthing'.*not a valid setting.*expected one of.* " + "['legacy', 'default', 'recommended', 'comprehensive']" + ) + with pytest.raises(ValueError, match=expected): + options.set("oddthing") def test_arg_bad_type(self): options = LoadPolicy() - expected = "must be a dict, or one of" + expected = r"arg 'options' has unexpected type \, expected one of \(None \| str \| dict\)\." with pytest.raises(TypeError, match=expected): options.set((1, 2, 3)) From 220180d2f7416d5dc88c9ecfdcba4a4c36e25aa8 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 24 Feb 2025 16:22:21 +0000 Subject: [PATCH 07/27] Properly support merge_unique, and add equalise_cubes. --- lib/iris/_combine.py | 57 +++++++++++++++++--------- lib/iris/tests/unit/test_LoadPolicy.py | 6 --- 2 files changed, 37 insertions(+), 26 deletions(-) diff --git a/lib/iris/_combine.py b/lib/iris/_combine.py index 30dbd9d0d3..bfe02e024b 100644 --- a/lib/iris/_combine.py +++ b/lib/iris/_combine.py @@ -14,7 +14,7 @@ from __future__ import annotations import threading -from typing import TYPE_CHECKING, List +from typing import TYPE_CHECKING, Any, Dict, List if TYPE_CHECKING: from iris.cube import Cube, CubeList @@ -35,11 +35,15 @@ class CombineOptions(threading.local): ----- The individual configurable options are : + * ``equalise_cubes_kwargs`` = (dict) + Specifies keywords for a :func:`iris.util.equalise_cubes` call, to be applied + before any merge/concatenate step. + * ``merge_concat_sequence`` = "m" / "c" / "cm" / "mc" Specifies whether to apply :meth:`~iris.cube.CubeList.merge`, or :meth:`~iris.cube.CubeList.concatenate` operations, or both, in either order. - * ``merge_uses_unique`` = True / False + * ``merge_unique`` = True / False When True, any merge operation will error if its result contains multiple identical cubes. Otherwise (unique=False), that is a permitted result. @@ -90,29 +94,40 @@ class CombineOptions(threading.local): # Useful constants #: Valid option names OPTION_KEYS = [ + "equalise_cubes_kwargs", # N.B. gets special treatment in options checking "merge_concat_sequence", + "merge_unique", "repeat_until_unchanged", ] # this is a list, so we can update it in an inheriting class _OPTIONS_ALLOWED_VALUES = { "merge_concat_sequence": ("", "m", "c", "mc", "cm"), + "merge_unique": (True, False), "repeat_until_unchanged": (False, True), } - #: Settings content - SETTINGS = { + #: Standard settings dictionaries + SETTINGS: Dict[str, Dict[str, Any]] = { "legacy": dict( + equalise_cubes_kwargs=None, merge_concat_sequence="m", + merge_unique=False, repeat_until_unchanged=False, ), "default": dict( + equalise_cubes_kwargs=None, merge_concat_sequence="m", + merge_unique=False, repeat_until_unchanged=False, ), "recommended": dict( + equalise_cubes_kwargs=None, merge_concat_sequence="mc", + merge_unique=False, repeat_until_unchanged=False, ), "comprehensive": dict( + equalise_cubes_kwargs={"apply_all": True}, merge_concat_sequence="mc", + merge_unique=False, repeat_until_unchanged=True, ), } @@ -128,13 +143,14 @@ def __setattr__(self, key, value): if key not in self.OPTION_KEYS: raise KeyError(f"LoadPolicy object has no property '{key}'.") - allowed_values = self._OPTIONS_ALLOWED_VALUES[key] - if value not in allowed_values: - msg = ( - f"{value!r} is not a valid setting for LoadPolicy.{key} : " - f"must be one of '{allowed_values}'." - ) - raise ValueError(msg) + if key != "equalise_cubes_kwargs": + allowed_values = self._OPTIONS_ALLOWED_VALUES[key] + if value not in allowed_values: + msg = ( + f"{value!r} is not a valid setting for LoadPolicy.{key} : " + f"must be one of '{allowed_values}'." + ) + raise ValueError(msg) self.__dict__[key] = value @@ -157,7 +173,7 @@ def set(self, options: str | dict | None = None, **kwargs): """ if options is None: - options_dict = {} + options_dict: dict = {} elif isinstance(options, str): if options in self.SETTINGS: options_dict = self.SETTINGS[options] @@ -169,12 +185,6 @@ def set(self, options: str | dict | None = None, **kwargs): raise ValueError(msg) elif isinstance(options, dict): options_dict = options - else: - msg = ( # type: ignore[unreachable] - f"arg 'options' has unexpected type {type(options)!r}, " - f"expected one of (None | str | dict)." - ) # type: ignore[unreachable] - raise TypeError(msg) # type: ignore[unreachable] # Override any options with keywords options_dict = options_dict.copy() # don't modify original @@ -189,7 +199,7 @@ def set(self, options: str | dict | None = None, **kwargs): setattr(self, key, value) def settings(self) -> dict: - """Return an options dict containing the current settings.""" + """Return a settings dict containing the current options settings.""" return {key: getattr(self, key) for key in self.OPTION_KEYS} def __repr__(self): @@ -234,7 +244,14 @@ def _combine_cubes(cubes: List[Cube], options: dict) -> CubeList: else: cubelist = CubeList(cubes) + eq_args = options.get("equalise_cubes_kwargs", None) + if eq_args: + from iris.util import equalise_cubes + + equalise_cubes(cubelist, **eq_args) + sequence = options["merge_concat_sequence"] + merge_unique = options.get("merge_unique", False) while True: n_original_cubes = len(cubelist) @@ -245,7 +262,7 @@ def _combine_cubes(cubes: List[Cube], options: dict) -> CubeList: # merge if requested # NOTE: this needs "unique=False" to make "iris.load()" work correctly. # TODO: make configurable via options. - cubelist = cubelist.merge(unique=False) + cubelist = cubelist.merge(unique=merge_unique) if sequence[-1] == "c": # concat if it comes last cubelist = cubelist.concatenate() diff --git a/lib/iris/tests/unit/test_LoadPolicy.py b/lib/iris/tests/unit/test_LoadPolicy.py index 7bc7130cf2..e531780dad 100644 --- a/lib/iris/tests/unit/test_LoadPolicy.py +++ b/lib/iris/tests/unit/test_LoadPolicy.py @@ -81,12 +81,6 @@ def test_arg_bad_string(self): with pytest.raises(ValueError, match=expected): options.set("oddthing") - def test_arg_bad_type(self): - options = LoadPolicy() - expected = r"arg 'options' has unexpected type \, expected one of \(None \| str \| dict\)\." - with pytest.raises(TypeError, match=expected): - options.set((1, 2, 3)) - def test_kwargs(self): options = LoadPolicy() assert options.settings()["merge_concat_sequence"] == "m" From 78a041d1e45cf0afdb75dbb0f2f65280537546c5 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 24 Feb 2025 16:50:41 +0000 Subject: [PATCH 08/27] Allow LoadPolicy.context with no settings arg. --- lib/iris/loading.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/iris/loading.py b/lib/iris/loading.py index eaa86c1a7b..c4e1db50ca 100644 --- a/lib/iris/loading.py +++ b/lib/iris/loading.py @@ -292,7 +292,7 @@ class LoadPolicy(CombineOptions): modified for the duration of a code block. In addition to controlling the "combine" operations during loading, LoadPolicy also - provides a ``support_multiple_references`` option to manage the detection and + provides the ``support_multiple_references`` option, to manage the detection and handling of cases where a hybrid coordinate has multiple reference fields : for example, a UM file which contains a series of fields describing a time-varying orography. @@ -346,12 +346,12 @@ class LoadPolicy(CombineOptions): } @contextlib.contextmanager - def context(self, settings: str | dict, **kwargs): + def context(self, settings: str | dict | None = None, **kwargs): """Return a context manager applying given options changes during a scope. Parameters ---------- - settings : str or dict + settings : str or dict, optional A settings name or options dictionary, as for :meth:`~LoadPolicy.set`. kwargs : dict Option values, as for :meth:`~LoadPolicy.set`. From 45e29447389e11144fc22f40808d231655fd17a2 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 24 Feb 2025 17:19:17 +0000 Subject: [PATCH 09/27] Turn off varying-reference support for 'legacy' load setting only. --- lib/iris/loading.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/iris/loading.py b/lib/iris/loading.py index c4e1db50ca..72571e312c 100644 --- a/lib/iris/loading.py +++ b/lib/iris/loading.py @@ -298,7 +298,8 @@ class LoadPolicy(CombineOptions): orography. The ``support_multiple_references`` option takes the value ``True`` or ``False`` to - enable or disable this. The default is ``True``. + enable or disable this. The default is ``True`` for all standard settings except + "legacy". .. note :: @@ -339,9 +340,11 @@ class LoadPolicy(CombineOptions): + [("support_multiple_references", (True, False))] ) - # Settings dicts are as for CombineOptions, but all have multiple load refs enabled + # Settings dicts are as for CombineOptions, but with a multiple-load value added SETTINGS = { - key: dict(list(settings.items()) + [("support_multiple_references", True)]) + key: dict( + list(settings.items()) + [("support_multiple_references", key != "legacy")] + ) for key, settings in CombineOptions.SETTINGS.items() } From 3769b17df989b8acb8941c7c7fc4afeea4d29a65 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 24 Feb 2025 17:49:30 +0000 Subject: [PATCH 10/27] Implement combine_cube as util, and redirect test. --- .../unit/{ => util}/test_combine_cubes.py | 10 +-- lib/iris/util.py | 82 ++++++++++++++++++- 2 files changed, 82 insertions(+), 10 deletions(-) rename lib/iris/tests/unit/{ => util}/test_combine_cubes.py (89%) diff --git a/lib/iris/tests/unit/test_combine_cubes.py b/lib/iris/tests/unit/util/test_combine_cubes.py similarity index 89% rename from lib/iris/tests/unit/test_combine_cubes.py rename to lib/iris/tests/unit/util/test_combine_cubes.py index 107b7d4a4f..b5630193c9 100644 --- a/lib/iris/tests/unit/test_combine_cubes.py +++ b/lib/iris/tests/unit/util/test_combine_cubes.py @@ -13,8 +13,8 @@ import pytest from iris import LoadPolicy -from iris._combine import _combine_cubes from iris.tests.unit.fileformats.test_load_functions import cu +from iris.util import combine_cubes @pytest.fixture(params=list(LoadPolicy.SETTINGS.keys())) @@ -23,14 +23,6 @@ def options(request): return request.param # Return the name of the attribute to test. -# Interface to convert settings-name / kwargs into an options dict, -# TODO: remove this wrapper when the API of "combine_cubes" is opened up. -def combine_cubes(cubes, settings_name="default", **kwargs): - options = LoadPolicy.SETTINGS[settings_name] - options.update(kwargs) - return _combine_cubes(cubes, options) - - class Test: def test_mergeable(self, options): c1, c2 = cu(t=1), cu(t=2) diff --git a/lib/iris/util.py b/lib/iris/util.py index 94cb077a2f..cef04eba77 100644 --- a/lib/iris/util.py +++ b/lib/iris/util.py @@ -15,7 +15,7 @@ import os.path import sys import tempfile -from typing import Literal +from typing import TYPE_CHECKING, List, Literal from warnings import warn import cf_units @@ -31,6 +31,9 @@ import iris.exceptions import iris.warnings +if TYPE_CHECKING: + from iris.cube import Cube, CubeList + def broadcast_to_shape(array, shape, dim_map, chunks=None): """Broadcast an array to a given shape. @@ -2343,3 +2346,80 @@ def _print_xml(doc): """ result = doc.toprettyxml(indent=" ") return result.replace(" ", "\n") + + +def combine_cubes( + cubes: List[Cube], + options: str | dict | None = None, + **kwargs, +) -> CubeList: + """Combine cubes, according to "combine options". + + Applies a combination of :meth:`~iris.util.equalise_cubes`, + :meth:`~iris.cube.CubeList.merge` and/or :meth:`~iris.cube.CubeList.concatenate` + steps to the given cubes, as determined by the given settings (from `options` and + `kwargs`). + + Parameters + ---------- + cubes : list of :class:`~iris.cube.Cube` + A list of cubes to combine. + + options : str or dict, optional + Either a standard "combine settings" name, i.e. one of the + :data:`iris.CombineOptions.SETTINGS_NAMES`, or a dictionary of + settings options, as described for :class:`~iris.CombineOptions`. + Defaults to the current value of :data:`iris.LOAD_POLICY.settings`. + + kwargs : dict + Individual option setting values, i.e. values for keys named in + :data:`iris.CombineOptions.OPTION_KEYS`, as described for + :meth:`~iris.CombineOptions.set`. These take precedence over those set by the + `options` arg. + + Returns + ------- + :class:`~iris.cube.CubeList` + + .. Note:: + A ``support_multiple_references`` keyword/property is treated as a valid + option, but it has *no* effect on :func:`combine_cubes` because this option + only acts during load operations. + + Examples + -------- + >>> results = combine_cubes(cubes) + >>> results = combine_cubes(cubes, options=CombineOptions("recommended")) + >>> results = combine_cubes(cubes, repeat_until_unchanged=True) + + """ + # TODO: somehow, provide a real + useful working code example + + from iris import LOAD_POLICY, CombineOptions + from iris._combine import _combine_cubes + + if options is None: + opts_dict = LOAD_POLICY.settings() + elif isinstance(options, str): + if options in CombineOptions.SETTINGS: + opts_dict = CombineOptions.SETTINGS[options] + else: + msg = ( + "Unrecognised settings name : expected one of " + f"{tuple(CombineOptions.SETTINGS)}." + ) + raise ValueError(msg) + elif isinstance(options, dict): + opts_dict = options + else: + msg = ( # type: ignore[unreachable] + f"arg 'options' has type {type(options)!r}, " + "expected one of (str | dict | None)" + ) + raise ValueError(msg) # type: ignore[unreachable] + + if kwargs is not None: + opts_dict = opts_dict.copy() # avoid changing original + opts_dict.update(kwargs) + + return _combine_cubes(cubes, opts_dict) From 63a24787e5de0b2fb7b85a19bdfc46e58d7baafe Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 24 Feb 2025 18:18:35 +0000 Subject: [PATCH 11/27] Small docs fixes. --- lib/iris/util.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/iris/util.py b/lib/iris/util.py index cef04eba77..31188a23cb 100644 --- a/lib/iris/util.py +++ b/lib/iris/util.py @@ -2369,7 +2369,8 @@ def combine_cubes( Either a standard "combine settings" name, i.e. one of the :data:`iris.CombineOptions.SETTINGS_NAMES`, or a dictionary of settings options, as described for :class:`~iris.CombineOptions`. - Defaults to the current value of :data:`iris.LOAD_POLICY.settings`. + Defaults to the current :meth:`~iris.CombineOptions.settings` of the + :data:`iris.LOAD_POLICY`. kwargs : dict Individual option setting values, i.e. values for keys named in @@ -2381,10 +2382,11 @@ def combine_cubes( ------- :class:`~iris.cube.CubeList` - .. Note:: - A ``support_multiple_references`` keyword/property is treated as a valid - option, but it has *no* effect on :func:`combine_cubes` because this option - only acts during load operations. + Notes + ----- + A ``support_multiple_references`` option will be accepted as valid, but will + have *no* effect on :func:`combine_cubes` because this option only acts during + load operations. Examples -------- From c313f9469df8c1ff558c6d9efeec10a0890dc8c5 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Tue, 25 Feb 2025 15:37:38 +0000 Subject: [PATCH 12/27] Add functioning doctest example for combine_cubes. --- lib/iris/util.py | 54 +++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 51 insertions(+), 3 deletions(-) diff --git a/lib/iris/util.py b/lib/iris/util.py index 31188a23cb..bc61bc1a23 100644 --- a/lib/iris/util.py +++ b/lib/iris/util.py @@ -2388,11 +2388,59 @@ def combine_cubes( have *no* effect on :func:`combine_cubes` because this option only acts during load operations. + Examples -------- - >>> results = combine_cubes(cubes) - >>> results = combine_cubes(cubes, options=CombineOptions("recommended")) - >>> results = combine_cubes(cubes, repeat_until_unchanged=True) + .. testsetup:: + + import numpy as np + from iris.cube import Cube, CubeList + from iris.coords import DimCoord + from iris.util import combine_cubes + + def testcube(timepts): + cube = Cube(np.array(timepts)) + cube.add_dim_coord( + DimCoord(timepts, standard_name="time", units="days since 1990-01-01"), + 0 + ) + return cube + + cubes = CubeList([testcube([1., 2]), testcube([13., 14, 15])]) + + >>> # Take a pair of sample cubes which can concatenate together + >>> print(cubes) + 0: unknown / (unknown) (time: 2) + 1: unknown / (unknown) (time: 3) + >>> print([cube.coord("time").points for cube in cubes]) + [array([1., 2.]), array([13., 14., 15.])] + + >>> # Show these do NOT combine with the "default" action, which only merges .. + >>> print(combine_cubes(cubes)) + 0: unknown / (unknown) (time: 2) + 1: unknown / (unknown) (time: 3) + >>> # ... however, they **do** combine if you enable concatenation + >>> print(combine_cubes(cubes, merge_concat_sequence="mc")) + 0: unknown / (unknown) (time: 5) + >>> # ... which may be controlled by various means + >>> iris.LOAD_POLICY.set("recommended") + >>> print(combine_cubes(cubes)) + 0: unknown / (unknown) (time: 5) + + >>> # Also, show how a differing attribute will block cube combination + >>> cubes[0].attributes["x"] = 3 + >>> print(combine_cubes(cubes)) + 0: unknown / (unknown) (time: 2) + 1: unknown / (unknown) (time: 3) + >>> # ... which can then be fixed by enabling attribute equalisation + >>> with iris.LOAD_POLICY.context(equalise_cubes_kwargs={"apply_all":True}): + ... print(combine_cubes(cubes)) + ... + 0: unknown / (unknown) (time: 5) + + >>> # .. BUT NOTE : this modifies the original input cubes + >>> print(cubes[0].attributes.get("x")) + None """ # TODO: somehow, provide a real + useful working code example From a79ea9dbc3bd240632565b13f9298fecb8743a8a Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Tue, 25 Feb 2025 17:34:10 +0000 Subject: [PATCH 13/27] Further doctest fixes. --- lib/iris/loading.py | 16 +++++++++++----- lib/iris/util.py | 6 ++++++ 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/lib/iris/loading.py b/lib/iris/loading.py index 72571e312c..3b6c2c3d57 100644 --- a/lib/iris/loading.py +++ b/lib/iris/loading.py @@ -312,23 +312,29 @@ class LoadPolicy(CombineOptions): .. testsetup:: from iris import LOAD_POLICY + loadpolicy_old_settings = LOAD_POLICY.settings() + + .. testcleanup:: + + # restore original settings, so as not to upset other tests + LOAD_POLICY.set(loadpolicy_old_settings) Examples -------- >>> LOAD_POLICY.set("legacy") >>> print(LOAD_POLICY) - LoadPolicy(support_multiple_references=False, merge_concat_sequence='m', repeat_until_unchanged=False) + LoadPolicy(equalise_cubes_kwargs=None, merge_concat_sequence='m', merge_unique=False, repeat_until_unchanged=False, support_multiple_references=False) >>> LOAD_POLICY.support_multiple_references = True >>> print(LOAD_POLICY) - LoadPolicy(support_multiple_references=True, merge_concat_sequence='m', repeat_until_unchanged=False) + LoadPolicy(equalise_cubes_kwargs=None, merge_concat_sequence='m', merge_unique=False, repeat_until_unchanged=False, support_multiple_references=True) >>> LOAD_POLICY.set(merge_concat_sequence="cm") >>> print(LOAD_POLICY) - LoadPolicy(support_multiple_references=True, merge_concat_sequence='cm', repeat_until_unchanged=False) + LoadPolicy(equalise_cubes_kwargs=None, merge_concat_sequence='cm', merge_unique=False, repeat_until_unchanged=False, support_multiple_references=True) >>> with LOAD_POLICY.context("comprehensive"): ... print(LOAD_POLICY) - LoadPolicy(support_multiple_references=True, merge_concat_sequence='mc', repeat_until_unchanged=True) + LoadPolicy(equalise_cubes_kwargs={'apply_all': True}, merge_concat_sequence='mc', merge_unique=False, repeat_until_unchanged=True, support_multiple_references=True) >>> print(LOAD_POLICY) - LoadPolicy(support_multiple_references=True, merge_concat_sequence='cm', repeat_until_unchanged=False) + LoadPolicy(equalise_cubes_kwargs=None, merge_concat_sequence='cm', merge_unique=False, repeat_until_unchanged=False, support_multiple_references=True) """ # Option keys are as for CombineOptions, plus the multiple-refs control diff --git a/lib/iris/util.py b/lib/iris/util.py index bc61bc1a23..6d2748e942 100644 --- a/lib/iris/util.py +++ b/lib/iris/util.py @@ -2407,6 +2407,12 @@ def testcube(timepts): return cube cubes = CubeList([testcube([1., 2]), testcube([13., 14, 15])]) + combinecubes_old_policysettings = iris.LOAD_POLICY.settings() + + .. testcleanup:: + + # restore old state to avoid upsetting other tests + iris.LOAD_POLICY.set(combinecubes_old_policysettings) >>> # Take a pair of sample cubes which can concatenate together >>> print(cubes) From 4c11653dbbb3721f3b8f759651fb68b306ae8770 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Wed, 26 Feb 2025 00:17:49 +0000 Subject: [PATCH 14/27] Add cubelist combine methods. --- lib/iris/cube.py | 28 +++++++++++++++++++++++++++ lib/iris/util.py | 50 +++++++++++++++++++++++++++--------------------- 2 files changed, 56 insertions(+), 22 deletions(-) diff --git a/lib/iris/cube.py b/lib/iris/cube.py index 0d3d6a5b83..3e2278fab5 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -637,6 +637,34 @@ def concatenate( check_derived_coords=check_derived_coords, ) + def combine(self, options: str | dict | None = None, **kwargs) -> CubeList: + from iris.util import combine_cubes + + return combine_cubes(self, options, **kwargs) + + def combine_cube(self, options: str | dict | None = None, **kwargs) -> CubeList: + result = self.combine() + n_cubes = len(result) + if n_cubes != 1: + from iris.util import _combine_options_asdict + + opts_dict = _combine_options_asdict(options) + merge_concat_sequence = opts_dict.get("merge_concat_sequence") + is_merge = ( + not isinstance(merge_concat_sequence, str) + or len(merge_concat_sequence) < 0 + or not merge_concat_sequence.endswith("m") + ) + err_type = ( + iris.exceptions.MergeError + if is_merge + else iris.exceptions.ConcatenateError + ) + msg = f"'combine' operation yielded {n_cubes} cubes, expected exactly 1." + raise err_type(msg) + + return result + def realise_data(self): """Fetch 'real' data for all cubes, in a shared calculation. diff --git a/lib/iris/util.py b/lib/iris/util.py index 6d2748e942..77a5edfc07 100644 --- a/lib/iris/util.py +++ b/lib/iris/util.py @@ -2348,6 +2348,33 @@ def _print_xml(doc): return result.replace(" ", "\n") +def _combine_options_asdict(options: str | dict | None) -> dict: + """Convert any valid combine options into an options dictionary.""" + from iris import LOAD_POLICY + + if options is None: + opts_dict = LOAD_POLICY.settings() + elif isinstance(options, dict): + opts_dict = options + elif isinstance(options, str): + if options in LOAD_POLICY.SETTINGS: + opts_dict = LOAD_POLICY.SETTINGS[options] + else: + msg = ( + "Unrecognised settings name : expected one of " + f"{tuple(LOAD_POLICY.SETTINGS)}." + ) + raise ValueError(msg) + else: + msg = ( # type: ignore[unreachable] + f"arg 'options' has type {type(options)!r}, " + "expected one of (str | dict | None)" + ) + raise ValueError(msg) # type: ignore[unreachable] + + return opts_dict + + def combine_cubes( cubes: List[Cube], options: str | dict | None = None, @@ -2450,30 +2477,9 @@ def testcube(timepts): """ # TODO: somehow, provide a real + useful working code example - - from iris import LOAD_POLICY, CombineOptions from iris._combine import _combine_cubes - if options is None: - opts_dict = LOAD_POLICY.settings() - elif isinstance(options, str): - if options in CombineOptions.SETTINGS: - opts_dict = CombineOptions.SETTINGS[options] - else: - msg = ( - "Unrecognised settings name : expected one of " - f"{tuple(CombineOptions.SETTINGS)}." - ) - raise ValueError(msg) - elif isinstance(options, dict): - opts_dict = options - else: - msg = ( # type: ignore[unreachable] - f"arg 'options' has type {type(options)!r}, " - "expected one of (str | dict | None)" - ) - raise ValueError(msg) # type: ignore[unreachable] - + opts_dict = _combine_options_asdict(options) if kwargs is not None: opts_dict = opts_dict.copy() # avoid changing original opts_dict.update(kwargs) From d2f4733483732e7243b6330331b894d0139d6eca Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Wed, 26 Feb 2025 12:47:41 +0000 Subject: [PATCH 15/27] Better documentation of CombineOptions and LoadPolicy settings. --- lib/iris/_combine.py | 40 +++++++++++++++++++++++++--------------- lib/iris/loading.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+), 15 deletions(-) diff --git a/lib/iris/_combine.py b/lib/iris/_combine.py index bfe02e024b..4c1c07234a 100644 --- a/lib/iris/_combine.py +++ b/lib/iris/_combine.py @@ -36,7 +36,7 @@ class CombineOptions(threading.local): The individual configurable options are : * ``equalise_cubes_kwargs`` = (dict) - Specifies keywords for a :func:`iris.util.equalise_cubes` call, to be applied + Specifies keywords for an :func:`iris.util.equalise_cubes` call, to be applied before any merge/concatenate step. * ``merge_concat_sequence`` = "m" / "c" / "cm" / "mc" @@ -58,29 +58,34 @@ class CombineOptions(threading.local): When enabled, the configured "combine" operation will be repeated until the result is stable (no more cubes are combined). - Several common sets of options are provided in :data:`~iris.LOAD_POLICY.SETTINGS` : - - * ``"legacy"`` - Produces loading behaviour identical to Iris versions < 3.11, i.e. before the - varying hybrid references were supported. + Several sets of combined options are grouped for convenience for selection by + "settings" name, one of :data:`CombineOptions.SETTINGS_NAMES`. + Note though that these different "settings" are partly determined by the needs of + loading support. See the discussion in the derived + :class:`~iris.loading.LoadPolicy` class. The choices are: * ``"default"`` - As "legacy" except that ``support_multiple_references=True``. This differs - from "legacy" only when multiple mergeable reference fields are encountered, - in which case incoming cubes are extended into the extra dimension, and a - concatenate step is added. + Apply a plain merge step only, i.e. ``merge_concat_sequence="m"``. + Other options are all "off". + + * ``"legacy"`` + Identical to "default". See :class:`iris.LoadPolicy` for where this makes + a difference. * ``"recommended"`` - Enables multiple reference handling, *and* applies a merge step followed by - a concatenate step. + In addition to the "merge" step, allow a following "concatenate", i.e. + ``merge_concat_sequence="mc"``. * ``"comprehensive"`` - Like "recommended", but will also *repeat* the merge+concatenate steps until no - further change is produced. + Like "recommended", uses ``merge_concat_sequence="mc"``, but now also + *repeats* the merge+concatenate steps until no further change is produced, + i.e. ``repeat_until_unchanged=True``. + Also applies a prior 'equalise_cubes' call, of the form + ``equalise_cubes(cubes, apply_all=True)``. .. note :: - The 'comprehensive' policy makes a maximum effort to reduce the number of + The "comprehensive" policy makes a maximum effort to reduce the number of cubes to a minimum. However, it still cannot combine cubes with a mixture of matching dimension and scalar coordinates. This may be supported at some later date, but for now is not possible without specific user actions. @@ -89,6 +94,11 @@ class CombineOptions(threading.local): See also : :ref:`controlling_merge`. + Examples + -------- + Please see :func:`iris.util.combine_cubes` for usage examples of the settings and + keywords described here. + """ # Useful constants diff --git a/lib/iris/loading.py b/lib/iris/loading.py index 3b6c2c3d57..115826731b 100644 --- a/lib/iris/loading.py +++ b/lib/iris/loading.py @@ -319,20 +319,50 @@ class LoadPolicy(CombineOptions): # restore original settings, so as not to upset other tests LOAD_POLICY.set(loadpolicy_old_settings) + + Notes + ----- + The ``SETTINGS`` options for the various + :data:`~iris._combine.CombineOptions.SETTINGS_NAMES` are as follows: + + * ``"legacy"`` + Produces loading behaviour identical to Iris versions < 3.11, i.e. before the + varying hybrid references were supported. + + * ``"default"`` + As "legacy" except that ``support_multiple_references=True``. This differs + from "legacy" only when multiple mergeable reference fields are encountered, + in which case incoming cubes are extended into the extra dimension, and a + concatenate step is added. + + * ``"recommended"`` + Enables multiple reference handling, *and* applies a merge step followed by + a concatenate step. + + * ``"comprehensive"`` + Like "recommended", uses the 'mc' merge-concatenate sequence, but now also + *repeats* the merge+concatenate steps until no further change is produced. + Also applies a prior 'equalise_cubes' call, of the form + ``equalise_cubes(cubes, apply_all=True)``. + Examples -------- >>> LOAD_POLICY.set("legacy") >>> print(LOAD_POLICY) LoadPolicy(equalise_cubes_kwargs=None, merge_concat_sequence='m', merge_unique=False, repeat_until_unchanged=False, support_multiple_references=False) + >>> >>> LOAD_POLICY.support_multiple_references = True >>> print(LOAD_POLICY) LoadPolicy(equalise_cubes_kwargs=None, merge_concat_sequence='m', merge_unique=False, repeat_until_unchanged=False, support_multiple_references=True) + >>> LOAD_POLICY.set(merge_concat_sequence="cm") >>> print(LOAD_POLICY) LoadPolicy(equalise_cubes_kwargs=None, merge_concat_sequence='cm', merge_unique=False, repeat_until_unchanged=False, support_multiple_references=True) + >>> with LOAD_POLICY.context("comprehensive"): ... print(LOAD_POLICY) LoadPolicy(equalise_cubes_kwargs={'apply_all': True}, merge_concat_sequence='mc', merge_unique=False, repeat_until_unchanged=True, support_multiple_references=True) + >>> >>> print(LOAD_POLICY) LoadPolicy(equalise_cubes_kwargs=None, merge_concat_sequence='cm', merge_unique=False, repeat_until_unchanged=False, support_multiple_references=True) """ From ea9d82f36fa43187fc1156c39071b02d26c4d897 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Wed, 26 Feb 2025 15:52:55 +0000 Subject: [PATCH 16/27] Recombine LoadPolicy into CombineOptions. --- lib/iris/__init__.py | 16 +- lib/iris/_combine.py | 160 +++++++++++++++--- lib/iris/fileformats/rules.py | 6 +- lib/iris/loading.py | 157 ----------------- lib/iris/tests/integration/test_trajectory.py | 4 +- .../varying_references/test_realdata_load.py | 4 +- .../test_roundtrip_time_varying_references.py | 8 +- lib/iris/tests/unit/combine/__init__.py | 5 + .../test_CombineOptions.py} | 48 +++--- lib/iris/util.py | 20 +-- 10 files changed, 195 insertions(+), 233 deletions(-) create mode 100644 lib/iris/tests/unit/combine/__init__.py rename lib/iris/tests/unit/{test_LoadPolicy.py => combine/test_CombineOptions.py} (78%) diff --git a/lib/iris/__init__.py b/lib/iris/__init__.py index d622bd18b0..d2b6249825 100644 --- a/lib/iris/__init__.py +++ b/lib/iris/__init__.py @@ -94,14 +94,13 @@ def callback(cube, field, filename): import threading from typing import Callable, Literal +from iris._combine import COMBINE_POLICY as _COMBINE_POLICY +from iris._combine import CombineOptions import iris._constraints import iris.config import iris.io from iris.io import save -from iris.loading import LOAD_POLICY as _LOAD_POLICY from iris.loading import ( - CombineOptions, - LoadPolicy, load, load_cube, load_cubes, @@ -111,8 +110,14 @@ def callback(cube, field, filename): # NOTE: we make an independent local 'LOAD_POLICY' definition here, just so that we # can ensure an entry for it in our API documentation page. -#: A control object containing the current file loading strategy options. -LOAD_POLICY = _LOAD_POLICY +#: An object to control default cube combination and loading options +COMBINE_POLICY = _COMBINE_POLICY + +#: An alias for the :class:`~iris._combine.CombineOptions` class. +LoadPolicy = CombineOptions + +#: An alias for the :data:`~iris.COMBINE_POLICY` object. +LOAD_POLICY = _COMBINE_POLICY from ._deprecation import IrisDeprecation, warn_deprecated @@ -132,6 +137,7 @@ def callback(cube, field, filename): # Restrict the names imported when using "from iris import *" __all__ = [ "AttributeConstraint", + "COMBINE_POLICY", "CombineOptions", "Constraint", "DATALESS", diff --git a/lib/iris/_combine.py b/lib/iris/_combine.py index 4c1c07234a..9e54f8c76f 100644 --- a/lib/iris/_combine.py +++ b/lib/iris/_combine.py @@ -13,6 +13,7 @@ from __future__ import annotations +import contextlib import threading from typing import TYPE_CHECKING, Any, Dict, List @@ -21,18 +22,21 @@ class CombineOptions(threading.local): - """A container for cube combination options. + """A control object for Iris loading and cube combination options. - Controls for generalised merge/concatenate options. These are used as controls for + Both the iris loading functions and the "combine_cubes" utility apply a number of + possible "cube combination" operations to a list of cubes, in a definite sequence, + all of which tend to combine cubes into a smaller number of larger or + higher-dimensional cubes. + + This object groups various control options for these behaviours, which apply to both the :func:`iris.util.combine_cubes` utility method and the core Iris loading - functions : see also :data:`iris.loading.LoadPolicy`. + functions "iris.load_xxx". - It specifies a number of possible operations which may be applied to a list of - cubes, in a definite sequence, all of which tend to combine cubes into a smaller - number of larger or higher-dimensional cubes. + The :class:`CombineOptions` class defines the allowed control options, while a + global singleton object :data:`iris.COMBINE_POLICY` holds the current global + default settings. - Notes - ----- The individual configurable options are : * ``equalise_cubes_kwargs`` = (dict) @@ -58,47 +62,95 @@ class CombineOptions(threading.local): When enabled, the configured "combine" operation will be repeated until the result is stable (no more cubes are combined). - Several sets of combined options are grouped for convenience for selection by - "settings" name, one of :data:`CombineOptions.SETTINGS_NAMES`. - Note though that these different "settings" are partly determined by the needs of - loading support. See the discussion in the derived - :class:`~iris.loading.LoadPolicy` class. The choices are: + * ``support_multiple_references`` = True / False + When enabled, support cases where a hybrid coordinate has multiple reference + fields : for example, a UM file which contains a series of fields describing a + time-varying orography. - * ``"default"`` + Alternatively, certain fixed combinations of options can be selected by a + "settings" name, one of :data:`CombineOptions.SETTINGS_NAMES` : + + * ``"legacy"`` Apply a plain merge step only, i.e. ``merge_concat_sequence="m"``. Other options are all "off". + This produces loading behaviour identical to Iris versions < 3.11, i.e. before + the varying hybrid references were supported. - * ``"legacy"`` - Identical to "default". See :class:`iris.LoadPolicy` for where this makes - a difference. + * ``"default"`` + As "legacy" except that ``support_multiple_references=True``. This differs + from "legacy" only when multiple mergeable reference fields are encountered, + in which case incoming cubes are extended into the extra dimension, and a + concatenate step is added. + Since the handling of multiple references affects only loading operations, + for the purposes of calls to :func:`~iris.util.combine_cubes`, this setting is + *identical* to "legacy". + + .. Warning:: + + The ``"default"`` setting **is** the initial default mode. + + This "fixes" loading for cases like the time-varying orography case + described. However, this setting is not strictly + backwards-compatible. If this causes problems, you can force identical + loading behaviour to earlier Iris versions (< v3.11) with + ``COMBINE_POLICY.set("legacy")`` or equivalent. * ``"recommended"`` In addition to the "merge" step, allow a following "concatenate", i.e. ``merge_concat_sequence="mc"``. * ``"comprehensive"`` - Like "recommended", uses ``merge_concat_sequence="mc"``, but now also + As for "recommended", uses ``merge_concat_sequence="mc"``, but now also *repeats* the merge+concatenate steps until no further change is produced, i.e. ``repeat_until_unchanged=True``. Also applies a prior 'equalise_cubes' call, of the form ``equalise_cubes(cubes, apply_all=True)``. - .. note :: + .. Note:: The "comprehensive" policy makes a maximum effort to reduce the number of cubes to a minimum. However, it still cannot combine cubes with a mixture of matching dimension and scalar coordinates. This may be supported at some later date, but for now is not possible without specific user actions. - .. Note :: + .. testsetup:: - See also : :ref:`controlling_merge`. + loadpolicy_old_settings = COMBINE_POLICY.settings() + + .. testcleanup:: + + # restore original settings, so as not to upset other tests + COMBINE_POLICY.set(loadpolicy_old_settings) Examples -------- - Please see :func:`iris.util.combine_cubes` for usage examples of the settings and - keywords described here. + Note: :data:`COMBINE_POLICY` is the global control object, which determines + the current default options for loading or :func:`iris.util.combine_cubes` calls. + For the latter case, however, control via argument and keywords is also available. + + .. Note:: + The name ``iris.LOAD_POLICY`` refers to the same thing as + ``iris.COMBINE_POLICY``, and is still usable, but no longer recommended. + + >>> COMBINE_POLICY.set("legacy") + >>> print(COMBINE_POLICY) + CombineOptions(equalise_cubes_kwargs=None, merge_concat_sequence='m', merge_unique=False, repeat_until_unchanged=False, support_multiple_references=False) + >>> + >>> COMBINE_POLICY.support_multiple_references = True + >>> print(COMBINE_POLICY) + CombineOptions(equalise_cubes_kwargs=None, merge_concat_sequence='m', merge_unique=False, repeat_until_unchanged=False, support_multiple_references=True) + + >>> COMBINE_POLICY.set(merge_concat_sequence="cm") + >>> print(COMBINE_POLICY) + CombineOptions(equalise_cubes_kwargs=None, merge_concat_sequence='cm', merge_unique=False, repeat_until_unchanged=False, support_multiple_references=True) + + >>> with COMBINE_POLICY.context("comprehensive"): + ... print(COMBINE_POLICY) + CombineOptions(equalise_cubes_kwargs={'apply_all': True}, merge_concat_sequence='mc', merge_unique=False, repeat_until_unchanged=True, support_multiple_references=True) + >>> + >>> print(COMBINE_POLICY) + CombineOptions(equalise_cubes_kwargs=None, merge_concat_sequence='cm', merge_unique=False, repeat_until_unchanged=False, support_multiple_references=True) """ # Useful constants @@ -108,11 +160,13 @@ class CombineOptions(threading.local): "merge_concat_sequence", "merge_unique", "repeat_until_unchanged", + "support_multiple_references", ] # this is a list, so we can update it in an inheriting class _OPTIONS_ALLOWED_VALUES = { "merge_concat_sequence": ("", "m", "c", "mc", "cm"), "merge_unique": (True, False), "repeat_until_unchanged": (False, True), + "support_multiple_references": (True, False), } #: Standard settings dictionaries SETTINGS: Dict[str, Dict[str, Any]] = { @@ -121,24 +175,28 @@ class CombineOptions(threading.local): merge_concat_sequence="m", merge_unique=False, repeat_until_unchanged=False, + support_multiple_references=False, ), "default": dict( equalise_cubes_kwargs=None, merge_concat_sequence="m", merge_unique=False, repeat_until_unchanged=False, + support_multiple_references=True, ), "recommended": dict( equalise_cubes_kwargs=None, merge_concat_sequence="mc", merge_unique=False, repeat_until_unchanged=False, + support_multiple_references=True, ), "comprehensive": dict( equalise_cubes_kwargs={"apply_all": True}, merge_concat_sequence="mc", merge_unique=False, repeat_until_unchanged=True, + support_multiple_references=True, ), } #: Valid settings names @@ -151,7 +209,7 @@ def __init__(self, options: str | dict | None = None, **kwargs): def __setattr__(self, key, value): if key not in self.OPTION_KEYS: - raise KeyError(f"LoadPolicy object has no property '{key}'.") + raise KeyError(f"CombineOptions object has no property '{key}'.") if key != "equalise_cubes_kwargs": allowed_values = self._OPTIONS_ALLOWED_VALUES[key] @@ -208,6 +266,52 @@ def set(self, options: str | dict | None = None, **kwargs): for key, value in options_dict.items(): setattr(self, key, value) + @contextlib.contextmanager + def context(self, settings: str | dict | None = None, **kwargs): + """Return a context manager applying given options changes during a scope. + + Parameters + ---------- + settings : str or dict, optional + A settings name or options dictionary, as for :meth:`~LoadPolicy.set`. + kwargs : dict + Option values, as for :meth:`~LoadPolicy.set`. + + Examples + -------- + .. testsetup:: + + import iris + from iris import COMBINE_POLICY, sample_data_path + + >>> # Show how a CombineOptions acts in the context of a load operation + >>> path = sample_data_path("time_varying_hybrid_height", "*.pp") + >>> + >>> # Show that "legacy" load behaviour allows merge but not concatenate + >>> with COMBINE_POLICY.context("legacy"): + ... cubes = iris.load(path, "x_wind") + >>> print(cubes) + 0: x_wind / (m s-1) (time: 2; model_level_number: 5; latitude: 144; longitude: 192) + 1: x_wind / (m s-1) (time: 12; model_level_number: 5; latitude: 144; longitude: 192) + 2: x_wind / (m s-1) (model_level_number: 5; latitude: 144; longitude: 192) + >>> + >>> # Show how "recommended" behaviour enables concatenation also + >>> with COMBINE_POLICY.context("recommended"): + ... cubes = iris.load(path, "x_wind") + >>> print(cubes) + 0: x_wind / (m s-1) (model_level_number: 5; time: 15; latitude: 144; longitude: 192) + """ + # Save the current state + saved_settings = self.settings() + + # Apply the new options and execute the context + try: + self.set(settings, **kwargs) + yield + finally: + # Re-establish the former state + self.set(saved_settings) + def settings(self) -> dict: """Return a settings dict containing the current options settings.""" return {key: getattr(self, key) for key in self.OPTION_KEYS} @@ -287,9 +391,9 @@ def _combine_cubes(cubes: List[Cube], options: dict) -> CubeList: def _combine_load_cubes(cubes: List[Cube]) -> CubeList: # A special version to call _combine_cubes while also implementing the # _MULTIREF_DETECTION behaviour - from iris import LOAD_POLICY + from iris import COMBINE_POLICY - options = LOAD_POLICY.settings() + options = COMBINE_POLICY.settings() if ( options["support_multiple_references"] and "c" not in options["merge_concat_sequence"] @@ -301,3 +405,7 @@ def _combine_load_cubes(cubes: List[Cube]) -> CubeList: options["merge_concat_sequence"] += "c" return _combine_cubes(cubes, options) + + +#: An object to control default cube combination and loading options +COMBINE_POLICY = CombineOptions() diff --git a/lib/iris/fileformats/rules.py b/lib/iris/fileformats/rules.py index 2a1a74f374..d24f38d3f3 100644 --- a/lib/iris/fileformats/rules.py +++ b/lib/iris/fileformats/rules.py @@ -230,11 +230,11 @@ def _ensure_aligned(regrid_cache, src_cube, target_cube): # single, distinct dimension. # PP-MOD: first promote any scalar coords when needed as dims for target_coord in target_dimcoords: - from iris import LOAD_POLICY + from iris import COMBINE_POLICY if ( not target_cube.coord_dims(target_coord) - and LOAD_POLICY.support_multiple_references + and COMBINE_POLICY.support_multiple_references ): # The chosen coord is not a dimcoord in the target (yet) # Make it one with 'new_axis' @@ -400,7 +400,7 @@ def __init__(self): # A single global object (per thread) to record whether multiple reference fields # (e.g. time-dependent orography, or surface pressure fields) have been detected during # the latest load operation. -# This is used purely to implement the iris.LOAD_POLICY.multiref_triggers_concatenate +# This is used purely to implement the iris.COMBINE_POLICY.multiref_triggers_concatenate # functionality. _MULTIREF_DETECTION = MultipleReferenceFieldDetector() diff --git a/lib/iris/loading.py b/lib/iris/loading.py index 115826731b..03a395ca8d 100644 --- a/lib/iris/loading.py +++ b/lib/iris/loading.py @@ -4,7 +4,6 @@ # See LICENSE in the root of the repository for full licensing details. """Iris general file loading mechanism.""" -import contextlib import itertools from typing import Iterable @@ -277,159 +276,3 @@ def load_raw(uris, constraints=None, callback=None): with _raw_structured_loading(): return _load_collection(uris, constraints, callback).cubes() - - -from iris._combine import CombineOptions - - -class LoadPolicy(CombineOptions): - """A control object for Iris loading options. - - Incorporates all the settings of a :class:`~iris.CombineOptions`, and adds an - additional ``support_multiple_references`` option. - - Also adds :meth:`~iris.LoadPolicy.context`, allowing loading behaviours to be - modified for the duration of a code block. - - In addition to controlling the "combine" operations during loading, LoadPolicy also - provides the ``support_multiple_references`` option, to manage the detection and - handling of cases where a hybrid coordinate has multiple reference fields : for - example, a UM file which contains a series of fields describing a time-varying - orography. - - The ``support_multiple_references`` option takes the value ``True`` or ``False`` to - enable or disable this. The default is ``True`` for all standard settings except - "legacy". - - .. note :: - - The default behaviour will "fix" loading for cases like the time-varying - orography case described above. However, this is not strictly - backwards-compatible. If this causes problems, you can force identical loading - behaviour to earlier Iris versions with ``LOAD_POLICY.set("legacy")`` or - equivalent. - - .. testsetup:: - - from iris import LOAD_POLICY - loadpolicy_old_settings = LOAD_POLICY.settings() - - .. testcleanup:: - - # restore original settings, so as not to upset other tests - LOAD_POLICY.set(loadpolicy_old_settings) - - - Notes - ----- - The ``SETTINGS`` options for the various - :data:`~iris._combine.CombineOptions.SETTINGS_NAMES` are as follows: - - * ``"legacy"`` - Produces loading behaviour identical to Iris versions < 3.11, i.e. before the - varying hybrid references were supported. - - * ``"default"`` - As "legacy" except that ``support_multiple_references=True``. This differs - from "legacy" only when multiple mergeable reference fields are encountered, - in which case incoming cubes are extended into the extra dimension, and a - concatenate step is added. - - * ``"recommended"`` - Enables multiple reference handling, *and* applies a merge step followed by - a concatenate step. - - * ``"comprehensive"`` - Like "recommended", uses the 'mc' merge-concatenate sequence, but now also - *repeats* the merge+concatenate steps until no further change is produced. - Also applies a prior 'equalise_cubes' call, of the form - ``equalise_cubes(cubes, apply_all=True)``. - - Examples - -------- - >>> LOAD_POLICY.set("legacy") - >>> print(LOAD_POLICY) - LoadPolicy(equalise_cubes_kwargs=None, merge_concat_sequence='m', merge_unique=False, repeat_until_unchanged=False, support_multiple_references=False) - >>> - >>> LOAD_POLICY.support_multiple_references = True - >>> print(LOAD_POLICY) - LoadPolicy(equalise_cubes_kwargs=None, merge_concat_sequence='m', merge_unique=False, repeat_until_unchanged=False, support_multiple_references=True) - - >>> LOAD_POLICY.set(merge_concat_sequence="cm") - >>> print(LOAD_POLICY) - LoadPolicy(equalise_cubes_kwargs=None, merge_concat_sequence='cm', merge_unique=False, repeat_until_unchanged=False, support_multiple_references=True) - - >>> with LOAD_POLICY.context("comprehensive"): - ... print(LOAD_POLICY) - LoadPolicy(equalise_cubes_kwargs={'apply_all': True}, merge_concat_sequence='mc', merge_unique=False, repeat_until_unchanged=True, support_multiple_references=True) - >>> - >>> print(LOAD_POLICY) - LoadPolicy(equalise_cubes_kwargs=None, merge_concat_sequence='cm', merge_unique=False, repeat_until_unchanged=False, support_multiple_references=True) - """ - - # Option keys are as for CombineOptions, plus the multiple-refs control - OPTION_KEYS = CombineOptions.OPTION_KEYS + ["support_multiple_references"] - - # Allowed values are as for CombineOptions, plus boolean values for multiple-refs - _OPTIONS_ALLOWED_VALUES = dict( - list(CombineOptions._OPTIONS_ALLOWED_VALUES.items()) - + [("support_multiple_references", (True, False))] - ) - - # Settings dicts are as for CombineOptions, but with a multiple-load value added - SETTINGS = { - key: dict( - list(settings.items()) + [("support_multiple_references", key != "legacy")] - ) - for key, settings in CombineOptions.SETTINGS.items() - } - - @contextlib.contextmanager - def context(self, settings: str | dict | None = None, **kwargs): - """Return a context manager applying given options changes during a scope. - - Parameters - ---------- - settings : str or dict, optional - A settings name or options dictionary, as for :meth:`~LoadPolicy.set`. - kwargs : dict - Option values, as for :meth:`~LoadPolicy.set`. - - Examples - -------- - .. testsetup:: - - import iris - from iris import LOAD_POLICY, sample_data_path - - >>> # Show how a CombineOptions acts in the context of a load operation - >>> path = sample_data_path("time_varying_hybrid_height", "*.pp") - >>> - >>> # Show that "legacy" load behaviour allows merge but not concatenate - >>> with LOAD_POLICY.context("legacy"): - ... cubes = iris.load(path, "x_wind") - >>> print(cubes) - 0: x_wind / (m s-1) (time: 2; model_level_number: 5; latitude: 144; longitude: 192) - 1: x_wind / (m s-1) (time: 12; model_level_number: 5; latitude: 144; longitude: 192) - 2: x_wind / (m s-1) (model_level_number: 5; latitude: 144; longitude: 192) - >>> - >>> # Show how "recommended" behaviour enables concatenation also - >>> with LOAD_POLICY.context("recommended"): - ... cubes = iris.load(path, "x_wind") - >>> print(cubes) - 0: x_wind / (m s-1) (model_level_number: 5; time: 15; latitude: 144; longitude: 192) - """ - # Save the current state - saved_settings = self.settings() - - # Apply the new options and execute the context - try: - self.set(settings, **kwargs) - yield - finally: - # Re-establish the former state - self.set(saved_settings) - - -#: A control object containing the current file loading strategy options. -LOAD_POLICY = LoadPolicy() diff --git a/lib/iris/tests/integration/test_trajectory.py b/lib/iris/tests/integration/test_trajectory.py index f56970f9fa..e54e56b840 100644 --- a/lib/iris/tests/integration/test_trajectory.py +++ b/lib/iris/tests/integration/test_trajectory.py @@ -11,7 +11,7 @@ import numpy as np import iris -from iris import LOAD_POLICY +from iris import COMBINE_POLICY from iris._lazy_data import as_lazy_data from iris.analysis.trajectory import Trajectory from iris.analysis.trajectory import interpolate as traj_interpolate @@ -24,7 +24,7 @@ def setUp(self): # Load the COLPEX data => TZYX path = tests.get_data_path(["PP", "COLPEX", "theta_and_orog_subset.pp"]) # Fix to ignore time-varying orography, for the purposes of these tests - with LOAD_POLICY.context(support_multiple_references=False): + with COMBINE_POLICY.context(support_multiple_references=False): cube = iris.load_cube(path, "air_potential_temperature") cube.coord("grid_latitude").bounds = None cube.coord("grid_longitude").bounds = None diff --git a/lib/iris/tests/integration/varying_references/test_realdata_load.py b/lib/iris/tests/integration/varying_references/test_realdata_load.py index edf2b00824..8cf8bf2c80 100644 --- a/lib/iris/tests/integration/varying_references/test_realdata_load.py +++ b/lib/iris/tests/integration/varying_references/test_realdata_load.py @@ -7,7 +7,7 @@ import pytest import iris -from iris import LOAD_POLICY, sample_data_path +from iris import COMBINE_POLICY, sample_data_path @pytest.fixture(params=["default", "recommended", "legacy"]) @@ -18,7 +18,7 @@ def load_policy(request): def test_load_pp_timevarying_orography(load_policy): testdata_dirpath = sample_data_path("time_varying_hybrid_height", "*.pp") - with LOAD_POLICY.context(load_policy): + with COMBINE_POLICY.context(load_policy): cubes = iris.load(testdata_dirpath) n_cubes = len(cubes) diff --git a/lib/iris/tests/integration/varying_references/test_roundtrip_time_varying_references.py b/lib/iris/tests/integration/varying_references/test_roundtrip_time_varying_references.py index 0ad4b5a941..fce343719f 100644 --- a/lib/iris/tests/integration/varying_references/test_roundtrip_time_varying_references.py +++ b/lib/iris/tests/integration/varying_references/test_roundtrip_time_varying_references.py @@ -16,7 +16,7 @@ import pytest import iris -from iris import LOAD_POLICY +from iris import COMBINE_POLICY from iris.aux_factory import HybridHeightFactory, HybridPressureFactory from iris.coord_systems import GeogCS from iris.coords import AuxCoord, DimCoord @@ -240,7 +240,7 @@ def zcoord_type(request): return request.param -@pytest.fixture(params=[f"{name}_policy" for name in LOAD_POLICY.SETTINGS]) +@pytest.fixture(params=[f"{name}_policy" for name in COMBINE_POLICY.SETTINGS]) def load_policy(request): return request.param @@ -265,7 +265,7 @@ def test_roundtrip(file_extension, time_dependence, zcoord_type, load_policy, tm iris.save(data, filepath) policy_name = load_policy.split("_")[0] - with LOAD_POLICY.context(policy_name): + with COMBINE_POLICY.context(policy_name): # NOTE: this is default, but "legacy" mode would fail readback = iris.load(filepath) @@ -309,7 +309,7 @@ def test_split_netcdf_roundtrip(zcoord_type, load_policy, tmp_path): iris.save(field_cube, path) # load back with the chosen policy. - with LOAD_POLICY.context(policy_name): + with COMBINE_POLICY.context(policy_name): readback = iris.load(result_paths) n_cubes = len(readback) diff --git a/lib/iris/tests/unit/combine/__init__.py b/lib/iris/tests/unit/combine/__init__.py new file mode 100644 index 0000000000..ebaa1381a1 --- /dev/null +++ b/lib/iris/tests/unit/combine/__init__.py @@ -0,0 +1,5 @@ +# Copyright Iris contributors +# +# This file is part of Iris and is released under the BSD license. +# See LICENSE in the root of the repository for full licensing details. +"""Unit tests for the :mod:`iris._combine` module.""" diff --git a/lib/iris/tests/unit/test_LoadPolicy.py b/lib/iris/tests/unit/combine/test_CombineOptions.py similarity index 78% rename from lib/iris/tests/unit/test_LoadPolicy.py rename to lib/iris/tests/unit/combine/test_CombineOptions.py index e531780dad..b7cca68e1d 100644 --- a/lib/iris/tests/unit/test_LoadPolicy.py +++ b/lib/iris/tests/unit/combine/test_CombineOptions.py @@ -2,27 +2,27 @@ # # This file is part of Iris and is released under the BSD license. # See LICENSE in the root of the repository for full licensing details. -"""Unit tests for the :mod:`iris.io.loading.LoadPolicy` package.""" +"""Unit tests for the :mod:`iris._combine.CombineOptions` class.""" from unittest import mock import pytest -from iris import LoadPolicy +from iris import CombineOptions class TestInit: def test_init_empty(self): # Check how a bare init works - options = LoadPolicy() - assert options.settings() == LoadPolicy.SETTINGS["default"] + options = CombineOptions() + assert options.settings() == CombineOptions.SETTINGS["default"] def test_init_args_kwargs(self): # Check that init with args, kwargs equates to a pair of set() calls. - with mock.patch("iris.LoadPolicy.set") as mock_set: + with mock.patch("iris.CombineOptions.set") as mock_set: test_option = mock.sentinel.option test_kwargs = {"junk": "invalid"} - LoadPolicy(options=test_option, **test_kwargs) + CombineOptions(options=test_option, **test_kwargs) assert mock_set.call_args_list == [ mock.call("default"), mock.call(test_option, **test_kwargs), @@ -33,11 +33,11 @@ class Test_settings: """The .settings() returns a dict full of the settings.""" def test_settings(self): - options = LoadPolicy() + options = CombineOptions() settings = options.settings() assert isinstance(settings, dict) - assert list(settings.keys()) == LoadPolicy.OPTION_KEYS - for key in LoadPolicy.OPTION_KEYS: + assert list(settings.keys()) == CombineOptions.OPTION_KEYS + for key in CombineOptions.OPTION_KEYS: assert settings[key] == getattr(options, key) @@ -45,13 +45,13 @@ class Test_set: """Check the .set(arg, **kwargs) behaviour.""" def test_empty(self): - options = LoadPolicy() + options = CombineOptions() orig_settings = options.settings() options.set() assert options.settings() == orig_settings def test_arg_dict(self): - options = LoadPolicy() + options = CombineOptions() assert options.settings()["merge_concat_sequence"] == "m" assert options.settings()["repeat_until_unchanged"] is False options.set({"merge_concat_sequence": "c", "repeat_until_unchanged": True}) @@ -59,7 +59,7 @@ def test_arg_dict(self): assert options.settings()["repeat_until_unchanged"] is True def test_arg_string(self): - options = LoadPolicy() + options = CombineOptions() assert options.settings()["merge_concat_sequence"] == "m" assert options.settings()["repeat_until_unchanged"] is False options.set("comprehensive") @@ -67,13 +67,13 @@ def test_arg_string(self): assert options.settings()["repeat_until_unchanged"] is True def test_arg_bad_dict(self): - options = LoadPolicy() + options = CombineOptions() expected = "Unknown options.*'junk'.* : valid options are" with pytest.raises(ValueError, match=expected): options.set({"junk": "invalid"}) def test_arg_bad_string(self): - options = LoadPolicy() + options = CombineOptions() expected = ( r"arg 'options'='oddthing'.*not a valid setting.*expected one of.* " "['legacy', 'default', 'recommended', 'comprehensive']" @@ -82,7 +82,7 @@ def test_arg_bad_string(self): options.set("oddthing") def test_kwargs(self): - options = LoadPolicy() + options = CombineOptions() assert options.settings()["merge_concat_sequence"] == "m" assert options.settings()["repeat_until_unchanged"] is False options.set(merge_concat_sequence="c", repeat_until_unchanged=True) @@ -91,7 +91,7 @@ def test_kwargs(self): def test_arg_kwargs(self): # Show that kwargs override arg - options = LoadPolicy( + options = CombineOptions( support_multiple_references=False, merge_concat_sequence="", repeat_until_unchanged=False, @@ -104,7 +104,7 @@ def test_arg_kwargs(self): assert options.repeat_until_unchanged is True def test_bad_kwarg(self): - options = LoadPolicy() + options = CombineOptions() expected = "Unknown options.*'junk'.* : valid options are" with pytest.raises(ValueError, match=expected): options.set({"junk": "invalid"}) @@ -114,28 +114,28 @@ class Test_AttributeAccess: """Check operation of direct property access (with ".").""" def test_getattr(self): - options = LoadPolicy(merge_concat_sequence="m") + options = CombineOptions(merge_concat_sequence="m") assert options.merge_concat_sequence == "m" def test_getattr_badname(self): - options = LoadPolicy() - expected = "'LoadPolicy' object has no attribute 'unknown'" + options = CombineOptions() + expected = "'CombineOptions' object has no attribute 'unknown'" with pytest.raises(AttributeError, match=expected): options.unknown def test_setattr(self): - options = LoadPolicy(merge_concat_sequence="m") + options = CombineOptions(merge_concat_sequence="m") options.merge_concat_sequence = "mc" assert options.merge_concat_sequence == "mc" def test_setattr_badname(self): - options = LoadPolicy() - expected = "LoadPolicy object has no property 'anyold_property'" + options = CombineOptions() + expected = "CombineOptions object has no property 'anyold_property'" with pytest.raises(KeyError, match=expected): options.anyold_property = "x" def test_setattr_badvalue(self): - options = LoadPolicy() + options = CombineOptions() expected = "'mcm' is not a valid.*merge_concat_sequence : must be one of" with pytest.raises(ValueError, match=expected): options.merge_concat_sequence = "mcm" diff --git a/lib/iris/util.py b/lib/iris/util.py index 77a5edfc07..ed480d002c 100644 --- a/lib/iris/util.py +++ b/lib/iris/util.py @@ -2350,19 +2350,19 @@ def _print_xml(doc): def _combine_options_asdict(options: str | dict | None) -> dict: """Convert any valid combine options into an options dictionary.""" - from iris import LOAD_POLICY + from iris import COMBINE_POLICY if options is None: - opts_dict = LOAD_POLICY.settings() + opts_dict = COMBINE_POLICY.settings() elif isinstance(options, dict): opts_dict = options elif isinstance(options, str): - if options in LOAD_POLICY.SETTINGS: - opts_dict = LOAD_POLICY.SETTINGS[options] + if options in COMBINE_POLICY.SETTINGS: + opts_dict = COMBINE_POLICY.SETTINGS[options] else: msg = ( "Unrecognised settings name : expected one of " - f"{tuple(LOAD_POLICY.SETTINGS)}." + f"{tuple(COMBINE_POLICY.SETTINGS)}." ) raise ValueError(msg) else: @@ -2397,7 +2397,7 @@ def combine_cubes( :data:`iris.CombineOptions.SETTINGS_NAMES`, or a dictionary of settings options, as described for :class:`~iris.CombineOptions`. Defaults to the current :meth:`~iris.CombineOptions.settings` of the - :data:`iris.LOAD_POLICY`. + :data:`iris.COMBINE_POLICY`. kwargs : dict Individual option setting values, i.e. values for keys named in @@ -2434,12 +2434,12 @@ def testcube(timepts): return cube cubes = CubeList([testcube([1., 2]), testcube([13., 14, 15])]) - combinecubes_old_policysettings = iris.LOAD_POLICY.settings() + combinecubes_old_policysettings = iris.COMBINE_POLICY.settings() .. testcleanup:: # restore old state to avoid upsetting other tests - iris.LOAD_POLICY.set(combinecubes_old_policysettings) + iris.COMBINE_POLICY.set(combinecubes_old_policysettings) >>> # Take a pair of sample cubes which can concatenate together >>> print(cubes) @@ -2456,7 +2456,7 @@ def testcube(timepts): >>> print(combine_cubes(cubes, merge_concat_sequence="mc")) 0: unknown / (unknown) (time: 5) >>> # ... which may be controlled by various means - >>> iris.LOAD_POLICY.set("recommended") + >>> iris.COMBINE_POLICY.set("recommended") >>> print(combine_cubes(cubes)) 0: unknown / (unknown) (time: 5) @@ -2466,7 +2466,7 @@ def testcube(timepts): 0: unknown / (unknown) (time: 2) 1: unknown / (unknown) (time: 3) >>> # ... which can then be fixed by enabling attribute equalisation - >>> with iris.LOAD_POLICY.context(equalise_cubes_kwargs={"apply_all":True}): + >>> with iris.COMBINE_POLICY.context(equalise_cubes_kwargs={"apply_all":True}): ... print(combine_cubes(cubes)) ... 0: unknown / (unknown) (time: 5) From 75929b47267d6cdb605af13981c65c56ebae6fb5 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Wed, 26 Feb 2025 17:24:45 +0000 Subject: [PATCH 17/27] Fix doctest. --- lib/iris/_combine.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/iris/_combine.py b/lib/iris/_combine.py index 9e54f8c76f..f0e66191b4 100644 --- a/lib/iris/_combine.py +++ b/lib/iris/_combine.py @@ -115,6 +115,7 @@ class CombineOptions(threading.local): .. testsetup:: + from iris import COMBINE_POLICY loadpolicy_old_settings = COMBINE_POLICY.settings() .. testcleanup:: From e92b6e020039935c639ea440e064214992499a2c Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Thu, 27 Feb 2025 00:03:20 +0000 Subject: [PATCH 18/27] Rework CombineOptions.set() tests to cover context() method also. --- .../tests/unit/combine/test_CombineOptions.py | 152 +++++++++++++----- 1 file changed, 113 insertions(+), 39 deletions(-) diff --git a/lib/iris/tests/unit/combine/test_CombineOptions.py b/lib/iris/tests/unit/combine/test_CombineOptions.py index b7cca68e1d..4c3e791f0c 100644 --- a/lib/iris/tests/unit/combine/test_CombineOptions.py +++ b/lib/iris/tests/unit/combine/test_CombineOptions.py @@ -41,30 +41,126 @@ def test_settings(self): assert settings[key] == getattr(options, key) -class Test_set: - """Check the .set(arg, **kwargs) behaviour.""" - - def test_empty(self): +def options_checks(options, checks): + # Check (parts of) options against a dictionary of "expected" values. + settings = options.settings() + return all(settings[key] == value for key, value in checks.items()) + + +class Test_set_and_context: + """Check the .set(arg, **kwargs) and .context(arg, **kwargs) behaviours.""" + + @staticmethod + def do_check( + op_arg=None, + op_kwargs=None, + before_checks=None, + after_checks=None, + initial_options=None, + op_is_set=True, + ): + """Generic test routine check method. + + Perform an operation(op_arg, **op_kwargs) and test (partial) options state + before and after. If provided, can also start from a non-default + 'initial_options' state. + + Used to generalise between the .set() and .context() calls. + In the case of .context(), the 'after' is within the block, and the 'before' + state should always be restored again afterwards. + """ + if initial_options is not None: + options = initial_options + else: + options = CombineOptions() + + op_kwargs = op_kwargs or {} + + if before_checks is not None: + assert options_checks(options, before_checks) + + if op_is_set: + # do "set" check + options.set(op_arg, **op_kwargs) + assert options_checks(options, after_checks) + else: + # do "context" checks + with options.context(op_arg, **op_kwargs): + assert options_checks(options, after_checks) + assert options_checks(options, before_checks) + + @pytest.fixture(params=["set", "context"]) + def op_is_set(self, request): + """Parametrise a test over both .set() and and .context() calls.""" + return request.param == "set" + + def test_empty_set(self): + # More or less, just check that an empty set() call is OK. options = CombineOptions() orig_settings = options.settings() options.set() assert options.settings() == orig_settings - def test_arg_dict(self): + def test_empty_context(self): + # More or less, just check that an empty context() call is OK. options = CombineOptions() - assert options.settings()["merge_concat_sequence"] == "m" - assert options.settings()["repeat_until_unchanged"] is False - options.set({"merge_concat_sequence": "c", "repeat_until_unchanged": True}) - assert options.settings()["merge_concat_sequence"] == "c" - assert options.settings()["repeat_until_unchanged"] is True + orig_settings = options.settings() + with options.context(): + assert options.settings() == orig_settings + + def test_arg_dict(self, op_is_set): + expect_before = {"merge_concat_sequence": "m", "repeat_until_unchanged": False} + set_arg = {"merge_concat_sequence": "c", "repeat_until_unchanged": True} + expect_after = {"merge_concat_sequence": "c", "repeat_until_unchanged": True} + self.do_check( + op_arg=set_arg, + before_checks=expect_before, + after_checks=expect_after, + op_is_set=op_is_set, + ) - def test_arg_string(self): - options = CombineOptions() - assert options.settings()["merge_concat_sequence"] == "m" - assert options.settings()["repeat_until_unchanged"] is False - options.set("comprehensive") - assert options.settings()["merge_concat_sequence"] == "mc" - assert options.settings()["repeat_until_unchanged"] is True + def test_arg_string(self, op_is_set): + expect_before = {"merge_concat_sequence": "m", "repeat_until_unchanged": False} + set_arg = "comprehensive" + expect_after = {"merge_concat_sequence": "mc", "repeat_until_unchanged": True} + self.do_check( + op_arg=set_arg, + before_checks=expect_before, + after_checks=expect_after, + op_is_set=op_is_set, + ) + + def test_kwargs(self, op_is_set): + expect_before = {"merge_concat_sequence": "m", "repeat_until_unchanged": False} + set_arg = {"merge_concat_sequence": "c", "repeat_until_unchanged": True} + expect_after = {"merge_concat_sequence": "c", "repeat_until_unchanged": True} + self.do_check( + op_arg=set_arg, + before_checks=expect_before, + after_checks=expect_after, + op_is_set=op_is_set, + ) + + def test_arg_kwargs(self, op_is_set): + # Show that kwargs override arg + initial_options = CombineOptions( + merge_concat_sequence="m", + repeat_until_unchanged=False, + ) + expect_before = {"merge_concat_sequence": "m", "repeat_until_unchanged": False} + # NOTE: the arg changes the sequence from "m" to "c" ... + set_arg = dict(merge_concat_sequence="c", repeat_until_unchanged=True) + # .. but the keyword overrides that to "mc" + set_kwargs = dict(merge_concat_sequence="mc") + expect_after = {"merge_concat_sequence": "mc", "repeat_until_unchanged": True} + self.do_check( + initial_options=initial_options, + before_checks=expect_before, + op_arg=set_arg, + op_kwargs=set_kwargs, + after_checks=expect_after, + op_is_set=op_is_set, + ) def test_arg_bad_dict(self): options = CombineOptions() @@ -81,28 +177,6 @@ def test_arg_bad_string(self): with pytest.raises(ValueError, match=expected): options.set("oddthing") - def test_kwargs(self): - options = CombineOptions() - assert options.settings()["merge_concat_sequence"] == "m" - assert options.settings()["repeat_until_unchanged"] is False - options.set(merge_concat_sequence="c", repeat_until_unchanged=True) - assert options.settings()["merge_concat_sequence"] == "c" - assert options.settings()["repeat_until_unchanged"] is True - - def test_arg_kwargs(self): - # Show that kwargs override arg - options = CombineOptions( - support_multiple_references=False, - merge_concat_sequence="", - repeat_until_unchanged=False, - ) - options.set( - dict(merge_concat_sequence="c", repeat_until_unchanged=True), - merge_concat_sequence="mc", - ) - assert options.merge_concat_sequence == "mc" - assert options.repeat_until_unchanged is True - def test_bad_kwarg(self): options = CombineOptions() expected = "Unknown options.*'junk'.* : valid options are" From 923e2c86578289c5a1f4920c0e61a6f264e5aed3 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Thu, 27 Feb 2025 00:54:03 +0000 Subject: [PATCH 19/27] Add tests for cubelist combine functions (and fix). --- lib/iris/cube.py | 20 ++------- lib/iris/tests/unit/cube/test_CubeList.py | 49 +++++++++++++++++++++++ 2 files changed, 52 insertions(+), 17 deletions(-) diff --git a/lib/iris/cube.py b/lib/iris/cube.py index 3e2278fab5..5161dcfaa1 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -643,27 +643,13 @@ def combine(self, options: str | dict | None = None, **kwargs) -> CubeList: return combine_cubes(self, options, **kwargs) def combine_cube(self, options: str | dict | None = None, **kwargs) -> CubeList: - result = self.combine() + result = self.combine(options, **kwargs) n_cubes = len(result) if n_cubes != 1: - from iris.util import _combine_options_asdict - - opts_dict = _combine_options_asdict(options) - merge_concat_sequence = opts_dict.get("merge_concat_sequence") - is_merge = ( - not isinstance(merge_concat_sequence, str) - or len(merge_concat_sequence) < 0 - or not merge_concat_sequence.endswith("m") - ) - err_type = ( - iris.exceptions.MergeError - if is_merge - else iris.exceptions.ConcatenateError - ) msg = f"'combine' operation yielded {n_cubes} cubes, expected exactly 1." - raise err_type(msg) + raise ValueError(msg) - return result + return result[0] def realise_data(self): """Fetch 'real' data for all cubes, in a shared calculation. diff --git a/lib/iris/tests/unit/cube/test_CubeList.py b/lib/iris/tests/unit/cube/test_CubeList.py index 62e63e6694..ddd1ebc2bc 100644 --- a/lib/iris/tests/unit/cube/test_CubeList.py +++ b/lib/iris/tests/unit/cube/test_CubeList.py @@ -719,3 +719,52 @@ def test__repr_html_(mocker): # "CubeListRepresentation(cubelist).repr_html()" was called exactly once, with no args mock.call() ] + + +class Test_combine__apis: + """Confirm that CubeList.combine/combine_cube just call combine_cubes.""" + + def mock_combine_cubes(self): + def mock_call(cubes, options=None, **kwargs): + self.call_params = [cubes, options, kwargs] + return cubes # used to test effect of different cases + + return mock_call + + def test_combine(self): + cubelist = CubeList([]) + arg = mock.sentinel.options_arg + kwargs = dict( + key_test_1=1, + key_test_2=2, + ) + with mock.patch("iris.util.combine_cubes", self.mock_combine_cubes()): + result = cubelist.combine(arg, **kwargs) + assert self.call_params == [cubelist, arg, kwargs] + assert result == cubelist + + @pytest.mark.parametrize("ncubes", [0, 1, 2], ids=["nocubes", "onecube", "ncubes"]) + def test_combine_cube(self, ncubes): + """In this case, also check behaviour for result of <1 =1 >1 cubes.""" + cubelist = CubeList( + [Cube([0], long_name=f"cube_{i_cube})") for i_cube in range(ncubes)] + ) + arg = mock.sentinel.options_arg + kwargs = dict( + key_test_1=1, + key_test_2=2, + ) + if ncubes == 1: + with mock.patch("iris.util.combine_cubes", self.mock_combine_cubes()): + result = cubelist.combine_cube(arg, **kwargs) + assert self.call_params == [cubelist, arg, kwargs] + assert result == cubelist[0] + else: + if ncubes == 0: + msg = "'combine' operation yielded 0 cubes, expected exactly 1" + else: + msg = f"'combine' operation yielded {ncubes} cubes, expected exactly 1" + + with mock.patch("iris.util.combine_cubes", self.mock_combine_cubes()): + with pytest.raises(ValueError, match=msg): + result = cubelist.combine_cube(arg, **kwargs) From c8823c8da1d9d45cf5458aa82659eb8064a4c047 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Thu, 27 Feb 2025 14:08:52 +0000 Subject: [PATCH 20/27] Add tests for individual combine control keywords. --- .../tests/unit/util/test_combine_cubes.py | 87 +++++++++++++++---- 1 file changed, 72 insertions(+), 15 deletions(-) diff --git a/lib/iris/tests/unit/util/test_combine_cubes.py b/lib/iris/tests/unit/util/test_combine_cubes.py index b5630193c9..5cb2af0da4 100644 --- a/lib/iris/tests/unit/util/test_combine_cubes.py +++ b/lib/iris/tests/unit/util/test_combine_cubes.py @@ -13,52 +13,53 @@ import pytest from iris import LoadPolicy +from iris.exceptions import DuplicateDataError from iris.tests.unit.fileformats.test_load_functions import cu from iris.util import combine_cubes @pytest.fixture(params=list(LoadPolicy.SETTINGS.keys())) -def options(request): +def settings(request): # N.B. "request" is a standard PyTest fixture return request.param # Return the name of the attribute to test. -class Test: - def test_mergeable(self, options): +class Test_settings: + def test_mergeable(self, settings): c1, c2 = cu(t=1), cu(t=2) c12 = cu(t=(1, 2)) input_cubes = [c1, c2] - result = combine_cubes(input_cubes, options) + result = combine_cubes(input_cubes, settings) expected = [c12] # same in all cases assert result == expected - def test_catable(self, options): + def test_catable(self, settings): c1, c2 = cu(t=(1, 2)), cu(t=(3, 4)) c12 = cu(t=(1, 2, 3, 4)) input_cubes = [c1, c2] - result = combine_cubes(input_cubes, options) + result = combine_cubes(input_cubes, settings) expected = { - "legacy": [c1, c2], # standard options can't do this .. + "legacy": [c1, c2], # standard settings can't do this .. "default": [c1, c2], "recommended": [c12], # .. but it works if you enable concatenate "comprehensive": [c12], - }[options] + }[settings] assert result == expected - def test_cat_enables_merge(self, options): + def test_cat_enables_merge(self, settings): c1, c2 = cu(t=(1, 2), z=1), cu(t=(3, 4, 5), z=1) c3, c4 = cu(t=(1, 2, 3), z=2), cu(t=(4, 5), z=2) c1234 = cu(t=(1, 2, 3, 4, 5), z=(1, 2)) c12 = cu(t=(1, 2, 3, 4, 5), z=1) c34 = cu(t=(1, 2, 3, 4, 5), z=2) input_cubes = [c1, c2, c3, c4] - result = combine_cubes(input_cubes, options) + result = combine_cubes(input_cubes, settings) expected = { "legacy": input_cubes, "default": input_cubes, "recommended": [c12, c34], # standard "mc" sequence can't do this one.. "comprehensive": [c1234], # .. but works if you repeat - }[options] + }[settings] assert result == expected def test_cat_enables_merge__custom(self): @@ -69,14 +70,70 @@ def test_cat_enables_merge__custom(self): result = combine_cubes(input_cubes, merge_concat_sequence="cm") assert result == [c1234] - def test_nocombine_overlapping(self, options): + def test_nocombine_overlapping(self, settings): c1, c2 = cu(t=(1, 3)), cu(t=(2, 4)) input_cubes = [c1, c2] - result = combine_cubes(input_cubes, options) + result = combine_cubes(input_cubes, settings) assert result == input_cubes # same in all cases : can't do this - def test_nocombine_dim_scalar(self, options): + def test_nocombine_dim_scalar(self, settings): c1, c2 = cu(t=(1,)), cu(t=2) input_cubes = [c1, c2] - result = combine_cubes(input_cubes, options) + result = combine_cubes(input_cubes, settings) assert result == input_cubes # can't do this at present + + +class Test_options: + """Test the individual combine options keywords.""" + + def test_equalise_cubes_kwargs(self): + # two cubes will merge .. + cubes = [cu(t=1), cu(t=2)] + # .. but prevent by adding an attribute on one + cubes[0].attributes["x"] = 3 + # won't combine.. + result = combine_cubes(cubes) + assert len(result) == 2 + # ..but will if you enable attribute equalisation + result = combine_cubes( + cubes, equalise_cubes_kwargs={"equalise_attributes": True} + ) + assert len(result) == 1 + + def test_merge_concat_sequence(self): + # cubes require concat, merge won't work + cubes = [cu(t=[1, 2]), cu(t=[3, 4])] + result = combine_cubes(cubes) + assert len(result) == 2 + # .. but will if you put concat in the sequence + result = combine_cubes(cubes, merge_concat_sequence="c") + assert len(result) == 1 + + def test_merge_unique(self): + # two identical cubes + cubes = [cu("myname"), cu("myname")] + # the combine call (with merge) is OK if we *don't* insist on unique cubes + combine_cubes(cubes) + # .. but it errors if we *do* insist on uniqueness + msg = "Duplicate 'myname' cube" + with pytest.raises(DuplicateDataError, match=msg): + combine_cubes(cubes, merge_unique=True) + + def test_repeat_until_unchanged(self): + # construct a case that will only merge once it was previously concatenated + cubes = [ + cu(t=[1, 2, 3], z=1), + cu(t=[4, 5], z=1), + cu(t=[1, 2], z=2), + cu(t=[3, 4, 5], z=2), + ] + result = combine_cubes(cubes, merge_concat_sequence="mc") + assert len(result) == 2 + result = combine_cubes( + cubes, merge_concat_sequence="mc", repeat_until_unchanged=True + ) + assert len(result) == 1 + + # NOTE: "test_support_multiple_references" -- not tested here + # this may be too hard + # it is adequately tested in tests/integration/varying_references From cc74cdaa3608144f2e9aecd4bbd8dd965b7e4c09 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Thu, 27 Feb 2025 15:24:19 +0000 Subject: [PATCH 21/27] Added whatsnew, and minimal links in other docs sections. --- docs/src/userguide/loading_iris_cubes.rst | 6 ++++++ docs/src/whatsnew/latest.rst | 24 +++++++++++++++++++++-- lib/iris/__init__.py | 3 ++- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/docs/src/userguide/loading_iris_cubes.rst b/docs/src/userguide/loading_iris_cubes.rst index b71f033c30..82d3193517 100644 --- a/docs/src/userguide/loading_iris_cubes.rst +++ b/docs/src/userguide/loading_iris_cubes.rst @@ -15,6 +15,12 @@ Iris will attempt to return **as few cubes as possible** by collecting together multiple fields with a shared standard name into a single multidimensional cube. +.. hint:: + + There are some hints at :class:`iris.CombineOptions` on how Iris works to load + fewer and larger cubes, along with user options that can aid in controlling the + process. + The :py:func:`iris.load` function automatically recognises the format of the given files and attempts to produce Iris Cubes from their contents. diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index c7e820922d..0e7aa078ec 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -30,6 +30,26 @@ This document explains the changes made to Iris for this release ✨ Features =========== +#. `@pp-mo`_ renamed the :class:`iris.LoadPolicy` as :class:`iris.CombineOptions` and + :data:`iris.LOAD_POLICY` as :data:`iris.COMBINE_POLICY`, though the original names + remain functional (and refer to the same things) for now. + (:issue:`6203`, :pull:`6334`) + +#. `@pp-mo`_ added new :meth:`~iris.cube.CubeList.combine` and + :meth:`~iris.cube.CubeList.combine_cube` methods of a :class:`~iris.cube.CubeList` + as an alternative way of accessing the :func:`~iris.util.combine_cubes` mechanism. + (:issue:`6203`, :pull:`6334`) + +#. `@pp-mo`_ added a new utility function :func:`~iris.util.combine_cubes`, to give + general public access to the combine merge/concatenate mechanism introduced for + generalised loading support via :class:`iris.LoadPolicy` in the Iris 3.11 release. + (:issue:`6203`, :pull:`6334`) + +#. `@pp-mo`_ overhauled the :class:`iris.LoadPolicy` facility by adding a new + ``equalise_cubes_kwarg`` keyword, enabling it to call the + :func:`~iris.util.equalise_cubes` utility function as one of its processing stages. + (:issue:`6203`, :pull:`6334`) + #. `@pp-mo`_ added a new utility function :func:`~iris.util.equalise_cubes`, to help with aligning cubes so they can merge / concatenate. (:issue:`6248`, :pull:`6257`) @@ -48,11 +68,11 @@ This document explains the changes made to Iris for this release However, :meth:`~iris.cube.Cube.transpose` will work, as will :meth:`~iris.cube.Cube.copy`. Note that, ``cube.copy(data=iris.DATALESS)`` will provide a dataless copy of a cube. (:issue:`4447`, :pull:`6253`) - + #. `@ESadek-MO`_ added the :mod:`iris.quickplot` ``footer`` kwarg to render text in the bottom right of the plot figure. (:issue:`6247`, :pull:`6332`) - + 🐛 Bugs Fixed ============= diff --git a/lib/iris/__init__.py b/lib/iris/__init__.py index d2b6249825..ad90c5549d 100644 --- a/lib/iris/__init__.py +++ b/lib/iris/__init__.py @@ -16,7 +16,8 @@ The :func:`load` function provides a simple way to explore data from the interactive Python prompt. It will convert the source data into :class:`Cubes `, and combine those cubes into -higher-dimensional cubes where possible. +higher-dimensional cubes where possible +(for which, please see :class:`iris.CombineOptions`). The :func:`load_cube` and :func:`load_cubes` functions are similar to :func:`load`, but they raise an exception if the number of cubes is not From 8d09bc24c2b56240f3171980afd2d7a4cb67d1e7 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 3 Mar 2025 00:40:15 +0000 Subject: [PATCH 22/27] Review changes: docs improvements; docstrings for cubelist combine methods. --- docs/src/userguide/loading_iris_cubes.rst | 7 ++-- lib/iris/__init__.py | 9 ++++- lib/iris/_combine.py | 6 ++- lib/iris/cube.py | 46 +++++++++++++++++++++++ 4 files changed, 61 insertions(+), 7 deletions(-) diff --git a/docs/src/userguide/loading_iris_cubes.rst b/docs/src/userguide/loading_iris_cubes.rst index 82d3193517..cbba4da39f 100644 --- a/docs/src/userguide/loading_iris_cubes.rst +++ b/docs/src/userguide/loading_iris_cubes.rst @@ -17,9 +17,10 @@ into a single multidimensional cube. .. hint:: - There are some hints at :class:`iris.CombineOptions` on how Iris works to load - fewer and larger cubes, along with user options that can aid in controlling the - process. + There are details at :class:`iris.CombineOptions` on how Iris works to load + fewer and larger cubes : The :data:`iris.COMBINE_POLICY` object allows the user to + control how cubes are combined during the loading process. See the documentation + of the :class:`iris.CombineOptions` class for details. The :py:func:`iris.load` function automatically recognises the format of the given files and attempts to produce Iris Cubes from their contents. diff --git a/lib/iris/__init__.py b/lib/iris/__init__.py index ad90c5549d..d141dbdb5f 100644 --- a/lib/iris/__init__.py +++ b/lib/iris/__init__.py @@ -16,8 +16,13 @@ The :func:`load` function provides a simple way to explore data from the interactive Python prompt. It will convert the source data into :class:`Cubes `, and combine those cubes into -higher-dimensional cubes where possible -(for which, please see :class:`iris.CombineOptions`). +higher-dimensional cubes where possible. + +.. note:: + + User control of the 'combine' process is provided via a specific + :class:`iris.CombineOptions` object called :data:`iris.COMBINE_POLICY`. + See the :class:`iris.CombineOptions` class for details. The :func:`load_cube` and :func:`load_cubes` functions are similar to :func:`load`, but they raise an exception if the number of cubes is not diff --git a/lib/iris/_combine.py b/lib/iris/_combine.py index f0e66191b4..0a760189df 100644 --- a/lib/iris/_combine.py +++ b/lib/iris/_combine.py @@ -39,9 +39,10 @@ class CombineOptions(threading.local): The individual configurable options are : - * ``equalise_cubes_kwargs`` = (dict) + * ``equalise_cubes_kwargs`` = (dict or None) Specifies keywords for an :func:`iris.util.equalise_cubes` call, to be applied - before any merge/concatenate step. + before any merge/concatenate step. If ``None``, or empty, no equalisation step + is performed. * ``merge_concat_sequence`` = "m" / "c" / "cm" / "mc" Specifies whether to apply :meth:`~iris.cube.CubeList.merge`, or @@ -361,6 +362,7 @@ def _combine_cubes(cubes: List[Cube], options: dict) -> CubeList: eq_args = options.get("equalise_cubes_kwargs", None) if eq_args: + # Skip missing (or empty) arg, as no effect : see `equalise_cubes`. from iris.util import equalise_cubes equalise_cubes(cubelist, **eq_args) diff --git a/lib/iris/cube.py b/lib/iris/cube.py index 5161dcfaa1..551acef8c1 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -638,11 +638,57 @@ def concatenate( ) def combine(self, options: str | dict | None = None, **kwargs) -> CubeList: + """Combine cubes, as with :func:`iris.util.combine_cubes`. + + Parameters + ---------- + options : str or dict, optional + Either a standard "combine settings" name, i.e. one of the + :data:`iris.CombineOptions.SETTINGS_NAMES`, or a dictionary of + settings options, as described for :class:`~iris.CombineOptions`. + Defaults to the current :meth:`~iris.CombineOptions.settings` of the + :data:`iris.COMBINE_POLICY`. + + kwargs : dict + Individual option setting values, i.e. values for keys named in + :data:`iris.CombineOptions.OPTION_KEYS`, as described for + :meth:`~iris.CombineOptions.set`. + These take precedence over those set by the `options` arg. + + Returns + ------- + :class:`CubeList` + + """ from iris.util import combine_cubes return combine_cubes(self, options, **kwargs) def combine_cube(self, options: str | dict | None = None, **kwargs) -> CubeList: + """Combine to a single cube, with :func:`iris.util.combine_cubes`. + + As :meth:`combine`, but raises a ValueError if the result is not a single cube. + + Parameters + ---------- + options : str or dict, optional + Either a standard "combine settings" name, i.e. one of the + :data:`iris.CombineOptions.SETTINGS_NAMES`, or a dictionary of + settings options, as described for :class:`~iris.CombineOptions`. + Defaults to the current :meth:`~iris.CombineOptions.settings` of the + :data:`iris.COMBINE_POLICY`. + + kwargs : dict + Individual option setting values, i.e. values for keys named in + :data:`iris.CombineOptions.OPTION_KEYS`, as described for + :meth:`~iris.CombineOptions.set`. + These take precedence over those set by the `options` arg. + + Returns + ------- + Cube + + """ result = self.combine(options, **kwargs) n_cubes = len(result) if n_cubes != 1: From 7f14f68498ab9ce085312dc7ca48c1c132137a63 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Mon, 3 Mar 2025 09:04:05 +0000 Subject: [PATCH 23/27] Tiny formatting correction. --- lib/iris/cube.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/iris/cube.py b/lib/iris/cube.py index 2b8246d983..77191c3a9a 100644 --- a/lib/iris/cube.py +++ b/lib/iris/cube.py @@ -686,7 +686,7 @@ def combine_cube(self, options: str | dict | None = None, **kwargs) -> CubeList: Returns ------- - Cube + :class:`Cube` """ result = self.combine(options, **kwargs) From 857a4f56e47a7bd3fddb3920f12f00d8c64215f8 Mon Sep 17 00:00:00 2001 From: Patrick Peglar Date: Thu, 6 Mar 2025 18:01:05 +0000 Subject: [PATCH 24/27] Review changes: docs explain not to assign to iris.COMBINE_POLICY. --- lib/iris/_combine.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/iris/_combine.py b/lib/iris/_combine.py index 0a760189df..cc4a60e3bc 100644 --- a/lib/iris/_combine.py +++ b/lib/iris/_combine.py @@ -132,8 +132,18 @@ class CombineOptions(threading.local): .. Note:: - The name ``iris.LOAD_POLICY`` refers to the same thing as - ``iris.COMBINE_POLICY``, and is still usable, but no longer recommended. + The ``iris.COMBINE_POLICY`` can be adjusted by either: + + 1. calling ``iris.COMBINE_POLICY.set()``, or + 2. using ``with COMBINE_POLICY.context(): ...``, or + 3. assigning a property ``COMBINE_POLICY.