diff --git a/docs/src/userguide/loading_iris_cubes.rst b/docs/src/userguide/loading_iris_cubes.rst index b71f033c30..cbba4da39f 100644 --- a/docs/src/userguide/loading_iris_cubes.rst +++ b/docs/src/userguide/loading_iris_cubes.rst @@ -15,6 +15,13 @@ Iris will attempt to return **as few cubes as possible** by collecting together multiple fields with a shared standard name into a single multidimensional cube. +.. hint:: + + There are details at :class:`iris.CombineOptions` on how Iris works to load + fewer and larger cubes : The :data:`iris.COMBINE_POLICY` object allows the user to + control how cubes are combined during the loading process. See the documentation + of the :class:`iris.CombineOptions` class for details. + The :py:func:`iris.load` function automatically recognises the format of the given files and attempts to produce Iris Cubes from their contents. diff --git a/docs/src/whatsnew/latest.rst b/docs/src/whatsnew/latest.rst index 07915ad040..b243796082 100644 --- a/docs/src/whatsnew/latest.rst +++ b/docs/src/whatsnew/latest.rst @@ -30,6 +30,26 @@ This document explains the changes made to Iris for this release ✨ Features =========== +#. `@pp-mo`_ renamed the :class:`iris.LoadPolicy` as :class:`iris.CombineOptions` and + :data:`iris.LOAD_POLICY` as :data:`iris.COMBINE_POLICY`, though the original names + remain functional (and refer to the same things) for now. + (:issue:`6203`, :pull:`6334`) + +#. `@pp-mo`_ added new :meth:`~iris.cube.CubeList.combine` and + :meth:`~iris.cube.CubeList.combine_cube` methods of a :class:`~iris.cube.CubeList` + as an alternative way of accessing the :func:`~iris.util.combine_cubes` mechanism. + (:issue:`6203`, :pull:`6334`) + +#. `@pp-mo`_ added a new utility function :func:`~iris.util.combine_cubes`, to give + general public access to the combine merge/concatenate mechanism introduced for + generalised loading support via :class:`iris.LoadPolicy` in the Iris 3.11 release. + (:issue:`6203`, :pull:`6334`) + +#. `@pp-mo`_ overhauled the :class:`iris.LoadPolicy` facility by adding a new + ``equalise_cubes_kwarg`` keyword, enabling it to call the + :func:`~iris.util.equalise_cubes` utility function as one of its processing stages. + (:issue:`6203`, :pull:`6334`) + #. `@pp-mo`_ added a new utility function :func:`~iris.util.equalise_cubes`, to help with aligning cubes so they can merge / concatenate. (:issue:`6248`, :pull:`6257`) @@ -48,11 +68,11 @@ This document explains the changes made to Iris for this release However, :meth:`~iris.cube.Cube.transpose` will work, as will :meth:`~iris.cube.Cube.copy`. Note that, ``cube.copy(data=iris.DATALESS)`` will provide a dataless copy of a cube. (:issue:`4447`, :pull:`6253`) - + #. `@ESadek-MO`_ added the :mod:`iris.quickplot` ``footer`` kwarg to render text in the bottom right of the plot figure. (:issue:`6247`, :pull:`6332`) - + 🐛 Bugs Fixed ============= diff --git a/lib/iris/__init__.py b/lib/iris/__init__.py index d622bd18b0..d141dbdb5f 100644 --- a/lib/iris/__init__.py +++ b/lib/iris/__init__.py @@ -18,6 +18,12 @@ :class:`Cubes `, and combine those cubes into higher-dimensional cubes where possible. +.. note:: + + User control of the 'combine' process is provided via a specific + :class:`iris.CombineOptions` object called :data:`iris.COMBINE_POLICY`. + See the :class:`iris.CombineOptions` class for details. + The :func:`load_cube` and :func:`load_cubes` functions are similar to :func:`load`, but they raise an exception if the number of cubes is not what was expected. They are more useful in scripts, where they can @@ -94,14 +100,13 @@ def callback(cube, field, filename): import threading from typing import Callable, Literal +from iris._combine import COMBINE_POLICY as _COMBINE_POLICY +from iris._combine import CombineOptions import iris._constraints import iris.config import iris.io from iris.io import save -from iris.loading import LOAD_POLICY as _LOAD_POLICY from iris.loading import ( - CombineOptions, - LoadPolicy, load, load_cube, load_cubes, @@ -111,8 +116,14 @@ def callback(cube, field, filename): # NOTE: we make an independent local 'LOAD_POLICY' definition here, just so that we # can ensure an entry for it in our API documentation page. -#: A control object containing the current file loading strategy options. -LOAD_POLICY = _LOAD_POLICY +#: An object to control default cube combination and loading options +COMBINE_POLICY = _COMBINE_POLICY + +#: An alias for the :class:`~iris._combine.CombineOptions` class. +LoadPolicy = CombineOptions + +#: An alias for the :data:`~iris.COMBINE_POLICY` object. +LOAD_POLICY = _COMBINE_POLICY from ._deprecation import IrisDeprecation, warn_deprecated @@ -132,6 +143,7 @@ def callback(cube, field, filename): # Restrict the names imported when using "from iris import *" __all__ = [ "AttributeConstraint", + "COMBINE_POLICY", "CombineOptions", "Constraint", "DATALESS", diff --git a/lib/iris/_combine.py b/lib/iris/_combine.py index 7b01dfc87e..5afffe7309 100644 --- a/lib/iris/_combine.py +++ b/lib/iris/_combine.py @@ -11,124 +11,214 @@ publicly available. """ +from __future__ import annotations + import contextlib import threading -from typing import Mapping - +from typing import TYPE_CHECKING, Any, Dict, List -class CombineOptions(threading.local): - """A container for cube combination options. +if TYPE_CHECKING: + from iris.cube import Cube, CubeList - Controls for generalised merge/concatenate options. - Also controls the detection and handling of cases where a hybrid coordinate - uses multiple reference fields during loading : for example, a UM file which - contains a series of fields describing time-varying orography. +class CombineOptions(threading.local): + """A control object for Iris loading and cube combination options. - Options can be set directly, or via :meth:`~iris.LoadPolicy.set`, or changed for - the scope of a code block with :meth:`~iris.LoadPolicy.context`. + Both the iris loading functions and the "combine_cubes" utility apply a number of + possible "cube combination" operations to a list of cubes, in a definite sequence, + all of which tend to combine cubes into a smaller number of larger or + higher-dimensional cubes. - .. note :: + This object groups various control options for these behaviours, which apply to + both the :func:`iris.util.combine_cubes` utility method and the core Iris loading + functions "iris.load_xxx". - The default behaviour will "fix" loading for cases like the time-varying - orography case described above. However, this is not strictly - backwards-compatible. If this causes problems, you can force identical loading - behaviour to earlier Iris versions with ``LOAD_POLICY.set("legacy")`` or - equivalent. + The :class:`CombineOptions` class defines the allowed control options, while a + global singleton object :data:`iris.COMBINE_POLICY` holds the current global + default settings. - .. testsetup:: + The individual configurable options are : - from iris import LOAD_POLICY + * ``equalise_cubes_kwargs`` = (dict or None) + Specifies keywords for an :func:`iris.util.equalise_cubes` call, to be applied + before any merge/concatenate step. If ``None``, or empty, no equalisation step + is performed. - Notes - ----- - The individual configurable options are : + * ``merge_concat_sequence`` = "m" / "c" / "cm" / "mc" + Specifies whether to apply :meth:`~iris.cube.CubeList.merge`, or + :meth:`~iris.cube.CubeList.concatenate` operations, or both, in either order. - * ``support_multiple_references`` = True / False - When enabled, the presence of multiple aux-factory reference cubes, which merge - to define a extra dimension, will add that dimension to the loaded cubes. - This is essential for correct support of time-dependent hybrid coordinates (i.e. - aux factories) when loading from fields-based data (e.g. PP or GRIB). - For example (notably) time-dependent orography in UM data on hybrid-heights. + * ``merge_unique`` = True / False + When True, any merge operation will error if its result contains multiple + identical cubes. Otherwise (unique=False), that is a permitted result. - In addition, when such multiple references are detected, an extra concatenate - step is added to the 'merge_concat_sequence' (see below), if none is already - configured there. + .. Note:: - * ``merge_concat_sequence`` = "m" / "c" / "cm" / "mc" - Specifies whether to merge, or concatenate, or both in either order. - This is the "combine" operation which is applied to loaded data. + By default, in a normal :meth:`~iris.cube.CubeList.merge` operation on a + :class:`~iris.cube.CubeList`, ``unique`` defaults to ``True``. + For loading operations, however, the default is ``unique=False``, as this + produces the intended behaviour when loading with multiple constraints. * ``repeat_until_unchanged`` = True / False When enabled, the configured "combine" operation will be repeated until the result is stable (no more cubes are combined). - Several common sets of options are provided in :data:`~iris.LOAD_POLICY.SETTINGS` : + * ``support_multiple_references`` = True / False + When enabled, support cases where a hybrid coordinate has multiple reference + fields : for example, a UM file which contains a series of fields describing a + time-varying orography. + + Alternatively, certain fixed combinations of options can be selected by a + "settings" name, one of :data:`CombineOptions.SETTINGS_NAMES` : * ``"legacy"`` - Produces loading behaviour identical to Iris versions < 3.11, i.e. before the - varying hybrid references were supported. + Apply a plain merge step only, i.e. ``merge_concat_sequence="m"``. + Other options are all "off". + This produces loading behaviour identical to Iris versions < 3.11, i.e. before + the varying hybrid references were supported. * ``"default"`` As "legacy" except that ``support_multiple_references=True``. This differs from "legacy" only when multiple mergeable reference fields are encountered, in which case incoming cubes are extended into the extra dimension, and a concatenate step is added. + Since the handling of multiple references affects only loading operations, + for the purposes of calls to :func:`~iris.util.combine_cubes`, this setting is + *identical* to "legacy". + + .. Warning:: + + The ``"default"`` setting **is** the initial default mode. + + This "fixes" loading for cases like the time-varying orography case + described. However, this setting is not strictly + backwards-compatible. If this causes problems, you can force identical + loading behaviour to earlier Iris versions (< v3.11) with + ``COMBINE_POLICY.set("legacy")`` or equivalent. * ``"recommended"`` - Enables multiple reference handling, *and* applies a merge step followed by - a concatenate step. + In addition to the "merge" step, allow a following "concatenate", i.e. + ``merge_concat_sequence="mc"``. * ``"comprehensive"`` - Like "recommended", but will also *repeat* the merge+concatenate steps until no - further change is produced. + As for "recommended", uses ``merge_concat_sequence="mc"``, but now also + *repeats* the merge+concatenate steps until no further change is produced, + i.e. ``repeat_until_unchanged=True``. + Also applies a prior 'equalise_cubes' call, of the form + ``equalise_cubes(cubes, apply_all=True)``. - .. note :: + .. Note:: - The 'comprehensive' policy makes a maximum effort to reduce the number of + The "comprehensive" policy makes a maximum effort to reduce the number of cubes to a minimum. However, it still cannot combine cubes with a mixture of matching dimension and scalar coordinates. This may be supported at some later date, but for now is not possible without specific user actions. - .. Note :: + .. testsetup:: - See also : :ref:`controlling_merge`. + from iris import COMBINE_POLICY + loadpolicy_old_settings = COMBINE_POLICY.settings() + + .. testcleanup:: + + # restore original settings, so as not to upset other tests + COMBINE_POLICY.set(loadpolicy_old_settings) + + Examples + -------- + Note: :data:`COMBINE_POLICY` is the global control object, which determines + the current default options for loading or :func:`iris.util.combine_cubes` calls. + For the latter case, however, control via argument and keywords is also available. + + .. Note:: + + The ``iris.COMBINE_POLICY`` can be adjusted by either: + + 1. calling ``iris.COMBINE_POLICY.set()``, or + 2. using ``with COMBINE_POLICY.context(): ...``, or + 3. assigning a property ``COMBINE_POLICY.