Caching during load

lib/iris/_lazy_data.py

@@ -47,17 +47,19 @@ def is_lazy_data(data):
     return result
 
 
-def _optimum_chunksize(chunks, shape, limit=None, dtype=np.dtype("f4")):
+def _optimum_chunksize_internals(
+    chunks, shape, limit=None, dtype=np.dtype("f4")
+):
     """
     Reduce or increase an initial chunk shape to get close to a chosen ideal
     size, while prioritising the splitting of the earlier (outer) dimensions
     and keeping intact the later (inner) ones.
 
     Args:
 
-    * chunks (tuple of int, or None):
+    * chunks (iterable of int, or None):
         Pre-existing chunk shape of the target data : None if unknown.
-    * shape (tuple of int):
+    * shape (iterable of int):
         The full array shape of the target data.
     * limit (int):
         The 'ideal' target chunk size, in bytes.  Default from dask.config.
@@ -146,6 +148,30 @@ def _optimum_chunksize(chunks, shape, limit=None, dtype=np.dtype("f4")):
     return tuple(result)
 
 
+_optimum_chunksize_cache = {}
+
+
+@wraps(_optimum_chunksize_internals)
+def _optimum_chunksize(chunks, shape, limit=None, dtype=np.dtype("f4")):
+
+    key = tuple(
+        [
+            tuple(chunks),
+            tuple(shape),
+            limit,
+            dtype,
+            dask.config.get("array.chunk-size"),
+        ]
+    )
+
+    if key not in _optimum_chunksize_cache:
+        _optimum_chunksize_cache[key] = _optimum_chunksize_internals(
+            chunks, shape, limit, dtype
+        )
+
+    return _optimum_chunksize_cache[key]
+
+
 def as_lazy_data(data, chunks=None, asarray=False):
     """
     Convert the input array `data` to a dask array.

lib/iris/coords.py

@@ -2921,6 +2921,25 @@ def xml_element(self, doc):
         return element
 
 
+_dim_coord_cache = {}
+
+
+def dim_coord_from_regular(*args, **kwargs):
+    # Throughout this function we treat the coord_system specially as it is
+    # mutable and therefore not hashable. It is assumed to be identical to any
+    # coord_system with the same __repr__ output.
+    coord_system = kwargs.pop("coord_system", None)
+    key = (args, tuple(kwargs.items()), repr(coord_system))
+
+    # Check for cache hit
+    if key not in _dim_coord_cache:
+        _dim_coord_cache[key] = DimCoord.from_regular(
+            *args, coord_system=coord_system, **kwargs
+        )
+
+    return _dim_coord_cache[key].copy()
+
+
 class AuxCoord(Coord):
     """
     A CF auxiliary coordinate.

lib/iris/fileformats/pp_load_rules.py

@@ -9,12 +9,13 @@
 # SciTools/iris-code-generators:tools/gen_rules.py
 
 import calendar
+from functools import wraps
 
 import cf_units
 import numpy as np
 
 from iris.aux_factory import HybridHeightFactory, HybridPressureFactory
-from iris.coords import AuxCoord, CellMethod, DimCoord
+from iris.coords import AuxCoord, CellMethod, DimCoord, dim_coord_from_regular
 from iris.fileformats._pp_lbproc_pairs import LBPROC_MAP
 from iris.fileformats.rules import (
     ConversionMetadata,
@@ -514,7 +515,7 @@ def _new_coord_and_dims(
 _HOURS_UNIT = cf_units.Unit("hours")
 
 
-def _epoch_date_hours(epoch_hours_unit, datetime):
+def _epoch_date_hours_internals(epoch_hours_unit, datetime):
     """
     Return an 'hours since epoch' number for a date.
 
@@ -589,6 +590,22 @@ def _epoch_date_hours(epoch_hours_unit, datetime):
     return epoch_hours
 
 
+_epoch_date_hours_cache = {}
+
+
+@wraps(_epoch_date_hours_internals)
+def _epoch_date_hours(epoch_hours_unit, datetime):
+
+    key = (epoch_hours_unit, datetime)
+
+    if key not in _epoch_date_hours_cache:
+        _epoch_date_hours_cache[key] = _epoch_date_hours_internals(
+            epoch_hours_unit, datetime
+        )
+
+    return _epoch_date_hours_cache[key]
+
+
 def _convert_time_coords(
     lbcode,
     lbtim,
@@ -1120,7 +1137,7 @@ def _all_other_rules(f):
     ):
         dim_coords_and_dims.append(
             (
-                DimCoord.from_regular(
+                dim_coord_from_regular(
                     f.bzx,
                     f.bdx,
                     f.lbnpt,
@@ -1141,7 +1158,7 @@ def _all_other_rules(f):
     ):
         dim_coords_and_dims.append(
             (
-                DimCoord.from_regular(
+                dim_coord_from_regular(
                     f.bzx,
                     f.bdx,
                     f.lbnpt,
@@ -1163,7 +1180,7 @@ def _all_other_rules(f):
     ):
         dim_coords_and_dims.append(
             (
-                DimCoord.from_regular(
+                dim_coord_from_regular(
                     f.bzy,
                     f.bdy,
                     f.lbrow,
@@ -1183,7 +1200,7 @@ def _all_other_rules(f):
     ):
         dim_coords_and_dims.append(
             (
-                DimCoord.from_regular(
+                dim_coord_from_regular(
                     f.bzy,
                     f.bdy,
                     f.lbrow,
@@ -1270,7 +1287,7 @@ def _all_other_rules(f):
     ):
         dim_coords_and_dims.append(
             (
-                DimCoord.from_regular(
+                dim_coord_from_regular(
                     f.bzx,
                     f.bdx,
                     f.lbnpt,
@@ -1380,7 +1397,7 @@ def _all_other_rules(f):
     ):
         dim_coords_and_dims.append(
             (
-                DimCoord.from_regular(
+                dim_coord_from_regular(
                     f.bzx, f.bdx, f.lbnpt, long_name="site_number", units="1"
                 ),
                 1,