Skip to content

Caching during load #4716

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jamesp merged 13 commits into from
May 10, 2022
Merged

Caching during load #4716

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
eeae96d
Caching
Apr 26, 2022
5214d07
Move _optimum_chunksize docstring
Apr 26, 2022
31c7e91
Should be copying coords because they're mutable
Apr 26, 2022
87920d7
Handle failure of datetime comparisons gracefully
Apr 26, 2022
6e386a1
Review fixes and more careful checks on mutable objects
Apr 27, 2022
6f9bcc0
coord_system doesn't have 'copy' but __repr__ covers the whole thing …
Apr 27, 2022
bade0a4
Review fixes
Apr 28, 2022
3016e35
Revert coord_system changes
Apr 28, 2022
ef7de0f
Still need to handle tuple issue
Apr 28, 2022
a6405c5
What's new
Apr 28, 2022
580aa71
Fix prompted by test failure
Apr 28, 2022
4696598
Limit cache size
May 5, 2022
2a791dd
Work around comparable datetime issue
May 5, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 14 additions & 3 deletions lib/iris/_lazy_data.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

"""

from functools import wraps
from functools import lru_cache, wraps

import dask
import dask.array as da
Expand Down Expand Up @@ -55,9 +55,9 @@ def _optimum_chunksize(chunks, shape, limit=None, dtype=np.dtype("f4")):

Args:

* chunks (tuple of int, or None):
* chunks (iterable of int, or None):
Pre-existing chunk shape of the target data : None if unknown.
* shape (tuple of int):
* shape (iterable of int):
The full array shape of the target data.
* limit (int):
The 'ideal' target chunk size, in bytes. Default from dask.config.
Expand All @@ -83,6 +83,17 @@ def _optimum_chunksize(chunks, shape, limit=None, dtype=np.dtype("f4")):
"chunks = [c[0] for c in normalise_chunks('auto', ...)]".

"""
if isinstance(chunks, list):
chunks = tuple(chunks)
if isinstance(shape, list):
shape = tuple(shape)
return _optimum_chunksize_internals(chunks, shape, limit, dtype)


@lru_cache
def _optimum_chunksize_internals(
chunks, shape, limit=None, dtype=np.dtype("f4")
):
# Set the chunksize limit.
if limit is None:
# Fetch the default 'optimal' chunksize from the dask config.
Expand Down
18 changes: 18 additions & 0 deletions lib/iris/coords.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2921,6 +2921,24 @@ def xml_element(self, doc):
return element


_dim_coord_cache = {}


def dim_coord_from_regular(*args, **kwargs):
coord_system = kwargs.pop("coord_system", None)
key = (args, tuple(kwargs.items()))
if key in _dim_coord_cache:
cached_coord, cached_coord_system = _dim_coord_cache[key]
if coord_system == cached_coord_system:
return cached_coord
else:
new_coord = DimCoord.from_regular(
*args, coord_system=coord_system, **kwargs
)
_dim_coord_cache[key] = (new_coord, coord_system)
return new_coord


class AuxCoord(Coord):
"""
A CF auxiliary coordinate.
Expand Down
16 changes: 9 additions & 7 deletions lib/iris/fileformats/pp_load_rules.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,13 @@
# SciTools/iris-code-generators:tools/gen_rules.py

import calendar
from functools import lru_cache

import cf_units
import numpy as np

from iris.aux_factory import HybridHeightFactory, HybridPressureFactory
from iris.coords import AuxCoord, CellMethod, DimCoord
from iris.coords import AuxCoord, CellMethod, DimCoord, dim_coord_from_regular
from iris.fileformats._pp_lbproc_pairs import LBPROC_MAP
from iris.fileformats.rules import (
ConversionMetadata,
Expand Down Expand Up @@ -514,6 +515,7 @@ def _new_coord_and_dims(
_HOURS_UNIT = cf_units.Unit("hours")


@lru_cache
def _epoch_date_hours(epoch_hours_unit, datetime):
"""
Return an 'hours since epoch' number for a date.
Expand Down Expand Up @@ -1120,7 +1122,7 @@ def _all_other_rules(f):
):
dim_coords_and_dims.append(
(
DimCoord.from_regular(
dim_coord_from_regular(
f.bzx,
f.bdx,
f.lbnpt,
Expand All @@ -1141,7 +1143,7 @@ def _all_other_rules(f):
):
dim_coords_and_dims.append(
(
DimCoord.from_regular(
dim_coord_from_regular(
f.bzx,
f.bdx,
f.lbnpt,
Expand All @@ -1163,7 +1165,7 @@ def _all_other_rules(f):
):
dim_coords_and_dims.append(
(
DimCoord.from_regular(
dim_coord_from_regular(
f.bzy,
f.bdy,
f.lbrow,
Expand All @@ -1183,7 +1185,7 @@ def _all_other_rules(f):
):
dim_coords_and_dims.append(
(
DimCoord.from_regular(
dim_coord_from_regular(
f.bzy,
f.bdy,
f.lbrow,
Expand Down Expand Up @@ -1270,7 +1272,7 @@ def _all_other_rules(f):
):
dim_coords_and_dims.append(
(
DimCoord.from_regular(
dim_coord_from_regular(
f.bzx,
f.bdx,
f.lbnpt,
Expand Down Expand Up @@ -1380,7 +1382,7 @@ def _all_other_rules(f):
):
dim_coords_and_dims.append(
(
DimCoord.from_regular(
dim_coord_from_regular(
f.bzx, f.bdx, f.lbnpt, long_name="site_number", units="1"
),
1,
Expand Down