Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Read AMR domains lazily #4734

Merged
merged 38 commits into from
Dec 6, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
6bd9592
Read AMR domains lazily
cphyc Nov 6, 2023
1117bb9
Statically compute hydro offsets
cphyc Nov 6, 2023
8231e1d
Cache read_fluid_file_descriptor
cphyc Nov 6, 2023
7fc18ee
Delay as much as possible reading AMR
cphyc Nov 6, 2023
1266f13
Use os.path.getsize ot measure file len
cphyc Nov 6, 2023
31c3eb5
Fix typo
cphyc Nov 6, 2023
0b4b4a8
Trust more conservative Hilbert indexing for fast chunking
cphyc Nov 6, 2023
26c01c3
Do not trigger oct_handler construction
cphyc Nov 6, 2023
23f3f28
Handle case where reading is a bit too optimistic
cphyc Nov 6, 2023
ad4648f
Use own version of the oct handler for stream dataset
cphyc Nov 6, 2023
a02cb16
Use oct_handler to refine selection is already created
cphyc Nov 6, 2023
d2139eb
Reduce verbosity
cphyc Nov 6, 2023
4ac917d
oct_handler should be a read-only property
cphyc Nov 6, 2023
378a103
Implement oct_handler for ART
cphyc Nov 6, 2023
aaefe7c
Fix broken test
cphyc Nov 6, 2023
62f923b
Make hilbert lookup faster
cphyc Nov 8, 2023
52162fe
Raise an error if we get weird results with max_level
cphyc Nov 8, 2023
02749b4
Fix max level
cphyc Nov 8, 2023
bc92ac8
Remove print statement
cphyc Nov 8, 2023
a833f06
For convex regions, do an early break when the selector fully contain…
cphyc Nov 8, 2023
1ebbb2c
Info specific to the choice of selector
cphyc Nov 8, 2023
4348492
Fix issue when there are no selected regions
cphyc Nov 8, 2023
bd1c02e
Fix typing
cphyc Nov 8, 2023
7a0d486
Infer dtype from record length
cphyc Nov 8, 2023
b1a013d
Quote np.ndarray typing for numpy<1.22
cphyc Nov 8, 2023
71a074a
Remove double "#"
cphyc Nov 27, 2023
14f3e83
Do not create single-use variable
cphyc Nov 27, 2023
8ddb5d5
Move state diagram out of function to prevent costly alloc/dealloc
cphyc Nov 27, 2023
a7302ed
Remove comment that pertains to broken code
cphyc Nov 27, 2023
19996e3
Remove arbitrary limitation to LRU cache size
cphyc Nov 27, 2023
1e08c62
Make attributes private
cphyc Nov 27, 2023
c7d6703
Name function more explicitly
cphyc Nov 27, 2023
526b7a5
Lazy-read particle files
cphyc Dec 6, 2023
685961f
Improve typing
cphyc Dec 6, 2023
a778a94
Remove duplicated desc
cphyc Dec 6, 2023
502ce2a
Reuse detection of gravity fields
cphyc Dec 6, 2023
51ce380
Move properties to ParticleFileHandler
cphyc Dec 6, 2023
bcf3447
Fix typing
cphyc Dec 6, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions yt/data_objects/index_subobjects/octree_subset.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import abc
from contextlib import contextmanager
from functools import cached_property
from itertools import product, repeat
Expand Down Expand Up @@ -33,7 +34,7 @@ def cc_cache_func(self, dobj):
return cc_cache_func


class OctreeSubset(YTSelectionContainer):
class OctreeSubset(YTSelectionContainer, abc.ABC):
_spatial = True
_num_ghost_zones = 0
_type_name = "octree_subset"
Expand All @@ -51,12 +52,17 @@ def __init__(self, base_region, domain, ds, num_zones=2, num_ghost_zones=0):
self.domain_id = domain.domain_id
self.ds = domain.ds
self._index = self.ds.index
self.oct_handler = domain.oct_handler
self._last_mask = None
self._last_selector_id = None
self.base_region = base_region
self.base_selector = base_region.selector

@property
@abc.abstractmethod
def oct_handler(self):
# In charge of returning the oct_handler
pass

def __getitem__(self, key):
tr = super().__getitem__(key)
try:
Expand Down
10 changes: 9 additions & 1 deletion yt/frontends/art/data_structures.py
Original file line number Diff line number Diff line change
Expand Up @@ -727,6 +727,10 @@ def _is_valid(cls, filename: str, *args, **kwargs) -> bool:


class ARTDomainSubset(OctreeSubset):
@property
def oct_handler(self):
return self.domain.oct_handler

def fill(self, content, ftfields, selector):
"""
This is called from IOHandler. It takes content
Expand Down Expand Up @@ -798,7 +802,11 @@ def __init__(self, ds, nvar, oct_handler, domain_id):
self._level_count = None
self._level_oct_offsets = None
self._level_child_offsets = None
self.oct_handler = oct_handler
self._oct_handler = oct_handler

@property
def oct_handler(self):
return self._oct_handler

@property
def level_count(self):
Expand Down
6 changes: 5 additions & 1 deletion yt/frontends/artio/data_structures.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ def __init__(self, base_region, sfc_start, sfc_end, oct_handler, ds):
self.field_parameters = {}
self.sfc_start = sfc_start
self.sfc_end = sfc_end
self.oct_handler = oct_handler
self._oct_handler = oct_handler
self.ds = ds
self._last_mask = None
self._last_selector_id = None
Expand All @@ -43,6 +43,10 @@ def __init__(self, base_region, sfc_start, sfc_end, oct_handler, ds):
self.base_region = base_region
self.base_selector = base_region.selector

@property
def oct_handler(self):
return self._oct_handler

@property
def min_ind(self):
return self.sfc_start
Expand Down
Loading