Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Migrate indexing and broadcasting logic to xarray.namedarray (Part 1) #8856

Draft
wants to merge 21 commits into
base: backend-indexing
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
b78795d
Refactor _in_memory property in Variable and NamedArray classes
andersy005 Mar 19, 2024
054d308
migrate *broadcast_ methods to namedarray
andersy005 Mar 19, 2024
4cf1deb
Fix return statement in NamedArray._validate_indexers method
andersy005 Mar 19, 2024
2502c76
migrate __getitem__ from Variable to NamedArray
andersy005 Mar 19, 2024
7055503
Merge branch 'main' into introduce-indexing-into-namedarray
andersy005 Mar 20, 2024
5cd75d2
Add typing annotations to expanded_indexer and as_indexable functions
andersy005 Mar 20, 2024
6183eff
rename broadcast_variables to broadcast_namedarrays
andersy005 Mar 20, 2024
1ff5dec
Refactor _broadcast_compat_namedarrays and broadcast_namedarrays func…
andersy005 Mar 20, 2024
b1f6d3e
update type hints
andersy005 Mar 20, 2024
05edb0c
try replacing set_dims with a combination of expand_dims and broadcas…
andersy005 Mar 22, 2024
32495e0
Merge branch 'main' into introduce-indexing-into-namedarray
andersy005 Mar 22, 2024
b81b451
temporary enable CI triggers on feature branch
andersy005 Apr 10, 2024
10c133b
add `.oindex` and `.vindex` to `BackendArray` (#8885)
andersy005 Apr 17, 2024
e96e70e
Merge branch 'main' into backend-indexing
andersy005 Apr 30, 2024
7b42c53
Merge branch 'backend-indexing' into introduce-indexing-into-namedarray
andersy005 Apr 30, 2024
535835c
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Apr 30, 2024
79cfe4d
Merge remote-tracking branch 'origin/main' into backend-indexing
andersy005 Apr 30, 2024
245c3db
Enable explicit use of key tuples (instead of *Indexer objects) in in…
andersy005 May 3, 2024
2fd3b8b
Merge remote-tracking branch 'origin/main' into backend-indexing
andersy005 May 3, 2024
9355a06
Merge branch 'backend-indexing' into introduce-indexing-into-namedarray
andersy005 May 3, 2024
099fdfe
Merge branch 'backend-indexing' into introduce-indexing-into-namedarray
andersy005 May 12, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 2 additions & 3 deletions xarray/core/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,7 +119,7 @@
IndexVariable,
Variable,
as_variable,
broadcast_variables,
broadcast_namedarrays,
calculate_dimensions,
)
from xarray.namedarray.parallelcompat import get_chunked_array_type, guess_chunkmanager
Expand Down Expand Up @@ -291,7 +291,6 @@ def _maybe_chunk(
chunked_array_type: str | ChunkManagerEntrypoint | None = None,
from_array_kwargs=None,
):

from xarray.namedarray.daskmanager import DaskManager

if chunks is not None:
Expand Down Expand Up @@ -7168,7 +7167,7 @@ def to_dataarray(
from xarray.core.dataarray import DataArray

data_vars = [self.variables[k] for k in self.data_vars]
broadcast_vars = broadcast_variables(*data_vars)
broadcast_vars = broadcast_namedarrays(*data_vars)
data = duck_array_ops.stack([b.data for b in broadcast_vars], axis=0)

dims = (dim,) + broadcast_vars[0].dims
Expand Down
4 changes: 2 additions & 2 deletions xarray/core/indexing.py
Original file line number Diff line number Diff line change
Expand Up @@ -207,7 +207,7 @@ def map_index_queries(
return merged


def expanded_indexer(key, ndim):
def expanded_indexer(key, ndim: int) -> tuple[Any, ...]:
"""Given a key for indexing an ndarray, return an equivalent key which is a
tuple with length equal to the number of dimensions.

Expand Down Expand Up @@ -882,7 +882,7 @@ def __setitem__(self, indexer: _IndexerKey, value: Any) -> None:
self.array[indexer] = value


def as_indexable(array):
def as_indexable(array: Any):
"""
This function always returns a ExplicitlyIndexed subclass,
so that the vectorized indexing is always possible with the returned
Expand Down
4 changes: 2 additions & 2 deletions xarray/core/missing.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
from xarray.core.options import _get_keep_attrs
from xarray.core.types import Interp1dOptions, InterpOptions
from xarray.core.utils import OrderedSet, is_scalar
from xarray.core.variable import Variable, broadcast_variables
from xarray.core.variable import Variable, broadcast_namedarrays
from xarray.namedarray.parallelcompat import get_chunked_array_type
from xarray.namedarray.pycompat import is_chunked_array

Expand Down Expand Up @@ -623,7 +623,7 @@ def interp(var, indexes_coords, method: InterpOptions, **kwargs):
# target dimensions
dims = list(indexes_coords)
x, new_x = zip(*[indexes_coords[d] for d in dims])
destination = broadcast_variables(*new_x)
destination = broadcast_namedarrays(*new_x)

# transpose to make the interpolated axis to the last position
broadcast_dims = [d for d in var.dims if d not in dims]
Expand Down
44 changes: 1 addition & 43 deletions xarray/core/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@
KeysView,
Mapping,
MutableMapping,
MutableSet,
ValuesView,
)
from enum import Enum
Expand All @@ -75,6 +74,7 @@
import pandas as pd

from xarray.namedarray.utils import ( # noqa: F401
OrderedSet,
ReprObject,
drop_missing_dims,
either_dict_or_kwargs,
Expand Down Expand Up @@ -521,48 +521,6 @@ def __len__(self) -> int:
return len(self._keys)


class OrderedSet(MutableSet[T]):
"""A simple ordered set.

The API matches the builtin set, but it preserves insertion order of elements, like
a dict. Note that, unlike in an OrderedDict, equality tests are not order-sensitive.
"""

_d: dict[T, None]

__slots__ = ("_d",)

def __init__(self, values: Iterable[T] | None = None):
self._d = {}
if values is not None:
self.update(values)

# Required methods for MutableSet

def __contains__(self, value: Hashable) -> bool:
return value in self._d

def __iter__(self) -> Iterator[T]:
return iter(self._d)

def __len__(self) -> int:
return len(self._d)

def add(self, value: T) -> None:
self._d[value] = None

def discard(self, value: T) -> None:
del self._d[value]

# Additional methods

def update(self, values: Iterable[T]) -> None:
self._d.update(dict.fromkeys(values))

def __repr__(self) -> str:
return f"{type(self).__name__}({list(self)!r})"


class NdimSizeLenMixin:
"""Mixin class that extends a class that defines a ``shape`` property to
one that also defines ``ndim``, ``size`` and ``__len__``.
Expand Down
Loading
Loading