From 931d7ea4d5dc947353bca53d092d4284704d2f1e Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Wed, 5 Mar 2025 15:10:15 +0000 Subject: [PATCH 01/58] refactor: Add `_compliant` sub-package Follow-up to https://github.com/narwhals-dev/narwhals/pull/2119#issuecomment-2700860171 --- narwhals/_compliant/__init__.py | 23 +++ narwhals/_compliant/dataframe.py | 52 ++++++ narwhals/_compliant/expr.py | 214 +++++++++++++++++++++ narwhals/_compliant/namespace.py | 26 +++ narwhals/_compliant/selectors.py | 307 +++++++++++++++++++++++++++++++ narwhals/_compliant/series.py | 20 ++ narwhals/_compliant/typing.py | 17 ++ 7 files changed, 659 insertions(+) create mode 100644 narwhals/_compliant/__init__.py create mode 100644 narwhals/_compliant/dataframe.py create mode 100644 narwhals/_compliant/expr.py create mode 100644 narwhals/_compliant/namespace.py create mode 100644 narwhals/_compliant/selectors.py create mode 100644 narwhals/_compliant/series.py create mode 100644 narwhals/_compliant/typing.py diff --git a/narwhals/_compliant/__init__.py b/narwhals/_compliant/__init__.py new file mode 100644 index 0000000000..cb86be5205 --- /dev/null +++ b/narwhals/_compliant/__init__.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from narwhals._compliant.dataframe import CompliantDataFrame +from narwhals._compliant.dataframe import CompliantLazyFrame +from narwhals._compliant.expr import CompliantExpr +from narwhals._compliant.namespace import CompliantNamespace +from narwhals._compliant.selectors import CompliantSelector +from narwhals._compliant.selectors import CompliantSelectorNamespace +from narwhals._compliant.selectors import EagerSelectorNamespace +from narwhals._compliant.selectors import LazySelectorNamespace +from narwhals._compliant.series import CompliantSeries + +__all__ = [ + "CompliantDataFrame", + "CompliantExpr", + "CompliantLazyFrame", + "CompliantNamespace", + "CompliantSelector", + "CompliantSelectorNamespace", + "CompliantSeries", + "EagerSelectorNamespace", + "LazySelectorNamespace", +] diff --git a/narwhals/_compliant/dataframe.py b/narwhals/_compliant/dataframe.py new file mode 100644 index 0000000000..a02069310e --- /dev/null +++ b/narwhals/_compliant/dataframe.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING +from typing import Any +from typing import Iterator +from typing import Mapping +from typing import Protocol +from typing import Sequence + +from narwhals._compliant.typing import CompliantSeriesT_co + +if TYPE_CHECKING: + from typing_extensions import Self + + from narwhals.dtypes import DType + +__all__ = ["CompliantDataFrame", "CompliantLazyFrame"] + + +class CompliantDataFrame(Protocol[CompliantSeriesT_co]): + def __narwhals_dataframe__(self) -> Self: ... + def __narwhals_namespace__(self) -> Any: ... + def simple_select( + self, *column_names: str + ) -> Self: ... # `select` where all args are column names. + def aggregate(self, *exprs: Any) -> Self: + ... # `select` where all args are aggregations or literals + # (so, no broadcasting is necessary). + + @property + def columns(self) -> Sequence[str]: ... + @property + def schema(self) -> Mapping[str, DType]: ... + def get_column(self, name: str) -> CompliantSeriesT_co: ... + def iter_columns(self) -> Iterator[CompliantSeriesT_co]: ... + + +class CompliantLazyFrame(Protocol): + def __narwhals_lazyframe__(self) -> Self: ... + def __narwhals_namespace__(self) -> Any: ... + def simple_select( + self, *column_names: str + ) -> Self: ... # `select` where all args are column names. + def aggregate(self, *exprs: Any) -> Self: + ... # `select` where all args are aggregations or literals + # (so, no broadcasting is necessary). + + @property + def columns(self) -> Sequence[str]: ... + @property + def schema(self) -> Mapping[str, DType]: ... + def _iter_columns(self) -> Iterator[Any]: ... diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py new file mode 100644 index 0000000000..2c86fbeb1d --- /dev/null +++ b/narwhals/_compliant/expr.py @@ -0,0 +1,214 @@ +from __future__ import annotations + +import sys +from typing import TYPE_CHECKING +from typing import Any +from typing import Callable +from typing import Literal +from typing import Sequence + +from narwhals._compliant.typing import CompliantFrameT +from narwhals._compliant.typing import CompliantSeriesT_co +from narwhals.utils import deprecated +from narwhals.utils import unstable + +if not TYPE_CHECKING: + if sys.version_info >= (3, 9): + from typing import Protocol as Protocol38 + else: + from typing import Generic as Protocol38 +else: + # TODO @dangotbanned: Remove after dropping `3.8` (#2084) + # - https://github.com/narwhals-dev/narwhals/pull/2064#discussion_r1965921386 + from typing import Protocol as Protocol38 + +if TYPE_CHECKING: + from typing import Mapping + + from typing_extensions import Self + + from narwhals._compliant.namespace import CompliantNamespace + from narwhals._compliant.series import CompliantSeries + from narwhals._expression_parsing import ExprKind + from narwhals.dtypes import DType + from narwhals.utils import Implementation + from narwhals.utils import Version + +__all__ = ["CompliantExpr"] + + +class CompliantExpr(Protocol38[CompliantFrameT, CompliantSeriesT_co]): + _implementation: Implementation + _backend_version: tuple[int, ...] + _version: Version + _evaluate_output_names: Callable[[CompliantFrameT], Sequence[str]] + _alias_output_names: Callable[[Sequence[str]], Sequence[str]] | None + _depth: int + _function_name: str + + def __call__(self, df: CompliantFrameT) -> Sequence[CompliantSeriesT_co]: ... + def __narwhals_expr__(self) -> None: ... + def __narwhals_namespace__( + self, + ) -> CompliantNamespace[CompliantFrameT, CompliantSeriesT_co]: ... + def is_null(self) -> Self: ... + def abs(self) -> Self: ... + def all(self) -> Self: ... + def any(self) -> Self: ... + def alias(self, name: str) -> Self: ... + def cast(self, dtype: DType | type[DType]) -> Self: ... + def count(self) -> Self: ... + def min(self) -> Self: ... + def max(self) -> Self: ... + def arg_min(self) -> Self: ... + def arg_max(self) -> Self: ... + def arg_true(self) -> Self: ... + def mean(self) -> Self: ... + def sum(self) -> Self: ... + def median(self) -> Self: ... + def skew(self) -> Self: ... + def std(self, *, ddof: int) -> Self: ... + def var(self, *, ddof: int) -> Self: ... + def n_unique(self) -> Self: ... + def null_count(self) -> Self: ... + def drop_nulls(self) -> Self: ... + def fill_null( + self, + value: Any | None, + strategy: Literal["forward", "backward"] | None, + limit: int | None, + ) -> Self: ... + def diff(self) -> Self: ... + def unique(self) -> Self: ... + def len(self) -> Self: ... + def round(self, decimals: int) -> Self: ... + def mode(self) -> Self: ... + def head(self, n: int) -> Self: ... + def tail(self, n: int) -> Self: ... + def shift(self, n: int) -> Self: ... + def is_finite(self) -> Self: ... + def is_nan(self) -> Self: ... + def is_unique(self) -> Self: ... + def is_first_distinct(self) -> Self: ... + def is_last_distinct(self) -> Self: ... + def cum_sum(self, *, reverse: bool) -> Self: ... + def cum_count(self, *, reverse: bool) -> Self: ... + def cum_min(self, *, reverse: bool) -> Self: ... + def cum_max(self, *, reverse: bool) -> Self: ... + def cum_prod(self, *, reverse: bool) -> Self: ... + def is_in(self, other: Any) -> Self: ... + def sort(self, *, descending: bool, nulls_last: bool) -> Self: ... + def rank( + self, + method: Literal["average", "min", "max", "dense", "ordinal"], + *, + descending: bool, + ) -> Self: ... + def replace_strict( + self, + old: Sequence[Any] | Mapping[Any, Any], + new: Sequence[Any], + *, + return_dtype: DType | type[DType] | None, + ) -> Self: ... + def over(self: Self, keys: Sequence[str], kind: ExprKind) -> Self: ... + def sample( + self, + n: int | None, + *, + fraction: float | None, + with_replacement: bool, + seed: int | None, + ) -> Self: ... + def quantile( + self, + quantile: float, + interpolation: Literal["nearest", "higher", "lower", "midpoint", "linear"], + ) -> Self: ... + def map_batches( + self, + function: Callable[[CompliantSeries], CompliantExpr[Any, Any]], + return_dtype: DType | type[DType] | None, + ) -> Self: ... + + @property + def str(self) -> Any: ... + @property + def name(self) -> Any: ... + @property + def dt(self) -> Any: ... + @property + def cat(self) -> Any: ... + @property + def list(self) -> Any: ... + + @unstable + def ewm_mean( + self, + *, + com: float | None, + span: float | None, + half_life: float | None, + alpha: float | None, + adjust: bool, + min_samples: int, + ignore_nulls: bool, + ) -> Self: ... + + @unstable + def rolling_sum( + self, + window_size: int, + *, + min_samples: int | None, + center: bool, + ) -> Self: ... + + @unstable + def rolling_mean( + self, + window_size: int, + *, + min_samples: int | None, + center: bool, + ) -> Self: ... + + @unstable + def rolling_var( + self, + window_size: int, + *, + min_samples: int | None, + center: bool, + ddof: int, + ) -> Self: ... + + @unstable + def rolling_std( + self, + window_size: int, + *, + min_samples: int | None, + center: bool, + ddof: int, + ) -> Self: ... + + @deprecated("Since `1.22.0`") + def gather_every(self, n: int, offset: int) -> Self: ... + def __and__(self, other: Any) -> Self: ... + def __or__(self, other: Any) -> Self: ... + def __add__(self, other: Any) -> Self: ... + def __sub__(self, other: Any) -> Self: ... + def __mul__(self, other: Any) -> Self: ... + def __floordiv__(self, other: Any) -> Self: ... + def __truediv__(self, other: Any) -> Self: ... + def __mod__(self, other: Any) -> Self: ... + def __pow__(self, other: Any) -> Self: ... + def __gt__(self, other: Any) -> Self: ... + def __ge__(self, other: Any) -> Self: ... + def __lt__(self, other: Any) -> Self: ... + def __le__(self, other: Any) -> Self: ... + def __invert__(self) -> Self: ... + def broadcast( + self, kind: Literal[ExprKind.AGGREGATION, ExprKind.LITERAL] + ) -> Self: ... diff --git a/narwhals/_compliant/namespace.py b/narwhals/_compliant/namespace.py new file mode 100644 index 0000000000..893a2faecb --- /dev/null +++ b/narwhals/_compliant/namespace.py @@ -0,0 +1,26 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING +from typing import Any +from typing import Protocol + +from narwhals._compliant.typing import CompliantFrameT +from narwhals._compliant.typing import CompliantSeriesT_co + +if TYPE_CHECKING: + from narwhals._compliant.expr import CompliantExpr + from narwhals._compliant.selectors import CompliantSelectorNamespace + from narwhals.dtypes import DType + +__all__ = ["CompliantNamespace"] + + +class CompliantNamespace(Protocol[CompliantFrameT, CompliantSeriesT_co]): + def col( + self, *column_names: str + ) -> CompliantExpr[CompliantFrameT, CompliantSeriesT_co]: ... + def lit( + self, value: Any, dtype: DType | None + ) -> CompliantExpr[CompliantFrameT, CompliantSeriesT_co]: ... + @property + def selectors(self) -> CompliantSelectorNamespace[Any, Any]: ... diff --git a/narwhals/_compliant/selectors.py b/narwhals/_compliant/selectors.py new file mode 100644 index 0000000000..292c47429d --- /dev/null +++ b/narwhals/_compliant/selectors.py @@ -0,0 +1,307 @@ +"""Almost entirely complete, generic `selectors` implementation.""" + +from __future__ import annotations + +import re +from functools import partial +from typing import TYPE_CHECKING +from typing import Any +from typing import Callable +from typing import Collection +from typing import Iterable +from typing import Iterator +from typing import Sequence +from typing import TypeVar +from typing import overload + +from narwhals._compliant.expr import CompliantExpr +from narwhals.utils import _parse_time_unit_and_time_zone +from narwhals.utils import dtype_matches_time_unit_and_time_zone +from narwhals.utils import get_column_names +from narwhals.utils import import_dtypes_module +from narwhals.utils import is_compliant_dataframe +from narwhals.utils import is_tracks_depth + +if not TYPE_CHECKING: # pragma: no cover + # TODO @dangotbanned: Remove after dropping `3.8` (#2084) + # - https://github.com/narwhals-dev/narwhals/pull/2064#discussion_r1965921386 + import sys + + if sys.version_info >= (3, 9): + from typing import Protocol + else: + from typing import Generic + + Protocol = Generic +else: # pragma: no cover + from typing import Protocol + +if TYPE_CHECKING: + from datetime import timezone + + from typing_extensions import Self + from typing_extensions import TypeAlias + from typing_extensions import TypeIs + + from narwhals._compliant.dataframe import CompliantDataFrame + from narwhals._compliant.dataframe import CompliantLazyFrame + from narwhals._compliant.series import CompliantSeries + from narwhals.dtypes import DType + from narwhals.typing import TimeUnit + from narwhals.utils import Implementation + from narwhals.utils import Version + +__all__ = [ + "CompliantSelector", + "CompliantSelectorNamespace", + "EagerSelectorNamespace", + "EvalNames", + "EvalSeries", + "LazySelectorNamespace", +] + + +SeriesT = TypeVar("SeriesT", bound="CompliantSeries") +FrameT = TypeVar("FrameT", bound="CompliantDataFrame[Any] | CompliantLazyFrame") +DataFrameT = TypeVar("DataFrameT", bound="CompliantDataFrame[Any]") +LazyFrameT = TypeVar("LazyFrameT", bound="CompliantLazyFrame") +SelectorOrExpr: TypeAlias = ( + "CompliantSelector[FrameT, SeriesT] | CompliantExpr[FrameT, SeriesT]" +) +EvalSeries: TypeAlias = Callable[[FrameT], Sequence[SeriesT]] +EvalNames: TypeAlias = Callable[[FrameT], Sequence[str]] + + +class CompliantSelectorNamespace(Protocol[FrameT, SeriesT]): + _implementation: Implementation + _backend_version: tuple[int, ...] + _version: Version + + def _selector( + self, + call: EvalSeries[FrameT, SeriesT], + evaluate_output_names: EvalNames[FrameT], + /, + ) -> CompliantSelector[FrameT, SeriesT]: ... + + def _iter_columns(self, df: FrameT, /) -> Iterator[SeriesT]: ... + + def _iter_schema(self, df: FrameT, /) -> Iterator[tuple[str, DType]]: + for ser in self._iter_columns(df): + yield ser.name, ser.dtype + + def _iter_columns_dtypes(self, df: FrameT, /) -> Iterator[tuple[SeriesT, DType]]: + # NOTE: Defined to be overridden for lazy + # - Their `SeriesT` is a **native** object + # - `.dtype` won't return a `nw.DType` (or maybe anything) for lazy backends + # - See (https://github.com/narwhals-dev/narwhals/issues/2044) + for ser in self._iter_columns(df): + yield ser, ser.dtype + + def _iter_columns_names(self, df: FrameT, /) -> Iterator[tuple[SeriesT, str]]: + yield from zip(self._iter_columns(df), df.columns) + + def _is_dtype( + self: CompliantSelectorNamespace[FrameT, SeriesT], dtype: type[DType], / + ) -> CompliantSelector[FrameT, SeriesT]: + def series(df: FrameT) -> Sequence[SeriesT]: + return [ + ser for ser, tp in self._iter_columns_dtypes(df) if isinstance(tp, dtype) + ] + + def names(df: FrameT) -> Sequence[str]: + return [name for name, tp in self._iter_schema(df) if isinstance(tp, dtype)] + + return self._selector(series, names) + + def by_dtype( + self: Self, dtypes: Collection[DType | type[DType]] + ) -> CompliantSelector[FrameT, SeriesT]: + def series(df: FrameT) -> Sequence[SeriesT]: + return [ser for ser, tp in self._iter_columns_dtypes(df) if tp in dtypes] + + def names(df: FrameT) -> Sequence[str]: + return [name for name, tp in self._iter_schema(df) if tp in dtypes] + + return self._selector(series, names) + + def matches(self: Self, pattern: str) -> CompliantSelector[FrameT, SeriesT]: + p = re.compile(pattern) + + def series(df: FrameT) -> Sequence[SeriesT]: + if is_compliant_dataframe(df) and not self._implementation.is_duckdb(): + return [df.get_column(col) for col in df.columns if p.search(col)] + + return [ser for ser, name in self._iter_columns_names(df) if p.search(name)] + + def names(df: FrameT) -> Sequence[str]: + return [col for col in df.columns if p.search(col)] + + return self._selector(series, names) + + def numeric(self: Self) -> CompliantSelector[FrameT, SeriesT]: + def series(df: FrameT) -> Sequence[SeriesT]: + return [ser for ser, tp in self._iter_columns_dtypes(df) if tp.is_numeric()] + + def names(df: FrameT) -> Sequence[str]: + return [name for name, tp in self._iter_schema(df) if tp.is_numeric()] + + return self._selector(series, names) + + def categorical(self: Self) -> CompliantSelector[FrameT, SeriesT]: + return self._is_dtype(import_dtypes_module(self._version).Categorical) + + def string(self: Self) -> CompliantSelector[FrameT, SeriesT]: + return self._is_dtype(import_dtypes_module(self._version).String) + + def boolean(self: Self) -> CompliantSelector[FrameT, SeriesT]: + return self._is_dtype(import_dtypes_module(self._version).Boolean) + + def all(self: Self) -> CompliantSelector[FrameT, SeriesT]: + def series(df: FrameT) -> Sequence[SeriesT]: + return list(self._iter_columns(df)) + + return self._selector(series, get_column_names) + + def datetime( + self: Self, + time_unit: TimeUnit | Iterable[TimeUnit] | None, + time_zone: str | timezone | Iterable[str | timezone | None] | None, + ) -> CompliantSelector[FrameT, SeriesT]: + time_units, time_zones = _parse_time_unit_and_time_zone(time_unit, time_zone) + matches = partial( + dtype_matches_time_unit_and_time_zone, + dtypes=import_dtypes_module(version=self._version), + time_units=time_units, + time_zones=time_zones, + ) + + def series(df: FrameT) -> Sequence[SeriesT]: + return [ser for ser, tp in self._iter_columns_dtypes(df) if matches(tp)] + + def names(df: FrameT) -> Sequence[str]: + return [name for name, tp in self._iter_schema(df) if matches(tp)] + + return self._selector(series, names) + + +class EagerSelectorNamespace( + CompliantSelectorNamespace[DataFrameT, SeriesT], Protocol[DataFrameT, SeriesT] +): + def _iter_columns(self, df: DataFrameT, /) -> Iterator[SeriesT]: + yield from df.iter_columns() + + +class LazySelectorNamespace( + CompliantSelectorNamespace[LazyFrameT, SeriesT], Protocol[LazyFrameT, SeriesT] +): + def _iter_schema(self, df: LazyFrameT) -> Iterator[tuple[str, DType]]: + yield from df.schema.items() + + def _iter_columns(self, df: LazyFrameT) -> Iterator[SeriesT]: + yield from df._iter_columns() + + def _iter_columns_dtypes(self, df: LazyFrameT, /) -> Iterator[tuple[SeriesT, DType]]: + yield from zip(self._iter_columns(df), df.schema.values()) + + +class CompliantSelector(CompliantExpr[FrameT, SeriesT], Protocol[FrameT, SeriesT]): + @property + def selectors(self) -> CompliantSelectorNamespace[FrameT, SeriesT]: + return self.__narwhals_namespace__().selectors + + def _to_expr(self: Self) -> CompliantExpr[FrameT, SeriesT]: ... + + def _is_selector( + self: Self, other: Self | CompliantExpr[FrameT, SeriesT] + ) -> TypeIs[CompliantSelector[FrameT, SeriesT]]: + return isinstance(other, type(self)) + + @overload + def __sub__(self: Self, other: Self) -> Self: ... + @overload + def __sub__( + self: Self, other: CompliantExpr[FrameT, SeriesT] + ) -> CompliantExpr[FrameT, SeriesT]: ... + def __sub__( + self: Self, other: SelectorOrExpr[FrameT, SeriesT] + ) -> SelectorOrExpr[FrameT, SeriesT]: + if self._is_selector(other): + + def series(df: FrameT) -> Sequence[SeriesT]: + lhs_names, rhs_names = _eval_lhs_rhs(df, self, other) + return [ + x for x, name in zip(self(df), lhs_names) if name not in rhs_names + ] + + def names(df: FrameT) -> Sequence[str]: + lhs_names, rhs_names = _eval_lhs_rhs(df, self, other) + return [x for x in lhs_names if x not in rhs_names] + + return self.selectors._selector(series, names) + else: + return self._to_expr() - other + + @overload + def __or__(self: Self, other: Self) -> Self: ... + @overload + def __or__( + self: Self, other: CompliantExpr[FrameT, SeriesT] + ) -> CompliantExpr[FrameT, SeriesT]: ... + def __or__( + self: Self, other: SelectorOrExpr[FrameT, SeriesT] + ) -> SelectorOrExpr[FrameT, SeriesT]: + if self._is_selector(other): + + def names(df: FrameT) -> Sequence[SeriesT]: + lhs_names, rhs_names = _eval_lhs_rhs(df, self, other) + return [ + *(x for x, name in zip(self(df), lhs_names) if name not in rhs_names), + *other(df), + ] + + def series(df: FrameT) -> Sequence[str]: + lhs_names, rhs_names = _eval_lhs_rhs(df, self, other) + return [*(x for x in lhs_names if x not in rhs_names), *rhs_names] + + return self.selectors._selector(names, series) + else: + return self._to_expr() | other + + @overload + def __and__(self: Self, other: Self) -> Self: ... + @overload + def __and__( + self: Self, other: CompliantExpr[FrameT, SeriesT] + ) -> CompliantExpr[FrameT, SeriesT]: ... + def __and__( + self: Self, other: SelectorOrExpr[FrameT, SeriesT] + ) -> SelectorOrExpr[FrameT, SeriesT]: + if self._is_selector(other): + + def series(df: FrameT) -> Sequence[SeriesT]: + lhs_names, rhs_names = _eval_lhs_rhs(df, self, other) + return [x for x, name in zip(self(df), lhs_names) if name in rhs_names] + + def names(df: FrameT) -> Sequence[str]: + lhs_names, rhs_names = _eval_lhs_rhs(df, self, other) + return [x for x in lhs_names if x in rhs_names] + + return self.selectors._selector(series, names) + else: + return self._to_expr() & other + + def __invert__(self: Self) -> CompliantSelector[FrameT, SeriesT]: + return self.selectors.all() - self # type: ignore[no-any-return] + + def __repr__(self: Self) -> str: # pragma: no cover + s = f"depth={self._depth}, " if is_tracks_depth(self._implementation) else "" + return f"{type(self).__name__}({s}function_name={self._function_name})" + + +def _eval_lhs_rhs( + df: CompliantDataFrame[Any] | CompliantLazyFrame, + lhs: CompliantExpr[Any, Any], + rhs: CompliantExpr[Any, Any], +) -> tuple[Sequence[str], Sequence[str]]: + return lhs._evaluate_output_names(df), rhs._evaluate_output_names(df) diff --git a/narwhals/_compliant/series.py b/narwhals/_compliant/series.py new file mode 100644 index 0000000000..4002baf995 --- /dev/null +++ b/narwhals/_compliant/series.py @@ -0,0 +1,20 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING +from typing import Protocol + +if TYPE_CHECKING: + from typing_extensions import Self + + from narwhals.dtypes import DType + +__all__ = ["CompliantSeries"] + + +class CompliantSeries(Protocol): + @property + def dtype(self) -> DType: ... + @property + def name(self) -> str: ... + def __narwhals_series__(self) -> CompliantSeries: ... + def alias(self, name: str) -> Self: ... diff --git a/narwhals/_compliant/typing.py b/narwhals/_compliant/typing.py new file mode 100644 index 0000000000..4d967969e2 --- /dev/null +++ b/narwhals/_compliant/typing.py @@ -0,0 +1,17 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING +from typing import Any +from typing import TypeVar + +if TYPE_CHECKING: + from narwhals._compliant.dataframe import CompliantDataFrame + from narwhals._compliant.dataframe import CompliantLazyFrame + from narwhals._compliant.series import CompliantSeries + +CompliantSeriesT_co = TypeVar( + "CompliantSeriesT_co", bound="CompliantSeries", covariant=True +) +CompliantFrameT = TypeVar( + "CompliantFrameT", bound="CompliantDataFrame[Any] | CompliantLazyFrame" +) From cef630e147743984b7eb846a2240bf2ba4c0df2f Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Wed, 5 Mar 2025 15:41:58 +0000 Subject: [PATCH 02/58] refactor: migrate imports, delete `_selectors` --- narwhals/_arrow/selectors.py | 8 +- narwhals/_compliant/__init__.py | 6 + narwhals/_compliant/typing.py | 8 + narwhals/_dask/selectors.py | 8 +- narwhals/_duckdb/selectors.py | 8 +- narwhals/_pandas_like/selectors.py | 8 +- narwhals/_selectors.py | 298 ----------------------------- narwhals/_spark_like/selectors.py | 8 +- narwhals/typing.py | 283 ++------------------------- 9 files changed, 47 insertions(+), 588 deletions(-) delete mode 100644 narwhals/_selectors.py diff --git a/narwhals/_arrow/selectors.py b/narwhals/_arrow/selectors.py index d9c74be112..eb86def160 100644 --- a/narwhals/_arrow/selectors.py +++ b/narwhals/_arrow/selectors.py @@ -3,16 +3,16 @@ from typing import TYPE_CHECKING from narwhals._arrow.expr import ArrowExpr -from narwhals._selectors import CompliantSelector -from narwhals._selectors import EagerSelectorNamespace +from narwhals._compliant import CompliantSelector +from narwhals._compliant import EagerSelectorNamespace if TYPE_CHECKING: from typing_extensions import Self from narwhals._arrow.dataframe import ArrowDataFrame from narwhals._arrow.series import ArrowSeries - from narwhals._selectors import EvalNames - from narwhals._selectors import EvalSeries + from narwhals._compliant.selectors import EvalNames + from narwhals._compliant.selectors import EvalSeries from narwhals.utils import _FullContext diff --git a/narwhals/_compliant/__init__.py b/narwhals/_compliant/__init__.py index cb86be5205..b1fd9b61a2 100644 --- a/narwhals/_compliant/__init__.py +++ b/narwhals/_compliant/__init__.py @@ -9,15 +9,21 @@ from narwhals._compliant.selectors import EagerSelectorNamespace from narwhals._compliant.selectors import LazySelectorNamespace from narwhals._compliant.series import CompliantSeries +from narwhals._compliant.typing import CompliantFrameT +from narwhals._compliant.typing import CompliantSeriesT_co +from narwhals._compliant.typing import IntoCompliantExpr __all__ = [ "CompliantDataFrame", "CompliantExpr", + "CompliantFrameT", "CompliantLazyFrame", "CompliantNamespace", "CompliantSelector", "CompliantSelectorNamespace", "CompliantSeries", + "CompliantSeriesT_co", "EagerSelectorNamespace", + "IntoCompliantExpr", "LazySelectorNamespace", ] diff --git a/narwhals/_compliant/typing.py b/narwhals/_compliant/typing.py index 4d967969e2..f02dc92569 100644 --- a/narwhals/_compliant/typing.py +++ b/narwhals/_compliant/typing.py @@ -5,13 +5,21 @@ from typing import TypeVar if TYPE_CHECKING: + from typing_extensions import TypeAlias + from narwhals._compliant.dataframe import CompliantDataFrame from narwhals._compliant.dataframe import CompliantLazyFrame + from narwhals._compliant.expr import CompliantExpr from narwhals._compliant.series import CompliantSeries +__all__ = ["CompliantFrameT", "CompliantSeriesT_co", "IntoCompliantExpr"] + CompliantSeriesT_co = TypeVar( "CompliantSeriesT_co", bound="CompliantSeries", covariant=True ) CompliantFrameT = TypeVar( "CompliantFrameT", bound="CompliantDataFrame[Any] | CompliantLazyFrame" ) +IntoCompliantExpr: TypeAlias = ( + "CompliantExpr[CompliantFrameT, CompliantSeriesT_co] | CompliantSeriesT_co" +) diff --git a/narwhals/_dask/selectors.py b/narwhals/_dask/selectors.py index 9533721d49..505d16c1b9 100644 --- a/narwhals/_dask/selectors.py +++ b/narwhals/_dask/selectors.py @@ -2,9 +2,9 @@ from typing import TYPE_CHECKING +from narwhals._compliant import CompliantSelector +from narwhals._compliant import LazySelectorNamespace from narwhals._dask.expr import DaskExpr -from narwhals._selectors import CompliantSelector -from narwhals._selectors import LazySelectorNamespace if TYPE_CHECKING: try: @@ -14,9 +14,9 @@ from typing_extensions import Self + from narwhals._compliant.selectors import EvalNames + from narwhals._compliant.selectors import EvalSeries from narwhals._dask.dataframe import DaskLazyFrame - from narwhals._selectors import EvalNames - from narwhals._selectors import EvalSeries from narwhals.utils import _FullContext diff --git a/narwhals/_duckdb/selectors.py b/narwhals/_duckdb/selectors.py index 0e54fd3c76..983e7cb37e 100644 --- a/narwhals/_duckdb/selectors.py +++ b/narwhals/_duckdb/selectors.py @@ -2,17 +2,17 @@ from typing import TYPE_CHECKING +from narwhals._compliant import CompliantSelector +from narwhals._compliant import LazySelectorNamespace from narwhals._duckdb.expr import DuckDBExpr -from narwhals._selectors import CompliantSelector -from narwhals._selectors import LazySelectorNamespace if TYPE_CHECKING: import duckdb from typing_extensions import Self + from narwhals._compliant.selectors import EvalNames + from narwhals._compliant.selectors import EvalSeries from narwhals._duckdb.dataframe import DuckDBLazyFrame - from narwhals._selectors import EvalNames - from narwhals._selectors import EvalSeries from narwhals.utils import _FullContext diff --git a/narwhals/_pandas_like/selectors.py b/narwhals/_pandas_like/selectors.py index bdf5cf33cd..2d7566682c 100644 --- a/narwhals/_pandas_like/selectors.py +++ b/narwhals/_pandas_like/selectors.py @@ -2,19 +2,19 @@ from typing import TYPE_CHECKING +from narwhals._compliant import CompliantSelector +from narwhals._compliant import EagerSelectorNamespace from narwhals._pandas_like.dataframe import PandasLikeDataFrame from narwhals._pandas_like.expr import PandasLikeExpr from narwhals._pandas_like.series import PandasLikeSeries -from narwhals._selectors import CompliantSelector -from narwhals._selectors import EagerSelectorNamespace if TYPE_CHECKING: from typing_extensions import Self + from narwhals._compliant.selectors import EvalNames + from narwhals._compliant.selectors import EvalSeries from narwhals._pandas_like.dataframe import PandasLikeDataFrame from narwhals._pandas_like.series import PandasLikeSeries - from narwhals._selectors import EvalNames - from narwhals._selectors import EvalSeries from narwhals.utils import _FullContext diff --git a/narwhals/_selectors.py b/narwhals/_selectors.py deleted file mode 100644 index 639b0eb740..0000000000 --- a/narwhals/_selectors.py +++ /dev/null @@ -1,298 +0,0 @@ -"""Almost entirely complete, generic `selectors` implementation.""" - -from __future__ import annotations - -import re -from functools import partial -from typing import TYPE_CHECKING -from typing import Any -from typing import Callable -from typing import Collection -from typing import Iterable -from typing import Iterator -from typing import Sequence -from typing import TypeVar -from typing import overload - -from narwhals.typing import CompliantExpr -from narwhals.utils import _parse_time_unit_and_time_zone -from narwhals.utils import dtype_matches_time_unit_and_time_zone -from narwhals.utils import get_column_names -from narwhals.utils import import_dtypes_module -from narwhals.utils import is_compliant_dataframe -from narwhals.utils import is_tracks_depth - -if not TYPE_CHECKING: # pragma: no cover - # TODO @dangotbanned: Remove after dropping `3.8` (#2084) - # - https://github.com/narwhals-dev/narwhals/pull/2064#discussion_r1965921386 - import sys - - if sys.version_info >= (3, 9): - from typing import Protocol - else: - from typing import Generic - - Protocol = Generic -else: # pragma: no cover - from typing import Protocol - -if TYPE_CHECKING: - from datetime import timezone - - from typing_extensions import Self - from typing_extensions import TypeAlias - from typing_extensions import TypeIs - - from narwhals.dtypes import DType - from narwhals.typing import CompliantDataFrame - from narwhals.typing import CompliantLazyFrame - from narwhals.typing import CompliantSeries - from narwhals.typing import TimeUnit - from narwhals.utils import Implementation - from narwhals.utils import Version - - -SeriesT = TypeVar("SeriesT", bound="CompliantSeries") -FrameT = TypeVar("FrameT", bound="CompliantDataFrame[Any] | CompliantLazyFrame") -DataFrameT = TypeVar("DataFrameT", bound="CompliantDataFrame[Any]") -LazyFrameT = TypeVar("LazyFrameT", bound="CompliantLazyFrame") -SelectorOrExpr: TypeAlias = ( - "CompliantSelector[FrameT, SeriesT] | CompliantExpr[FrameT, SeriesT]" -) -EvalSeries: TypeAlias = Callable[[FrameT], Sequence[SeriesT]] -EvalNames: TypeAlias = Callable[[FrameT], Sequence[str]] - - -class CompliantSelectorNamespace(Protocol[FrameT, SeriesT]): - _implementation: Implementation - _backend_version: tuple[int, ...] - _version: Version - - def _selector( - self, - call: EvalSeries[FrameT, SeriesT], - evaluate_output_names: EvalNames[FrameT], - /, - ) -> CompliantSelector[FrameT, SeriesT]: ... - - def _iter_columns(self, df: FrameT, /) -> Iterator[SeriesT]: ... - - def _iter_schema(self, df: FrameT, /) -> Iterator[tuple[str, DType]]: - for ser in self._iter_columns(df): - yield ser.name, ser.dtype - - def _iter_columns_dtypes(self, df: FrameT, /) -> Iterator[tuple[SeriesT, DType]]: - # NOTE: Defined to be overridden for lazy - # - Their `SeriesT` is a **native** object - # - `.dtype` won't return a `nw.DType` (or maybe anything) for lazy backends - # - See (https://github.com/narwhals-dev/narwhals/issues/2044) - for ser in self._iter_columns(df): - yield ser, ser.dtype - - def _iter_columns_names(self, df: FrameT, /) -> Iterator[tuple[SeriesT, str]]: - yield from zip(self._iter_columns(df), df.columns) - - def _is_dtype( - self: CompliantSelectorNamespace[FrameT, SeriesT], dtype: type[DType], / - ) -> CompliantSelector[FrameT, SeriesT]: - def series(df: FrameT) -> Sequence[SeriesT]: - return [ - ser for ser, tp in self._iter_columns_dtypes(df) if isinstance(tp, dtype) - ] - - def names(df: FrameT) -> Sequence[str]: - return [name for name, tp in self._iter_schema(df) if isinstance(tp, dtype)] - - return self._selector(series, names) - - def by_dtype( - self: Self, dtypes: Collection[DType | type[DType]] - ) -> CompliantSelector[FrameT, SeriesT]: - def series(df: FrameT) -> Sequence[SeriesT]: - return [ser for ser, tp in self._iter_columns_dtypes(df) if tp in dtypes] - - def names(df: FrameT) -> Sequence[str]: - return [name for name, tp in self._iter_schema(df) if tp in dtypes] - - return self._selector(series, names) - - def matches(self: Self, pattern: str) -> CompliantSelector[FrameT, SeriesT]: - p = re.compile(pattern) - - def series(df: FrameT) -> Sequence[SeriesT]: - if is_compliant_dataframe(df) and not self._implementation.is_duckdb(): - return [df.get_column(col) for col in df.columns if p.search(col)] - - return [ser for ser, name in self._iter_columns_names(df) if p.search(name)] - - def names(df: FrameT) -> Sequence[str]: - return [col for col in df.columns if p.search(col)] - - return self._selector(series, names) - - def numeric(self: Self) -> CompliantSelector[FrameT, SeriesT]: - def series(df: FrameT) -> Sequence[SeriesT]: - return [ser for ser, tp in self._iter_columns_dtypes(df) if tp.is_numeric()] - - def names(df: FrameT) -> Sequence[str]: - return [name for name, tp in self._iter_schema(df) if tp.is_numeric()] - - return self._selector(series, names) - - def categorical(self: Self) -> CompliantSelector[FrameT, SeriesT]: - return self._is_dtype(import_dtypes_module(self._version).Categorical) - - def string(self: Self) -> CompliantSelector[FrameT, SeriesT]: - return self._is_dtype(import_dtypes_module(self._version).String) - - def boolean(self: Self) -> CompliantSelector[FrameT, SeriesT]: - return self._is_dtype(import_dtypes_module(self._version).Boolean) - - def all(self: Self) -> CompliantSelector[FrameT, SeriesT]: - def series(df: FrameT) -> Sequence[SeriesT]: - return list(self._iter_columns(df)) - - return self._selector(series, get_column_names) - - def datetime( - self: Self, - time_unit: TimeUnit | Iterable[TimeUnit] | None, - time_zone: str | timezone | Iterable[str | timezone | None] | None, - ) -> CompliantSelector[FrameT, SeriesT]: - time_units, time_zones = _parse_time_unit_and_time_zone(time_unit, time_zone) - matches = partial( - dtype_matches_time_unit_and_time_zone, - dtypes=import_dtypes_module(version=self._version), - time_units=time_units, - time_zones=time_zones, - ) - - def series(df: FrameT) -> Sequence[SeriesT]: - return [ser for ser, tp in self._iter_columns_dtypes(df) if matches(tp)] - - def names(df: FrameT) -> Sequence[str]: - return [name for name, tp in self._iter_schema(df) if matches(tp)] - - return self._selector(series, names) - - -class EagerSelectorNamespace( - CompliantSelectorNamespace[DataFrameT, SeriesT], Protocol[DataFrameT, SeriesT] -): - def _iter_columns(self, df: DataFrameT, /) -> Iterator[SeriesT]: - yield from df.iter_columns() - - -class LazySelectorNamespace( - CompliantSelectorNamespace[LazyFrameT, SeriesT], Protocol[LazyFrameT, SeriesT] -): - def _iter_schema(self, df: LazyFrameT) -> Iterator[tuple[str, DType]]: - yield from df.schema.items() - - def _iter_columns(self, df: LazyFrameT) -> Iterator[SeriesT]: - yield from df._iter_columns() - - def _iter_columns_dtypes(self, df: LazyFrameT, /) -> Iterator[tuple[SeriesT, DType]]: - yield from zip(self._iter_columns(df), df.schema.values()) - - -class CompliantSelector(CompliantExpr[FrameT, SeriesT], Protocol[FrameT, SeriesT]): - @property - def selectors(self) -> CompliantSelectorNamespace[FrameT, SeriesT]: - return self.__narwhals_namespace__().selectors - - def _to_expr(self: Self) -> CompliantExpr[FrameT, SeriesT]: ... - - def _is_selector( - self: Self, other: Self | CompliantExpr[FrameT, SeriesT] - ) -> TypeIs[CompliantSelector[FrameT, SeriesT]]: - return isinstance(other, type(self)) - - @overload - def __sub__(self: Self, other: Self) -> Self: ... - @overload - def __sub__( - self: Self, other: CompliantExpr[FrameT, SeriesT] - ) -> CompliantExpr[FrameT, SeriesT]: ... - def __sub__( - self: Self, other: SelectorOrExpr[FrameT, SeriesT] - ) -> SelectorOrExpr[FrameT, SeriesT]: - if self._is_selector(other): - - def series(df: FrameT) -> Sequence[SeriesT]: - lhs_names, rhs_names = _eval_lhs_rhs(df, self, other) - return [ - x for x, name in zip(self(df), lhs_names) if name not in rhs_names - ] - - def names(df: FrameT) -> Sequence[str]: - lhs_names, rhs_names = _eval_lhs_rhs(df, self, other) - return [x for x in lhs_names if x not in rhs_names] - - return self.selectors._selector(series, names) - else: - return self._to_expr() - other - - @overload - def __or__(self: Self, other: Self) -> Self: ... - @overload - def __or__( - self: Self, other: CompliantExpr[FrameT, SeriesT] - ) -> CompliantExpr[FrameT, SeriesT]: ... - def __or__( - self: Self, other: SelectorOrExpr[FrameT, SeriesT] - ) -> SelectorOrExpr[FrameT, SeriesT]: - if self._is_selector(other): - - def names(df: FrameT) -> Sequence[SeriesT]: - lhs_names, rhs_names = _eval_lhs_rhs(df, self, other) - return [ - *(x for x, name in zip(self(df), lhs_names) if name not in rhs_names), - *other(df), - ] - - def series(df: FrameT) -> Sequence[str]: - lhs_names, rhs_names = _eval_lhs_rhs(df, self, other) - return [*(x for x in lhs_names if x not in rhs_names), *rhs_names] - - return self.selectors._selector(names, series) - else: - return self._to_expr() | other - - @overload - def __and__(self: Self, other: Self) -> Self: ... - @overload - def __and__( - self: Self, other: CompliantExpr[FrameT, SeriesT] - ) -> CompliantExpr[FrameT, SeriesT]: ... - def __and__( - self: Self, other: SelectorOrExpr[FrameT, SeriesT] - ) -> SelectorOrExpr[FrameT, SeriesT]: - if self._is_selector(other): - - def series(df: FrameT) -> Sequence[SeriesT]: - lhs_names, rhs_names = _eval_lhs_rhs(df, self, other) - return [x for x, name in zip(self(df), lhs_names) if name in rhs_names] - - def names(df: FrameT) -> Sequence[str]: - lhs_names, rhs_names = _eval_lhs_rhs(df, self, other) - return [x for x in lhs_names if x in rhs_names] - - return self.selectors._selector(series, names) - else: - return self._to_expr() & other - - def __invert__(self: Self) -> CompliantSelector[FrameT, SeriesT]: - return self.selectors.all() - self # type: ignore[no-any-return] - - def __repr__(self: Self) -> str: # pragma: no cover - s = f"depth={self._depth}, " if is_tracks_depth(self._implementation) else "" - return f"{type(self).__name__}({s}function_name={self._function_name})" - - -def _eval_lhs_rhs( - df: CompliantDataFrame[Any] | CompliantLazyFrame, - lhs: CompliantExpr[Any, Any], - rhs: CompliantExpr[Any, Any], -) -> tuple[Sequence[str], Sequence[str]]: - return lhs._evaluate_output_names(df), rhs._evaluate_output_names(df) diff --git a/narwhals/_spark_like/selectors.py b/narwhals/_spark_like/selectors.py index eb7ab72fae..1e3cc01a6e 100644 --- a/narwhals/_spark_like/selectors.py +++ b/narwhals/_spark_like/selectors.py @@ -2,16 +2,16 @@ from typing import TYPE_CHECKING -from narwhals._selectors import CompliantSelector -from narwhals._selectors import LazySelectorNamespace +from narwhals._compliant import CompliantSelector +from narwhals._compliant import LazySelectorNamespace from narwhals._spark_like.expr import SparkLikeExpr if TYPE_CHECKING: from pyspark.sql import Column from typing_extensions import Self - from narwhals._selectors import EvalNames - from narwhals._selectors import EvalSeries + from narwhals._compliant.selectors import EvalNames + from narwhals._compliant.selectors import EvalSeries from narwhals._spark_like.dataframe import SparkLikeLazyFrame from narwhals.utils import _FullContext diff --git a/narwhals/typing.py b/narwhals/typing.py index b908e67112..39b483750c 100644 --- a/narwhals/typing.py +++ b/narwhals/typing.py @@ -1,48 +1,32 @@ from __future__ import annotations -import sys from typing import TYPE_CHECKING from typing import Any -from typing import Callable -from typing import Iterator from typing import Literal -from typing import Mapping from typing import Protocol -from typing import Sequence from typing import TypeVar from typing import Union -from narwhals.utils import deprecated -from narwhals.utils import unstable - -if not TYPE_CHECKING: - if sys.version_info >= (3, 9): - from typing import Protocol as Protocol38 - else: - from typing import Generic as Protocol38 -else: - # TODO @dangotbanned: Remove after dropping `3.8` (#2084) - # - https://github.com/narwhals-dev/narwhals/pull/2064#discussion_r1965921386 - from typing import Protocol as Protocol38 +from narwhals._compliant import CompliantDataFrame +from narwhals._compliant import CompliantExpr +from narwhals._compliant import CompliantFrameT +from narwhals._compliant import CompliantLazyFrame +from narwhals._compliant import CompliantNamespace +from narwhals._compliant import CompliantSeries +from narwhals._compliant import CompliantSeriesT_co +from narwhals._compliant import IntoCompliantExpr if TYPE_CHECKING: from types import ModuleType - from typing import Mapping import numpy as np - from typing_extensions import Self from typing_extensions import TypeAlias from narwhals import dtypes - from narwhals._expression_parsing import ExprKind - from narwhals._selectors import CompliantSelectorNamespace from narwhals.dataframe import DataFrame from narwhals.dataframe import LazyFrame - from narwhals.dtypes import DType from narwhals.expr import Expr from narwhals.series import Series - from narwhals.utils import Implementation - from narwhals.utils import Version # All dataframes supported by Narwhals have a # `columns` property. Their similarities don't extend @@ -60,256 +44,10 @@ class DataFrameLike(Protocol): def __dataframe__(self, *args: Any, **kwargs: Any) -> Any: ... -class CompliantSeries(Protocol): - @property - def dtype(self) -> DType: ... - @property - def name(self) -> str: ... - def __narwhals_series__(self) -> CompliantSeries: ... - def alias(self, name: str) -> Self: ... - - -CompliantSeriesT_co = TypeVar( - "CompliantSeriesT_co", bound=CompliantSeries, covariant=True -) - - -class CompliantDataFrame(Protocol[CompliantSeriesT_co]): - def __narwhals_dataframe__(self) -> Self: ... - def __narwhals_namespace__(self) -> Any: ... - def simple_select( - self, *column_names: str - ) -> Self: ... # `select` where all args are column names. - def aggregate(self, *exprs: Any) -> Self: - ... # `select` where all args are aggregations or literals - # (so, no broadcasting is necessary). - - @property - def columns(self) -> Sequence[str]: ... - @property - def schema(self) -> Mapping[str, DType]: ... - def get_column(self, name: str) -> CompliantSeriesT_co: ... - def iter_columns(self) -> Iterator[CompliantSeriesT_co]: ... - - -class CompliantLazyFrame(Protocol): - def __narwhals_lazyframe__(self) -> Self: ... - def __narwhals_namespace__(self) -> Any: ... - def simple_select( - self, *column_names: str - ) -> Self: ... # `select` where all args are column names. - def aggregate(self, *exprs: Any) -> Self: - ... # `select` where all args are aggregations or literals - # (so, no broadcasting is necessary). - - @property - def columns(self) -> Sequence[str]: ... - @property - def schema(self) -> Mapping[str, DType]: ... - def _iter_columns(self) -> Iterator[Any]: ... - - -CompliantFrameT = TypeVar( - "CompliantFrameT", bound="CompliantDataFrame[Any] | CompliantLazyFrame" -) - - -class CompliantExpr(Protocol38[CompliantFrameT, CompliantSeriesT_co]): - _implementation: Implementation - _backend_version: tuple[int, ...] - _version: Version - _evaluate_output_names: Callable[[CompliantFrameT], Sequence[str]] - _alias_output_names: Callable[[Sequence[str]], Sequence[str]] | None - _depth: int - _function_name: str - - def __call__(self, df: CompliantFrameT) -> Sequence[CompliantSeriesT_co]: ... - def __narwhals_expr__(self) -> None: ... - def __narwhals_namespace__( - self, - ) -> CompliantNamespace[CompliantFrameT, CompliantSeriesT_co]: ... - def is_null(self) -> Self: ... - def abs(self) -> Self: ... - def all(self) -> Self: ... - def any(self) -> Self: ... - def alias(self, name: str) -> Self: ... - def cast(self, dtype: DType | type[DType]) -> Self: ... - def count(self) -> Self: ... - def min(self) -> Self: ... - def max(self) -> Self: ... - def arg_min(self) -> Self: ... - def arg_max(self) -> Self: ... - def arg_true(self) -> Self: ... - def mean(self) -> Self: ... - def sum(self) -> Self: ... - def median(self) -> Self: ... - def skew(self) -> Self: ... - def std(self, *, ddof: int) -> Self: ... - def var(self, *, ddof: int) -> Self: ... - def n_unique(self) -> Self: ... - def null_count(self) -> Self: ... - def drop_nulls(self) -> Self: ... - def fill_null( - self, - value: Any | None, - strategy: Literal["forward", "backward"] | None, - limit: int | None, - ) -> Self: ... - def diff(self) -> Self: ... - def unique(self) -> Self: ... - def len(self) -> Self: ... - def round(self, decimals: int) -> Self: ... - def mode(self) -> Self: ... - def head(self, n: int) -> Self: ... - def tail(self, n: int) -> Self: ... - def shift(self, n: int) -> Self: ... - def is_finite(self) -> Self: ... - def is_nan(self) -> Self: ... - def is_unique(self) -> Self: ... - def is_first_distinct(self) -> Self: ... - def is_last_distinct(self) -> Self: ... - def cum_sum(self, *, reverse: bool) -> Self: ... - def cum_count(self, *, reverse: bool) -> Self: ... - def cum_min(self, *, reverse: bool) -> Self: ... - def cum_max(self, *, reverse: bool) -> Self: ... - def cum_prod(self, *, reverse: bool) -> Self: ... - def is_in(self, other: Any) -> Self: ... - def sort(self, *, descending: bool, nulls_last: bool) -> Self: ... - def rank( - self, - method: Literal["average", "min", "max", "dense", "ordinal"], - *, - descending: bool, - ) -> Self: ... - def replace_strict( - self, - old: Sequence[Any] | Mapping[Any, Any], - new: Sequence[Any], - *, - return_dtype: DType | type[DType] | None, - ) -> Self: ... - def over(self: Self, keys: Sequence[str], kind: ExprKind) -> Self: ... - def sample( - self, - n: int | None, - *, - fraction: float | None, - with_replacement: bool, - seed: int | None, - ) -> Self: ... - def quantile( - self, - quantile: float, - interpolation: Literal["nearest", "higher", "lower", "midpoint", "linear"], - ) -> Self: ... - def map_batches( - self, - function: Callable[[CompliantSeries], CompliantExpr[Any, Any]], - return_dtype: DType | type[DType] | None, - ) -> Self: ... - - @property - def str(self) -> Any: ... - @property - def name(self) -> Any: ... - @property - def dt(self) -> Any: ... - @property - def cat(self) -> Any: ... - @property - def list(self) -> Any: ... - - @unstable - def ewm_mean( - self, - *, - com: float | None, - span: float | None, - half_life: float | None, - alpha: float | None, - adjust: bool, - min_samples: int, - ignore_nulls: bool, - ) -> Self: ... - - @unstable - def rolling_sum( - self, - window_size: int, - *, - min_samples: int | None, - center: bool, - ) -> Self: ... - - @unstable - def rolling_mean( - self, - window_size: int, - *, - min_samples: int | None, - center: bool, - ) -> Self: ... - - @unstable - def rolling_var( - self, - window_size: int, - *, - min_samples: int | None, - center: bool, - ddof: int, - ) -> Self: ... - - @unstable - def rolling_std( - self, - window_size: int, - *, - min_samples: int | None, - center: bool, - ddof: int, - ) -> Self: ... - - @deprecated("Since `1.22.0`") - def gather_every(self, n: int, offset: int) -> Self: ... - def __and__(self, other: Any) -> Self: ... - def __or__(self, other: Any) -> Self: ... - def __add__(self, other: Any) -> Self: ... - def __sub__(self, other: Any) -> Self: ... - def __mul__(self, other: Any) -> Self: ... - def __floordiv__(self, other: Any) -> Self: ... - def __truediv__(self, other: Any) -> Self: ... - def __mod__(self, other: Any) -> Self: ... - def __pow__(self, other: Any) -> Self: ... - def __gt__(self, other: Any) -> Self: ... - def __ge__(self, other: Any) -> Self: ... - def __lt__(self, other: Any) -> Self: ... - def __le__(self, other: Any) -> Self: ... - def __invert__(self) -> Self: ... - def broadcast( - self, kind: Literal[ExprKind.AGGREGATION, ExprKind.LITERAL] - ) -> Self: ... - - -class CompliantNamespace(Protocol[CompliantFrameT, CompliantSeriesT_co]): - def col( - self, *column_names: str - ) -> CompliantExpr[CompliantFrameT, CompliantSeriesT_co]: ... - def lit( - self, value: Any, dtype: DType | None - ) -> CompliantExpr[CompliantFrameT, CompliantSeriesT_co]: ... - @property - def selectors(self) -> CompliantSelectorNamespace[Any, Any]: ... - - class SupportsNativeNamespace(Protocol): def __native_namespace__(self) -> ModuleType: ... -IntoCompliantExpr: TypeAlias = ( - "CompliantExpr[CompliantFrameT, CompliantSeriesT_co] | CompliantSeriesT_co" -) - IntoExpr: TypeAlias = Union["Expr", str, "Series[Any]"] """Anything which can be converted to an expression. @@ -501,11 +239,16 @@ class DTypes: __all__ = [ "CompliantDataFrame", + "CompliantExpr", + "CompliantFrameT", "CompliantLazyFrame", + "CompliantNamespace", "CompliantSeries", + "CompliantSeriesT_co", "DataFrameT", "Frame", "FrameT", + "IntoCompliantExpr", "IntoDataFrame", "IntoDataFrameT", "IntoExpr", From ca2ca2494f9bed5c03a544c6eef5e44da1f3ed16 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Wed, 5 Mar 2025 16:00:10 +0000 Subject: [PATCH 03/58] refactor: Export `Eval(Names|Series)` - Long-term, should probably be defined in `nw.typing` - Or just generally used in parts of each backend impl https://github.com/narwhals-dev/narwhals/pull/2064#discussion_r1976486891 --- narwhals/_arrow/selectors.py | 4 ++-- narwhals/_compliant/__init__.py | 4 ++++ narwhals/_dask/selectors.py | 4 ++-- narwhals/_duckdb/selectors.py | 4 ++-- narwhals/_pandas_like/selectors.py | 4 ++-- narwhals/_spark_like/selectors.py | 4 ++-- 6 files changed, 14 insertions(+), 10 deletions(-) diff --git a/narwhals/_arrow/selectors.py b/narwhals/_arrow/selectors.py index eb86def160..4cf50535db 100644 --- a/narwhals/_arrow/selectors.py +++ b/narwhals/_arrow/selectors.py @@ -11,8 +11,8 @@ from narwhals._arrow.dataframe import ArrowDataFrame from narwhals._arrow.series import ArrowSeries - from narwhals._compliant.selectors import EvalNames - from narwhals._compliant.selectors import EvalSeries + from narwhals._compliant import EvalNames + from narwhals._compliant import EvalSeries from narwhals.utils import _FullContext diff --git a/narwhals/_compliant/__init__.py b/narwhals/_compliant/__init__.py index b1fd9b61a2..f158bbec5c 100644 --- a/narwhals/_compliant/__init__.py +++ b/narwhals/_compliant/__init__.py @@ -7,6 +7,8 @@ from narwhals._compliant.selectors import CompliantSelector from narwhals._compliant.selectors import CompliantSelectorNamespace from narwhals._compliant.selectors import EagerSelectorNamespace +from narwhals._compliant.selectors import EvalNames +from narwhals._compliant.selectors import EvalSeries from narwhals._compliant.selectors import LazySelectorNamespace from narwhals._compliant.series import CompliantSeries from narwhals._compliant.typing import CompliantFrameT @@ -24,6 +26,8 @@ "CompliantSeries", "CompliantSeriesT_co", "EagerSelectorNamespace", + "EvalNames", + "EvalSeries", "IntoCompliantExpr", "LazySelectorNamespace", ] diff --git a/narwhals/_dask/selectors.py b/narwhals/_dask/selectors.py index 505d16c1b9..1d3b311352 100644 --- a/narwhals/_dask/selectors.py +++ b/narwhals/_dask/selectors.py @@ -14,8 +14,8 @@ from typing_extensions import Self - from narwhals._compliant.selectors import EvalNames - from narwhals._compliant.selectors import EvalSeries + from narwhals._compliant import EvalNames + from narwhals._compliant import EvalSeries from narwhals._dask.dataframe import DaskLazyFrame from narwhals.utils import _FullContext diff --git a/narwhals/_duckdb/selectors.py b/narwhals/_duckdb/selectors.py index 983e7cb37e..908fa41f3b 100644 --- a/narwhals/_duckdb/selectors.py +++ b/narwhals/_duckdb/selectors.py @@ -10,8 +10,8 @@ import duckdb from typing_extensions import Self - from narwhals._compliant.selectors import EvalNames - from narwhals._compliant.selectors import EvalSeries + from narwhals._compliant import EvalNames + from narwhals._compliant import EvalSeries from narwhals._duckdb.dataframe import DuckDBLazyFrame from narwhals.utils import _FullContext diff --git a/narwhals/_pandas_like/selectors.py b/narwhals/_pandas_like/selectors.py index 2d7566682c..69109f0cec 100644 --- a/narwhals/_pandas_like/selectors.py +++ b/narwhals/_pandas_like/selectors.py @@ -11,8 +11,8 @@ if TYPE_CHECKING: from typing_extensions import Self - from narwhals._compliant.selectors import EvalNames - from narwhals._compliant.selectors import EvalSeries + from narwhals._compliant import EvalNames + from narwhals._compliant import EvalSeries from narwhals._pandas_like.dataframe import PandasLikeDataFrame from narwhals._pandas_like.series import PandasLikeSeries from narwhals.utils import _FullContext diff --git a/narwhals/_spark_like/selectors.py b/narwhals/_spark_like/selectors.py index 1e3cc01a6e..48a92329a1 100644 --- a/narwhals/_spark_like/selectors.py +++ b/narwhals/_spark_like/selectors.py @@ -10,8 +10,8 @@ from pyspark.sql import Column from typing_extensions import Self - from narwhals._compliant.selectors import EvalNames - from narwhals._compliant.selectors import EvalSeries + from narwhals._compliant import EvalNames + from narwhals._compliant import EvalSeries from narwhals._spark_like.dataframe import SparkLikeLazyFrame from narwhals.utils import _FullContext From 9cc3ce8b9fa4b6d4d732a18d28be3280285b7aff Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Wed, 5 Mar 2025 16:06:38 +0000 Subject: [PATCH 04/58] feat(DRAFT): Add placeholder `(Eager|Lazy)Expr` protocols https://github.com/narwhals-dev/narwhals/issues/2044#issuecomment-2674262833 --- narwhals/_compliant/expr.py | 15 +++++++++++++++ narwhals/_compliant/typing.py | 10 +++++++++- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index 2c86fbeb1d..7b84a69199 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -7,7 +7,9 @@ from typing import Literal from typing import Sequence +from narwhals._compliant.typing import CompliantDataFrameT from narwhals._compliant.typing import CompliantFrameT +from narwhals._compliant.typing import CompliantLazyFrameT from narwhals._compliant.typing import CompliantSeriesT_co from narwhals.utils import deprecated from narwhals.utils import unstable @@ -212,3 +214,16 @@ def __invert__(self) -> Self: ... def broadcast( self, kind: Literal[ExprKind.AGGREGATION, ExprKind.LITERAL] ) -> Self: ... + + +class EagerExpr( + CompliantExpr[CompliantDataFrameT, CompliantSeriesT_co], + Protocol38[CompliantDataFrameT, CompliantSeriesT_co], +): ... + + +# NOTE: See (https://github.com/narwhals-dev/narwhals/issues/2044#issuecomment-2674262833) +class LazyExpr( + CompliantExpr[CompliantLazyFrameT, CompliantSeriesT_co], + Protocol38[CompliantLazyFrameT, CompliantSeriesT_co], +): ... diff --git a/narwhals/_compliant/typing.py b/narwhals/_compliant/typing.py index f02dc92569..9754141dfd 100644 --- a/narwhals/_compliant/typing.py +++ b/narwhals/_compliant/typing.py @@ -12,7 +12,13 @@ from narwhals._compliant.expr import CompliantExpr from narwhals._compliant.series import CompliantSeries -__all__ = ["CompliantFrameT", "CompliantSeriesT_co", "IntoCompliantExpr"] +__all__ = [ + "CompliantDataFrameT", + "CompliantFrameT", + "CompliantLazyFrameT", + "CompliantSeriesT_co", + "IntoCompliantExpr", +] CompliantSeriesT_co = TypeVar( "CompliantSeriesT_co", bound="CompliantSeries", covariant=True @@ -20,6 +26,8 @@ CompliantFrameT = TypeVar( "CompliantFrameT", bound="CompliantDataFrame[Any] | CompliantLazyFrame" ) +CompliantDataFrameT = TypeVar("CompliantDataFrameT", bound="CompliantDataFrame[Any]") +CompliantLazyFrameT = TypeVar("CompliantLazyFrameT", bound="CompliantLazyFrame") IntoCompliantExpr: TypeAlias = ( "CompliantExpr[CompliantFrameT, CompliantSeriesT_co] | CompliantSeriesT_co" ) From 53413aef4bcfb922849d7847926057cd477cbe66 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Wed, 5 Mar 2025 16:37:57 +0000 Subject: [PATCH 05/58] ci: ignore coverage https://github.com/narwhals-dev/narwhals/actions/runs/13680592205/job/38251688439 --- narwhals/_compliant/dataframe.py | 4 ++-- narwhals/_compliant/expr.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/narwhals/_compliant/dataframe.py b/narwhals/_compliant/dataframe.py index a02069310e..8ea87b14b4 100644 --- a/narwhals/_compliant/dataframe.py +++ b/narwhals/_compliant/dataframe.py @@ -23,7 +23,7 @@ def __narwhals_namespace__(self) -> Any: ... def simple_select( self, *column_names: str ) -> Self: ... # `select` where all args are column names. - def aggregate(self, *exprs: Any) -> Self: + def aggregate(self, *exprs: Any) -> Self: # pragma: no cover ... # `select` where all args are aggregations or literals # (so, no broadcasting is necessary). @@ -41,7 +41,7 @@ def __narwhals_namespace__(self) -> Any: ... def simple_select( self, *column_names: str ) -> Self: ... # `select` where all args are column names. - def aggregate(self, *exprs: Any) -> Self: + def aggregate(self, *exprs: Any) -> Self: # pragma: no cover ... # `select` where all args are aggregations or literals # (so, no broadcasting is necessary). diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index 7b84a69199..2dbdc2dd65 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -14,12 +14,12 @@ from narwhals.utils import deprecated from narwhals.utils import unstable -if not TYPE_CHECKING: +if not TYPE_CHECKING: # pragma: no cover if sys.version_info >= (3, 9): from typing import Protocol as Protocol38 else: from typing import Generic as Protocol38 -else: +else: # pragma: no cover # TODO @dangotbanned: Remove after dropping `3.8` (#2084) # - https://github.com/narwhals-dev/narwhals/pull/2064#discussion_r1965921386 from typing import Protocol as Protocol38 From 2bf05fba2f46bf5b2e62dc46d47dbb691ff1806f Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Thu, 6 Mar 2025 15:43:45 +0000 Subject: [PATCH 06/58] refactor: avoid adding `typing.IntoCompliantExpr` https://github.com/narwhals-dev/narwhals/pull/2149#discussion_r1981760892 --- narwhals/dataframe.py | 2 +- narwhals/typing.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/narwhals/dataframe.py b/narwhals/dataframe.py index d89a9dbedb..e5a2a580c8 100644 --- a/narwhals/dataframe.py +++ b/narwhals/dataframe.py @@ -49,10 +49,10 @@ from typing_extensions import ParamSpec from typing_extensions import Self + from narwhals._compliant import IntoCompliantExpr from narwhals.group_by import GroupBy from narwhals.group_by import LazyGroupBy from narwhals.series import Series - from narwhals.typing import IntoCompliantExpr from narwhals.typing import IntoDataFrame from narwhals.typing import IntoExpr from narwhals.typing import IntoFrame diff --git a/narwhals/typing.py b/narwhals/typing.py index 39b483750c..b9469f831b 100644 --- a/narwhals/typing.py +++ b/narwhals/typing.py @@ -14,7 +14,6 @@ from narwhals._compliant import CompliantNamespace from narwhals._compliant import CompliantSeries from narwhals._compliant import CompliantSeriesT_co -from narwhals._compliant import IntoCompliantExpr if TYPE_CHECKING: from types import ModuleType @@ -248,7 +247,6 @@ class DTypes: "DataFrameT", "Frame", "FrameT", - "IntoCompliantExpr", "IntoDataFrame", "IntoDataFrameT", "IntoExpr", From 7fb4914fb5a846447372267861b140a39571a081 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Thu, 6 Mar 2025 15:47:00 +0000 Subject: [PATCH 07/58] refactor: avoid adding `typing.CompliantFrameT` https://github.com/narwhals-dev/narwhals/pull/2149#discussion_r1981760892 --- narwhals/_expression_parsing.py | 2 +- narwhals/typing.py | 2 -- narwhals/utils.py | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/narwhals/_expression_parsing.py b/narwhals/_expression_parsing.py index 96a6f91441..a2be44a7b4 100644 --- a/narwhals/_expression_parsing.py +++ b/narwhals/_expression_parsing.py @@ -26,11 +26,11 @@ from typing_extensions import TypeIs from narwhals._arrow.expr import ArrowExpr + from narwhals._compliant import CompliantFrameT from narwhals._pandas_like.expr import PandasLikeExpr from narwhals.expr import Expr from narwhals.typing import CompliantDataFrame from narwhals.typing import CompliantExpr - from narwhals.typing import CompliantFrameT from narwhals.typing import CompliantLazyFrame from narwhals.typing import CompliantNamespace from narwhals.typing import CompliantSeries diff --git a/narwhals/typing.py b/narwhals/typing.py index b9469f831b..06c28af42d 100644 --- a/narwhals/typing.py +++ b/narwhals/typing.py @@ -9,7 +9,6 @@ from narwhals._compliant import CompliantDataFrame from narwhals._compliant import CompliantExpr -from narwhals._compliant import CompliantFrameT from narwhals._compliant import CompliantLazyFrame from narwhals._compliant import CompliantNamespace from narwhals._compliant import CompliantSeries @@ -239,7 +238,6 @@ class DTypes: __all__ = [ "CompliantDataFrame", "CompliantExpr", - "CompliantFrameT", "CompliantLazyFrame", "CompliantNamespace", "CompliantSeries", diff --git a/narwhals/utils.py b/narwhals/utils.py index 43844171b8..450a0c12e8 100644 --- a/narwhals/utils.py +++ b/narwhals/utils.py @@ -52,13 +52,13 @@ from typing_extensions import TypeAlias from typing_extensions import TypeIs + from narwhals._compliant import CompliantFrameT from narwhals.dataframe import DataFrame from narwhals.dataframe import LazyFrame from narwhals.dtypes import DType from narwhals.series import Series from narwhals.typing import CompliantDataFrame from narwhals.typing import CompliantExpr - from narwhals.typing import CompliantFrameT from narwhals.typing import CompliantLazyFrame from narwhals.typing import CompliantSeries from narwhals.typing import CompliantSeriesT_co From 51820a14c0aa7cd82a4f897fcc6743bc26151568 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Thu, 6 Mar 2025 15:48:45 +0000 Subject: [PATCH 08/58] refactor: avoid adding `typing.CompliantSeriesT_co` https://github.com/narwhals-dev/narwhals/pull/2149#discussion_r1981760892 --- narwhals/_expression_parsing.py | 2 +- narwhals/typing.py | 2 -- narwhals/utils.py | 2 +- 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/narwhals/_expression_parsing.py b/narwhals/_expression_parsing.py index a2be44a7b4..3699bc0649 100644 --- a/narwhals/_expression_parsing.py +++ b/narwhals/_expression_parsing.py @@ -27,6 +27,7 @@ from narwhals._arrow.expr import ArrowExpr from narwhals._compliant import CompliantFrameT + from narwhals._compliant import CompliantSeriesT_co from narwhals._pandas_like.expr import PandasLikeExpr from narwhals.expr import Expr from narwhals.typing import CompliantDataFrame @@ -34,7 +35,6 @@ from narwhals.typing import CompliantLazyFrame from narwhals.typing import CompliantNamespace from narwhals.typing import CompliantSeries - from narwhals.typing import CompliantSeriesT_co from narwhals.typing import IntoExpr from narwhals.typing import _1DArray diff --git a/narwhals/typing.py b/narwhals/typing.py index 06c28af42d..971ccb3e4a 100644 --- a/narwhals/typing.py +++ b/narwhals/typing.py @@ -12,7 +12,6 @@ from narwhals._compliant import CompliantLazyFrame from narwhals._compliant import CompliantNamespace from narwhals._compliant import CompliantSeries -from narwhals._compliant import CompliantSeriesT_co if TYPE_CHECKING: from types import ModuleType @@ -241,7 +240,6 @@ class DTypes: "CompliantLazyFrame", "CompliantNamespace", "CompliantSeries", - "CompliantSeriesT_co", "DataFrameT", "Frame", "FrameT", diff --git a/narwhals/utils.py b/narwhals/utils.py index 450a0c12e8..8090c1cedb 100644 --- a/narwhals/utils.py +++ b/narwhals/utils.py @@ -53,6 +53,7 @@ from typing_extensions import TypeIs from narwhals._compliant import CompliantFrameT + from narwhals._compliant import CompliantSeriesT_co from narwhals.dataframe import DataFrame from narwhals.dataframe import LazyFrame from narwhals.dtypes import DType @@ -61,7 +62,6 @@ from narwhals.typing import CompliantExpr from narwhals.typing import CompliantLazyFrame from narwhals.typing import CompliantSeries - from narwhals.typing import CompliantSeriesT_co from narwhals.typing import DataFrameLike from narwhals.typing import DTypes from narwhals.typing import IntoSeriesT From 8f1c5ca76360a116772cb956cc4a071519311c12 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Thu, 6 Mar 2025 15:52:45 +0000 Subject: [PATCH 09/58] refactor: avoid adding `typing.CompliantNamespace` https://github.com/narwhals-dev/narwhals/pull/2149#discussion_r1981760892 --- narwhals/_arrow/namespace.py | 2 +- narwhals/_dask/namespace.py | 2 +- narwhals/_duckdb/namespace.py | 2 +- narwhals/_expression_parsing.py | 2 +- narwhals/_pandas_like/namespace.py | 2 +- narwhals/_spark_like/namespace.py | 2 +- narwhals/expr.py | 2 +- narwhals/functions.py | 2 +- narwhals/typing.py | 2 -- 9 files changed, 8 insertions(+), 10 deletions(-) diff --git a/narwhals/_arrow/namespace.py b/narwhals/_arrow/namespace.py index 7d80310352..68ace016d8 100644 --- a/narwhals/_arrow/namespace.py +++ b/narwhals/_arrow/namespace.py @@ -25,9 +25,9 @@ from narwhals._arrow.utils import horizontal_concat from narwhals._arrow.utils import nulls_like from narwhals._arrow.utils import vertical_concat +from narwhals._compliant import CompliantNamespace from narwhals._expression_parsing import combine_alias_output_names from narwhals._expression_parsing import combine_evaluate_output_names -from narwhals.typing import CompliantNamespace from narwhals.utils import Implementation from narwhals.utils import exclude_column_names from narwhals.utils import get_column_names diff --git a/narwhals/_dask/namespace.py b/narwhals/_dask/namespace.py index eddbe7925f..3ebdbfd00b 100644 --- a/narwhals/_dask/namespace.py +++ b/narwhals/_dask/namespace.py @@ -14,6 +14,7 @@ import dask.dataframe as dd import pandas as pd +from narwhals._compliant import CompliantNamespace from narwhals._dask.dataframe import DaskLazyFrame from narwhals._dask.expr import DaskExpr from narwhals._dask.selectors import DaskSelectorNamespace @@ -24,7 +25,6 @@ from narwhals._dask.utils import validate_comparand from narwhals._expression_parsing import combine_alias_output_names from narwhals._expression_parsing import combine_evaluate_output_names -from narwhals.typing import CompliantNamespace from narwhals.utils import Implementation from narwhals.utils import exclude_column_names from narwhals.utils import get_column_names diff --git a/narwhals/_duckdb/namespace.py b/narwhals/_duckdb/namespace.py index f8780fbc44..c71507a8ef 100644 --- a/narwhals/_duckdb/namespace.py +++ b/narwhals/_duckdb/namespace.py @@ -16,6 +16,7 @@ from duckdb.typing import BIGINT from duckdb.typing import VARCHAR +from narwhals._compliant import CompliantNamespace from narwhals._duckdb.expr import DuckDBExpr from narwhals._duckdb.selectors import DuckDBSelectorNamespace from narwhals._duckdb.utils import lit @@ -23,7 +24,6 @@ from narwhals._duckdb.utils import narwhals_to_native_dtype from narwhals._expression_parsing import combine_alias_output_names from narwhals._expression_parsing import combine_evaluate_output_names -from narwhals.typing import CompliantNamespace from narwhals.utils import Implementation from narwhals.utils import exclude_column_names from narwhals.utils import get_column_names diff --git a/narwhals/_expression_parsing.py b/narwhals/_expression_parsing.py index 3699bc0649..8667b633e2 100644 --- a/narwhals/_expression_parsing.py +++ b/narwhals/_expression_parsing.py @@ -27,13 +27,13 @@ from narwhals._arrow.expr import ArrowExpr from narwhals._compliant import CompliantFrameT + from narwhals._compliant import CompliantNamespace from narwhals._compliant import CompliantSeriesT_co from narwhals._pandas_like.expr import PandasLikeExpr from narwhals.expr import Expr from narwhals.typing import CompliantDataFrame from narwhals.typing import CompliantExpr from narwhals.typing import CompliantLazyFrame - from narwhals.typing import CompliantNamespace from narwhals.typing import CompliantSeries from narwhals.typing import IntoExpr from narwhals.typing import _1DArray diff --git a/narwhals/_pandas_like/namespace.py b/narwhals/_pandas_like/namespace.py index 4b4994c6d0..ec2a696dbc 100644 --- a/narwhals/_pandas_like/namespace.py +++ b/narwhals/_pandas_like/namespace.py @@ -11,6 +11,7 @@ from typing import Literal from typing import Sequence +from narwhals._compliant import CompliantNamespace from narwhals._expression_parsing import combine_alias_output_names from narwhals._expression_parsing import combine_evaluate_output_names from narwhals._pandas_like.dataframe import PandasLikeDataFrame @@ -23,7 +24,6 @@ from narwhals._pandas_like.utils import extract_dataframe_comparand from narwhals._pandas_like.utils import horizontal_concat from narwhals._pandas_like.utils import vertical_concat -from narwhals.typing import CompliantNamespace from narwhals.utils import exclude_column_names from narwhals.utils import get_column_names from narwhals.utils import import_dtypes_module diff --git a/narwhals/_spark_like/namespace.py b/narwhals/_spark_like/namespace.py index 91cad2bbb6..60b370573c 100644 --- a/narwhals/_spark_like/namespace.py +++ b/narwhals/_spark_like/namespace.py @@ -12,6 +12,7 @@ from typing import Sequence from typing import cast +from narwhals._compliant import CompliantNamespace from narwhals._expression_parsing import combine_alias_output_names from narwhals._expression_parsing import combine_evaluate_output_names from narwhals._spark_like.dataframe import SparkLikeLazyFrame @@ -19,7 +20,6 @@ from narwhals._spark_like.selectors import SparkLikeSelectorNamespace from narwhals._spark_like.utils import maybe_evaluate_expr from narwhals._spark_like.utils import narwhals_to_native_dtype -from narwhals.typing import CompliantNamespace from narwhals.utils import exclude_column_names from narwhals.utils import get_column_names from narwhals.utils import passthrough_column_names diff --git a/narwhals/expr.py b/narwhals/expr.py index b829d3789b..649f6b4026 100644 --- a/narwhals/expr.py +++ b/narwhals/expr.py @@ -33,9 +33,9 @@ from typing_extensions import Self from typing_extensions import TypeAlias + from narwhals._compliant import CompliantNamespace from narwhals.dtypes import DType from narwhals.typing import CompliantExpr - from narwhals.typing import CompliantNamespace from narwhals.typing import IntoExpr PS = ParamSpec("PS") diff --git a/narwhals/functions.py b/narwhals/functions.py index a1fe424d3b..93c6e08397 100644 --- a/narwhals/functions.py +++ b/narwhals/functions.py @@ -42,12 +42,12 @@ import pyarrow as pa from typing_extensions import Self + from narwhals._compliant import CompliantNamespace from narwhals.dataframe import DataFrame from narwhals.dataframe import LazyFrame from narwhals.dtypes import DType from narwhals.series import Series from narwhals.typing import CompliantExpr - from narwhals.typing import CompliantNamespace from narwhals.typing import DTypeBackend from narwhals.typing import IntoDataFrameT from narwhals.typing import IntoExpr diff --git a/narwhals/typing.py b/narwhals/typing.py index 971ccb3e4a..406cdef0c8 100644 --- a/narwhals/typing.py +++ b/narwhals/typing.py @@ -10,7 +10,6 @@ from narwhals._compliant import CompliantDataFrame from narwhals._compliant import CompliantExpr from narwhals._compliant import CompliantLazyFrame -from narwhals._compliant import CompliantNamespace from narwhals._compliant import CompliantSeries if TYPE_CHECKING: @@ -238,7 +237,6 @@ class DTypes: "CompliantDataFrame", "CompliantExpr", "CompliantLazyFrame", - "CompliantNamespace", "CompliantSeries", "DataFrameT", "Frame", From 5bf2bdb60ea676b94b6a8cc65e23b3b174161243 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Thu, 6 Mar 2025 15:56:27 +0000 Subject: [PATCH 10/58] refactor: avoid adding `typing.CompliantExpr` https://github.com/narwhals-dev/narwhals/pull/2149#discussion_r1981760892 --- narwhals/_arrow/expr.py | 2 +- narwhals/_dask/expr.py | 2 +- narwhals/_duckdb/expr.py | 2 +- narwhals/_expression_parsing.py | 2 +- narwhals/_pandas_like/expr.py | 2 +- narwhals/_spark_like/expr.py | 2 +- narwhals/expr.py | 2 +- narwhals/functions.py | 2 +- narwhals/typing.py | 2 -- narwhals/utils.py | 2 +- 10 files changed, 9 insertions(+), 11 deletions(-) diff --git a/narwhals/_arrow/expr.py b/narwhals/_arrow/expr.py index f36a43e92d..4861866ccc 100644 --- a/narwhals/_arrow/expr.py +++ b/narwhals/_arrow/expr.py @@ -13,6 +13,7 @@ from narwhals._arrow.expr_name import ArrowExprNameNamespace from narwhals._arrow.expr_str import ArrowExprStringNamespace from narwhals._arrow.series import ArrowSeries +from narwhals._compliant import CompliantExpr from narwhals._expression_parsing import ExprKind from narwhals._expression_parsing import evaluate_output_names_and_aliases from narwhals._expression_parsing import is_scalar_like @@ -20,7 +21,6 @@ from narwhals.dependencies import get_numpy from narwhals.dependencies import is_numpy_array from narwhals.exceptions import ColumnNotFoundError -from narwhals.typing import CompliantExpr from narwhals.utils import Implementation from narwhals.utils import not_implemented diff --git a/narwhals/_dask/expr.py b/narwhals/_dask/expr.py index 5cb473773d..c322d6be82 100644 --- a/narwhals/_dask/expr.py +++ b/narwhals/_dask/expr.py @@ -8,6 +8,7 @@ from typing import Literal from typing import Sequence +from narwhals._compliant import CompliantExpr from narwhals._dask.expr_dt import DaskExprDateTimeNamespace from narwhals._dask.expr_name import DaskExprNameNamespace from narwhals._dask.expr_str import DaskExprStringNamespace @@ -20,7 +21,6 @@ from narwhals._pandas_like.utils import native_to_narwhals_dtype from narwhals.exceptions import ColumnNotFoundError from narwhals.exceptions import InvalidOperationError -from narwhals.typing import CompliantExpr from narwhals.utils import Implementation from narwhals.utils import generate_temporary_column_name from narwhals.utils import not_implemented diff --git a/narwhals/_duckdb/expr.py b/narwhals/_duckdb/expr.py index c47fa8bf66..ddc91a01ab 100644 --- a/narwhals/_duckdb/expr.py +++ b/narwhals/_duckdb/expr.py @@ -14,6 +14,7 @@ from duckdb import FunctionExpression from duckdb.typing import DuckDBPyType +from narwhals._compliant import CompliantExpr from narwhals._duckdb.expr_dt import DuckDBExprDateTimeNamespace from narwhals._duckdb.expr_list import DuckDBExprListNamespace from narwhals._duckdb.expr_name import DuckDBExprNameNamespace @@ -22,7 +23,6 @@ from narwhals._duckdb.utils import maybe_evaluate_expr from narwhals._duckdb.utils import narwhals_to_native_dtype from narwhals._expression_parsing import ExprKind -from narwhals.typing import CompliantExpr from narwhals.utils import Implementation from narwhals.utils import not_implemented diff --git a/narwhals/_expression_parsing.py b/narwhals/_expression_parsing.py index 8667b633e2..f985c3ff55 100644 --- a/narwhals/_expression_parsing.py +++ b/narwhals/_expression_parsing.py @@ -26,13 +26,13 @@ from typing_extensions import TypeIs from narwhals._arrow.expr import ArrowExpr + from narwhals._compliant import CompliantExpr from narwhals._compliant import CompliantFrameT from narwhals._compliant import CompliantNamespace from narwhals._compliant import CompliantSeriesT_co from narwhals._pandas_like.expr import PandasLikeExpr from narwhals.expr import Expr from narwhals.typing import CompliantDataFrame - from narwhals.typing import CompliantExpr from narwhals.typing import CompliantLazyFrame from narwhals.typing import CompliantSeries from narwhals.typing import IntoExpr diff --git a/narwhals/_pandas_like/expr.py b/narwhals/_pandas_like/expr.py index 14b4625a6e..77851de25b 100644 --- a/narwhals/_pandas_like/expr.py +++ b/narwhals/_pandas_like/expr.py @@ -8,6 +8,7 @@ from typing import Mapping from typing import Sequence +from narwhals._compliant import CompliantExpr from narwhals._expression_parsing import ExprKind from narwhals._expression_parsing import evaluate_output_names_and_aliases from narwhals._expression_parsing import is_elementary_expression @@ -23,7 +24,6 @@ from narwhals.dependencies import get_numpy from narwhals.dependencies import is_numpy_array from narwhals.exceptions import ColumnNotFoundError -from narwhals.typing import CompliantExpr if TYPE_CHECKING: from typing_extensions import Self diff --git a/narwhals/_spark_like/expr.py b/narwhals/_spark_like/expr.py index 7c48731443..6565da5a3d 100644 --- a/narwhals/_spark_like/expr.py +++ b/narwhals/_spark_like/expr.py @@ -9,6 +9,7 @@ from typing import Sequence from typing import cast +from narwhals._compliant import CompliantExpr from narwhals._expression_parsing import ExprKind from narwhals._spark_like.expr_dt import SparkLikeExprDateTimeNamespace from narwhals._spark_like.expr_list import SparkLikeExprListNamespace @@ -17,7 +18,6 @@ from narwhals._spark_like.utils import maybe_evaluate_expr from narwhals._spark_like.utils import narwhals_to_native_dtype from narwhals.dependencies import get_pyspark -from narwhals.typing import CompliantExpr from narwhals.utils import Implementation from narwhals.utils import not_implemented from narwhals.utils import parse_version diff --git a/narwhals/expr.py b/narwhals/expr.py index 649f6b4026..cb41ca683f 100644 --- a/narwhals/expr.py +++ b/narwhals/expr.py @@ -33,9 +33,9 @@ from typing_extensions import Self from typing_extensions import TypeAlias + from narwhals._compliant import CompliantExpr from narwhals._compliant import CompliantNamespace from narwhals.dtypes import DType - from narwhals.typing import CompliantExpr from narwhals.typing import IntoExpr PS = ParamSpec("PS") diff --git a/narwhals/functions.py b/narwhals/functions.py index 93c6e08397..db29692b98 100644 --- a/narwhals/functions.py +++ b/narwhals/functions.py @@ -42,12 +42,12 @@ import pyarrow as pa from typing_extensions import Self + from narwhals._compliant import CompliantExpr from narwhals._compliant import CompliantNamespace from narwhals.dataframe import DataFrame from narwhals.dataframe import LazyFrame from narwhals.dtypes import DType from narwhals.series import Series - from narwhals.typing import CompliantExpr from narwhals.typing import DTypeBackend from narwhals.typing import IntoDataFrameT from narwhals.typing import IntoExpr diff --git a/narwhals/typing.py b/narwhals/typing.py index 406cdef0c8..f941d60297 100644 --- a/narwhals/typing.py +++ b/narwhals/typing.py @@ -8,7 +8,6 @@ from typing import Union from narwhals._compliant import CompliantDataFrame -from narwhals._compliant import CompliantExpr from narwhals._compliant import CompliantLazyFrame from narwhals._compliant import CompliantSeries @@ -235,7 +234,6 @@ class DTypes: __all__ = [ "CompliantDataFrame", - "CompliantExpr", "CompliantLazyFrame", "CompliantSeries", "DataFrameT", diff --git a/narwhals/utils.py b/narwhals/utils.py index 8090c1cedb..9f8e9c5f30 100644 --- a/narwhals/utils.py +++ b/narwhals/utils.py @@ -52,6 +52,7 @@ from typing_extensions import TypeAlias from typing_extensions import TypeIs + from narwhals._compliant import CompliantExpr from narwhals._compliant import CompliantFrameT from narwhals._compliant import CompliantSeriesT_co from narwhals.dataframe import DataFrame @@ -59,7 +60,6 @@ from narwhals.dtypes import DType from narwhals.series import Series from narwhals.typing import CompliantDataFrame - from narwhals.typing import CompliantExpr from narwhals.typing import CompliantLazyFrame from narwhals.typing import CompliantSeries from narwhals.typing import DataFrameLike From 075228b83c4bb6c7a37bb0b35bb0c4e6a6b7c6b2 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Thu, 6 Mar 2025 17:21:57 +0000 Subject: [PATCH 11/58] fix(typing): Resolve `NativeExpr`-related issues Fixes #2044 --- narwhals/_compliant/__init__.py | 2 + narwhals/_compliant/expr.py | 21 +++-- narwhals/_compliant/namespace.py | 8 +- narwhals/_compliant/selectors.py | 125 ++++++++++++++++-------------- narwhals/_compliant/typing.py | 12 ++- narwhals/_dask/expr.py | 2 +- narwhals/_dask/namespace.py | 2 +- narwhals/_duckdb/expr.py | 2 +- narwhals/_duckdb/namespace.py | 2 +- narwhals/_duckdb/selectors.py | 7 +- narwhals/_expression_parsing.py | 5 +- narwhals/_spark_like/expr.py | 2 +- narwhals/_spark_like/namespace.py | 2 +- narwhals/_spark_like/selectors.py | 7 +- narwhals/utils.py | 5 +- 15 files changed, 115 insertions(+), 89 deletions(-) diff --git a/narwhals/_compliant/__init__.py b/narwhals/_compliant/__init__.py index f158bbec5c..f0ead106c6 100644 --- a/narwhals/_compliant/__init__.py +++ b/narwhals/_compliant/__init__.py @@ -12,6 +12,7 @@ from narwhals._compliant.selectors import LazySelectorNamespace from narwhals._compliant.series import CompliantSeries from narwhals._compliant.typing import CompliantFrameT +from narwhals._compliant.typing import CompliantSeriesOrNativeExprT_co from narwhals._compliant.typing import CompliantSeriesT_co from narwhals._compliant.typing import IntoCompliantExpr @@ -24,6 +25,7 @@ "CompliantSelector", "CompliantSelectorNamespace", "CompliantSeries", + "CompliantSeriesOrNativeExprT_co", "CompliantSeriesT_co", "EagerSelectorNamespace", "EvalNames", diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index 2dbdc2dd65..6bee83804a 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -5,12 +5,15 @@ from typing import Any from typing import Callable from typing import Literal +from typing import Protocol from typing import Sequence from narwhals._compliant.typing import CompliantDataFrameT from narwhals._compliant.typing import CompliantFrameT from narwhals._compliant.typing import CompliantLazyFrameT +from narwhals._compliant.typing import CompliantSeriesOrNativeExprT_co from narwhals._compliant.typing import CompliantSeriesT_co +from narwhals._compliant.typing import NativeExprT_co from narwhals.utils import deprecated from narwhals.utils import unstable @@ -39,7 +42,13 @@ __all__ = ["CompliantExpr"] -class CompliantExpr(Protocol38[CompliantFrameT, CompliantSeriesT_co]): +# NOTE: Only common methods for lazy expr-like objects +class NativeExpr(Protocol): + def between(self, *args: Any, **kwds: Any) -> Any: ... + def isin(self, *args: Any, **kwds: Any) -> Any: ... + + +class CompliantExpr(Protocol38[CompliantFrameT, CompliantSeriesOrNativeExprT_co]): _implementation: Implementation _backend_version: tuple[int, ...] _version: Version @@ -48,11 +57,13 @@ class CompliantExpr(Protocol38[CompliantFrameT, CompliantSeriesT_co]): _depth: int _function_name: str - def __call__(self, df: CompliantFrameT) -> Sequence[CompliantSeriesT_co]: ... + def __call__( + self, df: CompliantFrameT + ) -> Sequence[CompliantSeriesOrNativeExprT_co]: ... def __narwhals_expr__(self) -> None: ... def __narwhals_namespace__( self, - ) -> CompliantNamespace[CompliantFrameT, CompliantSeriesT_co]: ... + ) -> CompliantNamespace[CompliantFrameT, CompliantSeriesOrNativeExprT_co]: ... def is_null(self) -> Self: ... def abs(self) -> Self: ... def all(self) -> Self: ... @@ -224,6 +235,6 @@ class EagerExpr( # NOTE: See (https://github.com/narwhals-dev/narwhals/issues/2044#issuecomment-2674262833) class LazyExpr( - CompliantExpr[CompliantLazyFrameT, CompliantSeriesT_co], - Protocol38[CompliantLazyFrameT, CompliantSeriesT_co], + CompliantExpr[CompliantLazyFrameT, NativeExprT_co], + Protocol38[CompliantLazyFrameT, NativeExprT_co], ): ... diff --git a/narwhals/_compliant/namespace.py b/narwhals/_compliant/namespace.py index 893a2faecb..a2b07fd591 100644 --- a/narwhals/_compliant/namespace.py +++ b/narwhals/_compliant/namespace.py @@ -5,7 +5,7 @@ from typing import Protocol from narwhals._compliant.typing import CompliantFrameT -from narwhals._compliant.typing import CompliantSeriesT_co +from narwhals._compliant.typing import CompliantSeriesOrNativeExprT_co if TYPE_CHECKING: from narwhals._compliant.expr import CompliantExpr @@ -15,12 +15,12 @@ __all__ = ["CompliantNamespace"] -class CompliantNamespace(Protocol[CompliantFrameT, CompliantSeriesT_co]): +class CompliantNamespace(Protocol[CompliantFrameT, CompliantSeriesOrNativeExprT_co]): def col( self, *column_names: str - ) -> CompliantExpr[CompliantFrameT, CompliantSeriesT_co]: ... + ) -> CompliantExpr[CompliantFrameT, CompliantSeriesOrNativeExprT_co]: ... def lit( self, value: Any, dtype: DType | None - ) -> CompliantExpr[CompliantFrameT, CompliantSeriesT_co]: ... + ) -> CompliantExpr[CompliantFrameT, CompliantSeriesOrNativeExprT_co]: ... @property def selectors(self) -> CompliantSelectorNamespace[Any, Any]: ... diff --git a/narwhals/_compliant/selectors.py b/narwhals/_compliant/selectors.py index 292c47429d..969d4f460c 100644 --- a/narwhals/_compliant/selectors.py +++ b/narwhals/_compliant/selectors.py @@ -45,6 +45,7 @@ from narwhals._compliant.dataframe import CompliantDataFrame from narwhals._compliant.dataframe import CompliantLazyFrame + from narwhals._compliant.expr import NativeExpr from narwhals._compliant.series import CompliantSeries from narwhals.dtypes import DType from narwhals.typing import TimeUnit @@ -61,50 +62,46 @@ ] +SeriesOrExprT = TypeVar("SeriesOrExprT", bound="CompliantSeries | NativeExpr") SeriesT = TypeVar("SeriesT", bound="CompliantSeries") +ExprT = TypeVar("ExprT", bound="NativeExpr") FrameT = TypeVar("FrameT", bound="CompliantDataFrame[Any] | CompliantLazyFrame") DataFrameT = TypeVar("DataFrameT", bound="CompliantDataFrame[Any]") LazyFrameT = TypeVar("LazyFrameT", bound="CompliantLazyFrame") SelectorOrExpr: TypeAlias = ( - "CompliantSelector[FrameT, SeriesT] | CompliantExpr[FrameT, SeriesT]" + "CompliantSelector[FrameT, SeriesOrExprT] | CompliantExpr[FrameT, SeriesOrExprT]" ) -EvalSeries: TypeAlias = Callable[[FrameT], Sequence[SeriesT]] +EvalSeries: TypeAlias = Callable[[FrameT], Sequence[SeriesOrExprT]] EvalNames: TypeAlias = Callable[[FrameT], Sequence[str]] -class CompliantSelectorNamespace(Protocol[FrameT, SeriesT]): +class CompliantSelectorNamespace(Protocol[FrameT, SeriesOrExprT]): _implementation: Implementation _backend_version: tuple[int, ...] _version: Version def _selector( self, - call: EvalSeries[FrameT, SeriesT], + call: EvalSeries[FrameT, SeriesOrExprT], evaluate_output_names: EvalNames[FrameT], /, - ) -> CompliantSelector[FrameT, SeriesT]: ... + ) -> CompliantSelector[FrameT, SeriesOrExprT]: ... - def _iter_columns(self, df: FrameT, /) -> Iterator[SeriesT]: ... + def _iter_columns(self, df: FrameT, /) -> Iterator[SeriesOrExprT]: ... - def _iter_schema(self, df: FrameT, /) -> Iterator[tuple[str, DType]]: - for ser in self._iter_columns(df): - yield ser.name, ser.dtype + def _iter_schema(self, df: FrameT, /) -> Iterator[tuple[str, DType]]: ... - def _iter_columns_dtypes(self, df: FrameT, /) -> Iterator[tuple[SeriesT, DType]]: - # NOTE: Defined to be overridden for lazy - # - Their `SeriesT` is a **native** object - # - `.dtype` won't return a `nw.DType` (or maybe anything) for lazy backends - # - See (https://github.com/narwhals-dev/narwhals/issues/2044) - for ser in self._iter_columns(df): - yield ser, ser.dtype + def _iter_columns_dtypes( + self, df: FrameT, / + ) -> Iterator[tuple[SeriesOrExprT, DType]]: ... - def _iter_columns_names(self, df: FrameT, /) -> Iterator[tuple[SeriesT, str]]: + def _iter_columns_names(self, df: FrameT, /) -> Iterator[tuple[SeriesOrExprT, str]]: yield from zip(self._iter_columns(df), df.columns) def _is_dtype( - self: CompliantSelectorNamespace[FrameT, SeriesT], dtype: type[DType], / - ) -> CompliantSelector[FrameT, SeriesT]: - def series(df: FrameT) -> Sequence[SeriesT]: + self: CompliantSelectorNamespace[FrameT, SeriesOrExprT], dtype: type[DType], / + ) -> CompliantSelector[FrameT, SeriesOrExprT]: + def series(df: FrameT) -> Sequence[SeriesOrExprT]: return [ ser for ser, tp in self._iter_columns_dtypes(df) if isinstance(tp, dtype) ] @@ -116,8 +113,8 @@ def names(df: FrameT) -> Sequence[str]: def by_dtype( self: Self, dtypes: Collection[DType | type[DType]] - ) -> CompliantSelector[FrameT, SeriesT]: - def series(df: FrameT) -> Sequence[SeriesT]: + ) -> CompliantSelector[FrameT, SeriesOrExprT]: + def series(df: FrameT) -> Sequence[SeriesOrExprT]: return [ser for ser, tp in self._iter_columns_dtypes(df) if tp in dtypes] def names(df: FrameT) -> Sequence[str]: @@ -125,10 +122,10 @@ def names(df: FrameT) -> Sequence[str]: return self._selector(series, names) - def matches(self: Self, pattern: str) -> CompliantSelector[FrameT, SeriesT]: + def matches(self: Self, pattern: str) -> CompliantSelector[FrameT, SeriesOrExprT]: p = re.compile(pattern) - def series(df: FrameT) -> Sequence[SeriesT]: + def series(df: FrameT) -> Sequence[SeriesOrExprT]: if is_compliant_dataframe(df) and not self._implementation.is_duckdb(): return [df.get_column(col) for col in df.columns if p.search(col)] @@ -139,8 +136,8 @@ def names(df: FrameT) -> Sequence[str]: return self._selector(series, names) - def numeric(self: Self) -> CompliantSelector[FrameT, SeriesT]: - def series(df: FrameT) -> Sequence[SeriesT]: + def numeric(self: Self) -> CompliantSelector[FrameT, SeriesOrExprT]: + def series(df: FrameT) -> Sequence[SeriesOrExprT]: return [ser for ser, tp in self._iter_columns_dtypes(df) if tp.is_numeric()] def names(df: FrameT) -> Sequence[str]: @@ -148,17 +145,17 @@ def names(df: FrameT) -> Sequence[str]: return self._selector(series, names) - def categorical(self: Self) -> CompliantSelector[FrameT, SeriesT]: + def categorical(self: Self) -> CompliantSelector[FrameT, SeriesOrExprT]: return self._is_dtype(import_dtypes_module(self._version).Categorical) - def string(self: Self) -> CompliantSelector[FrameT, SeriesT]: + def string(self: Self) -> CompliantSelector[FrameT, SeriesOrExprT]: return self._is_dtype(import_dtypes_module(self._version).String) - def boolean(self: Self) -> CompliantSelector[FrameT, SeriesT]: + def boolean(self: Self) -> CompliantSelector[FrameT, SeriesOrExprT]: return self._is_dtype(import_dtypes_module(self._version).Boolean) - def all(self: Self) -> CompliantSelector[FrameT, SeriesT]: - def series(df: FrameT) -> Sequence[SeriesT]: + def all(self: Self) -> CompliantSelector[FrameT, SeriesOrExprT]: + def series(df: FrameT) -> Sequence[SeriesOrExprT]: return list(self._iter_columns(df)) return self._selector(series, get_column_names) @@ -167,7 +164,7 @@ def datetime( self: Self, time_unit: TimeUnit | Iterable[TimeUnit] | None, time_zone: str | timezone | Iterable[str | timezone | None] | None, - ) -> CompliantSelector[FrameT, SeriesT]: + ) -> CompliantSelector[FrameT, SeriesOrExprT]: time_units, time_zones = _parse_time_unit_and_time_zone(time_unit, time_zone) matches = partial( dtype_matches_time_unit_and_time_zone, @@ -176,7 +173,7 @@ def datetime( time_zones=time_zones, ) - def series(df: FrameT) -> Sequence[SeriesT]: + def series(df: FrameT) -> Sequence[SeriesOrExprT]: return [ser for ser, tp in self._iter_columns_dtypes(df) if matches(tp)] def names(df: FrameT) -> Sequence[str]: @@ -186,49 +183,61 @@ def names(df: FrameT) -> Sequence[str]: class EagerSelectorNamespace( - CompliantSelectorNamespace[DataFrameT, SeriesT], Protocol[DataFrameT, SeriesT] + CompliantSelectorNamespace[DataFrameT, SeriesT], + Protocol[DataFrameT, SeriesT], ): + def _iter_schema(self, df: DataFrameT, /) -> Iterator[tuple[str, DType]]: + for ser in self._iter_columns(df): + yield ser.name, ser.dtype + def _iter_columns(self, df: DataFrameT, /) -> Iterator[SeriesT]: yield from df.iter_columns() + def _iter_columns_dtypes(self, df: DataFrameT, /) -> Iterator[tuple[SeriesT, DType]]: + for ser in self._iter_columns(df): + yield ser, ser.dtype + class LazySelectorNamespace( - CompliantSelectorNamespace[LazyFrameT, SeriesT], Protocol[LazyFrameT, SeriesT] + CompliantSelectorNamespace[LazyFrameT, ExprT], + Protocol[LazyFrameT, ExprT], ): def _iter_schema(self, df: LazyFrameT) -> Iterator[tuple[str, DType]]: yield from df.schema.items() - def _iter_columns(self, df: LazyFrameT) -> Iterator[SeriesT]: + def _iter_columns(self, df: LazyFrameT) -> Iterator[ExprT]: yield from df._iter_columns() - def _iter_columns_dtypes(self, df: LazyFrameT, /) -> Iterator[tuple[SeriesT, DType]]: + def _iter_columns_dtypes(self, df: LazyFrameT, /) -> Iterator[tuple[ExprT, DType]]: yield from zip(self._iter_columns(df), df.schema.values()) -class CompliantSelector(CompliantExpr[FrameT, SeriesT], Protocol[FrameT, SeriesT]): +class CompliantSelector( + CompliantExpr[FrameT, SeriesOrExprT], Protocol[FrameT, SeriesOrExprT] +): @property - def selectors(self) -> CompliantSelectorNamespace[FrameT, SeriesT]: + def selectors(self) -> CompliantSelectorNamespace[FrameT, SeriesOrExprT]: return self.__narwhals_namespace__().selectors - def _to_expr(self: Self) -> CompliantExpr[FrameT, SeriesT]: ... + def _to_expr(self: Self) -> CompliantExpr[FrameT, SeriesOrExprT]: ... def _is_selector( - self: Self, other: Self | CompliantExpr[FrameT, SeriesT] - ) -> TypeIs[CompliantSelector[FrameT, SeriesT]]: + self: Self, other: Self | CompliantExpr[FrameT, SeriesOrExprT] + ) -> TypeIs[CompliantSelector[FrameT, SeriesOrExprT]]: return isinstance(other, type(self)) @overload def __sub__(self: Self, other: Self) -> Self: ... @overload def __sub__( - self: Self, other: CompliantExpr[FrameT, SeriesT] - ) -> CompliantExpr[FrameT, SeriesT]: ... + self: Self, other: CompliantExpr[FrameT, SeriesOrExprT] + ) -> CompliantExpr[FrameT, SeriesOrExprT]: ... def __sub__( - self: Self, other: SelectorOrExpr[FrameT, SeriesT] - ) -> SelectorOrExpr[FrameT, SeriesT]: + self: Self, other: SelectorOrExpr[FrameT, SeriesOrExprT] + ) -> SelectorOrExpr[FrameT, SeriesOrExprT]: if self._is_selector(other): - def series(df: FrameT) -> Sequence[SeriesT]: + def series(df: FrameT) -> Sequence[SeriesOrExprT]: lhs_names, rhs_names = _eval_lhs_rhs(df, self, other) return [ x for x, name in zip(self(df), lhs_names) if name not in rhs_names @@ -246,14 +255,14 @@ def names(df: FrameT) -> Sequence[str]: def __or__(self: Self, other: Self) -> Self: ... @overload def __or__( - self: Self, other: CompliantExpr[FrameT, SeriesT] - ) -> CompliantExpr[FrameT, SeriesT]: ... + self: Self, other: CompliantExpr[FrameT, SeriesOrExprT] + ) -> CompliantExpr[FrameT, SeriesOrExprT]: ... def __or__( - self: Self, other: SelectorOrExpr[FrameT, SeriesT] - ) -> SelectorOrExpr[FrameT, SeriesT]: + self: Self, other: SelectorOrExpr[FrameT, SeriesOrExprT] + ) -> SelectorOrExpr[FrameT, SeriesOrExprT]: if self._is_selector(other): - def names(df: FrameT) -> Sequence[SeriesT]: + def names(df: FrameT) -> Sequence[SeriesOrExprT]: lhs_names, rhs_names = _eval_lhs_rhs(df, self, other) return [ *(x for x, name in zip(self(df), lhs_names) if name not in rhs_names), @@ -272,14 +281,14 @@ def series(df: FrameT) -> Sequence[str]: def __and__(self: Self, other: Self) -> Self: ... @overload def __and__( - self: Self, other: CompliantExpr[FrameT, SeriesT] - ) -> CompliantExpr[FrameT, SeriesT]: ... + self: Self, other: CompliantExpr[FrameT, SeriesOrExprT] + ) -> CompliantExpr[FrameT, SeriesOrExprT]: ... def __and__( - self: Self, other: SelectorOrExpr[FrameT, SeriesT] - ) -> SelectorOrExpr[FrameT, SeriesT]: + self: Self, other: SelectorOrExpr[FrameT, SeriesOrExprT] + ) -> SelectorOrExpr[FrameT, SeriesOrExprT]: if self._is_selector(other): - def series(df: FrameT) -> Sequence[SeriesT]: + def series(df: FrameT) -> Sequence[SeriesOrExprT]: lhs_names, rhs_names = _eval_lhs_rhs(df, self, other) return [x for x, name in zip(self(df), lhs_names) if name in rhs_names] @@ -291,7 +300,7 @@ def names(df: FrameT) -> Sequence[str]: else: return self._to_expr() & other - def __invert__(self: Self) -> CompliantSelector[FrameT, SeriesT]: + def __invert__(self: Self) -> CompliantSelector[FrameT, SeriesOrExprT]: return self.selectors.all() - self # type: ignore[no-any-return] def __repr__(self: Self) -> str: # pragma: no cover diff --git a/narwhals/_compliant/typing.py b/narwhals/_compliant/typing.py index 9754141dfd..0d03ea6757 100644 --- a/narwhals/_compliant/typing.py +++ b/narwhals/_compliant/typing.py @@ -10,6 +10,7 @@ from narwhals._compliant.dataframe import CompliantDataFrame from narwhals._compliant.dataframe import CompliantLazyFrame from narwhals._compliant.expr import CompliantExpr + from narwhals._compliant.expr import NativeExpr from narwhals._compliant.series import CompliantSeries __all__ = [ @@ -19,15 +20,18 @@ "CompliantSeriesT_co", "IntoCompliantExpr", ] - +NativeExprT_co = TypeVar("NativeExprT_co", bound="NativeExpr", covariant=True) CompliantSeriesT_co = TypeVar( "CompliantSeriesT_co", bound="CompliantSeries", covariant=True ) +CompliantSeriesOrNativeExprT_co = TypeVar( + "CompliantSeriesOrNativeExprT_co", + bound="CompliantSeries | NativeExpr", + covariant=True, +) CompliantFrameT = TypeVar( "CompliantFrameT", bound="CompliantDataFrame[Any] | CompliantLazyFrame" ) CompliantDataFrameT = TypeVar("CompliantDataFrameT", bound="CompliantDataFrame[Any]") CompliantLazyFrameT = TypeVar("CompliantLazyFrameT", bound="CompliantLazyFrame") -IntoCompliantExpr: TypeAlias = ( - "CompliantExpr[CompliantFrameT, CompliantSeriesT_co] | CompliantSeriesT_co" -) +IntoCompliantExpr: TypeAlias = "CompliantExpr[CompliantFrameT, CompliantSeriesOrNativeExprT_co] | CompliantSeriesOrNativeExprT_co" diff --git a/narwhals/_dask/expr.py b/narwhals/_dask/expr.py index c322d6be82..fefe8ecb5f 100644 --- a/narwhals/_dask/expr.py +++ b/narwhals/_dask/expr.py @@ -39,7 +39,7 @@ from narwhals.utils import Version -class DaskExpr(CompliantExpr["DaskLazyFrame", "dx.Series"]): # pyright: ignore[reportInvalidTypeArguments] (#2044) +class DaskExpr(CompliantExpr["DaskLazyFrame", "dx.Series"]): _implementation: Implementation = Implementation.DASK def __init__( diff --git a/narwhals/_dask/namespace.py b/narwhals/_dask/namespace.py index 3ebdbfd00b..10744d6bb0 100644 --- a/narwhals/_dask/namespace.py +++ b/narwhals/_dask/namespace.py @@ -42,7 +42,7 @@ import dask_expr as dx -class DaskNamespace(CompliantNamespace[DaskLazyFrame, "dx.Series"]): # pyright: ignore[reportInvalidTypeArguments] (#2044) +class DaskNamespace(CompliantNamespace[DaskLazyFrame, "dx.Series"]): _implementation: Implementation = Implementation.DASK @property diff --git a/narwhals/_duckdb/expr.py b/narwhals/_duckdb/expr.py index ddc91a01ab..8b0679fbbb 100644 --- a/narwhals/_duckdb/expr.py +++ b/narwhals/_duckdb/expr.py @@ -36,7 +36,7 @@ from narwhals.utils import Version -class DuckDBExpr(CompliantExpr["DuckDBLazyFrame", "duckdb.Expression"]): # type: ignore[type-var] +class DuckDBExpr(CompliantExpr["DuckDBLazyFrame", "duckdb.Expression"]): _implementation = Implementation.DUCKDB _depth = 0 # Unused, just for compatibility with CompliantExpr diff --git a/narwhals/_duckdb/namespace.py b/narwhals/_duckdb/namespace.py index c71507a8ef..f245c74e27 100644 --- a/narwhals/_duckdb/namespace.py +++ b/narwhals/_duckdb/namespace.py @@ -38,7 +38,7 @@ from narwhals.utils import Version -class DuckDBNamespace(CompliantNamespace["DuckDBLazyFrame", "duckdb.Expression"]): # type: ignore[type-var] +class DuckDBNamespace(CompliantNamespace["DuckDBLazyFrame", "duckdb.Expression"]): _implementation: Implementation = Implementation.DUCKDB def __init__( diff --git a/narwhals/_duckdb/selectors.py b/narwhals/_duckdb/selectors.py index 908fa41f3b..77b5bc2d15 100644 --- a/narwhals/_duckdb/selectors.py +++ b/narwhals/_duckdb/selectors.py @@ -17,11 +17,11 @@ class DuckDBSelectorNamespace( - LazySelectorNamespace["DuckDBLazyFrame", "duckdb.Expression"] # type: ignore[type-var] + LazySelectorNamespace["DuckDBLazyFrame", "duckdb.Expression"] ): def _selector( self, - call: EvalSeries[DuckDBLazyFrame, duckdb.Expression], # type: ignore[type-var] + call: EvalSeries[DuckDBLazyFrame, duckdb.Expression], evaluate_output_names: EvalNames[DuckDBLazyFrame], /, ) -> DuckDBSelector: @@ -41,8 +41,7 @@ def __init__(self: Self, context: _FullContext, /) -> None: class DuckDBSelector( # type: ignore[misc] - CompliantSelector["DuckDBLazyFrame", "duckdb.Expression"], # type: ignore[type-var] - DuckDBExpr, + CompliantSelector["DuckDBLazyFrame", "duckdb.Expression"], DuckDBExpr ): def _to_expr(self: Self) -> DuckDBExpr: return DuckDBExpr( diff --git a/narwhals/_expression_parsing.py b/narwhals/_expression_parsing.py index f985c3ff55..57bbb71c14 100644 --- a/narwhals/_expression_parsing.py +++ b/narwhals/_expression_parsing.py @@ -29,6 +29,7 @@ from narwhals._compliant import CompliantExpr from narwhals._compliant import CompliantFrameT from narwhals._compliant import CompliantNamespace + from narwhals._compliant import CompliantSeriesOrNativeExprT_co from narwhals._compliant import CompliantSeriesT_co from narwhals._pandas_like.expr import PandasLikeExpr from narwhals.expr import Expr @@ -280,11 +281,11 @@ def alias_output_names(names: Sequence[str]) -> Sequence[str]: def extract_compliant( - plx: CompliantNamespace[CompliantFrameT, CompliantSeriesT_co], + plx: CompliantNamespace[CompliantFrameT, CompliantSeriesOrNativeExprT_co], other: Any, *, str_as_lit: bool, -) -> CompliantExpr[CompliantFrameT, CompliantSeriesT_co] | object: +) -> CompliantExpr[CompliantFrameT, CompliantSeriesOrNativeExprT_co] | object: if is_expr(other): return other._to_compliant_expr(plx) if isinstance(other, str) and not str_as_lit: diff --git a/narwhals/_spark_like/expr.py b/narwhals/_spark_like/expr.py index 6565da5a3d..53c8b9c69d 100644 --- a/narwhals/_spark_like/expr.py +++ b/narwhals/_spark_like/expr.py @@ -32,7 +32,7 @@ from narwhals.utils import Version -class SparkLikeExpr(CompliantExpr["SparkLikeLazyFrame", "Column"]): # type: ignore[type-var] # (#2044) +class SparkLikeExpr(CompliantExpr["SparkLikeLazyFrame", "Column"]): _depth = 0 # Unused, just for compatibility with CompliantExpr def __init__( diff --git a/narwhals/_spark_like/namespace.py b/narwhals/_spark_like/namespace.py index 60b370573c..21d9746712 100644 --- a/narwhals/_spark_like/namespace.py +++ b/narwhals/_spark_like/namespace.py @@ -34,7 +34,7 @@ from narwhals.utils import Version -class SparkLikeNamespace(CompliantNamespace["SparkLikeLazyFrame", "Column"]): # type: ignore[type-var] # (#2044) +class SparkLikeNamespace(CompliantNamespace["SparkLikeLazyFrame", "Column"]): def __init__( self: Self, *, diff --git a/narwhals/_spark_like/selectors.py b/narwhals/_spark_like/selectors.py index 48a92329a1..66654304da 100644 --- a/narwhals/_spark_like/selectors.py +++ b/narwhals/_spark_like/selectors.py @@ -16,11 +16,10 @@ from narwhals.utils import _FullContext -# NOTE: See issue regarding ignores (#2044) -class SparkLikeSelectorNamespace(LazySelectorNamespace["SparkLikeLazyFrame", "Column"]): # type: ignore[type-var] +class SparkLikeSelectorNamespace(LazySelectorNamespace["SparkLikeLazyFrame", "Column"]): def _selector( self, - call: EvalSeries[SparkLikeLazyFrame, Column], # type: ignore[type-var] + call: EvalSeries[SparkLikeLazyFrame, Column], evaluate_output_names: EvalNames[SparkLikeLazyFrame], /, ) -> SparkLikeSelector: @@ -40,7 +39,7 @@ def __init__(self: Self, context: _FullContext, /) -> None: self._implementation = context._implementation -class SparkLikeSelector(CompliantSelector["SparkLikeLazyFrame", "Column"], SparkLikeExpr): # type: ignore[type-var, misc] +class SparkLikeSelector(CompliantSelector["SparkLikeLazyFrame", "Column"], SparkLikeExpr): # type: ignore[misc] def _to_expr(self: Self) -> SparkLikeExpr: return SparkLikeExpr( self._call, diff --git a/narwhals/utils.py b/narwhals/utils.py index 9f8e9c5f30..798c6f5221 100644 --- a/narwhals/utils.py +++ b/narwhals/utils.py @@ -54,6 +54,7 @@ from narwhals._compliant import CompliantExpr from narwhals._compliant import CompliantFrameT + from narwhals._compliant import CompliantSeriesOrNativeExprT_co from narwhals._compliant import CompliantSeriesT_co from narwhals.dataframe import DataFrame from narwhals.dataframe import LazyFrame @@ -1393,8 +1394,8 @@ def is_compliant_series(obj: Any) -> TypeIs[CompliantSeries]: def is_compliant_expr( - obj: CompliantExpr[CompliantFrameT, CompliantSeriesT_co] | Any, -) -> TypeIs[CompliantExpr[CompliantFrameT, CompliantSeriesT_co]]: + obj: CompliantExpr[CompliantFrameT, CompliantSeriesOrNativeExprT_co] | Any, +) -> TypeIs[CompliantExpr[CompliantFrameT, CompliantSeriesOrNativeExprT_co]]: return hasattr(obj, "__narwhals_expr__") From a0fe52a61b80df952c2bbdc3de001ad138cc84aa Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Thu, 6 Mar 2025 17:37:26 +0000 Subject: [PATCH 12/58] refactor(typing): spec out and reuse `LazyExpr` --- narwhals/_compliant/__init__.py | 2 ++ narwhals/_compliant/expr.py | 21 ++++++++++++++++++++- narwhals/_dask/expr.py | 23 ++--------------------- narwhals/_duckdb/expr.py | 23 ++--------------------- narwhals/_spark_like/expr.py | 23 ++--------------------- 5 files changed, 28 insertions(+), 64 deletions(-) diff --git a/narwhals/_compliant/__init__.py b/narwhals/_compliant/__init__.py index f0ead106c6..19434c1a4c 100644 --- a/narwhals/_compliant/__init__.py +++ b/narwhals/_compliant/__init__.py @@ -3,6 +3,7 @@ from narwhals._compliant.dataframe import CompliantDataFrame from narwhals._compliant.dataframe import CompliantLazyFrame from narwhals._compliant.expr import CompliantExpr +from narwhals._compliant.expr import LazyExpr from narwhals._compliant.namespace import CompliantNamespace from narwhals._compliant.selectors import CompliantSelector from narwhals._compliant.selectors import CompliantSelectorNamespace @@ -31,5 +32,6 @@ "EvalNames", "EvalSeries", "IntoCompliantExpr", + "LazyExpr", "LazySelectorNamespace", ] diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index 6bee83804a..7de58ef11c 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -15,6 +15,7 @@ from narwhals._compliant.typing import CompliantSeriesT_co from narwhals._compliant.typing import NativeExprT_co from narwhals.utils import deprecated +from narwhals.utils import not_implemented from narwhals.utils import unstable if not TYPE_CHECKING: # pragma: no cover @@ -237,4 +238,22 @@ class EagerExpr( class LazyExpr( CompliantExpr[CompliantLazyFrameT, NativeExprT_co], Protocol38[CompliantLazyFrameT, NativeExprT_co], -): ... +): + arg_min = not_implemented() + arg_max = not_implemented() + arg_true = not_implemented() + head = not_implemented() + tail = not_implemented() + mode = not_implemented() + sort = not_implemented() + rank = not_implemented() + sample = not_implemented() + map_batches = not_implemented() + ewm_mean = not_implemented() + rolling_sum = not_implemented() + rolling_mean = not_implemented() + rolling_var = not_implemented() + rolling_std = not_implemented() + gather_every = not_implemented() + replace_strict = not_implemented() + cat = not_implemented() # pyright: ignore[reportAssignmentType] diff --git a/narwhals/_dask/expr.py b/narwhals/_dask/expr.py index fefe8ecb5f..ca53db8bc0 100644 --- a/narwhals/_dask/expr.py +++ b/narwhals/_dask/expr.py @@ -8,7 +8,7 @@ from typing import Literal from typing import Sequence -from narwhals._compliant import CompliantExpr +from narwhals._compliant import LazyExpr from narwhals._dask.expr_dt import DaskExprDateTimeNamespace from narwhals._dask.expr_name import DaskExprNameNamespace from narwhals._dask.expr_str import DaskExprStringNamespace @@ -39,7 +39,7 @@ from narwhals.utils import Version -class DaskExpr(CompliantExpr["DaskLazyFrame", "dx.Series"]): +class DaskExpr(LazyExpr["DaskLazyFrame", "dx.Series"]): _implementation: Implementation = Implementation.DASK def __init__( @@ -603,23 +603,4 @@ def dt(self: Self) -> DaskExprDateTimeNamespace: def name(self: Self) -> DaskExprNameNamespace: return DaskExprNameNamespace(self) - arg_min = not_implemented() - arg_max = not_implemented() - arg_true = not_implemented() - head = not_implemented() - tail = not_implemented() - mode = not_implemented() - sort = not_implemented() - rank = not_implemented() - sample = not_implemented() - map_batches = not_implemented() - ewm_mean = not_implemented() - rolling_sum = not_implemented() - rolling_mean = not_implemented() - rolling_var = not_implemented() - rolling_std = not_implemented() - gather_every = not_implemented() - replace_strict = not_implemented() - - cat = not_implemented() # pyright: ignore[reportAssignmentType] list = not_implemented() # pyright: ignore[reportAssignmentType] diff --git a/narwhals/_duckdb/expr.py b/narwhals/_duckdb/expr.py index 8b0679fbbb..492c0f11ed 100644 --- a/narwhals/_duckdb/expr.py +++ b/narwhals/_duckdb/expr.py @@ -14,7 +14,7 @@ from duckdb import FunctionExpression from duckdb.typing import DuckDBPyType -from narwhals._compliant import CompliantExpr +from narwhals._compliant import LazyExpr from narwhals._duckdb.expr_dt import DuckDBExprDateTimeNamespace from narwhals._duckdb.expr_list import DuckDBExprListNamespace from narwhals._duckdb.expr_name import DuckDBExprNameNamespace @@ -36,7 +36,7 @@ from narwhals.utils import Version -class DuckDBExpr(CompliantExpr["DuckDBLazyFrame", "duckdb.Expression"]): +class DuckDBExpr(LazyExpr["DuckDBLazyFrame", "duckdb.Expression"]): _implementation = Implementation.DUCKDB _depth = 0 # Unused, just for compatibility with CompliantExpr @@ -484,22 +484,6 @@ def name(self: Self) -> DuckDBExprNameNamespace: def list(self: Self) -> DuckDBExprListNamespace: return DuckDBExprListNamespace(self) - arg_min = not_implemented() - arg_max = not_implemented() - arg_true = not_implemented() - head = not_implemented() - tail = not_implemented() - mode = not_implemented() - sort = not_implemented() - rank = not_implemented() - sample = not_implemented() - map_batches = not_implemented() - ewm_mean = not_implemented() - rolling_sum = not_implemented() - rolling_mean = not_implemented() - rolling_var = not_implemented() - rolling_std = not_implemented() - gather_every = not_implemented() drop_nulls = not_implemented() diff = not_implemented() unique = not_implemented() @@ -512,7 +496,4 @@ def list(self: Self) -> DuckDBExprListNamespace: cum_min = not_implemented() cum_max = not_implemented() cum_prod = not_implemented() - replace_strict = not_implemented() over = not_implemented() - - cat = not_implemented() # pyright: ignore[reportAssignmentType] diff --git a/narwhals/_spark_like/expr.py b/narwhals/_spark_like/expr.py index 53c8b9c69d..9cd7980dc3 100644 --- a/narwhals/_spark_like/expr.py +++ b/narwhals/_spark_like/expr.py @@ -9,7 +9,7 @@ from typing import Sequence from typing import cast -from narwhals._compliant import CompliantExpr +from narwhals._compliant import LazyExpr from narwhals._expression_parsing import ExprKind from narwhals._spark_like.expr_dt import SparkLikeExprDateTimeNamespace from narwhals._spark_like.expr_list import SparkLikeExprListNamespace @@ -32,7 +32,7 @@ from narwhals.utils import Version -class SparkLikeExpr(CompliantExpr["SparkLikeLazyFrame", "Column"]): +class SparkLikeExpr(LazyExpr["SparkLikeLazyFrame", "Column"]): _depth = 0 # Unused, just for compatibility with CompliantExpr def __init__( @@ -528,22 +528,6 @@ def dt(self: Self) -> SparkLikeExprDateTimeNamespace: def list(self: Self) -> SparkLikeExprListNamespace: return SparkLikeExprListNamespace(self) - arg_min = not_implemented() - arg_max = not_implemented() - arg_true = not_implemented() - head = not_implemented() - tail = not_implemented() - mode = not_implemented() - sort = not_implemented() - rank = not_implemented() - sample = not_implemented() - map_batches = not_implemented() - ewm_mean = not_implemented() - rolling_sum = not_implemented() - rolling_mean = not_implemented() - rolling_var = not_implemented() - rolling_std = not_implemented() - gather_every = not_implemented() drop_nulls = not_implemented() diff = not_implemented() unique = not_implemented() @@ -555,8 +539,5 @@ def list(self: Self) -> SparkLikeExprListNamespace: cum_min = not_implemented() cum_max = not_implemented() cum_prod = not_implemented() - replace_strict = not_implemented() fill_null = not_implemented() quantile = not_implemented() - - cat = not_implemented() # pyright: ignore[reportAssignmentType] From 23fdf8f20bddee3e3be87fb74f05598c682859ca Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Thu, 6 Mar 2025 17:40:08 +0000 Subject: [PATCH 13/58] be quiet `mypy`! ```log error: All protocol members must have explicitly declared types [misc] ``` --- narwhals/_compliant/expr.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index 7de58ef11c..79d4f947fe 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -239,21 +239,21 @@ class LazyExpr( CompliantExpr[CompliantLazyFrameT, NativeExprT_co], Protocol38[CompliantLazyFrameT, NativeExprT_co], ): - arg_min = not_implemented() - arg_max = not_implemented() - arg_true = not_implemented() - head = not_implemented() - tail = not_implemented() - mode = not_implemented() - sort = not_implemented() - rank = not_implemented() - sample = not_implemented() - map_batches = not_implemented() - ewm_mean = not_implemented() - rolling_sum = not_implemented() - rolling_mean = not_implemented() - rolling_var = not_implemented() - rolling_std = not_implemented() - gather_every = not_implemented() - replace_strict = not_implemented() - cat = not_implemented() # pyright: ignore[reportAssignmentType] + arg_min: not_implemented = not_implemented() + arg_max: not_implemented = not_implemented() + arg_true: not_implemented = not_implemented() + head: not_implemented = not_implemented() + tail: not_implemented = not_implemented() + mode: not_implemented = not_implemented() + sort: not_implemented = not_implemented() + rank: not_implemented = not_implemented() + sample: not_implemented = not_implemented() + map_batches: not_implemented = not_implemented() + ewm_mean: not_implemented = not_implemented() + rolling_sum: not_implemented = not_implemented() + rolling_mean: not_implemented = not_implemented() + rolling_var: not_implemented = not_implemented() + rolling_std: not_implemented = not_implemented() + gather_every: not_implemented = not_implemented() + replace_strict: not_implemented = not_implemented() + cat: not_implemented = not_implemented() # pyright: ignore[reportAssignmentType] From d58e2c9826c67e012b5847f3451b4b4e61874b68 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Thu, 6 Mar 2025 20:29:49 +0000 Subject: [PATCH 14/58] feat(DRAFT): `EagerExpr` from `_expression_parsing.py` Beginning to merge parts of (#2055) --- narwhals/_compliant/dataframe.py | 29 ++++++ narwhals/_compliant/expr.py | 160 ++++++++++++++++++++++++++++++- narwhals/_compliant/namespace.py | 8 ++ narwhals/_compliant/series.py | 24 +++++ narwhals/_compliant/typing.py | 5 + 5 files changed, 221 insertions(+), 5 deletions(-) diff --git a/narwhals/_compliant/dataframe.py b/narwhals/_compliant/dataframe.py index 8ea87b14b4..0a760d5a95 100644 --- a/narwhals/_compliant/dataframe.py +++ b/narwhals/_compliant/dataframe.py @@ -6,16 +6,22 @@ from typing import Mapping from typing import Protocol from typing import Sequence +from typing import TypeVar from narwhals._compliant.typing import CompliantSeriesT_co +from narwhals._compliant.typing import EagerSeriesT if TYPE_CHECKING: from typing_extensions import Self + from typing_extensions import TypeIs + from narwhals._compliant.expr import EagerExpr from narwhals.dtypes import DType __all__ = ["CompliantDataFrame", "CompliantLazyFrame"] +T = TypeVar("T") + class CompliantDataFrame(Protocol[CompliantSeriesT_co]): def __narwhals_dataframe__(self) -> Self: ... @@ -50,3 +56,26 @@ def columns(self) -> Sequence[str]: ... @property def schema(self) -> Mapping[str, DType]: ... def _iter_columns(self) -> Iterator[Any]: ... + + +class EagerDataFrame(CompliantDataFrame[EagerSeriesT], Protocol[EagerSeriesT]): # pyright: ignore[reportInvalidTypeVarUse] + def _maybe_evaluate_expr( + self, expr: EagerExpr[EagerDataFrame[EagerSeriesT], EagerSeriesT] | T, / + ) -> EagerSeriesT | T: + if is_eager_expr(expr): + result: Sequence[EagerSeriesT] = expr(self) + if len(result) > 1: + msg = ( + "Multi-output expressions (e.g. `nw.all()` or `nw.col('a', 'b')`) " + "are not supported in this context" + ) + raise ValueError(msg) + return result[0] + return expr + + +# NOTE: DON'T CHANGE THIS EITHER +def is_eager_expr( + obj: EagerExpr[Any, EagerSeriesT] | Any, +) -> TypeIs[EagerExpr[Any, EagerSeriesT]]: + return hasattr(obj, "__narwhals_expr__") diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index 79d4f947fe..f601426707 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -1,6 +1,8 @@ from __future__ import annotations import sys +from functools import partial +from operator import methodcaller from typing import TYPE_CHECKING from typing import Any from typing import Callable @@ -8,12 +10,13 @@ from typing import Protocol from typing import Sequence -from narwhals._compliant.typing import CompliantDataFrameT from narwhals._compliant.typing import CompliantFrameT from narwhals._compliant.typing import CompliantLazyFrameT from narwhals._compliant.typing import CompliantSeriesOrNativeExprT_co -from narwhals._compliant.typing import CompliantSeriesT_co +from narwhals._compliant.typing import EagerDataFrameT +from narwhals._compliant.typing import EagerSeriesT from narwhals._compliant.typing import NativeExprT_co +from narwhals._expression_parsing import evaluate_output_names_and_aliases from narwhals.utils import deprecated from narwhals.utils import not_implemented from narwhals.utils import unstable @@ -39,6 +42,7 @@ from narwhals.dtypes import DType from narwhals.utils import Implementation from narwhals.utils import Version + from narwhals.utils import _FullContext __all__ = ["CompliantExpr"] @@ -229,9 +233,155 @@ def broadcast( class EagerExpr( - CompliantExpr[CompliantDataFrameT, CompliantSeriesT_co], - Protocol38[CompliantDataFrameT, CompliantSeriesT_co], -): ... + CompliantExpr[EagerDataFrameT, EagerSeriesT], + Protocol38[EagerDataFrameT, EagerSeriesT], +): + _depth: int + _function_name: str + _evaluate_output_names: Any + _alias_output_names: Any + _call_kwargs: dict[str, Any] + + @property + def _series(self) -> type[EagerSeriesT]: ... + + def __init__( + self: Self, + call: Callable[[EagerDataFrameT], Sequence[EagerSeriesT]], + *, + depth: int, + function_name: str, + evaluate_output_names: Callable[[EagerDataFrameT], Sequence[str]], + alias_output_names: Callable[[Sequence[str]], Sequence[str]] | None, + implementation: Implementation, + backend_version: tuple[int, ...], + version: Version, + call_kwargs: dict[str, Any] | None = None, + ) -> None: ... + + @classmethod + def _from_callable( + cls, + func: Callable[[EagerDataFrameT], Sequence[EagerSeriesT]], + *, + depth: int, + function_name: str, + evaluate_output_names: Callable[[EagerDataFrameT], Sequence[str]], + alias_output_names: Callable[[Sequence[str]], Sequence[str]] | None, + context: _FullContext, + call_kwargs: dict[str, Any] | None = None, + ) -> Self: + return cls( + func, + depth=depth, + function_name=function_name, + evaluate_output_names=evaluate_output_names, + alias_output_names=alias_output_names, + implementation=context._implementation, + backend_version=context._backend_version, + version=context._version, + call_kwargs=call_kwargs, + ) + + @classmethod + def _from_series(cls, series: EagerSeriesT, *, context: _FullContext) -> Self: + return cls( + lambda _df: [series], + depth=0, + function_name="series", + evaluate_output_names=lambda _df: [series.name], + alias_output_names=None, + implementation=context._implementation, + backend_version=context._backend_version, + version=context._version, + ) + + # https://github.com/narwhals-dev/narwhals/blob/35cef0b1e2c892fb24aa730902b08b6994008c18/narwhals/_protocols.py#L135 + def _reuse_series_implementation( + self: EagerExpr[EagerDataFrameT, EagerSeriesT], + attr: str, + *, + returns_scalar: bool = False, + call_kwargs: dict[str, Any] | None = None, + **expressifiable_args: Any, + ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + func = partial( + self._reuse_series_inner, + method_name=attr, + returns_scalar=returns_scalar, + call_kwargs=call_kwargs or {}, + expressifiable_args=expressifiable_args, + ) + return self._from_callable( + func, + depth=self._depth + 1, + function_name=f"{self._function_name}->{attr}", + evaluate_output_names=self._evaluate_output_names, + alias_output_names=self._alias_output_names, + call_kwargs=call_kwargs, + context=self, + ) + + # For PyArrow.Series, we return Python Scalars (like Polars does) instead of PyArrow Scalars. + # However, when working with expressions, we keep everything PyArrow-native. + def _reuse_series_extra_kwargs( + self, *, returns_scalar: bool = False + ) -> dict[str, Any]: + return {} + + def _reuse_series_inner( + self, + df: EagerDataFrameT, + *, + method_name: str, + returns_scalar: bool, + call_kwargs: dict[str, Any], + expressifiable_args: dict[str, Any], + ) -> Sequence[EagerSeriesT]: + kwargs = { + **call_kwargs, + **{ + arg_name: df._maybe_evaluate_expr(arg_value) + for arg_name, arg_value in expressifiable_args.items() + }, + } + method = methodcaller( + method_name, + **self._reuse_series_extra_kwargs(returns_scalar=returns_scalar), + **kwargs, + ) + out: Sequence[EagerSeriesT] = [ + series._from_scalar(method(series)) if returns_scalar else method(series) + for series in self(df) + ] + _, aliases = evaluate_output_names_and_aliases(self, df, []) + if [s.name for s in out] != list(aliases): # pragma: no cover + msg = ( + f"Safety assertion failed, please report a bug to https://github.com/narwhals-dev/narwhals/issues\n" + f"Expression aliases: {aliases}\n" + f"Series names: {[s.name for s in out]}" + ) + raise AssertionError(msg) + return out + + def _reuse_series_namespace_implementation( + self: EagerExpr[EagerDataFrameT, EagerSeriesT], + series_namespace: str, + attr: str, + **kwargs: Any, + ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._from_callable( + lambda df: [ + getattr(getattr(series, series_namespace), attr)(**kwargs) + for series in self(df) + ], + depth=self._depth + 1, + function_name=f"{self._function_name}->{series_namespace}.{attr}", + evaluate_output_names=self._evaluate_output_names, + alias_output_names=self._alias_output_names, + call_kwargs={**self._call_kwargs, **kwargs}, + context=self, + ) # NOTE: See (https://github.com/narwhals-dev/narwhals/issues/2044#issuecomment-2674262833) diff --git a/narwhals/_compliant/namespace.py b/narwhals/_compliant/namespace.py index a2b07fd591..16a6d553f6 100644 --- a/narwhals/_compliant/namespace.py +++ b/narwhals/_compliant/namespace.py @@ -4,8 +4,10 @@ from typing import Any from typing import Protocol +from narwhals._compliant.typing import CompliantDataFrameT from narwhals._compliant.typing import CompliantFrameT from narwhals._compliant.typing import CompliantSeriesOrNativeExprT_co +from narwhals._compliant.typing import CompliantSeriesT_co if TYPE_CHECKING: from narwhals._compliant.expr import CompliantExpr @@ -24,3 +26,9 @@ def lit( ) -> CompliantExpr[CompliantFrameT, CompliantSeriesOrNativeExprT_co]: ... @property def selectors(self) -> CompliantSelectorNamespace[Any, Any]: ... + + +class EagerNamespace( + CompliantNamespace[CompliantDataFrameT, CompliantSeriesT_co], + Protocol[CompliantDataFrameT, CompliantSeriesT_co], +): ... diff --git a/narwhals/_compliant/series.py b/narwhals/_compliant/series.py index 4002baf995..63badefaa4 100644 --- a/narwhals/_compliant/series.py +++ b/narwhals/_compliant/series.py @@ -1,15 +1,24 @@ from __future__ import annotations from typing import TYPE_CHECKING +from typing import Any +from typing import Iterable from typing import Protocol +from typing import TypeVar if TYPE_CHECKING: from typing_extensions import Self from narwhals.dtypes import DType + from narwhals.typing import NativeSeries + from narwhals.utils import Implementation + from narwhals.utils import Version + from narwhals.utils import _FullContext __all__ = ["CompliantSeries"] +NativeSeriesT_co = TypeVar("NativeSeriesT_co", bound="NativeSeries", covariant=True) + class CompliantSeries(Protocol): @property @@ -18,3 +27,18 @@ def dtype(self) -> DType: ... def name(self) -> str: ... def __narwhals_series__(self) -> CompliantSeries: ... def alias(self, name: str) -> Self: ... + + +class EagerSeries(CompliantSeries, Protocol[NativeSeriesT_co]): + _native_series: NativeSeriesT_co + _implementation: Implementation + _backend_version: tuple[int, ...] + _version: Version + + def _from_scalar(self, value: Any) -> Self: + return self._from_iterable([value], name=self.name, context=self) + + @classmethod + def _from_iterable( + cls: type[Self], data: Iterable[Any], name: str, *, context: _FullContext + ) -> Self: ... diff --git a/narwhals/_compliant/typing.py b/narwhals/_compliant/typing.py index 0d03ea6757..c71e336782 100644 --- a/narwhals/_compliant/typing.py +++ b/narwhals/_compliant/typing.py @@ -9,9 +9,11 @@ from narwhals._compliant.dataframe import CompliantDataFrame from narwhals._compliant.dataframe import CompliantLazyFrame + from narwhals._compliant.dataframe import EagerDataFrame from narwhals._compliant.expr import CompliantExpr from narwhals._compliant.expr import NativeExpr from narwhals._compliant.series import CompliantSeries + from narwhals._compliant.series import EagerSeries __all__ = [ "CompliantDataFrameT", @@ -35,3 +37,6 @@ CompliantDataFrameT = TypeVar("CompliantDataFrameT", bound="CompliantDataFrame[Any]") CompliantLazyFrameT = TypeVar("CompliantLazyFrameT", bound="CompliantLazyFrame") IntoCompliantExpr: TypeAlias = "CompliantExpr[CompliantFrameT, CompliantSeriesOrNativeExprT_co] | CompliantSeriesOrNativeExprT_co" + +EagerDataFrameT = TypeVar("EagerDataFrameT", bound="EagerDataFrame[Any]") +EagerSeriesT = TypeVar("EagerSeriesT", bound="EagerSeries[Any]") From 6cb725c75a38c5108bd8625b1d54d5fa64b2c8b2 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Thu, 6 Mar 2025 20:34:10 +0000 Subject: [PATCH 15/58] feat: Generic dunders --- narwhals/_compliant/expr.py | 74 +++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index f601426707..d04bcfb370 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -383,6 +383,80 @@ def _reuse_series_namespace_implementation( context=self, ) + def __eq__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: # type: ignore[override] + return self._reuse_series_implementation("__eq__", other=other) + + def __ne__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: # type: ignore[override] + return self._reuse_series_implementation("__ne__", other=other) + + def __ge__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("__ge__", other=other) + + def __gt__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("__gt__", other=other) + + def __le__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("__le__", other=other) + + def __lt__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("__lt__", other=other) + + def __and__( + self, other: Self | bool | Any + ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("__and__", other=other) + + def __or__( + self, other: Self | bool | Any + ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("__or__", other=other) + + def __add__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("__add__", other=other) + + def __sub__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("__sub__", other=other) + + def __rsub__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self.alias("literal")._reuse_series_implementation("__rsub__", other=other) + + def __mul__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("__mul__", other=other) + + def __truediv__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("__truediv__", other=other) + + def __rtruediv__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self.alias("literal")._reuse_series_implementation( + "__rtruediv__", other=other + ) + + def __floordiv__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("__floordiv__", other=other) + + def __rfloordiv__( + self, other: Self | Any + ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self.alias("literal")._reuse_series_implementation( + "__rfloordiv__", other=other + ) + + def __pow__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("__pow__", other=other) + + def __rpow__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self.alias("literal")._reuse_series_implementation("__rpow__", other=other) + + def __mod__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("__mod__", other=other) + + def __rmod__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self.alias("literal")._reuse_series_implementation("__rmod__", other=other) + + # Unary + def __invert__(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("__invert__") + # NOTE: See (https://github.com/narwhals-dev/narwhals/issues/2044#issuecomment-2674262833) class LazyExpr( From bc937822fcf12c8bc1b140e97427c449981a895b Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Thu, 6 Mar 2025 20:39:35 +0000 Subject: [PATCH 16/58] chore(typing): fix variance issues --- narwhals/_compliant/dataframe.py | 9 +++++---- narwhals/_compliant/series.py | 5 ++++- narwhals/_compliant/typing.py | 1 + 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/narwhals/_compliant/dataframe.py b/narwhals/_compliant/dataframe.py index 0a760d5a95..a192f2b647 100644 --- a/narwhals/_compliant/dataframe.py +++ b/narwhals/_compliant/dataframe.py @@ -10,6 +10,7 @@ from narwhals._compliant.typing import CompliantSeriesT_co from narwhals._compliant.typing import EagerSeriesT +from narwhals._compliant.typing import EagerSeriesT_co if TYPE_CHECKING: from typing_extensions import Self @@ -58,12 +59,12 @@ def schema(self) -> Mapping[str, DType]: ... def _iter_columns(self) -> Iterator[Any]: ... -class EagerDataFrame(CompliantDataFrame[EagerSeriesT], Protocol[EagerSeriesT]): # pyright: ignore[reportInvalidTypeVarUse] +class EagerDataFrame(CompliantDataFrame[EagerSeriesT_co], Protocol[EagerSeriesT_co]): def _maybe_evaluate_expr( - self, expr: EagerExpr[EagerDataFrame[EagerSeriesT], EagerSeriesT] | T, / - ) -> EagerSeriesT | T: + self, expr: EagerExpr[EagerDataFrame[EagerSeriesT_co], EagerSeriesT_co] | T, / + ) -> EagerSeriesT_co | T: if is_eager_expr(expr): - result: Sequence[EagerSeriesT] = expr(self) + result: Sequence[EagerSeriesT_co] = expr(self) if len(result) > 1: msg = ( "Multi-output expressions (e.g. `nw.all()` or `nw.col('a', 'b')`) " diff --git a/narwhals/_compliant/series.py b/narwhals/_compliant/series.py index 63badefaa4..dc8ca3b490 100644 --- a/narwhals/_compliant/series.py +++ b/narwhals/_compliant/series.py @@ -30,11 +30,14 @@ def alias(self, name: str) -> Self: ... class EagerSeries(CompliantSeries, Protocol[NativeSeriesT_co]): - _native_series: NativeSeriesT_co + _native_series: Any _implementation: Implementation _backend_version: tuple[int, ...] _version: Version + @property + def native(self) -> NativeSeriesT_co: ... + def _from_scalar(self, value: Any) -> Self: return self._from_iterable([value], name=self.name, context=self) diff --git a/narwhals/_compliant/typing.py b/narwhals/_compliant/typing.py index c71e336782..fb9aa1beb0 100644 --- a/narwhals/_compliant/typing.py +++ b/narwhals/_compliant/typing.py @@ -40,3 +40,4 @@ EagerDataFrameT = TypeVar("EagerDataFrameT", bound="EagerDataFrame[Any]") EagerSeriesT = TypeVar("EagerSeriesT", bound="EagerSeries[Any]") +EagerSeriesT_co = TypeVar("EagerSeriesT_co", bound="EagerSeries[Any]", covariant=True) From 7d487267a74e8cee3b1fb272b72b87e4fb29957e Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Thu, 6 Mar 2025 20:44:21 +0000 Subject: [PATCH 17/58] feat(DRAFT): add a couple more methods --- narwhals/_compliant/expr.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index d04bcfb370..622a2cadb0 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -457,6 +457,14 @@ def __rmod__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT def __invert__(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: return self._reuse_series_implementation("__invert__") + def cast( + self, dtype: DType | type[DType] + ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("cast", dtype=dtype) + + def null_count(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("null_count", returns_scalar=True) + # NOTE: See (https://github.com/narwhals-dev/narwhals/issues/2044#issuecomment-2674262833) class LazyExpr( From 75b33c4a4dcf840607cf7a0d3b6299385fcfb143 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Thu, 6 Mar 2025 22:29:50 +0000 Subject: [PATCH 18/58] feat: Spec-out all generic parts of `EagerExpr` Wild how much of this is identical and easy to share https://github.com/narwhals-dev/narwhals/pull/2149#discussion_r1984030218 --- narwhals/_compliant/expr.py | 287 ++++++++++++++++++++++++++++++- narwhals/_compliant/namespace.py | 14 +- narwhals/_compliant/series.py | 1 + pyproject.toml | 2 + 4 files changed, 294 insertions(+), 10 deletions(-) diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index 622a2cadb0..7521a909b7 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -7,9 +7,11 @@ from typing import Any from typing import Callable from typing import Literal +from typing import Mapping from typing import Protocol from typing import Sequence +from narwhals._compliant.namespace import CompliantNamespace from narwhals._compliant.typing import CompliantFrameT from narwhals._compliant.typing import CompliantLazyFrameT from narwhals._compliant.typing import CompliantSeriesOrNativeExprT_co @@ -17,6 +19,7 @@ from narwhals._compliant.typing import EagerSeriesT from narwhals._compliant.typing import NativeExprT_co from narwhals._expression_parsing import evaluate_output_names_and_aliases +from narwhals.dtypes import DType from narwhals.utils import deprecated from narwhals.utils import not_implemented from narwhals.utils import unstable @@ -37,6 +40,7 @@ from typing_extensions import Self from narwhals._compliant.namespace import CompliantNamespace + from narwhals._compliant.namespace import EagerNamespace from narwhals._compliant.series import CompliantSeries from narwhals._expression_parsing import ExprKind from narwhals.dtypes import DType @@ -236,6 +240,7 @@ class EagerExpr( CompliantExpr[EagerDataFrameT, EagerSeriesT], Protocol38[EagerDataFrameT, EagerSeriesT], ): + _call: Callable[[EagerDataFrameT], Sequence[EagerSeriesT]] _depth: int _function_name: str _evaluate_output_names: Any @@ -259,6 +264,15 @@ def __init__( call_kwargs: dict[str, Any] | None = None, ) -> None: ... + def __call__(self, df: EagerDataFrameT) -> Sequence[EagerSeriesT]: + return self._call(df) + + def __repr__(self) -> str: # pragma: no cover + return f"{type(self).__name__}(depth={self._depth}, function_name={self._function_name})" + + def __narwhals_namespace__(self) -> EagerNamespace[EagerDataFrameT, EagerSeriesT]: ... + def __narwhals_expr__(self) -> None: ... + @classmethod def _from_callable( cls, @@ -383,6 +397,34 @@ def _reuse_series_namespace_implementation( context=self, ) + def broadcast(self, kind: Literal[ExprKind.AGGREGATION, ExprKind.LITERAL]) -> Self: + # Mark the resulting Series with `_broadcast = True`. + # Then, when extracting native objects, `extract_native` will + # know what to do. + def func(df: EagerDataFrameT) -> list[EagerSeriesT]: + results = [] + for result in self(df): + result._broadcast = True + results.append(result) + return results + + return type(self)( + func, + depth=self._depth, + function_name=self._function_name, + evaluate_output_names=self._evaluate_output_names, + alias_output_names=self._alias_output_names, + backend_version=self._backend_version, + implementation=self._implementation, + version=self._version, + call_kwargs=self._call_kwargs, + ) + + def cast( + self, dtype: DType | type[DType] + ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("cast", dtype=dtype) + def __eq__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: # type: ignore[override] return self._reuse_series_implementation("__eq__", other=other) @@ -457,14 +499,249 @@ def __rmod__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT def __invert__(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: return self._reuse_series_implementation("__invert__") - def cast( - self, dtype: DType | type[DType] - ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: - return self._reuse_series_implementation("cast", dtype=dtype) - + # Reductions def null_count(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: return self._reuse_series_implementation("null_count", returns_scalar=True) + def n_unique(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("n_unique", returns_scalar=True) + + def sum(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("sum", returns_scalar=True) + + def mean(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("mean", returns_scalar=True) + + def median(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("median", returns_scalar=True) + + def std(self, *, ddof: int) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation( + "std", returns_scalar=True, call_kwargs={"ddof": ddof} + ) + + def var(self, *, ddof: int) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation( + "var", returns_scalar=True, call_kwargs={"ddof": ddof} + ) + + def skew(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("skew", returns_scalar=True) + + def any(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("any", returns_scalar=True) + + def all(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("all", returns_scalar=True) + + def max(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("max", returns_scalar=True) + + def mix(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("min", returns_scalar=True) + + def arg_min(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("arg_min", returns_scalar=True) + + def arg_max(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("arg_max", returns_scalar=True) + + # Other + + def clip( + self, lower_bound: Any, upper_bound: Any + ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation( + "clip", lower_bound=lower_bound, upper_bound=upper_bound + ) + + def is_null(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("is_null") + + def is_nan(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("is_nan") + + def fill_null( + self, + value: Any | None, + strategy: Literal["forward", "backward"] | None, + limit: int | None, + ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation( + "fill_null", value=value, strategy=strategy, limit=limit + ) + + def is_in(self, other: Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("is_in", other="other") + + def arg_true(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("arg_true") + + # NOTE: `ewm_mean` not implemented `pyarrow` + + def filter(self, *predicates: Self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + plx = self.__narwhals_namespace__() + other = plx.all_horizontal(*predicates) + return self._reuse_series_implementation("filter", other=other) + + def drop_nulls(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("drop_nulls") + + def replace_strict( + self, + old: Sequence[Any] | Mapping[Any, Any], + new: Sequence[Any], + *, + return_dtype: DType | type[DType] | None, + ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation( + "replace_strict", old=old, new=new, return_dtype=return_dtype + ) + + def sort( + self, *, descending: bool, nulls_last: bool + ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation( + "sort", descending=descending, nulls_last=nulls_last + ) + + def abs(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("abs") + + def unique(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("unique", maintain_order=False) + + def diff(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("diff") + + # NOTE: `shift` differs + + def sample( + self, + n: int | None, + *, + fraction: float | None, + with_replacement: bool, + seed: int | None, + ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation( + "sample", n=n, fraction=fraction, with_replacement=with_replacement, seed=seed + ) + + def alias(self: Self, name: str) -> Self: + def alias_output_names(names: Sequence[str]) -> Sequence[str]: + if len(names) != 1: + msg = f"Expected function with single output, found output names: {names}" + raise ValueError(msg) + return [name] + + # Define this one manually, so that we can + # override `output_names` and not increase depth + return type(self)( + lambda df: [series.alias(name) for series in self(df)], + depth=self._depth, + function_name=self._function_name, + evaluate_output_names=self._evaluate_output_names, + alias_output_names=alias_output_names, + backend_version=self._backend_version, + implementation=self._implementation, + version=self._version, + call_kwargs=self._call_kwargs, + ) + + # NOTE: `over` differs + + def is_unique(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("is_unique") + + def is_first_distinct(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("is_first_distinct") + + def is_last_distinct(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("is_last_distinct") + + def quantile( + self, + quantile: float, + interpolation: Literal["nearest", "higher", "lower", "midpoint", "linear"], + ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation( + "quantile", + quantile=quantile, + interpolation=interpolation, + returns_scalar=True, + ) + + def head(self, n: int) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("head", n=n) + + def tail(self, n: int) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("tail", n=n) + + def round(self, decimals: int) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("round", decimals=decimals) + + def len(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("len", returns_scalar=True) + + def gather_every( + self, n: int, offset: int + ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("gather_every", n=n, offset=offset) + + def mode(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("mode") + + # NOTE: `map_batches` differs + + def is_finite(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation("is_finite") + + # NOTE: `cum_(sum|count|min|max|prod)` differ + + def rolling_mean( + self, window_size: int, *, min_samples: int | None, center: bool + ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation( + "rolling_mean", + window_size=window_size, + min_samples=min_samples, + center=center, + ) + + def rolling_std( + self, window_size: int, *, min_samples: int | None, center: bool, ddof: int + ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation( + "rolling_std", + window_size=window_size, + min_samples=min_samples, + center=center, + ddof=ddof, + ) + + def rolling_sum( + self, window_size: int, *, min_samples: int | None, center: bool + ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation( + "rolling_sum", window_size=window_size, min_samples=min_samples, center=center + ) + + def rolling_var( + self, window_size: int, *, min_samples: int | None, center: bool, ddof: int + ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + return self._reuse_series_implementation( + "rolling_var", + window_size=window_size, + min_samples=min_samples, + center=center, + ddof=ddof, + ) + + # NOTE: `rank` differs + + # NOTE: All namespaces differ + # NOTE: See (https://github.com/narwhals-dev/narwhals/issues/2044#issuecomment-2674262833) class LazyExpr( diff --git a/narwhals/_compliant/namespace.py b/narwhals/_compliant/namespace.py index 16a6d553f6..6433784d1d 100644 --- a/narwhals/_compliant/namespace.py +++ b/narwhals/_compliant/namespace.py @@ -4,13 +4,14 @@ from typing import Any from typing import Protocol -from narwhals._compliant.typing import CompliantDataFrameT from narwhals._compliant.typing import CompliantFrameT from narwhals._compliant.typing import CompliantSeriesOrNativeExprT_co -from narwhals._compliant.typing import CompliantSeriesT_co +from narwhals._compliant.typing import EagerDataFrameT +from narwhals._compliant.typing import EagerSeriesT if TYPE_CHECKING: from narwhals._compliant.expr import CompliantExpr + from narwhals._compliant.expr import EagerExpr from narwhals._compliant.selectors import CompliantSelectorNamespace from narwhals.dtypes import DType @@ -29,6 +30,9 @@ def selectors(self) -> CompliantSelectorNamespace[Any, Any]: ... class EagerNamespace( - CompliantNamespace[CompliantDataFrameT, CompliantSeriesT_co], - Protocol[CompliantDataFrameT, CompliantSeriesT_co], -): ... + CompliantNamespace[EagerDataFrameT, EagerSeriesT], + Protocol[EagerDataFrameT, EagerSeriesT], +): + def all_horizontal( + self, *exprs: EagerExpr[EagerDataFrameT, EagerSeriesT] + ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: ... diff --git a/narwhals/_compliant/series.py b/narwhals/_compliant/series.py index dc8ca3b490..0dcc44c1a9 100644 --- a/narwhals/_compliant/series.py +++ b/narwhals/_compliant/series.py @@ -34,6 +34,7 @@ class EagerSeries(CompliantSeries, Protocol[NativeSeriesT_co]): _implementation: Implementation _backend_version: tuple[int, ...] _version: Version + _broadcast: bool @property def native(self) -> NativeSeriesT_co: ... diff --git a/pyproject.toml b/pyproject.toml index 21a252f26a..5706d27944 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -230,6 +230,8 @@ omit = [ 'narwhals/_spark_like/*', # we don't run these in every environment 'tests/ibis_test.py', + # Remove after finishing eager sub-protocols + 'narwhals/_compliant/*', ] exclude_also = [ "if sys.version_info() <", From 5d766c27eab6ac27f35e607470447a72fd556540 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 7 Mar 2025 10:52:35 +0000 Subject: [PATCH 19/58] revert: remove unused `_series` property - Thought I was going to need in `_reuse_series_*` - Turns out, I only needed a ref to instance methods --- narwhals/_compliant/expr.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index 7521a909b7..830780e44d 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -247,9 +247,6 @@ class EagerExpr( _alias_output_names: Any _call_kwargs: dict[str, Any] - @property - def _series(self) -> type[EagerSeriesT]: ... - def __init__( self: Self, call: Callable[[EagerDataFrameT], Sequence[EagerSeriesT]], From a066213f16180e520b6d9427f8ed09c0b42bb254 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 7 Mar 2025 10:54:15 +0000 Subject: [PATCH 20/58] feat: add `EagerExpr.from_column_(names|indices)` They can use the same signature by not passing in `Implementation` and handling that internally --- narwhals/_compliant/expr.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index 830780e44d..97597df300 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -307,6 +307,22 @@ def _from_series(cls, series: EagerSeriesT, *, context: _FullContext) -> Self: version=context._version, ) + @classmethod + def from_column_names( + cls, + evaluate_column_names: Callable[[EagerDataFrameT], Sequence[str]], + /, + *, + function_name: str, + context: _FullContext, + ) -> Self: ... + @classmethod + def from_column_indices( + cls, + *column_indices: int, + context: _FullContext, + ) -> Self: ... + # https://github.com/narwhals-dev/narwhals/blob/35cef0b1e2c892fb24aa730902b08b6994008c18/narwhals/_protocols.py#L135 def _reuse_series_implementation( self: EagerExpr[EagerDataFrameT, EagerSeriesT], From 57ab13b7eea75801f5ae0431bdfe120e7b08d040 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 7 Mar 2025 11:14:44 +0000 Subject: [PATCH 21/58] chore: add some notes on `Eager(Namespace|Series)` --- narwhals/_compliant/namespace.py | 13 +++++++++++++ narwhals/_compliant/series.py | 9 +++++++++ 2 files changed, 22 insertions(+) diff --git a/narwhals/_compliant/namespace.py b/narwhals/_compliant/namespace.py index 6433784d1d..feae3bf2f5 100644 --- a/narwhals/_compliant/namespace.py +++ b/narwhals/_compliant/namespace.py @@ -33,6 +33,19 @@ class EagerNamespace( CompliantNamespace[EagerDataFrameT, EagerSeriesT], Protocol[EagerDataFrameT, EagerSeriesT], ): + # NOTE: Supporting moved ops + # - `self_create_expr_from_callable` -> `self._expr._from_callable` + # - `self_create_expr_from_series` -> `self._expr._from_series` + @property + def _expr(self) -> type[EagerExpr[EagerDataFrameT, EagerSeriesT]]: ... + + # NOTE: Supporting moved ops + # - `self._create_series_from_scalar` -> `EagerSeries()._from_scalar` + # - Was dependent on a `reference_series`, so is now an instance method + # - `._from_iterable` -> `self._series._from_iterable` + @property + def _series(self) -> type[EagerSeriesT]: ... + def all_horizontal( self, *exprs: EagerExpr[EagerDataFrameT, EagerSeriesT] ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: ... diff --git a/narwhals/_compliant/series.py b/narwhals/_compliant/series.py index 0dcc44c1a9..d3fb0a7069 100644 --- a/narwhals/_compliant/series.py +++ b/narwhals/_compliant/series.py @@ -39,6 +39,9 @@ class EagerSeries(CompliantSeries, Protocol[NativeSeriesT_co]): @property def native(self) -> NativeSeriesT_co: ... + # NOTE: `ArrowSeries` needs to intercept `value` w/ + # if self._backend_version < (13,) and hasattr(value, "as_py"): + # value = value.as_py() # noqa: ERA001 def _from_scalar(self, value: Any) -> Self: return self._from_iterable([value], name=self.name, context=self) @@ -46,3 +49,9 @@ def _from_scalar(self, value: Any) -> Self: def _from_iterable( cls: type[Self], data: Iterable[Any], name: str, *, context: _FullContext ) -> Self: ... + + # TODO @dangotbanned: replacing `Namespace._create_compliant_series`` + # - All usage within `*Expr.map_batches` + # - `PandasLikeExpr` uses that **once** + # - `ArrowExpr` uses **twice** + # - `PandasLikeDataFrame.with_row_index` uses the wrapped `utils` function once From e16e942115048745a18c2efbf20c5cc48778c57b Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 7 Mar 2025 11:25:08 +0000 Subject: [PATCH 22/58] update import/exports --- narwhals/_compliant/__init__.py | 8 ++++++++ narwhals/_compliant/dataframe.py | 2 +- narwhals/_compliant/expr.py | 2 +- narwhals/_compliant/namespace.py | 2 +- narwhals/_compliant/series.py | 2 +- 5 files changed, 12 insertions(+), 4 deletions(-) diff --git a/narwhals/_compliant/__init__.py b/narwhals/_compliant/__init__.py index 19434c1a4c..186a0657bb 100644 --- a/narwhals/_compliant/__init__.py +++ b/narwhals/_compliant/__init__.py @@ -2,9 +2,12 @@ from narwhals._compliant.dataframe import CompliantDataFrame from narwhals._compliant.dataframe import CompliantLazyFrame +from narwhals._compliant.dataframe import EagerDataFrame from narwhals._compliant.expr import CompliantExpr +from narwhals._compliant.expr import EagerExpr from narwhals._compliant.expr import LazyExpr from narwhals._compliant.namespace import CompliantNamespace +from narwhals._compliant.namespace import EagerNamespace from narwhals._compliant.selectors import CompliantSelector from narwhals._compliant.selectors import CompliantSelectorNamespace from narwhals._compliant.selectors import EagerSelectorNamespace @@ -12,6 +15,7 @@ from narwhals._compliant.selectors import EvalSeries from narwhals._compliant.selectors import LazySelectorNamespace from narwhals._compliant.series import CompliantSeries +from narwhals._compliant.series import EagerSeries from narwhals._compliant.typing import CompliantFrameT from narwhals._compliant.typing import CompliantSeriesOrNativeExprT_co from narwhals._compliant.typing import CompliantSeriesT_co @@ -28,7 +32,11 @@ "CompliantSeries", "CompliantSeriesOrNativeExprT_co", "CompliantSeriesT_co", + "EagerDataFrame", + "EagerExpr", + "EagerNamespace", "EagerSelectorNamespace", + "EagerSeries", "EvalNames", "EvalSeries", "IntoCompliantExpr", diff --git a/narwhals/_compliant/dataframe.py b/narwhals/_compliant/dataframe.py index a192f2b647..ba30f2515c 100644 --- a/narwhals/_compliant/dataframe.py +++ b/narwhals/_compliant/dataframe.py @@ -19,7 +19,7 @@ from narwhals._compliant.expr import EagerExpr from narwhals.dtypes import DType -__all__ = ["CompliantDataFrame", "CompliantLazyFrame"] +__all__ = ["CompliantDataFrame", "CompliantLazyFrame", "EagerDataFrame"] T = TypeVar("T") diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index 97597df300..b56c800a97 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -48,7 +48,7 @@ from narwhals.utils import Version from narwhals.utils import _FullContext -__all__ = ["CompliantExpr"] +__all__ = ["CompliantExpr", "EagerExpr", "LazyExpr", "NativeExpr"] # NOTE: Only common methods for lazy expr-like objects diff --git a/narwhals/_compliant/namespace.py b/narwhals/_compliant/namespace.py index feae3bf2f5..d8d6df0a7c 100644 --- a/narwhals/_compliant/namespace.py +++ b/narwhals/_compliant/namespace.py @@ -15,7 +15,7 @@ from narwhals._compliant.selectors import CompliantSelectorNamespace from narwhals.dtypes import DType -__all__ = ["CompliantNamespace"] +__all__ = ["CompliantNamespace", "EagerNamespace"] class CompliantNamespace(Protocol[CompliantFrameT, CompliantSeriesOrNativeExprT_co]): diff --git a/narwhals/_compliant/series.py b/narwhals/_compliant/series.py index d3fb0a7069..bb5337d4e6 100644 --- a/narwhals/_compliant/series.py +++ b/narwhals/_compliant/series.py @@ -15,7 +15,7 @@ from narwhals.utils import Version from narwhals.utils import _FullContext -__all__ = ["CompliantSeries"] +__all__ = ["CompliantSeries", "EagerSeries"] NativeSeriesT_co = TypeVar("NativeSeriesT_co", bound="NativeSeries", covariant=True) From 364d2ee815721f43aba49e7fa6a18f44f0e9303e Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 7 Mar 2025 11:45:12 +0000 Subject: [PATCH 23/58] fix(typing): Use `Self` I did things generic to match `_reuse_series*` - but I couldv'e used `Self` there :facepalm: --- narwhals/_compliant/expr.py | 155 ++++++++++++++++-------------------- 1 file changed, 69 insertions(+), 86 deletions(-) diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index b56c800a97..20454e38cc 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -325,13 +325,13 @@ def from_column_indices( # https://github.com/narwhals-dev/narwhals/blob/35cef0b1e2c892fb24aa730902b08b6994008c18/narwhals/_protocols.py#L135 def _reuse_series_implementation( - self: EagerExpr[EagerDataFrameT, EagerSeriesT], + self: Self, attr: str, *, returns_scalar: bool = False, call_kwargs: dict[str, Any] | None = None, **expressifiable_args: Any, - ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + ) -> Self: func = partial( self._reuse_series_inner, method_name=attr, @@ -392,11 +392,8 @@ def _reuse_series_inner( return out def _reuse_series_namespace_implementation( - self: EagerExpr[EagerDataFrameT, EagerSeriesT], - series_namespace: str, - attr: str, - **kwargs: Any, - ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + self: Self, series_namespace: str, attr: str, **kwargs: Any + ) -> Self: return self._from_callable( lambda df: [ getattr(getattr(series, series_namespace), attr)(**kwargs) @@ -433,145 +430,135 @@ def func(df: EagerDataFrameT) -> list[EagerSeriesT]: call_kwargs=self._call_kwargs, ) - def cast( - self, dtype: DType | type[DType] - ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def cast(self, dtype: DType | type[DType]) -> Self: return self._reuse_series_implementation("cast", dtype=dtype) - def __eq__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: # type: ignore[override] + def __eq__(self, other: Self | Any) -> Self: # type: ignore[override] return self._reuse_series_implementation("__eq__", other=other) - def __ne__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: # type: ignore[override] + def __ne__(self, other: Self | Any) -> Self: # type: ignore[override] return self._reuse_series_implementation("__ne__", other=other) - def __ge__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def __ge__(self, other: Self | Any) -> Self: return self._reuse_series_implementation("__ge__", other=other) - def __gt__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def __gt__(self, other: Self | Any) -> Self: return self._reuse_series_implementation("__gt__", other=other) - def __le__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def __le__(self, other: Self | Any) -> Self: return self._reuse_series_implementation("__le__", other=other) - def __lt__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def __lt__(self, other: Self | Any) -> Self: return self._reuse_series_implementation("__lt__", other=other) - def __and__( - self, other: Self | bool | Any - ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def __and__(self, other: Self | bool | Any) -> Self: return self._reuse_series_implementation("__and__", other=other) - def __or__( - self, other: Self | bool | Any - ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def __or__(self, other: Self | bool | Any) -> Self: return self._reuse_series_implementation("__or__", other=other) - def __add__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def __add__(self, other: Self | Any) -> Self: return self._reuse_series_implementation("__add__", other=other) - def __sub__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def __sub__(self, other: Self | Any) -> Self: return self._reuse_series_implementation("__sub__", other=other) - def __rsub__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def __rsub__(self, other: Self | Any) -> Self: return self.alias("literal")._reuse_series_implementation("__rsub__", other=other) - def __mul__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def __mul__(self, other: Self | Any) -> Self: return self._reuse_series_implementation("__mul__", other=other) - def __truediv__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def __truediv__(self, other: Self | Any) -> Self: return self._reuse_series_implementation("__truediv__", other=other) - def __rtruediv__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def __rtruediv__(self, other: Self | Any) -> Self: return self.alias("literal")._reuse_series_implementation( "__rtruediv__", other=other ) - def __floordiv__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def __floordiv__(self, other: Self | Any) -> Self: return self._reuse_series_implementation("__floordiv__", other=other) - def __rfloordiv__( - self, other: Self | Any - ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def __rfloordiv__(self, other: Self | Any) -> Self: return self.alias("literal")._reuse_series_implementation( "__rfloordiv__", other=other ) - def __pow__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def __pow__(self, other: Self | Any) -> Self: return self._reuse_series_implementation("__pow__", other=other) - def __rpow__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def __rpow__(self, other: Self | Any) -> Self: return self.alias("literal")._reuse_series_implementation("__rpow__", other=other) - def __mod__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def __mod__(self, other: Self | Any) -> Self: return self._reuse_series_implementation("__mod__", other=other) - def __rmod__(self, other: Self | Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def __rmod__(self, other: Self | Any) -> Self: return self.alias("literal")._reuse_series_implementation("__rmod__", other=other) # Unary - def __invert__(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def __invert__(self) -> Self: return self._reuse_series_implementation("__invert__") # Reductions - def null_count(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def null_count(self) -> Self: return self._reuse_series_implementation("null_count", returns_scalar=True) - def n_unique(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def n_unique(self) -> Self: return self._reuse_series_implementation("n_unique", returns_scalar=True) - def sum(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def sum(self) -> Self: return self._reuse_series_implementation("sum", returns_scalar=True) - def mean(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def mean(self) -> Self: return self._reuse_series_implementation("mean", returns_scalar=True) - def median(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def median(self) -> Self: return self._reuse_series_implementation("median", returns_scalar=True) - def std(self, *, ddof: int) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def std(self, *, ddof: int) -> Self: return self._reuse_series_implementation( "std", returns_scalar=True, call_kwargs={"ddof": ddof} ) - def var(self, *, ddof: int) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def var(self, *, ddof: int) -> Self: return self._reuse_series_implementation( "var", returns_scalar=True, call_kwargs={"ddof": ddof} ) - def skew(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def skew(self) -> Self: return self._reuse_series_implementation("skew", returns_scalar=True) - def any(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def any(self) -> Self: return self._reuse_series_implementation("any", returns_scalar=True) - def all(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def all(self) -> Self: return self._reuse_series_implementation("all", returns_scalar=True) - def max(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def max(self) -> Self: return self._reuse_series_implementation("max", returns_scalar=True) - def mix(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def mix(self) -> Self: return self._reuse_series_implementation("min", returns_scalar=True) - def arg_min(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def arg_min(self) -> Self: return self._reuse_series_implementation("arg_min", returns_scalar=True) - def arg_max(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def arg_max(self) -> Self: return self._reuse_series_implementation("arg_max", returns_scalar=True) # Other - def clip( - self, lower_bound: Any, upper_bound: Any - ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def clip(self, lower_bound: Any, upper_bound: Any) -> Self: return self._reuse_series_implementation( "clip", lower_bound=lower_bound, upper_bound=upper_bound ) - def is_null(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def is_null(self) -> Self: return self._reuse_series_implementation("is_null") - def is_nan(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def is_nan(self) -> Self: return self._reuse_series_implementation("is_nan") def fill_null( @@ -579,25 +566,25 @@ def fill_null( value: Any | None, strategy: Literal["forward", "backward"] | None, limit: int | None, - ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + ) -> Self: return self._reuse_series_implementation( "fill_null", value=value, strategy=strategy, limit=limit ) - def is_in(self, other: Any) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def is_in(self, other: Any) -> Self: return self._reuse_series_implementation("is_in", other="other") - def arg_true(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def arg_true(self) -> Self: return self._reuse_series_implementation("arg_true") # NOTE: `ewm_mean` not implemented `pyarrow` - def filter(self, *predicates: Self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def filter(self, *predicates: Self) -> Self: plx = self.__narwhals_namespace__() other = plx.all_horizontal(*predicates) return self._reuse_series_implementation("filter", other=other) - def drop_nulls(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def drop_nulls(self) -> Self: return self._reuse_series_implementation("drop_nulls") def replace_strict( @@ -606,25 +593,23 @@ def replace_strict( new: Sequence[Any], *, return_dtype: DType | type[DType] | None, - ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + ) -> Self: return self._reuse_series_implementation( "replace_strict", old=old, new=new, return_dtype=return_dtype ) - def sort( - self, *, descending: bool, nulls_last: bool - ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def sort(self, *, descending: bool, nulls_last: bool) -> Self: return self._reuse_series_implementation( "sort", descending=descending, nulls_last=nulls_last ) - def abs(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def abs(self) -> Self: return self._reuse_series_implementation("abs") - def unique(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def unique(self) -> Self: return self._reuse_series_implementation("unique", maintain_order=False) - def diff(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def diff(self) -> Self: return self._reuse_series_implementation("diff") # NOTE: `shift` differs @@ -636,7 +621,7 @@ def sample( fraction: float | None, with_replacement: bool, seed: int | None, - ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + ) -> Self: return self._reuse_series_implementation( "sample", n=n, fraction=fraction, with_replacement=with_replacement, seed=seed ) @@ -664,20 +649,20 @@ def alias_output_names(names: Sequence[str]) -> Sequence[str]: # NOTE: `over` differs - def is_unique(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def is_unique(self) -> Self: return self._reuse_series_implementation("is_unique") - def is_first_distinct(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def is_first_distinct(self) -> Self: return self._reuse_series_implementation("is_first_distinct") - def is_last_distinct(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def is_last_distinct(self) -> Self: return self._reuse_series_implementation("is_last_distinct") def quantile( self, quantile: float, interpolation: Literal["nearest", "higher", "lower", "midpoint", "linear"], - ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + ) -> Self: return self._reuse_series_implementation( "quantile", quantile=quantile, @@ -685,36 +670,34 @@ def quantile( returns_scalar=True, ) - def head(self, n: int) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def head(self, n: int) -> Self: return self._reuse_series_implementation("head", n=n) - def tail(self, n: int) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def tail(self, n: int) -> Self: return self._reuse_series_implementation("tail", n=n) - def round(self, decimals: int) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def round(self, decimals: int) -> Self: return self._reuse_series_implementation("round", decimals=decimals) - def len(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def len(self) -> Self: return self._reuse_series_implementation("len", returns_scalar=True) - def gather_every( - self, n: int, offset: int - ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def gather_every(self, n: int, offset: int) -> Self: return self._reuse_series_implementation("gather_every", n=n, offset=offset) - def mode(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def mode(self) -> Self: return self._reuse_series_implementation("mode") # NOTE: `map_batches` differs - def is_finite(self) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + def is_finite(self) -> Self: return self._reuse_series_implementation("is_finite") # NOTE: `cum_(sum|count|min|max|prod)` differ def rolling_mean( self, window_size: int, *, min_samples: int | None, center: bool - ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + ) -> Self: return self._reuse_series_implementation( "rolling_mean", window_size=window_size, @@ -724,7 +707,7 @@ def rolling_mean( def rolling_std( self, window_size: int, *, min_samples: int | None, center: bool, ddof: int - ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + ) -> Self: return self._reuse_series_implementation( "rolling_std", window_size=window_size, @@ -735,14 +718,14 @@ def rolling_std( def rolling_sum( self, window_size: int, *, min_samples: int | None, center: bool - ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + ) -> Self: return self._reuse_series_implementation( "rolling_sum", window_size=window_size, min_samples=min_samples, center=center ) def rolling_var( self, window_size: int, *, min_samples: int | None, center: bool, ddof: int - ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: + ) -> Self: return self._reuse_series_implementation( "rolling_var", window_size=window_size, From 484c997fca49de9b87b6ca49af2f2f0bf2b09027 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 7 Mar 2025 11:46:43 +0000 Subject: [PATCH 24/58] refactor(DRAFT): start implementing `Arrow*` --- narwhals/_arrow/dataframe.py | 3 ++- narwhals/_arrow/expr.py | 4 ++-- narwhals/_arrow/namespace.py | 12 ++++++++++-- narwhals/_arrow/series.py | 8 ++++++-- 4 files changed, 20 insertions(+), 7 deletions(-) diff --git a/narwhals/_arrow/dataframe.py b/narwhals/_arrow/dataframe.py index e0709a84ff..222d7fe334 100644 --- a/narwhals/_arrow/dataframe.py +++ b/narwhals/_arrow/dataframe.py @@ -17,6 +17,7 @@ from narwhals._arrow.utils import extract_dataframe_comparand from narwhals._arrow.utils import native_to_narwhals_dtype from narwhals._arrow.utils import select_rows +from narwhals._compliant import EagerDataFrame from narwhals._expression_parsing import ExprKind from narwhals._expression_parsing import evaluate_into_exprs from narwhals.dependencies import is_numpy_array_1d @@ -71,7 +72,7 @@ from narwhals.typing import CompliantLazyFrame -class ArrowDataFrame(CompliantDataFrame["ArrowSeries"], CompliantLazyFrame): +class ArrowDataFrame(EagerDataFrame["ArrowSeries"], CompliantLazyFrame): # --- not in the spec --- def __init__( self: Self, diff --git a/narwhals/_arrow/expr.py b/narwhals/_arrow/expr.py index 4861866ccc..dbde509763 100644 --- a/narwhals/_arrow/expr.py +++ b/narwhals/_arrow/expr.py @@ -13,7 +13,7 @@ from narwhals._arrow.expr_name import ArrowExprNameNamespace from narwhals._arrow.expr_str import ArrowExprStringNamespace from narwhals._arrow.series import ArrowSeries -from narwhals._compliant import CompliantExpr +from narwhals._compliant import EagerExpr from narwhals._expression_parsing import ExprKind from narwhals._expression_parsing import evaluate_output_names_and_aliases from narwhals._expression_parsing import is_scalar_like @@ -33,7 +33,7 @@ from narwhals.utils import Version -class ArrowExpr(CompliantExpr["ArrowDataFrame", ArrowSeries]): +class ArrowExpr(EagerExpr["ArrowDataFrame", ArrowSeries]): _implementation: Implementation = Implementation.PYARROW def __init__( diff --git a/narwhals/_arrow/namespace.py b/narwhals/_arrow/namespace.py index 68ace016d8..ef033f26d7 100644 --- a/narwhals/_arrow/namespace.py +++ b/narwhals/_arrow/namespace.py @@ -25,7 +25,7 @@ from narwhals._arrow.utils import horizontal_concat from narwhals._arrow.utils import nulls_like from narwhals._arrow.utils import vertical_concat -from narwhals._compliant import CompliantNamespace +from narwhals._compliant import EagerNamespace from narwhals._expression_parsing import combine_alias_output_names from narwhals._expression_parsing import combine_evaluate_output_names from narwhals.utils import Implementation @@ -48,7 +48,15 @@ _Scalar: TypeAlias = Any -class ArrowNamespace(CompliantNamespace[ArrowDataFrame, ArrowSeries]): +class ArrowNamespace(EagerNamespace[ArrowDataFrame, ArrowSeries]): + @property + def _expr(self) -> type[ArrowExpr]: + return ArrowExpr + + @property + def _series(self) -> type[ArrowSeries]: + return ArrowSeries + def _create_expr_from_callable( self: Self, func: Callable[[ArrowDataFrame], Sequence[ArrowSeries]], diff --git a/narwhals/_arrow/series.py b/narwhals/_arrow/series.py index 141c98fbf7..f6fc552cf5 100644 --- a/narwhals/_arrow/series.py +++ b/narwhals/_arrow/series.py @@ -25,8 +25,8 @@ from narwhals._arrow.utils import native_to_narwhals_dtype from narwhals._arrow.utils import nulls_like from narwhals._arrow.utils import pad_series +from narwhals._compliant import EagerSeries from narwhals.exceptions import InvalidOperationError -from narwhals.typing import CompliantSeries from narwhals.utils import Implementation from narwhals.utils import generate_temporary_column_name from narwhals.utils import import_dtypes_module @@ -94,7 +94,7 @@ def maybe_extract_py_scalar(value: Any, return_py_scalar: bool) -> Any: # noqa: return value -class ArrowSeries(CompliantSeries): +class ArrowSeries(EagerSeries["ArrowChunkedArray"]): def __init__( self: Self, native_series: ArrowChunkedArray, @@ -111,6 +111,10 @@ def __init__( validate_backend_version(self._implementation, self._backend_version) self._broadcast = False + @property + def native(self) -> ArrowChunkedArray: + return self._native_series + def _change_version(self: Self, version: Version) -> Self: return self.__class__( self._native_series, From 77b599a2f3b34d7bd144d887a07694a528bf2335 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 7 Mar 2025 11:53:29 +0000 Subject: [PATCH 25/58] fix: typo and add missing `count` --- narwhals/_compliant/expr.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index 20454e38cc..65a0c41363 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -511,6 +511,9 @@ def n_unique(self) -> Self: def sum(self) -> Self: return self._reuse_series_implementation("sum", returns_scalar=True) + def count(self) -> Self: + return self._reuse_series_implementation("count", returns_scalar=True) + def mean(self) -> Self: return self._reuse_series_implementation("mean", returns_scalar=True) @@ -539,7 +542,7 @@ def all(self) -> Self: def max(self) -> Self: return self._reuse_series_implementation("max", returns_scalar=True) - def mix(self) -> Self: + def min(self) -> Self: return self._reuse_series_implementation("min", returns_scalar=True) def arg_min(self) -> Self: From 7a1a653f77e0133742593daa7c6d927a14c149bb Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 7 Mar 2025 11:55:01 +0000 Subject: [PATCH 26/58] refactor(DRAFT): mostly ready `ArrowExpr` - Need to handle the signature changes - Also the other parts need to adapt to their protocols --- narwhals/_arrow/expr.py | 372 ++-------------------------------------- 1 file changed, 12 insertions(+), 360 deletions(-) diff --git a/narwhals/_arrow/expr.py b/narwhals/_arrow/expr.py index dbde509763..4bee2a81f9 100644 --- a/narwhals/_arrow/expr.py +++ b/narwhals/_arrow/expr.py @@ -4,7 +4,6 @@ from typing import Any from typing import Callable from typing import Literal -from typing import Mapping from typing import Sequence from narwhals._arrow.expr_cat import ArrowExprCatNamespace @@ -17,7 +16,6 @@ from narwhals._expression_parsing import ExprKind from narwhals._expression_parsing import evaluate_output_names_and_aliases from narwhals._expression_parsing import is_scalar_like -from narwhals._expression_parsing import reuse_series_implementation from narwhals.dependencies import get_numpy from narwhals.dependencies import is_numpy_array from narwhals.exceptions import ColumnNotFoundError @@ -58,34 +56,6 @@ def __init__( self._version = version self._call_kwargs = call_kwargs or {} - def __repr__(self: Self) -> str: # pragma: no cover - return f"ArrowExpr(depth={self._depth}, function_name={self._function_name}, " - - def __call__(self: Self, df: ArrowDataFrame) -> Sequence[ArrowSeries]: - return self._call(df) - - def broadcast(self, kind: Literal[ExprKind.AGGREGATION, ExprKind.LITERAL]) -> Self: - # Mark the resulting ArrowSeries with `_broadcast = True`. - # Then, when extracting native objects, `extract_native` will - # know what to do. - def func(df: ArrowDataFrame) -> list[ArrowSeries]: - results = [] - for result in self(df): - result._broadcast = True - results.append(result) - return results - - return self.__class__( - func, - depth=self._depth, - function_name=self._function_name, - evaluate_output_names=self._evaluate_output_names, - alias_output_names=self._alias_output_names, - backend_version=self._backend_version, - version=self._version, - call_kwargs=self._call_kwargs, - ) - @classmethod def from_column_names( cls: type[Self], @@ -164,264 +134,16 @@ def __narwhals_namespace__(self: Self) -> ArrowNamespace: def __narwhals_expr__(self: Self) -> None: ... - def __eq__(self: Self, other: ArrowExpr | Any) -> Self: # type: ignore[override] - return reuse_series_implementation(self, "__eq__", other=other) - - def __ne__(self: Self, other: ArrowExpr | Any) -> Self: # type: ignore[override] - return reuse_series_implementation(self, "__ne__", other=other) - - def __ge__(self: Self, other: ArrowExpr | Any) -> Self: - return reuse_series_implementation(self, "__ge__", other=other) - - def __gt__(self: Self, other: ArrowExpr | Any) -> Self: - return reuse_series_implementation(self, "__gt__", other=other) - - def __le__(self: Self, other: ArrowExpr | Any) -> Self: - return reuse_series_implementation(self, "__le__", other=other) - - def __lt__(self: Self, other: ArrowExpr | Any) -> Self: - return reuse_series_implementation(self, "__lt__", other=other) - - def __and__(self: Self, other: ArrowExpr | bool | Any) -> Self: - return reuse_series_implementation(self, "__and__", other=other) - - def __or__(self: Self, other: ArrowExpr | bool | Any) -> Self: - return reuse_series_implementation(self, "__or__", other=other) - - def __add__(self: Self, other: ArrowExpr | Any) -> Self: - return reuse_series_implementation(self, "__add__", other=other) - - def __sub__(self: Self, other: ArrowExpr | Any) -> Self: - return reuse_series_implementation(self, "__sub__", other=other) - - def __rsub__(self: Self, other: ArrowExpr | Any) -> Self: - return reuse_series_implementation(self.alias("literal"), "__rsub__", other=other) - - def __mul__(self: Self, other: ArrowExpr | Any) -> Self: - return reuse_series_implementation(self, "__mul__", other=other) - - def __pow__(self: Self, other: ArrowExpr | Any) -> Self: - return reuse_series_implementation(self, "__pow__", other=other) - - def __rpow__(self: Self, other: ArrowExpr | Any) -> Self: - return reuse_series_implementation(self.alias("literal"), "__rpow__", other=other) - - def __floordiv__(self: Self, other: ArrowExpr | Any) -> Self: - return reuse_series_implementation(self, "__floordiv__", other=other) - - def __rfloordiv__(self: Self, other: ArrowExpr | Any) -> Self: - return reuse_series_implementation( - self.alias("literal"), "__rfloordiv__", other=other - ) - - def __truediv__(self: Self, other: ArrowExpr | Any) -> Self: - return reuse_series_implementation(self, "__truediv__", other=other) - - def __rtruediv__(self: Self, other: ArrowExpr | Any) -> Self: - return reuse_series_implementation( - self.alias("literal"), "__rtruediv__", other=other - ) - - def __mod__(self: Self, other: ArrowExpr | Any) -> Self: - return reuse_series_implementation(self, "__mod__", other=other) - - def __rmod__(self: Self, other: ArrowExpr | Any) -> Self: - return reuse_series_implementation(self.alias("literal"), "__rmod__", other=other) - - def __invert__(self: Self) -> Self: - return reuse_series_implementation(self, "__invert__") - - def len(self: Self) -> Self: - return reuse_series_implementation(self, "len", returns_scalar=True) - - def filter(self: Self, *predicates: ArrowExpr) -> Self: - plx = self.__narwhals_namespace__() - other = plx.all_horizontal(*predicates) - return reuse_series_implementation(self, "filter", other=other) - - def mean(self: Self) -> Self: - return reuse_series_implementation(self, "mean", returns_scalar=True) - - def median(self: Self) -> Self: - return reuse_series_implementation(self, "median", returns_scalar=True) - - def count(self: Self) -> Self: - return reuse_series_implementation(self, "count", returns_scalar=True) - - def n_unique(self: Self) -> Self: - return reuse_series_implementation(self, "n_unique", returns_scalar=True) - - def std(self: Self, ddof: int) -> Self: - return reuse_series_implementation( - self, "std", call_kwargs={"ddof": ddof}, returns_scalar=True - ) - - def var(self: Self, ddof: int) -> Self: - return reuse_series_implementation( - self, "var", call_kwargs={"ddof": ddof}, returns_scalar=True - ) - - def skew(self: Self) -> Self: - return reuse_series_implementation(self, "skew", returns_scalar=True) - - def cast(self: Self, dtype: DType | type[DType]) -> Self: - return reuse_series_implementation(self, "cast", dtype=dtype) - - def abs(self: Self) -> Self: - return reuse_series_implementation(self, "abs") - - def diff(self: Self) -> Self: - return reuse_series_implementation(self, "diff") + def _reuse_series_extra_kwargs( + self, *, returns_scalar: bool = False + ) -> dict[str, Any]: + return {"_return_py_scalar": False} if returns_scalar else {} def cum_sum(self: Self, *, reverse: bool) -> Self: - return reuse_series_implementation(self, "cum_sum", reverse=reverse) - - def round(self: Self, decimals: int) -> Self: - return reuse_series_implementation(self, "round", decimals=decimals) - - def any(self: Self) -> Self: - return reuse_series_implementation(self, "any", returns_scalar=True) - - def min(self: Self) -> Self: - return reuse_series_implementation(self, "min", returns_scalar=True) - - def max(self: Self) -> Self: - return reuse_series_implementation(self, "max", returns_scalar=True) - - def arg_min(self: Self) -> Self: - return reuse_series_implementation(self, "arg_min", returns_scalar=True) - - def arg_max(self: Self) -> Self: - return reuse_series_implementation(self, "arg_max", returns_scalar=True) - - def all(self: Self) -> Self: - return reuse_series_implementation(self, "all", returns_scalar=True) - - def sum(self: Self) -> Self: - return reuse_series_implementation(self, "sum", returns_scalar=True) - - def drop_nulls(self: Self) -> Self: - return reuse_series_implementation(self, "drop_nulls") + return self._reuse_series_implementation("cum_sum", reverse=reverse) def shift(self: Self, n: int) -> Self: - return reuse_series_implementation(self, "shift", n=n) - - def alias(self: Self, name: str) -> Self: - def alias_output_names(names: Sequence[str]) -> Sequence[str]: - if len(names) != 1: - msg = f"Expected function with single output, found output names: {names}" - raise ValueError(msg) - return [name] - - # Define this one manually, so that we can - # override `output_names` and not increase depth - return self.__class__( - lambda df: [series.alias(name) for series in self._call(df)], - depth=self._depth, - function_name=self._function_name, - evaluate_output_names=self._evaluate_output_names, - alias_output_names=alias_output_names, - backend_version=self._backend_version, - version=self._version, - call_kwargs=self._call_kwargs, - ) - - def null_count(self: Self) -> Self: - return reuse_series_implementation(self, "null_count", returns_scalar=True) - - def is_null(self: Self) -> Self: - return reuse_series_implementation(self, "is_null") - - def is_nan(self: Self) -> Self: - return reuse_series_implementation(self, "is_nan") - - def head(self: Self, n: int) -> Self: - return reuse_series_implementation(self, "head", n=n) - - def tail(self: Self, n: int) -> Self: - return reuse_series_implementation(self, "tail", n=n) - - def is_in(self: Self, other: ArrowExpr | Any) -> Self: - return reuse_series_implementation(self, "is_in", other=other) - - def arg_true(self: Self) -> Self: - return reuse_series_implementation(self, "arg_true") - - def sample( - self: Self, - n: int | None, - *, - fraction: float | None, - with_replacement: bool, - seed: int | None, - ) -> Self: - return reuse_series_implementation( - self, - "sample", - n=n, - fraction=fraction, - with_replacement=with_replacement, - seed=seed, - ) - - def fill_null( - self: Self, - value: Self | Any | None, - strategy: Literal["forward", "backward"] | None, - limit: int | None, - ) -> Self: - return reuse_series_implementation( - self, "fill_null", value=value, strategy=strategy, limit=limit - ) - - def is_unique(self: Self) -> Self: - return reuse_series_implementation(self, "is_unique") - - def is_first_distinct(self: Self) -> Self: - return reuse_series_implementation(self, "is_first_distinct") - - def is_last_distinct(self: Self) -> Self: - return reuse_series_implementation(self, "is_last_distinct") - - def unique(self: Self) -> Self: - return reuse_series_implementation(self, "unique", maintain_order=False) - - def replace_strict( - self: Self, - old: Sequence[Any] | Mapping[Any, Any], - new: Sequence[Any], - *, - return_dtype: DType | type[DType] | None, - ) -> Self: - return reuse_series_implementation( - self, "replace_strict", old=old, new=new, return_dtype=return_dtype - ) - - def sort(self: Self, *, descending: bool, nulls_last: bool) -> Self: - return reuse_series_implementation( - self, "sort", descending=descending, nulls_last=nulls_last - ) - - def quantile( - self: Self, - quantile: float, - interpolation: Literal["nearest", "higher", "lower", "midpoint", "linear"], - ) -> Self: - return reuse_series_implementation( - self, - "quantile", - returns_scalar=True, - quantile=quantile, - interpolation=interpolation, - ) - - def gather_every(self: Self, n: int, offset: int) -> Self: - return reuse_series_implementation(self, "gather_every", n=n, offset=offset) - - def clip(self: Self, lower_bound: Any | None, upper_bound: Any | None) -> Self: - return reuse_series_implementation( - self, "clip", lower_bound=lower_bound, upper_bound=upper_bound - ) + return self._reuse_series_implementation("shift", n=n) def over(self: Self, keys: Sequence[str], kind: ExprKind) -> Self: if not is_scalar_like(kind): @@ -456,9 +178,6 @@ def func(df: ArrowDataFrame) -> list[ArrowSeries]: version=self._version, ) - def mode(self: Self) -> Self: - return reuse_series_implementation(self, "mode") - def map_batches( self: Self, function: Callable[[Any], Any], @@ -497,84 +216,17 @@ def func(df: ArrowDataFrame) -> list[ArrowSeries]: version=self._version, ) - def is_finite(self: Self) -> Self: - return reuse_series_implementation(self, "is_finite") - def cum_count(self: Self, *, reverse: bool) -> Self: - return reuse_series_implementation(self, "cum_count", reverse=reverse) + return self._reuse_series_implementation("cum_count", reverse=reverse) def cum_min(self: Self, *, reverse: bool) -> Self: - return reuse_series_implementation(self, "cum_min", reverse=reverse) + return self._reuse_series_implementation("cum_min", reverse=reverse) def cum_max(self: Self, *, reverse: bool) -> Self: - return reuse_series_implementation(self, "cum_max", reverse=reverse) + return self._reuse_series_implementation("cum_max", reverse=reverse) def cum_prod(self: Self, *, reverse: bool) -> Self: - return reuse_series_implementation(self, "cum_prod", reverse=reverse) - - def rolling_sum( - self: Self, - window_size: int, - *, - min_samples: int | None, - center: bool, - ) -> Self: - return reuse_series_implementation( - self, - "rolling_sum", - window_size=window_size, - min_samples=min_samples, - center=center, - ) - - def rolling_mean( - self: Self, - window_size: int, - *, - min_samples: int | None, - center: bool, - ) -> Self: - return reuse_series_implementation( - self, - "rolling_mean", - window_size=window_size, - min_samples=min_samples, - center=center, - ) - - def rolling_var( - self: Self, - window_size: int, - *, - min_samples: int | None, - center: bool, - ddof: int, - ) -> Self: - return reuse_series_implementation( - self, - "rolling_var", - window_size=window_size, - min_samples=min_samples, - center=center, - ddof=ddof, - ) - - def rolling_std( - self: Self, - window_size: int, - *, - min_samples: int | None, - center: bool, - ddof: int, - ) -> Self: - return reuse_series_implementation( - self, - "rolling_std", - window_size=window_size, - min_samples=min_samples, - center=center, - ddof=ddof, - ) + return self._reuse_series_implementation("cum_prod", reverse=reverse) def rank( self: Self, @@ -582,8 +234,8 @@ def rank( *, descending: bool, ) -> Self: - return reuse_series_implementation( - self, "rank", method=method, descending=descending + return self._reuse_series_implementation( + "rank", method=method, descending=descending ) ewm_mean = not_implemented() From afddd4c88fed2f1517b567fa199f03b664ebe335 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 7 Mar 2025 14:51:20 +0000 Subject: [PATCH 27/58] refactor(DRAFT): Possibly all `ArrowExpr`? Main source of errors atm are expr namespaces - still needs migrating --- narwhals/_arrow/dataframe.py | 2 +- narwhals/_arrow/expr.py | 18 ++--- narwhals/_arrow/namespace.py | 124 ++++++++----------------------- narwhals/_arrow/series.py | 20 ++--- narwhals/_compliant/__init__.py | 4 + narwhals/_compliant/expr.py | 8 +- narwhals/_compliant/namespace.py | 4 + narwhals/_expression_parsing.py | 11 ++- narwhals/dataframe.py | 5 ++ narwhals/utils.py | 11 +++ 10 files changed, 85 insertions(+), 122 deletions(-) diff --git a/narwhals/_arrow/dataframe.py b/narwhals/_arrow/dataframe.py index 222d7fe334..a858e0a9fa 100644 --- a/narwhals/_arrow/dataframe.py +++ b/narwhals/_arrow/dataframe.py @@ -778,7 +778,7 @@ def unique( keep_idx = self.simple_select(*subset).is_unique() plx = self.__narwhals_namespace__() - return self.filter(plx._create_expr_from_series(keep_idx)) + return self.filter(plx._expr._from_series(keep_idx)) def gather_every(self: Self, n: int, offset: int) -> Self: return self._from_native_frame( diff --git a/narwhals/_arrow/expr.py b/narwhals/_arrow/expr.py index 4bee2a81f9..5538c1e725 100644 --- a/narwhals/_arrow/expr.py +++ b/narwhals/_arrow/expr.py @@ -29,6 +29,7 @@ from narwhals._arrow.namespace import ArrowNamespace from narwhals.dtypes import DType from narwhals.utils import Version + from narwhals.utils import _FullContext class ArrowExpr(EagerExpr["ArrowDataFrame", ArrowSeries]): @@ -45,6 +46,7 @@ def __init__( backend_version: tuple[int, ...], version: Version, call_kwargs: dict[str, Any] | None = None, + implementation: Implementation | None = None, ) -> None: self._call = call self._depth = depth @@ -63,8 +65,7 @@ def from_column_names( /, *, function_name: str, - backend_version: tuple[int, ...], - version: Version, + context: _FullContext, ) -> Self: def func(df: ArrowDataFrame) -> list[ArrowSeries]: try: @@ -91,16 +92,13 @@ def func(df: ArrowDataFrame) -> list[ArrowSeries]: function_name=function_name, evaluate_output_names=evaluate_column_names, alias_output_names=None, - backend_version=backend_version, - version=version, + backend_version=context._backend_version, + version=context._version, ) @classmethod def from_column_indices( - cls: type[Self], - *column_indices: int, - backend_version: tuple[int, ...], - version: Version, + cls: type[Self], *column_indices: int, context: _FullContext ) -> Self: from narwhals._arrow.series import ArrowSeries @@ -121,8 +119,8 @@ def func(df: ArrowDataFrame) -> list[ArrowSeries]: function_name="nth", evaluate_output_names=lambda df: [df.columns[i] for i in column_indices], alias_output_names=None, - backend_version=backend_version, - version=version, + backend_version=context._backend_version, + version=context._version, ) def __narwhals_namespace__(self: Self) -> ArrowNamespace: diff --git a/narwhals/_arrow/namespace.py b/narwhals/_arrow/namespace.py index ef033f26d7..90958c913e 100644 --- a/narwhals/_arrow/namespace.py +++ b/narwhals/_arrow/namespace.py @@ -57,59 +57,7 @@ def _expr(self) -> type[ArrowExpr]: def _series(self) -> type[ArrowSeries]: return ArrowSeries - def _create_expr_from_callable( - self: Self, - func: Callable[[ArrowDataFrame], Sequence[ArrowSeries]], - *, - depth: int, - function_name: str, - evaluate_output_names: Callable[[ArrowDataFrame], Sequence[str]], - alias_output_names: Callable[[Sequence[str]], Sequence[str]] | None, - call_kwargs: dict[str, Any] | None = None, - ) -> ArrowExpr: - from narwhals._arrow.expr import ArrowExpr - - return ArrowExpr( - func, - depth=depth, - function_name=function_name, - evaluate_output_names=evaluate_output_names, - alias_output_names=alias_output_names, - backend_version=self._backend_version, - version=self._version, - call_kwargs=call_kwargs, - ) - - def _create_expr_from_series(self: Self, series: ArrowSeries) -> ArrowExpr: - from narwhals._arrow.expr import ArrowExpr - - return ArrowExpr( - lambda _df: [series], - depth=0, - function_name="series", - evaluate_output_names=lambda _df: [series.name], - alias_output_names=None, - backend_version=self._backend_version, - version=self._version, - ) - - def _create_series_from_scalar( - self: Self, value: Any, *, reference_series: ArrowSeries - ) -> ArrowSeries: - from narwhals._arrow.series import ArrowSeries - - if self._backend_version < (13,) and hasattr(value, "as_py"): - value = value.as_py() - return ArrowSeries._from_iterable( - [value], - name=reference_series.name, - backend_version=self._backend_version, - version=self._version, - ) - def _create_compliant_series(self: Self, value: Any) -> ArrowSeries: - from narwhals._arrow.series import ArrowSeries - return ArrowSeries( native_series=pa.chunked_array([value]), name="", @@ -127,39 +75,26 @@ def __init__( # --- selection --- def col(self: Self, *column_names: str) -> ArrowExpr: - from narwhals._arrow.expr import ArrowExpr - - return ArrowExpr.from_column_names( - passthrough_column_names(column_names), - function_name="col", - backend_version=self._backend_version, - version=self._version, + return self._expr.from_column_names( + passthrough_column_names(column_names), function_name="col", context=self ) def exclude(self: Self, excluded_names: Container[str]) -> ArrowExpr: - return ArrowExpr.from_column_names( + return self._expr.from_column_names( partial(exclude_column_names, names=excluded_names), function_name="exclude", - backend_version=self._backend_version, - version=self._version, + context=self, ) def nth(self: Self, *column_indices: int) -> ArrowExpr: - from narwhals._arrow.expr import ArrowExpr - - return ArrowExpr.from_column_indices( - *column_indices, backend_version=self._backend_version, version=self._version - ) + return self._expr.from_column_indices(*column_indices, context=self) def len(self: Self) -> ArrowExpr: # coverage bug? this is definitely hit - return ArrowExpr( # pragma: no cover + return self._expr( # pragma: no cover lambda df: [ ArrowSeries._from_iterable( - [len(df._native_frame)], - name="len", - backend_version=self._backend_version, - version=self._version, + [len(df._native_frame)], name="len", context=self ) ], depth=0, @@ -171,26 +106,20 @@ def len(self: Self) -> ArrowExpr: ) def all(self: Self) -> ArrowExpr: - return ArrowExpr.from_column_names( - get_column_names, - function_name="all", - backend_version=self._backend_version, - version=self._version, + return self._expr.from_column_names( + get_column_names, function_name="all", context=self ) def lit(self: Self, value: Any, dtype: DType | None) -> ArrowExpr: def _lit_arrow_series(_: ArrowDataFrame) -> ArrowSeries: arrow_series = ArrowSeries._from_iterable( - data=[value], - name="literal", - backend_version=self._backend_version, - version=self._version, + data=[value], name="literal", context=self ) if dtype: return arrow_series.cast(dtype) return arrow_series - return ArrowExpr( + return self._expr( lambda df: [_lit_arrow_series(df)], depth=0, function_name="lit", @@ -200,17 +129,20 @@ def _lit_arrow_series(_: ArrowDataFrame) -> ArrowSeries: version=self._version, ) - def all_horizontal(self: Self, *exprs: ArrowExpr) -> ArrowExpr: + # NOTE: Needs to be resolved in `EagerNamespace` + # Probably, by adding an `EagerExprT` typevar + def all_horizontal(self: Self, *exprs: ArrowExpr) -> ArrowExpr: # type: ignore[override] def func(df: ArrowDataFrame) -> list[ArrowSeries]: series = chain.from_iterable(expr(df) for expr in exprs) return [reduce(operator.and_, align_series_full_broadcast(*series))] - return self._create_expr_from_callable( + return self._expr._from_callable( func=func, depth=max(x._depth for x in exprs) + 1, function_name="all_horizontal", evaluate_output_names=combine_evaluate_output_names(*exprs), alias_output_names=combine_alias_output_names(*exprs), + context=self, ) def any_horizontal(self: Self, *exprs: ArrowExpr) -> ArrowExpr: @@ -218,12 +150,13 @@ def func(df: ArrowDataFrame) -> list[ArrowSeries]: series = chain.from_iterable(expr(df) for expr in exprs) return [reduce(operator.or_, align_series_full_broadcast(*series))] - return self._create_expr_from_callable( + return self._expr._from_callable( func=func, depth=max(x._depth for x in exprs) + 1, function_name="any_horizontal", evaluate_output_names=combine_evaluate_output_names(*exprs), alias_output_names=combine_alias_output_names(*exprs), + context=self, ) def sum_horizontal(self: Self, *exprs: ArrowExpr) -> ArrowExpr: @@ -232,12 +165,13 @@ def func(df: ArrowDataFrame) -> list[ArrowSeries]: series = (s.fill_null(0, strategy=None, limit=None) for s in it) return [reduce(operator.add, align_series_full_broadcast(*series))] - return self._create_expr_from_callable( + return self._expr._from_callable( func=func, depth=max(x._depth for x in exprs) + 1, function_name="sum_horizontal", evaluate_output_names=combine_evaluate_output_names(*exprs), alias_output_names=combine_alias_output_names(*exprs), + context=self, ) def mean_horizontal(self: Self, *exprs: ArrowExpr) -> IntoArrowExpr: @@ -253,12 +187,13 @@ def func(df: ArrowDataFrame) -> list[ArrowSeries]: ) return [reduce(operator.add, series) / reduce(operator.add, non_na)] - return self._create_expr_from_callable( + return self._expr._from_callable( func=func, depth=max(x._depth for x in exprs) + 1, function_name="mean_horizontal", evaluate_output_names=combine_evaluate_output_names(*exprs), alias_output_names=combine_alias_output_names(*exprs), + context=self, ) def min_horizontal(self: Self, *exprs: ArrowExpr) -> ArrowExpr: @@ -281,12 +216,13 @@ def func(df: ArrowDataFrame) -> list[ArrowSeries]: ) ] - return self._create_expr_from_callable( + return self._expr._from_callable( func=func, depth=max(x._depth for x in exprs) + 1, function_name="min_horizontal", evaluate_output_names=combine_evaluate_output_names(*exprs), alias_output_names=combine_alias_output_names(*exprs), + context=self, ) def max_horizontal(self: Self, *exprs: ArrowExpr) -> ArrowExpr: @@ -310,12 +246,13 @@ def func(df: ArrowDataFrame) -> list[ArrowSeries]: ) ] - return self._create_expr_from_callable( + return self._expr._from_callable( func=func, depth=max(x._depth for x in exprs) + 1, function_name="max_horizontal", evaluate_output_names=combine_evaluate_output_names(*exprs), alias_output_names=combine_alias_output_names(*exprs), + context=self, ) def concat( @@ -381,12 +318,13 @@ def func(df: ArrowDataFrame) -> list[ArrowSeries]: ) ] - return self._create_expr_from_callable( + return self._expr._from_callable( func=func, depth=max(x._depth for x in exprs) + 1, function_name="concat_str", evaluate_output_names=combine_evaluate_output_names(*exprs), alias_output_names=combine_alias_output_names(*exprs), + context=self, ) @@ -407,16 +345,13 @@ def __init__( self._version = version def __call__(self: Self, df: ArrowDataFrame) -> Sequence[ArrowSeries]: - plx = df.__narwhals_namespace__() condition = self._condition(df)[0] condition_native = condition._native_series if isinstance(self._then_value, ArrowExpr): value_series = self._then_value(df)[0] else: - value_series = plx._create_series_from_scalar( - self._then_value, reference_series=condition.alias("literal") - ) + value_series = condition.alias("literal")._from_scalar(self._then_value) value_series._broadcast = True value_series_native = extract_dataframe_comparand( len(df), value_series, self._backend_version @@ -474,6 +409,7 @@ def __init__( backend_version: tuple[int, ...], version: Version, call_kwargs: dict[str, Any] | None = None, + implementation: Implementation | None = None, ) -> None: self._backend_version = backend_version self._version = version diff --git a/narwhals/_arrow/series.py b/narwhals/_arrow/series.py index f6fc552cf5..d34d0951af 100644 --- a/narwhals/_arrow/series.py +++ b/narwhals/_arrow/series.py @@ -54,6 +54,7 @@ from narwhals.typing import _1DArray from narwhals.typing import _2DArray from narwhals.utils import Version + from narwhals.utils import _FullContext # TODO @dangotbanned: move into `_arrow.utils` @@ -140,16 +141,20 @@ def _from_iterable( data: Iterable[Any], name: str, *, - backend_version: tuple[int, ...], - version: Version, + context: _FullContext, ) -> Self: return cls( chunked_array([data]), name=name, - backend_version=backend_version, - version=version, + backend_version=context._backend_version, + version=context._version, ) + def _from_scalar(self, value: Any) -> Self: + if self._backend_version < (13,) and hasattr(value, "as_py"): + value = value.as_py() + return super()._from_scalar(value) + def __narwhals_namespace__(self: Self) -> ArrowNamespace: from narwhals._arrow.namespace import ArrowNamespace @@ -570,12 +575,7 @@ def arg_true(self: Self) -> Self: ser = self._native_series res = np.flatnonzero(ser) - return self._from_iterable( - res, - name=self.name, - backend_version=self._backend_version, - version=self._version, - ) + return self._from_iterable(res, name=self.name, context=self) def item(self: Self, index: int | None = None) -> Any: if index is None: diff --git a/narwhals/_compliant/__init__.py b/narwhals/_compliant/__init__.py index 186a0657bb..c4dbdcd9ef 100644 --- a/narwhals/_compliant/__init__.py +++ b/narwhals/_compliant/__init__.py @@ -19,6 +19,8 @@ from narwhals._compliant.typing import CompliantFrameT from narwhals._compliant.typing import CompliantSeriesOrNativeExprT_co from narwhals._compliant.typing import CompliantSeriesT_co +from narwhals._compliant.typing import EagerDataFrameT +from narwhals._compliant.typing import EagerSeriesT from narwhals._compliant.typing import IntoCompliantExpr __all__ = [ @@ -33,10 +35,12 @@ "CompliantSeriesOrNativeExprT_co", "CompliantSeriesT_co", "EagerDataFrame", + "EagerDataFrameT", "EagerExpr", "EagerNamespace", "EagerSelectorNamespace", "EagerSeries", + "EagerSeriesT", "EvalNames", "EvalSeries", "IntoCompliantExpr", diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index 65a0c41363..ba896a66dc 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -295,16 +295,16 @@ def _from_callable( ) @classmethod - def _from_series(cls, series: EagerSeriesT, *, context: _FullContext) -> Self: + def _from_series(cls, series: EagerSeriesT) -> Self: return cls( lambda _df: [series], depth=0, function_name="series", evaluate_output_names=lambda _df: [series.name], alias_output_names=None, - implementation=context._implementation, - backend_version=context._backend_version, - version=context._version, + implementation=series._implementation, + backend_version=series._backend_version, + version=series._version, ) @classmethod diff --git a/narwhals/_compliant/namespace.py b/narwhals/_compliant/namespace.py index d8d6df0a7c..49fb4915d2 100644 --- a/narwhals/_compliant/namespace.py +++ b/narwhals/_compliant/namespace.py @@ -8,6 +8,7 @@ from narwhals._compliant.typing import CompliantSeriesOrNativeExprT_co from narwhals._compliant.typing import EagerDataFrameT from narwhals._compliant.typing import EagerSeriesT +from narwhals.utils import deprecated if TYPE_CHECKING: from narwhals._compliant.expr import CompliantExpr @@ -49,3 +50,6 @@ def _series(self) -> type[EagerSeriesT]: ... def all_horizontal( self, *exprs: EagerExpr[EagerDataFrameT, EagerSeriesT] ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: ... + + @deprecated("ref'd in untyped code") + def _create_compliant_series(self, value: Any) -> EagerSeriesT: ... diff --git a/narwhals/_expression_parsing.py b/narwhals/_expression_parsing.py index 57bbb71c14..714a510f68 100644 --- a/narwhals/_expression_parsing.py +++ b/narwhals/_expression_parsing.py @@ -20,6 +20,7 @@ from narwhals.exceptions import ShapeError from narwhals.utils import Implementation from narwhals.utils import is_compliant_expr +from narwhals.utils import is_eager_namespace if TYPE_CHECKING: from typing_extensions import Never @@ -167,7 +168,7 @@ def func(df: CompliantDataFrame[Any]) -> Sequence[CompliantSeries]: ) out: list[CompliantSeries] = [ - plx._create_series_from_scalar( + plx._create_series_from_scalar( # type: ignore # noqa: PGH003 getattr(series, attr)(**extra_kwargs, **_kwargs), reference_series=series, # type: ignore[arg-type] ) @@ -185,7 +186,7 @@ def func(df: CompliantDataFrame[Any]) -> Sequence[CompliantSeries]: raise AssertionError(msg) return out - return plx._create_expr_from_callable( # type: ignore[return-value] + return plx._create_expr_from_callable( # type: ignore[return-value, union-attr] func, # type: ignore[arg-type] depth=expr._depth + 1, function_name=f"{expr._function_name}->{attr}", @@ -221,7 +222,7 @@ def reuse_series_namespace_implementation( kwargs: keyword arguments to pass to function. """ plx = expr.__narwhals_namespace__() - return plx._create_expr_from_callable( # type: ignore[return-value] + return plx._create_expr_from_callable( # type: ignore[return-value, union-attr] lambda df: [ getattr(getattr(series, series_namespace), attr)(**kwargs) for series in expr(df) # type: ignore[arg-type] @@ -291,8 +292,12 @@ def extract_compliant( if isinstance(other, str) and not str_as_lit: return plx.col(other) if is_narwhals_series(other): + if is_eager_namespace(plx): + return plx._expr._from_series(other._compliant_series) return plx._create_expr_from_series(other._compliant_series) # type: ignore[attr-defined] if is_numpy_array(other): + if is_eager_namespace(plx): + return plx._expr._from_series(plx._create_compliant_series(other)) series = plx._create_compliant_series(other) # type: ignore[attr-defined] return plx._create_expr_from_series(series) # type: ignore[attr-defined] return other diff --git a/narwhals/dataframe.py b/narwhals/dataframe.py index e5a2a580c8..d8f0fc09fd 100644 --- a/narwhals/dataframe.py +++ b/narwhals/dataframe.py @@ -33,6 +33,7 @@ from narwhals.utils import find_stacklevel from narwhals.utils import flatten from narwhals.utils import generate_repr +from narwhals.utils import is_eager_namespace from narwhals.utils import is_sequence_but_not_str from narwhals.utils import issue_deprecation_warning from narwhals.utils import parse_version @@ -429,6 +430,8 @@ def _extract_compliant(self: Self, arg: Any) -> Any: if isinstance(arg, BaseFrame): return arg._compliant_frame if isinstance(arg, Series): + if is_eager_namespace(plx): + return plx._expr._from_series(arg._compliant_series) return plx._create_expr_from_series(arg._compliant_series) if isinstance(arg, Expr): return arg._to_compliant_expr(self.__narwhals_namespace__()) @@ -443,6 +446,8 @@ def _extract_compliant(self: Self, arg: Any) -> Any: ) raise TypeError(msg) if is_numpy_array(arg): + if is_eager_namespace(plx): + return plx._expr._from_series(plx._create_compliant_series(arg)) return plx._create_expr_from_series(plx._create_compliant_series(arg)) raise InvalidIntoExprError.from_invalid_type(type(arg)) diff --git a/narwhals/utils.py b/narwhals/utils.py index 798c6f5221..462ca4d0b1 100644 --- a/narwhals/utils.py +++ b/narwhals/utils.py @@ -56,6 +56,9 @@ from narwhals._compliant import CompliantFrameT from narwhals._compliant import CompliantSeriesOrNativeExprT_co from narwhals._compliant import CompliantSeriesT_co + from narwhals._compliant import EagerDataFrameT + from narwhals._compliant import EagerNamespace + from narwhals._compliant import EagerSeriesT from narwhals.dataframe import DataFrame from narwhals.dataframe import LazyFrame from narwhals.dtypes import DType @@ -1399,6 +1402,14 @@ def is_compliant_expr( return hasattr(obj, "__narwhals_expr__") +# NOTE: Temporary - just to introduce a path for the Arrow WIP +def is_eager_namespace( + obj: EagerNamespace[EagerDataFrameT, EagerSeriesT] | Any, +) -> TypeIs[EagerNamespace[EagerDataFrameT, EagerSeriesT]]: + return type(obj).__name__ == "ArrowNamespace" + # return all(hasattr(obj, name) for name in ("selectors", "_expr", "_series")) # noqa: ERA001 + + def has_native_namespace(obj: Any) -> TypeIs[SupportsNativeNamespace]: return hasattr(obj, "__native_namespace__") From f87a629cbeb495d89f604f8742effbb796f95791 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 7 Mar 2025 14:58:22 +0000 Subject: [PATCH 28/58] refactor: migrate to `ArrowExpr._reuse_series_namespace_implementation` Only 3 errors left - all seem to be `Expr.is_in` --- narwhals/_arrow/expr_cat.py | 6 +-- narwhals/_arrow/expr_dt.py | 72 ++++++++++++++++++------------------ narwhals/_arrow/expr_list.py | 4 +- narwhals/_arrow/expr_str.py | 48 +++++++++++------------- 4 files changed, 60 insertions(+), 70 deletions(-) diff --git a/narwhals/_arrow/expr_cat.py b/narwhals/_arrow/expr_cat.py index 6a26ee97f6..5c8861e3ba 100644 --- a/narwhals/_arrow/expr_cat.py +++ b/narwhals/_arrow/expr_cat.py @@ -2,8 +2,6 @@ from typing import TYPE_CHECKING -from narwhals._expression_parsing import reuse_series_namespace_implementation - if TYPE_CHECKING: from typing_extensions import Self @@ -15,6 +13,6 @@ def __init__(self: Self, expr: ArrowExpr) -> None: self._compliant_expr = expr def get_categories(self: Self) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "cat", "get_categories" + return self._compliant_expr._reuse_series_namespace_implementation( + "cat", "get_categories" ) diff --git a/narwhals/_arrow/expr_dt.py b/narwhals/_arrow/expr_dt.py index 30d2e22c8f..88d026ceb5 100644 --- a/narwhals/_arrow/expr_dt.py +++ b/narwhals/_arrow/expr_dt.py @@ -2,8 +2,6 @@ from typing import TYPE_CHECKING -from narwhals._expression_parsing import reuse_series_namespace_implementation - if TYPE_CHECKING: from typing_extensions import Self @@ -16,92 +14,92 @@ def __init__(self: Self, expr: ArrowExpr) -> None: self._compliant_expr = expr def to_string(self: Self, format: str) -> ArrowExpr: # noqa: A002 - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "to_string", format=format + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "to_string", format=format ) def replace_time_zone(self: Self, time_zone: str | None) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "replace_time_zone", time_zone=time_zone + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "replace_time_zone", time_zone=time_zone ) def convert_time_zone(self: Self, time_zone: str) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "convert_time_zone", time_zone=time_zone + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "convert_time_zone", time_zone=time_zone ) def timestamp(self: Self, time_unit: TimeUnit) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "timestamp", time_unit=time_unit + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "timestamp", time_unit=time_unit ) def date(self: Self) -> ArrowExpr: - return reuse_series_namespace_implementation(self._compliant_expr, "dt", "date") + return self._compliant_expr._reuse_series_namespace_implementation("dt", "date") def year(self: Self) -> ArrowExpr: - return reuse_series_namespace_implementation(self._compliant_expr, "dt", "year") + return self._compliant_expr._reuse_series_namespace_implementation("dt", "year") def month(self: Self) -> ArrowExpr: - return reuse_series_namespace_implementation(self._compliant_expr, "dt", "month") + return self._compliant_expr._reuse_series_namespace_implementation("dt", "month") def day(self: Self) -> ArrowExpr: - return reuse_series_namespace_implementation(self._compliant_expr, "dt", "day") + return self._compliant_expr._reuse_series_namespace_implementation("dt", "day") def hour(self: Self) -> ArrowExpr: - return reuse_series_namespace_implementation(self._compliant_expr, "dt", "hour") + return self._compliant_expr._reuse_series_namespace_implementation("dt", "hour") def minute(self: Self) -> ArrowExpr: - return reuse_series_namespace_implementation(self._compliant_expr, "dt", "minute") + return self._compliant_expr._reuse_series_namespace_implementation("dt", "minute") def second(self: Self) -> ArrowExpr: - return reuse_series_namespace_implementation(self._compliant_expr, "dt", "second") + return self._compliant_expr._reuse_series_namespace_implementation("dt", "second") def millisecond(self: Self) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "millisecond" + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "millisecond" ) def microsecond(self: Self) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "microsecond" + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "microsecond" ) def nanosecond(self: Self) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "nanosecond" + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "nanosecond" ) def ordinal_day(self: Self) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "ordinal_day" + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "ordinal_day" ) def weekday(self: Self) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "weekday" + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "weekday" ) def total_minutes(self: Self) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "total_minutes" + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "total_minutes" ) def total_seconds(self: Self) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "total_seconds" + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "total_seconds" ) def total_milliseconds(self: Self) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "total_milliseconds" + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "total_milliseconds" ) def total_microseconds(self: Self) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "total_microseconds" + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "total_microseconds" ) def total_nanoseconds(self: Self) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "total_nanoseconds" + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "total_nanoseconds" ) diff --git a/narwhals/_arrow/expr_list.py b/narwhals/_arrow/expr_list.py index 8e8e4c1f00..90983c8b0a 100644 --- a/narwhals/_arrow/expr_list.py +++ b/narwhals/_arrow/expr_list.py @@ -2,8 +2,6 @@ from typing import TYPE_CHECKING -from narwhals._expression_parsing import reuse_series_namespace_implementation - if TYPE_CHECKING: from typing_extensions import Self @@ -15,4 +13,4 @@ def __init__(self: Self, expr: ArrowExpr) -> None: self._expr = expr def len(self: Self) -> ArrowExpr: - return reuse_series_namespace_implementation(self._expr, "list", "len") + return self._expr._reuse_series_namespace_implementation("list", "len") diff --git a/narwhals/_arrow/expr_str.py b/narwhals/_arrow/expr_str.py index 11ba75914b..c3d71898e4 100644 --- a/narwhals/_arrow/expr_str.py +++ b/narwhals/_arrow/expr_str.py @@ -2,8 +2,6 @@ from typing import TYPE_CHECKING -from narwhals._expression_parsing import reuse_series_namespace_implementation - if TYPE_CHECKING: from typing_extensions import Self @@ -15,15 +13,14 @@ def __init__(self: Self, expr: ArrowExpr) -> None: self._compliant_expr = expr def len_chars(self: Self) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "str", "len_chars" + return self._compliant_expr._reuse_series_namespace_implementation( + "str", "len_chars" ) def replace( self: Self, pattern: str, value: str, *, literal: bool, n: int ) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, + return self._compliant_expr._reuse_series_namespace_implementation( "str", "replace", pattern=pattern, @@ -33,8 +30,7 @@ def replace( ) def replace_all(self: Self, pattern: str, value: str, *, literal: bool) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, + return self._compliant_expr._reuse_series_namespace_implementation( "str", "replace_all", pattern=pattern, @@ -43,46 +39,46 @@ def replace_all(self: Self, pattern: str, value: str, *, literal: bool) -> Arrow ) def strip_chars(self: Self, characters: str | None) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "str", "strip_chars", characters=characters + return self._compliant_expr._reuse_series_namespace_implementation( + "str", "strip_chars", characters=characters ) def starts_with(self: Self, prefix: str) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "str", "starts_with", prefix=prefix + return self._compliant_expr._reuse_series_namespace_implementation( + "str", "starts_with", prefix=prefix ) def ends_with(self: Self, suffix: str) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "str", "ends_with", suffix=suffix + return self._compliant_expr._reuse_series_namespace_implementation( + "str", "ends_with", suffix=suffix ) def contains(self: Self, pattern: str, *, literal: bool) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "str", "contains", pattern=pattern, literal=literal + return self._compliant_expr._reuse_series_namespace_implementation( + "str", "contains", pattern=pattern, literal=literal ) def slice(self: Self, offset: int, length: int | None) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "str", "slice", offset=offset, length=length + return self._compliant_expr._reuse_series_namespace_implementation( + "str", "slice", offset=offset, length=length ) def split(self: Self, by: str) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "str", "split", by=by + return self._compliant_expr._reuse_series_namespace_implementation( + "str", "split", by=by ) def to_datetime(self: Self, format: str | None) -> ArrowExpr: # noqa: A002 - return reuse_series_namespace_implementation( - self._compliant_expr, "str", "to_datetime", format=format + return self._compliant_expr._reuse_series_namespace_implementation( + "str", "to_datetime", format=format ) def to_uppercase(self: Self) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "str", "to_uppercase" + return self._compliant_expr._reuse_series_namespace_implementation( + "str", "to_uppercase" ) def to_lowercase(self: Self) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "str", "to_lowercase" + return self._compliant_expr._reuse_series_namespace_implementation( + "str", "to_lowercase" ) From d509fc91fdf30bc0e0a7ea3b7616a1d99df68ece Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 7 Mar 2025 15:02:39 +0000 Subject: [PATCH 29/58] fix: fix typo lol --- narwhals/_compliant/expr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index ba896a66dc..3cda425e21 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -575,7 +575,7 @@ def fill_null( ) def is_in(self, other: Any) -> Self: - return self._reuse_series_implementation("is_in", other="other") + return self._reuse_series_implementation("is_in", other=other) def arg_true(self) -> Self: return self._reuse_series_implementation("arg_true") From 9d55d2a616113bc03162d0e09f4b5d5ab270ce3c Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 7 Mar 2025 15:32:44 +0000 Subject: [PATCH 30/58] chore: remove `ArrowExpr` parts from `_expression_parsing` Fully migrated the "reuse" stuff for `pyarrow` - next up `pandas`! --- narwhals/_expression_parsing.py | 61 ++++++--------------------------- 1 file changed, 11 insertions(+), 50 deletions(-) diff --git a/narwhals/_expression_parsing.py b/narwhals/_expression_parsing.py index 714a510f68..8794f2763b 100644 --- a/narwhals/_expression_parsing.py +++ b/narwhals/_expression_parsing.py @@ -18,7 +18,6 @@ from narwhals.dependencies import is_numpy_array from narwhals.exceptions import LengthChangingExprError from narwhals.exceptions import ShapeError -from narwhals.utils import Implementation from narwhals.utils import is_compliant_expr from narwhals.utils import is_eager_namespace @@ -26,7 +25,6 @@ from typing_extensions import Never from typing_extensions import TypeIs - from narwhals._arrow.expr import ArrowExpr from narwhals._compliant import CompliantExpr from narwhals._compliant import CompliantFrameT from narwhals._compliant import CompliantNamespace @@ -41,7 +39,6 @@ from narwhals.typing import _1DArray PandasLikeExprT = TypeVar("PandasLikeExprT", bound=PandasLikeExpr) - ArrowExprT = TypeVar("ArrowExprT", bound=ArrowExpr) T = TypeVar("T") @@ -105,34 +102,14 @@ def maybe_evaluate_expr( return expr -@overload def reuse_series_implementation( expr: PandasLikeExprT, attr: str, *, returns_scalar: bool = False, - **kwargs: Any, -) -> PandasLikeExprT: ... - - -@overload -def reuse_series_implementation( - expr: ArrowExprT, - attr: str, - *, - returns_scalar: bool = False, - **kwargs: Any, -) -> ArrowExprT: ... - - -def reuse_series_implementation( - expr: ArrowExprT | PandasLikeExprT, - attr: str, - *, - returns_scalar: bool = False, call_kwargs: dict[str, Any] | None = None, **expressifiable_args: Any, -) -> ArrowExprT | PandasLikeExprT: +) -> PandasLikeExprT: """Reuse Series implementation for expression. If Series.foo is already defined, and we'd like Expr.foo to be the same, we can @@ -159,18 +136,10 @@ def func(df: CompliantDataFrame[Any]) -> Sequence[CompliantSeries]: }, } - # For PyArrow.Series, we return Python Scalars (like Polars does) instead of PyArrow Scalars. - # However, when working with expressions, we keep everything PyArrow-native. - extra_kwargs = ( - {"_return_py_scalar": False} - if returns_scalar and expr._implementation is Implementation.PYARROW - else {} - ) - out: list[CompliantSeries] = [ - plx._create_series_from_scalar( # type: ignore # noqa: PGH003 - getattr(series, attr)(**extra_kwargs, **_kwargs), - reference_series=series, # type: ignore[arg-type] + plx._create_series_from_scalar( + getattr(series, attr)(**_kwargs), + reference_series=series, ) if returns_scalar else getattr(series, attr)(**_kwargs) @@ -186,30 +155,22 @@ def func(df: CompliantDataFrame[Any]) -> Sequence[CompliantSeries]: raise AssertionError(msg) return out - return plx._create_expr_from_callable( # type: ignore[return-value, union-attr] + return plx._create_expr_from_callable( # type: ignore[return-value] func, # type: ignore[arg-type] depth=expr._depth + 1, function_name=f"{expr._function_name}->{attr}", - evaluate_output_names=expr._evaluate_output_names, # type: ignore[arg-type] + evaluate_output_names=expr._evaluate_output_names, alias_output_names=expr._alias_output_names, call_kwargs=call_kwargs, ) -@overload -def reuse_series_namespace_implementation( - expr: ArrowExprT, series_namespace: str, attr: str, **kwargs: Any -) -> ArrowExprT: ... -@overload def reuse_series_namespace_implementation( - expr: PandasLikeExprT, series_namespace: str, attr: str, **kwargs: Any -) -> PandasLikeExprT: ... -def reuse_series_namespace_implementation( - expr: ArrowExprT | PandasLikeExprT, + expr: PandasLikeExprT, series_namespace: str, attr: str, **kwargs: Any, -) -> ArrowExprT | PandasLikeExprT: +) -> PandasLikeExprT: """Reuse Series implementation for expression. Just like `reuse_series_implementation`, but for e.g. `Expr.dt.foo` instead @@ -222,14 +183,14 @@ def reuse_series_namespace_implementation( kwargs: keyword arguments to pass to function. """ plx = expr.__narwhals_namespace__() - return plx._create_expr_from_callable( # type: ignore[return-value, union-attr] + return plx._create_expr_from_callable( # type: ignore[return-value] lambda df: [ getattr(getattr(series, series_namespace), attr)(**kwargs) - for series in expr(df) # type: ignore[arg-type] + for series in expr(df) ], depth=expr._depth + 1, function_name=f"{expr._function_name}->{series_namespace}.{attr}", - evaluate_output_names=expr._evaluate_output_names, # type: ignore[arg-type] + evaluate_output_names=expr._evaluate_output_names, alias_output_names=expr._alias_output_names, ) From f16ece2a0dd96af3889642f3428cd53c918e3d1a Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 7 Mar 2025 15:34:33 +0000 Subject: [PATCH 31/58] ci: get coverage for properties https://github.com/narwhals-dev/narwhals/actions/runs/13723769208/job/38385197942?pr=2149 --- narwhals/_arrow/namespace.py | 2 +- narwhals/_arrow/series.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/narwhals/_arrow/namespace.py b/narwhals/_arrow/namespace.py index 90958c913e..946f5917d0 100644 --- a/narwhals/_arrow/namespace.py +++ b/narwhals/_arrow/namespace.py @@ -58,7 +58,7 @@ def _series(self) -> type[ArrowSeries]: return ArrowSeries def _create_compliant_series(self: Self, value: Any) -> ArrowSeries: - return ArrowSeries( + return self._series( native_series=pa.chunked_array([value]), name="", backend_version=self._backend_version, diff --git a/narwhals/_arrow/series.py b/narwhals/_arrow/series.py index d34d0951af..ab049ed160 100644 --- a/narwhals/_arrow/series.py +++ b/narwhals/_arrow/series.py @@ -118,7 +118,7 @@ def native(self) -> ArrowChunkedArray: def _change_version(self: Self, version: Version) -> Self: return self.__class__( - self._native_series, + self.native, name=self._name, backend_version=self._backend_version, version=version, From 388f71555287508605fc9aa14a1dcd3ea0e4fce7 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 7 Mar 2025 16:25:26 +0000 Subject: [PATCH 32/58] refactor(DRAFT): start migrating `PandasLike*` --- narwhals/_pandas_like/dataframe.py | 7 ++++--- narwhals/_pandas_like/expr.py | 4 ++-- narwhals/_pandas_like/namespace.py | 12 ++++++++++-- narwhals/_pandas_like/series.py | 10 +++++++--- 4 files changed, 23 insertions(+), 10 deletions(-) diff --git a/narwhals/_pandas_like/dataframe.py b/narwhals/_pandas_like/dataframe.py index e958575461..6684a97dc7 100644 --- a/narwhals/_pandas_like/dataframe.py +++ b/narwhals/_pandas_like/dataframe.py @@ -10,6 +10,7 @@ import numpy as np +from narwhals._compliant import EagerDataFrame from narwhals._expression_parsing import evaluate_into_exprs from narwhals._pandas_like.series import PANDAS_TO_NUMPY_DTYPE_MISSING from narwhals._pandas_like.series import PandasLikeSeries @@ -26,6 +27,8 @@ from narwhals._pandas_like.utils import select_columns_by_name from narwhals.dependencies import is_numpy_array_1d from narwhals.exceptions import InvalidOperationError +from narwhals.typing import CompliantDataFrame +from narwhals.typing import CompliantLazyFrame from narwhals.utils import Implementation from narwhals.utils import check_column_exists from narwhals.utils import generate_temporary_column_name @@ -54,8 +57,6 @@ from narwhals.typing import _2DArray from narwhals.utils import Version -from narwhals.typing import CompliantDataFrame -from narwhals.typing import CompliantLazyFrame CLASSICAL_NUMPY_DTYPES: frozenset[np.dtype[Any]] = frozenset( [ @@ -83,7 +84,7 @@ ) -class PandasLikeDataFrame(CompliantDataFrame["PandasLikeSeries"], CompliantLazyFrame): +class PandasLikeDataFrame(EagerDataFrame["PandasLikeSeries"], CompliantLazyFrame): # --- not in the spec --- def __init__( self: Self, diff --git a/narwhals/_pandas_like/expr.py b/narwhals/_pandas_like/expr.py index 77851de25b..a62b34b27f 100644 --- a/narwhals/_pandas_like/expr.py +++ b/narwhals/_pandas_like/expr.py @@ -8,7 +8,7 @@ from typing import Mapping from typing import Sequence -from narwhals._compliant import CompliantExpr +from narwhals._compliant import EagerExpr from narwhals._expression_parsing import ExprKind from narwhals._expression_parsing import evaluate_output_names_and_aliases from narwhals._expression_parsing import is_elementary_expression @@ -69,7 +69,7 @@ def window_kwargs_to_pandas_equivalent( return pandas_kwargs -class PandasLikeExpr(CompliantExpr["PandasLikeDataFrame", PandasLikeSeries]): +class PandasLikeExpr(EagerExpr["PandasLikeDataFrame", PandasLikeSeries]): def __init__( self: Self, call: Callable[[PandasLikeDataFrame], Sequence[PandasLikeSeries]], diff --git a/narwhals/_pandas_like/namespace.py b/narwhals/_pandas_like/namespace.py index ec2a696dbc..f58cc19144 100644 --- a/narwhals/_pandas_like/namespace.py +++ b/narwhals/_pandas_like/namespace.py @@ -11,7 +11,7 @@ from typing import Literal from typing import Sequence -from narwhals._compliant import CompliantNamespace +from narwhals._compliant import EagerNamespace from narwhals._expression_parsing import combine_alias_output_names from narwhals._expression_parsing import combine_evaluate_output_names from narwhals._pandas_like.dataframe import PandasLikeDataFrame @@ -40,7 +40,15 @@ _Scalar: TypeAlias = Any -class PandasLikeNamespace(CompliantNamespace[PandasLikeDataFrame, PandasLikeSeries]): +class PandasLikeNamespace(EagerNamespace[PandasLikeDataFrame, PandasLikeSeries]): + @property + def _expr(self) -> type[PandasLikeExpr]: + return PandasLikeExpr + + @property + def _series(self) -> type[PandasLikeSeries]: + return PandasLikeSeries + @property def selectors(self: Self) -> PandasSelectorNamespace: return PandasSelectorNamespace(self) diff --git a/narwhals/_pandas_like/series.py b/narwhals/_pandas_like/series.py index 0f9eba7e2a..bde17c03f1 100644 --- a/narwhals/_pandas_like/series.py +++ b/narwhals/_pandas_like/series.py @@ -9,6 +9,7 @@ from typing import cast from typing import overload +from narwhals._compliant import EagerSeries from narwhals._pandas_like.series_cat import PandasLikeSeriesCatNamespace from narwhals._pandas_like.series_dt import PandasLikeSeriesDateTimeNamespace from narwhals._pandas_like.series_list import PandasLikeSeriesListNamespace @@ -24,7 +25,6 @@ from narwhals._pandas_like.utils import set_index from narwhals.dependencies import is_numpy_scalar from narwhals.exceptions import InvalidOperationError -from narwhals.typing import CompliantSeries from narwhals.utils import Implementation from narwhals.utils import import_dtypes_module from narwhals.utils import validate_backend_version @@ -90,7 +90,7 @@ } -class PandasLikeSeries(CompliantSeries): +class PandasLikeSeries(EagerSeries[Any]): def __init__( self: Self, native_series: Any, @@ -112,6 +112,10 @@ def __init__( # the length of the whole dataframe, we just extract the scalar. self._broadcast = False + @property + def native(self) -> Any: + return self._native_series + def __native_namespace__(self: Self) -> ModuleType: if self._implementation in { Implementation.PANDAS, @@ -139,7 +143,7 @@ def __getitem__(self: Self, idx: int | slice | Sequence[int]) -> Any | Self: def _change_version(self: Self, version: Version) -> Self: return self.__class__( - self._native_series, + self.native, implementation=self._implementation, backend_version=self._backend_version, version=version, From 4bc483fba8affefaf028953be02191d1364dee6f Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 7 Mar 2025 17:11:29 +0000 Subject: [PATCH 33/58] refactor(DRAFT): big progress on `Pandas*` --- narwhals/_expression_parsing.py | 10 +- narwhals/_pandas_like/expr.py | 413 ++--------------------------- narwhals/_pandas_like/namespace.py | 126 +++------ narwhals/_pandas_like/series.py | 17 +- 4 files changed, 67 insertions(+), 499 deletions(-) diff --git a/narwhals/_expression_parsing.py b/narwhals/_expression_parsing.py index 8794f2763b..726589bc0f 100644 --- a/narwhals/_expression_parsing.py +++ b/narwhals/_expression_parsing.py @@ -137,13 +137,13 @@ def func(df: CompliantDataFrame[Any]) -> Sequence[CompliantSeries]: } out: list[CompliantSeries] = [ - plx._create_series_from_scalar( + plx._create_series_from_scalar( # type: ignore # noqa: PGH003 getattr(series, attr)(**_kwargs), reference_series=series, ) if returns_scalar else getattr(series, attr)(**_kwargs) - for series in expr(df) # type: ignore[arg-type] + for series in expr(df) # type: ignore # noqa: PGH003 ] _, aliases = evaluate_output_names_and_aliases(expr, df, []) if [s.name for s in out] != list(aliases): # pragma: no cover @@ -155,8 +155,8 @@ def func(df: CompliantDataFrame[Any]) -> Sequence[CompliantSeries]: raise AssertionError(msg) return out - return plx._create_expr_from_callable( # type: ignore[return-value] - func, # type: ignore[arg-type] + return plx._create_expr_from_callable( # type: ignore # noqa: PGH003 + func, depth=expr._depth + 1, function_name=f"{expr._function_name}->{attr}", evaluate_output_names=expr._evaluate_output_names, @@ -183,7 +183,7 @@ def reuse_series_namespace_implementation( kwargs: keyword arguments to pass to function. """ plx = expr.__narwhals_namespace__() - return plx._create_expr_from_callable( # type: ignore[return-value] + return plx._create_expr_from_callable( # type: ignore # noqa: PGH003 lambda df: [ getattr(getattr(series, series_namespace), attr)(**kwargs) for series in expr(df) diff --git a/narwhals/_pandas_like/expr.py b/narwhals/_pandas_like/expr.py index a62b34b27f..2d605ba087 100644 --- a/narwhals/_pandas_like/expr.py +++ b/narwhals/_pandas_like/expr.py @@ -5,14 +5,12 @@ from typing import Any from typing import Callable from typing import Literal -from typing import Mapping from typing import Sequence from narwhals._compliant import EagerExpr from narwhals._expression_parsing import ExprKind from narwhals._expression_parsing import evaluate_output_names_and_aliases from narwhals._expression_parsing import is_elementary_expression -from narwhals._expression_parsing import reuse_series_implementation from narwhals._pandas_like.expr_cat import PandasLikeExprCatNamespace from narwhals._pandas_like.expr_dt import PandasLikeExprDateTimeNamespace from narwhals._pandas_like.expr_list import PandasLikeExprListNamespace @@ -33,6 +31,7 @@ from narwhals.dtypes import DType from narwhals.utils import Implementation from narwhals.utils import Version + from narwhals.utils import _FullContext WINDOW_FUNCTIONS_TO_PANDAS_EQUIVALENT = { "cum_sum": "cumsum", @@ -93,14 +92,6 @@ def __init__( self._version = version self._call_kwargs = call_kwargs or {} - def __call__(self: Self, df: PandasLikeDataFrame) -> Sequence[PandasLikeSeries]: - return self._call(df) - - def __repr__(self) -> str: # pragma: no cover - return ( - f"PandasLikeExpr(depth={self._depth}, function_name={self._function_name}, )" - ) - def __narwhals_namespace__(self: Self) -> PandasLikeNamespace: from narwhals._pandas_like.namespace import PandasLikeNamespace @@ -110,29 +101,6 @@ def __narwhals_namespace__(self: Self) -> PandasLikeNamespace: def __narwhals_expr__(self) -> None: ... - def broadcast(self, kind: Literal[ExprKind.AGGREGATION, ExprKind.LITERAL]) -> Self: - # Make the resulting PandasLikeSeries with `_broadcast=True`. Then, - # when extracting native objects, `align_and_extract_native` will - # know what to do. - def func(df: PandasLikeDataFrame) -> list[PandasLikeSeries]: - results = [] - for result in self(df): - result._broadcast = True - results.append(result) - return results - - return self.__class__( - func, - depth=self._depth, - function_name=self._function_name, - evaluate_output_names=self._evaluate_output_names, - alias_output_names=self._alias_output_names, - backend_version=self._backend_version, - version=self._version, - implementation=self._implementation, - call_kwargs=self._call_kwargs, - ) - @classmethod def from_column_names( cls: type[Self], @@ -140,9 +108,7 @@ def from_column_names( /, *, function_name: str, - implementation: Implementation, - backend_version: tuple[int, ...], - version: Version, + context: _FullContext, ) -> Self: def func(df: PandasLikeDataFrame) -> list[PandasLikeSeries]: try: @@ -170,18 +136,14 @@ def func(df: PandasLikeDataFrame) -> list[PandasLikeSeries]: function_name=function_name, evaluate_output_names=evaluate_column_names, alias_output_names=None, - implementation=implementation, - backend_version=backend_version, - version=version, + implementation=context._implementation, + backend_version=context._backend_version, + version=context._version, ) @classmethod def from_column_indices( - cls: type[Self], - *column_indices: int, - implementation: Implementation, - backend_version: tuple[int, ...], - version: Version, + cls: type[Self], *column_indices: int, context: _FullContext ) -> Self: def func(df: PandasLikeDataFrame) -> list[PandasLikeSeries]: return [ @@ -200,161 +162,11 @@ def func(df: PandasLikeDataFrame) -> list[PandasLikeSeries]: function_name="nth", evaluate_output_names=lambda df: [df.columns[i] for i in column_indices], alias_output_names=None, - implementation=implementation, - backend_version=backend_version, - version=version, + implementation=context._implementation, + backend_version=context._backend_version, + version=context._version, ) - def cast(self: Self, dtype: DType | type[DType]) -> Self: - return reuse_series_implementation(self, "cast", dtype=dtype) - - def __eq__(self: Self, other: PandasLikeExpr | Any) -> Self: # type: ignore[override] - return reuse_series_implementation(self, "__eq__", other=other) - - def __ne__(self: Self, other: PandasLikeExpr | Any) -> Self: # type: ignore[override] - return reuse_series_implementation(self, "__ne__", other=other) - - def __ge__(self: Self, other: PandasLikeExpr | Any) -> Self: - return reuse_series_implementation(self, "__ge__", other=other) - - def __gt__(self: Self, other: PandasLikeExpr | Any) -> Self: - return reuse_series_implementation(self, "__gt__", other=other) - - def __le__(self: Self, other: PandasLikeExpr | Any) -> Self: - return reuse_series_implementation(self, "__le__", other=other) - - def __lt__(self: Self, other: PandasLikeExpr | Any) -> Self: - return reuse_series_implementation(self, "__lt__", other=other) - - def __and__(self: Self, other: PandasLikeExpr | bool | Any) -> Self: - return reuse_series_implementation(self, "__and__", other=other) - - def __or__(self: Self, other: PandasLikeExpr | bool | Any) -> Self: - return reuse_series_implementation(self, "__or__", other=other) - - def __add__(self: Self, other: PandasLikeExpr | Any) -> Self: - return reuse_series_implementation(self, "__add__", other=other) - - def __sub__(self: Self, other: PandasLikeExpr | Any) -> Self: - return reuse_series_implementation(self, "__sub__", other=other) - - def __rsub__(self: Self, other: PandasLikeExpr | Any) -> Self: - return reuse_series_implementation(self.alias("literal"), "__rsub__", other=other) - - def __mul__(self: Self, other: PandasLikeExpr | Any) -> Self: - return reuse_series_implementation(self, "__mul__", other=other) - - def __truediv__(self: Self, other: PandasLikeExpr | Any) -> Self: - return reuse_series_implementation(self, "__truediv__", other=other) - - def __rtruediv__(self: Self, other: PandasLikeExpr | Any) -> Self: - return reuse_series_implementation( - self.alias("literal"), "__rtruediv__", other=other - ) - - def __floordiv__(self: Self, other: PandasLikeExpr | Any) -> Self: - return reuse_series_implementation(self, "__floordiv__", other=other) - - def __rfloordiv__(self: Self, other: PandasLikeExpr | Any) -> Self: - return reuse_series_implementation( - self.alias("literal"), "__rfloordiv__", other=other - ) - - def __pow__(self: Self, other: PandasLikeExpr | Any) -> Self: - return reuse_series_implementation(self, "__pow__", other=other) - - def __rpow__(self: Self, other: PandasLikeExpr | Any) -> Self: - return reuse_series_implementation(self.alias("literal"), "__rpow__", other=other) - - def __mod__(self: Self, other: PandasLikeExpr | Any) -> Self: - return reuse_series_implementation(self, "__mod__", other=other) - - def __rmod__(self: Self, other: PandasLikeExpr | Any) -> Self: - return reuse_series_implementation(self.alias("literal"), "__rmod__", other=other) - - # Unary - def __invert__(self: Self) -> Self: - return reuse_series_implementation(self, "__invert__") - - # Reductions - def null_count(self: Self) -> Self: - return reuse_series_implementation(self, "null_count", returns_scalar=True) - - def n_unique(self: Self) -> Self: - return reuse_series_implementation(self, "n_unique", returns_scalar=True) - - def sum(self: Self) -> Self: - return reuse_series_implementation(self, "sum", returns_scalar=True) - - def count(self: Self) -> Self: - return reuse_series_implementation(self, "count", returns_scalar=True) - - def mean(self: Self) -> Self: - return reuse_series_implementation(self, "mean", returns_scalar=True) - - def median(self: Self) -> Self: - return reuse_series_implementation(self, "median", returns_scalar=True) - - def std(self: Self, *, ddof: int) -> Self: - return reuse_series_implementation( - self, "std", returns_scalar=True, call_kwargs={"ddof": ddof} - ) - - def var(self: Self, *, ddof: int) -> Self: - return reuse_series_implementation( - self, "var", returns_scalar=True, call_kwargs={"ddof": ddof} - ) - - def skew(self: Self) -> Self: - return reuse_series_implementation(self, "skew", returns_scalar=True) - - def any(self: Self) -> Self: - return reuse_series_implementation(self, "any", returns_scalar=True) - - def all(self: Self) -> Self: - return reuse_series_implementation(self, "all", returns_scalar=True) - - def max(self: Self) -> Self: - return reuse_series_implementation(self, "max", returns_scalar=True) - - def min(self: Self) -> Self: - return reuse_series_implementation(self, "min", returns_scalar=True) - - def arg_min(self: Self) -> Self: - return reuse_series_implementation(self, "arg_min", returns_scalar=True) - - def arg_max(self: Self) -> Self: - return reuse_series_implementation(self, "arg_max", returns_scalar=True) - - # Other - - def clip(self: Self, lower_bound: Any, upper_bound: Any) -> Self: - return reuse_series_implementation( - self, "clip", lower_bound=lower_bound, upper_bound=upper_bound - ) - - def is_null(self: Self) -> Self: - return reuse_series_implementation(self, "is_null") - - def is_nan(self: Self) -> Self: - return reuse_series_implementation(self, "is_nan") - - def fill_null( - self: Self, - value: Self | Any | None, - strategy: Literal["forward", "backward"] | None, - limit: int | None, - ) -> Self: - return reuse_series_implementation( - self, "fill_null", value=value, strategy=strategy, limit=limit - ) - - def is_in(self: Self, other: Any) -> Self: - return reuse_series_implementation(self, "is_in", other=other) - - def arg_true(self: Self) -> Self: - return reuse_series_implementation(self, "arg_true") - def ewm_mean( self: Self, *, @@ -366,8 +178,7 @@ def ewm_mean( min_samples: int, ignore_nulls: bool, ) -> Self: - return reuse_series_implementation( - self, + return self._reuse_series_implementation( "ewm_mean", com=com, span=span, @@ -378,84 +189,13 @@ def ewm_mean( ignore_nulls=ignore_nulls, ) - def filter(self: Self, *predicates: PandasLikeExpr) -> Self: - plx = self.__narwhals_namespace__() - other = plx.all_horizontal(*predicates) - return reuse_series_implementation(self, "filter", other=other) - - def drop_nulls(self: Self) -> Self: - return reuse_series_implementation(self, "drop_nulls") - - def replace_strict( - self: Self, - old: Sequence[Any] | Mapping[Any, Any], - new: Sequence[Any], - *, - return_dtype: DType | type[DType] | None, - ) -> Self: - return reuse_series_implementation( - self, "replace_strict", old=old, new=new, return_dtype=return_dtype - ) - - def sort(self: Self, *, descending: bool, nulls_last: bool) -> Self: - return reuse_series_implementation( - self, "sort", descending=descending, nulls_last=nulls_last - ) - - def abs(self: Self) -> Self: - return reuse_series_implementation(self, "abs") - def cum_sum(self: Self, *, reverse: bool) -> Self: - return reuse_series_implementation( - self, "cum_sum", call_kwargs={"reverse": reverse} + return self._reuse_series_implementation( + "cum_sum", call_kwargs={"reverse": reverse} ) - def unique(self: Self) -> Self: - return reuse_series_implementation(self, "unique", maintain_order=False) - - def diff(self: Self) -> Self: - return reuse_series_implementation(self, "diff") - def shift(self: Self, n: int) -> Self: - return reuse_series_implementation(self, "shift", call_kwargs={"n": n}) - - def sample( - self: Self, - n: int | None, - *, - fraction: float | None, - with_replacement: bool, - seed: int | None, - ) -> Self: - return reuse_series_implementation( - self, - "sample", - n=n, - fraction=fraction, - with_replacement=with_replacement, - seed=seed, - ) - - def alias(self: Self, name: str) -> Self: - def alias_output_names(names: Sequence[str]) -> Sequence[str]: - if len(names) != 1: - msg = f"Expected function with single output, found output names: {names}" - raise ValueError(msg) - return [name] - - # Define this one manually, so that we can - # override `output_names` and not increase depth - return self.__class__( - lambda df: [series.alias(name) for series in self._call(df)], - depth=self._depth, - function_name=self._function_name, - evaluate_output_names=self._evaluate_output_names, - alias_output_names=alias_output_names, - implementation=self._implementation, - backend_version=self._backend_version, - version=self._version, - call_kwargs=self._call_kwargs, - ) + return self._reuse_series_implementation("shift", call_kwargs={"n": n}) def over(self: Self, partition_by: Sequence[str], kind: ExprKind) -> Self: if not is_elementary_expression(self): @@ -526,46 +266,6 @@ def func(df: PandasLikeDataFrame) -> list[PandasLikeSeries]: version=self._version, ) - def is_unique(self: Self) -> Self: - return reuse_series_implementation(self, "is_unique") - - def is_first_distinct(self: Self) -> Self: - return reuse_series_implementation(self, "is_first_distinct") - - def is_last_distinct(self: Self) -> Self: - return reuse_series_implementation(self, "is_last_distinct") - - def quantile( - self: Self, - quantile: float, - interpolation: Literal["nearest", "higher", "lower", "midpoint", "linear"], - ) -> Self: - return reuse_series_implementation( - self, - "quantile", - quantile=quantile, - interpolation=interpolation, - returns_scalar=True, - ) - - def head(self: Self, n: int) -> Self: - return reuse_series_implementation(self, "head", n=n) - - def tail(self: Self, n: int) -> Self: - return reuse_series_implementation(self, "tail", n=n) - - def round(self: Self, decimals: int) -> Self: - return reuse_series_implementation(self, "round", decimals=decimals) - - def len(self: Self) -> Self: - return reuse_series_implementation(self, "len", returns_scalar=True) - - def gather_every(self: Self, n: int, offset: int) -> Self: - return reuse_series_implementation(self, "gather_every", n=n, offset=offset) - - def mode(self: Self) -> Self: - return reuse_series_implementation(self, "mode") - def map_batches( self: Self, function: Callable[[Any], Any], @@ -599,91 +299,24 @@ def func(df: PandasLikeDataFrame) -> list[PandasLikeSeries]: version=self._version, ) - def is_finite(self: Self) -> Self: - return reuse_series_implementation(self, "is_finite") - def cum_count(self: Self, *, reverse: bool) -> Self: - return reuse_series_implementation( - self, "cum_count", call_kwargs={"reverse": reverse} + return self._reuse_series_implementation( + "cum_count", call_kwargs={"reverse": reverse} ) def cum_min(self: Self, *, reverse: bool) -> Self: - return reuse_series_implementation( - self, "cum_min", call_kwargs={"reverse": reverse} + return self._reuse_series_implementation( + "cum_min", call_kwargs={"reverse": reverse} ) def cum_max(self: Self, *, reverse: bool) -> Self: - return reuse_series_implementation( - self, "cum_max", call_kwargs={"reverse": reverse} + return self._reuse_series_implementation( + "cum_max", call_kwargs={"reverse": reverse} ) def cum_prod(self: Self, *, reverse: bool) -> Self: - return reuse_series_implementation( - self, "cum_prod", call_kwargs={"reverse": reverse} - ) - - def rolling_sum( - self: Self, - window_size: int, - *, - min_samples: int | None, - center: bool, - ) -> Self: - return reuse_series_implementation( - self, - "rolling_sum", - window_size=window_size, - min_samples=min_samples, - center=center, - ) - - def rolling_mean( - self: Self, - window_size: int, - *, - min_samples: int | None, - center: bool, - ) -> Self: - return reuse_series_implementation( - self, - "rolling_mean", - window_size=window_size, - min_samples=min_samples, - center=center, - ) - - def rolling_var( - self: Self, - window_size: int, - *, - min_samples: int | None, - center: bool, - ddof: int, - ) -> Self: - return reuse_series_implementation( - self, - "rolling_var", - window_size=window_size, - min_samples=min_samples, - center=center, - ddof=ddof, - ) - - def rolling_std( - self: Self, - window_size: int, - *, - min_samples: int | None, - center: bool, - ddof: int, - ) -> Self: - return reuse_series_implementation( - self, - "rolling_std", - window_size=window_size, - min_samples=min_samples, - center=center, - ddof=ddof, + return self._reuse_series_implementation( + "cum_prod", call_kwargs={"reverse": reverse} ) def rank( @@ -692,8 +325,8 @@ def rank( *, descending: bool, ) -> Self: - return reuse_series_implementation( - self, "rank", call_kwargs={"method": method, "descending": descending} + return self._reuse_series_implementation( + "rank", call_kwargs={"method": method, "descending": descending} ) @property diff --git a/narwhals/_pandas_like/namespace.py b/narwhals/_pandas_like/namespace.py index f58cc19144..1af7ee17a9 100644 --- a/narwhals/_pandas_like/namespace.py +++ b/narwhals/_pandas_like/namespace.py @@ -64,52 +64,6 @@ def __init__( self._backend_version = backend_version self._version = version - def _create_expr_from_callable( - self: Self, - func: Callable[[PandasLikeDataFrame], Sequence[PandasLikeSeries]], - *, - depth: int, - function_name: str, - evaluate_output_names: Callable[[PandasLikeDataFrame], Sequence[str]], - alias_output_names: Callable[[Sequence[str]], Sequence[str]] | None, - call_kwargs: dict[str, Any] | None = None, - ) -> PandasLikeExpr: - return PandasLikeExpr( - func, - depth=depth, - function_name=function_name, - evaluate_output_names=evaluate_output_names, - alias_output_names=alias_output_names, - implementation=self._implementation, - backend_version=self._backend_version, - version=self._version, - call_kwargs=call_kwargs, - ) - - def _create_series_from_scalar( - self: Self, value: Any, *, reference_series: PandasLikeSeries - ) -> PandasLikeSeries: - return PandasLikeSeries._from_iterable( - [value], - name=reference_series._native_series.name, - index=reference_series._native_series.index[0:1], - implementation=self._implementation, - backend_version=self._backend_version, - version=self._version, - ) - - def _create_expr_from_series(self: Self, series: PandasLikeSeries) -> PandasLikeExpr: - return PandasLikeExpr( - lambda _df: [series], - depth=0, - function_name="series", - evaluate_output_names=lambda _df: [series.name], - alias_output_names=None, - implementation=self._implementation, - backend_version=self._backend_version, - version=self._version, - ) - def _create_compliant_series(self: Self, value: Any) -> PandasLikeSeries: return create_compliant_series( value, @@ -120,49 +74,32 @@ def _create_compliant_series(self: Self, value: Any) -> PandasLikeSeries: # --- selection --- def col(self: Self, *column_names: str) -> PandasLikeExpr: - return PandasLikeExpr.from_column_names( - passthrough_column_names(column_names), - function_name="col", - implementation=self._implementation, - backend_version=self._backend_version, - version=self._version, + return self._expr.from_column_names( + passthrough_column_names(column_names), function_name="col", context=self ) def exclude(self: Self, excluded_names: Container[str]) -> PandasLikeExpr: - return PandasLikeExpr.from_column_names( + return self._expr.from_column_names( partial(exclude_column_names, names=excluded_names), function_name="exclude", - implementation=self._implementation, - backend_version=self._backend_version, - version=self._version, + context=self, ) def nth(self: Self, *column_indices: int) -> PandasLikeExpr: - return PandasLikeExpr.from_column_indices( - *column_indices, - implementation=self._implementation, - backend_version=self._backend_version, - version=self._version, - ) + return self._expr.from_column_indices(*column_indices, context=self) def all(self: Self) -> PandasLikeExpr: - return PandasLikeExpr.from_column_names( - get_column_names, - function_name="all", - implementation=self._implementation, - backend_version=self._backend_version, - version=self._version, + return self._expr.from_column_names( + get_column_names, function_name="all", context=self ) def lit(self: Self, value: Any, dtype: DType | None) -> PandasLikeExpr: def _lit_pandas_series(df: PandasLikeDataFrame) -> PandasLikeSeries: - pandas_series = PandasLikeSeries._from_iterable( + pandas_series = self._series._from_iterable( data=[value], name="literal", index=df._native_frame.index[0:1], - implementation=self._implementation, - backend_version=self._backend_version, - version=self._version, + context=self, ) if dtype: return pandas_series.cast(dtype) @@ -182,13 +119,8 @@ def _lit_pandas_series(df: PandasLikeDataFrame) -> PandasLikeSeries: def len(self: Self) -> PandasLikeExpr: return PandasLikeExpr( lambda df: [ - PandasLikeSeries._from_iterable( - [len(df._native_frame)], - name="len", - index=[0], - implementation=self._implementation, - backend_version=self._backend_version, - version=self._version, + self._series._from_iterable( + [len(df._native_frame)], name="len", index=[0], context=self ) ], depth=0, @@ -208,27 +140,31 @@ def func(df: PandasLikeDataFrame) -> list[PandasLikeSeries]: native_series = (s.fill_null(0, None, None) for s in series) return [reduce(operator.add, native_series)] - return self._create_expr_from_callable( + return self._expr._from_callable( func=func, depth=max(x._depth for x in exprs) + 1, function_name="sum_horizontal", evaluate_output_names=combine_evaluate_output_names(*exprs), alias_output_names=combine_alias_output_names(*exprs), + context=self, ) - def all_horizontal(self: Self, *exprs: PandasLikeExpr) -> PandasLikeExpr: + # NOTE: Needs to be resolved in `EagerNamespace` + # Probably, by adding an `EagerExprT` typevar + def all_horizontal(self: Self, *exprs: PandasLikeExpr) -> PandasLikeExpr: # type: ignore[override] def func(df: PandasLikeDataFrame) -> list[PandasLikeSeries]: series = align_series_full_broadcast( *(s for _expr in exprs for s in _expr(df)) ) return [reduce(operator.and_, series)] - return self._create_expr_from_callable( + return self._expr._from_callable( func=func, depth=max(x._depth for x in exprs) + 1, function_name="all_horizontal", evaluate_output_names=combine_evaluate_output_names(*exprs), alias_output_names=combine_alias_output_names(*exprs), + context=self, ) def any_horizontal(self: Self, *exprs: PandasLikeExpr) -> PandasLikeExpr: @@ -238,12 +174,13 @@ def func(df: PandasLikeDataFrame) -> list[PandasLikeSeries]: ) return [reduce(operator.or_, series)] - return self._create_expr_from_callable( + return self._expr._from_callable( func=func, depth=max(x._depth for x in exprs) + 1, function_name="any_horizontal", evaluate_output_names=combine_evaluate_output_names(*exprs), alias_output_names=combine_alias_output_names(*exprs), + context=self, ) def mean_horizontal(self: Self, *exprs: PandasLikeExpr) -> PandasLikeExpr: @@ -255,12 +192,13 @@ def func(df: PandasLikeDataFrame) -> list[PandasLikeSeries]: non_na = align_series_full_broadcast(*(1 - s.is_null() for s in expr_results)) return [reduce(operator.add, series) / reduce(operator.add, non_na)] - return self._create_expr_from_callable( + return self._expr._from_callable( func=func, depth=max(x._depth for x in exprs) + 1, function_name="mean_horizontal", evaluate_output_names=combine_evaluate_output_names(*exprs), alias_output_names=combine_alias_output_names(*exprs), + context=self, ) def min_horizontal(self: Self, *exprs: PandasLikeExpr) -> PandasLikeExpr: @@ -279,12 +217,13 @@ def func(df: PandasLikeDataFrame) -> list[PandasLikeSeries]: ).alias(series[0].name) ] - return self._create_expr_from_callable( + return self._expr._from_callable( func=func, depth=max(x._depth for x in exprs) + 1, function_name="min_horizontal", evaluate_output_names=combine_evaluate_output_names(*exprs), alias_output_names=combine_alias_output_names(*exprs), + context=self, ) def max_horizontal(self: Self, *exprs: PandasLikeExpr) -> PandasLikeExpr: @@ -303,12 +242,13 @@ def func(df: PandasLikeDataFrame) -> list[PandasLikeSeries]: ).alias(series[0].name) ] - return self._create_expr_from_callable( + return self._expr._from_callable( func=func, depth=max(x._depth for x in exprs) + 1, function_name="max_horizontal", evaluate_output_names=combine_evaluate_output_names(*exprs), alias_output_names=combine_alias_output_names(*exprs), + context=self, ) def concat( @@ -387,13 +327,11 @@ def func(df: PandasLikeDataFrame) -> list[PandasLikeSeries]: s.zip_with(~nm, "") for s, nm in zip(series, null_mask) ] - sep_array = init_value.__class__._from_iterable( + sep_array = init_value._from_iterable( data=[separator] * len(init_value), name="sep", index=init_value._native_series.index, - implementation=self._implementation, - backend_version=self._backend_version, - version=self._version, + context=self, ) separators = (sep_array.zip_with(~nm, "") for nm in null_mask[:-1]) result = reduce( @@ -404,12 +342,13 @@ def func(df: PandasLikeDataFrame) -> list[PandasLikeSeries]: return [result] - return self._create_expr_from_callable( + return self._expr._from_callable( func=func, depth=max(x._depth for x in exprs) + 1, function_name="concat_str", evaluate_output_names=combine_evaluate_output_names(*exprs), alias_output_names=combine_alias_output_names(*exprs), + context=self, ) @@ -432,16 +371,13 @@ def __init__( self._version = version def __call__(self: Self, df: PandasLikeDataFrame) -> Sequence[PandasLikeSeries]: - plx = df.__narwhals_namespace__() condition = self._condition(df)[0] condition_native = condition._native_series if isinstance(self._then_value, PandasLikeExpr): value_series = self._then_value(df)[0] else: - value_series = plx._create_series_from_scalar( - self._then_value, reference_series=condition.alias("literal") - ) + value_series = condition.alias("literal")._from_scalar(self._then_value) value_series._broadcast = True value_series_native = extract_dataframe_comparand( df._native_frame.index, value_series diff --git a/narwhals/_pandas_like/series.py b/narwhals/_pandas_like/series.py index bde17c03f1..d7fe2fd4f0 100644 --- a/narwhals/_pandas_like/series.py +++ b/narwhals/_pandas_like/series.py @@ -43,6 +43,7 @@ from narwhals.typing import _1DArray from narwhals.typing import _AnyDArray from narwhals.utils import Version + from narwhals.utils import _FullContext PANDAS_TO_NUMPY_DTYPE_NO_MISSING = { "Int64": "int64", @@ -162,22 +163,20 @@ def _from_iterable( cls: type[Self], data: Iterable[Any], name: str, - index: Any, *, - implementation: Implementation, - backend_version: tuple[int, ...], - version: Version, + context: _FullContext, + index: Any = None, # NOTE: Originally a liskov substitution principle violation ) -> Self: return cls( native_series_from_iterable( data, name=name, - index=index, - implementation=implementation, + index=index or [], + implementation=context._implementation, ), - implementation=implementation, - backend_version=backend_version, - version=version, + implementation=context._implementation, + backend_version=context._backend_version, + version=context._version, ) def __len__(self: Self) -> int: From 9d7fc16d47c7357df544347d0ccd73a49d15eff2 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 7 Mar 2025 17:17:37 +0000 Subject: [PATCH 34/58] refactor: migrate expr namespaces --- narwhals/_pandas_like/expr_cat.py | 8 +-- narwhals/_pandas_like/expr_dt.py | 72 +++++++++++----------- narwhals/_pandas_like/expr_list.py | 8 +-- narwhals/_pandas_like/expr_str.py | 98 ++++++++---------------------- 4 files changed, 65 insertions(+), 121 deletions(-) diff --git a/narwhals/_pandas_like/expr_cat.py b/narwhals/_pandas_like/expr_cat.py index 985ae52935..21fa2d7c0d 100644 --- a/narwhals/_pandas_like/expr_cat.py +++ b/narwhals/_pandas_like/expr_cat.py @@ -2,8 +2,6 @@ from typing import TYPE_CHECKING -from narwhals._expression_parsing import reuse_series_namespace_implementation - if TYPE_CHECKING: from typing_extensions import Self @@ -15,8 +13,6 @@ def __init__(self: Self, expr: PandasLikeExpr) -> None: self._compliant_expr = expr def get_categories(self: Self) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, - "cat", - "get_categories", + return self._compliant_expr._reuse_series_namespace_implementation( + "cat", "get_categories" ) diff --git a/narwhals/_pandas_like/expr_dt.py b/narwhals/_pandas_like/expr_dt.py index ed5ffe8f12..e37d575ac5 100644 --- a/narwhals/_pandas_like/expr_dt.py +++ b/narwhals/_pandas_like/expr_dt.py @@ -2,8 +2,6 @@ from typing import TYPE_CHECKING -from narwhals._expression_parsing import reuse_series_namespace_implementation - if TYPE_CHECKING: from typing_extensions import Self @@ -16,92 +14,92 @@ def __init__(self: Self, expr: PandasLikeExpr) -> None: self._compliant_expr = expr def date(self: Self) -> PandasLikeExpr: - return reuse_series_namespace_implementation(self._compliant_expr, "dt", "date") + return self._compliant_expr._reuse_series_namespace_implementation("dt", "date") def year(self: Self) -> PandasLikeExpr: - return reuse_series_namespace_implementation(self._compliant_expr, "dt", "year") + return self._compliant_expr._reuse_series_namespace_implementation("dt", "year") def month(self: Self) -> PandasLikeExpr: - return reuse_series_namespace_implementation(self._compliant_expr, "dt", "month") + return self._compliant_expr._reuse_series_namespace_implementation("dt", "month") def day(self: Self) -> PandasLikeExpr: - return reuse_series_namespace_implementation(self._compliant_expr, "dt", "day") + return self._compliant_expr._reuse_series_namespace_implementation("dt", "day") def hour(self: Self) -> PandasLikeExpr: - return reuse_series_namespace_implementation(self._compliant_expr, "dt", "hour") + return self._compliant_expr._reuse_series_namespace_implementation("dt", "hour") def minute(self: Self) -> PandasLikeExpr: - return reuse_series_namespace_implementation(self._compliant_expr, "dt", "minute") + return self._compliant_expr._reuse_series_namespace_implementation("dt", "minute") def second(self: Self) -> PandasLikeExpr: - return reuse_series_namespace_implementation(self._compliant_expr, "dt", "second") + return self._compliant_expr._reuse_series_namespace_implementation("dt", "second") def millisecond(self: Self) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "millisecond" + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "millisecond" ) def microsecond(self: Self) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "microsecond" + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "microsecond" ) def nanosecond(self: Self) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "nanosecond" + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "nanosecond" ) def ordinal_day(self: Self) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "ordinal_day" + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "ordinal_day" ) def weekday(self: Self) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "weekday" + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "weekday" ) def total_minutes(self: Self) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "total_minutes" + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "total_minutes" ) def total_seconds(self: Self) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "total_seconds" + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "total_seconds" ) def total_milliseconds(self: Self) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "total_milliseconds" + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "total_milliseconds" ) def total_microseconds(self: Self) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "total_microseconds" + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "total_microseconds" ) def total_nanoseconds(self: Self) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "total_nanoseconds" + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "total_nanoseconds" ) def to_string(self: Self, format: str) -> PandasLikeExpr: # noqa: A002 - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "to_string", format=format + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "to_string", format=format ) def replace_time_zone(self: Self, time_zone: str | None) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "replace_time_zone", time_zone=time_zone + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "replace_time_zone", time_zone=time_zone ) def convert_time_zone(self: Self, time_zone: str) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "convert_time_zone", time_zone=time_zone + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "convert_time_zone", time_zone=time_zone ) def timestamp(self: Self, time_unit: TimeUnit) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "dt", "timestamp", time_unit=time_unit + return self._compliant_expr._reuse_series_namespace_implementation( + "dt", "timestamp", time_unit=time_unit ) diff --git a/narwhals/_pandas_like/expr_list.py b/narwhals/_pandas_like/expr_list.py index 865f73a8ee..41ba1337fd 100644 --- a/narwhals/_pandas_like/expr_list.py +++ b/narwhals/_pandas_like/expr_list.py @@ -2,8 +2,6 @@ from typing import TYPE_CHECKING -from narwhals._expression_parsing import reuse_series_namespace_implementation - if TYPE_CHECKING: from typing_extensions import Self @@ -15,8 +13,4 @@ def __init__(self: Self, expr: PandasLikeExpr) -> None: self._expr = expr def len(self: Self) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._expr, - "list", - "len", - ) + return self._expr._reuse_series_namespace_implementation("list", "len") diff --git a/narwhals/_pandas_like/expr_str.py b/narwhals/_pandas_like/expr_str.py index 8f241515de..e349ffc757 100644 --- a/narwhals/_pandas_like/expr_str.py +++ b/narwhals/_pandas_like/expr_str.py @@ -2,8 +2,6 @@ from typing import TYPE_CHECKING -from narwhals._expression_parsing import reuse_series_namespace_implementation - if TYPE_CHECKING: from typing_extensions import Self @@ -14,108 +12,66 @@ class PandasLikeExprStringNamespace: def __init__(self: Self, expr: PandasLikeExpr) -> None: self._compliant_expr = expr - def len_chars( - self: Self, - ) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "str", "len_chars" + def len_chars(self: Self) -> PandasLikeExpr: + return self._compliant_expr._reuse_series_namespace_implementation( + "str", "len_chars" ) def replace( - self: Self, - pattern: str, - value: str, - *, - literal: bool, - n: int, + self: Self, pattern: str, value: str, *, literal: bool, n: int ) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, - "str", - "replace", - pattern=pattern, - value=value, - literal=literal, - n=n, + return self._compliant_expr._reuse_series_namespace_implementation( + "str", "replace", pattern=pattern, value=value, literal=literal, n=n ) def replace_all( - self: Self, - pattern: str, - value: str, - *, - literal: bool, + self: Self, pattern: str, value: str, *, literal: bool ) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, - "str", - "replace_all", - pattern=pattern, - value=value, - literal=literal, + return self._compliant_expr._reuse_series_namespace_implementation( + "str", "replace_all", pattern=pattern, value=value, literal=literal ) def strip_chars(self: Self, characters: str | None) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, - "str", - "strip_chars", - characters=characters, + return self._compliant_expr._reuse_series_namespace_implementation( + "str", "strip_chars", characters=characters ) def starts_with(self: Self, prefix: str) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, - "str", - "starts_with", - prefix=prefix, + return self._compliant_expr._reuse_series_namespace_implementation( + "str", "starts_with", prefix=prefix ) def ends_with(self: Self, suffix: str) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, - "str", - "ends_with", - suffix=suffix, + return self._compliant_expr._reuse_series_namespace_implementation( + "str", "ends_with", suffix=suffix ) def contains(self: Self, pattern: str, *, literal: bool) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, - "str", - "contains", - pattern=pattern, - literal=literal, + return self._compliant_expr._reuse_series_namespace_implementation( + "str", "contains", pattern=pattern, literal=literal ) def slice(self: Self, offset: int, length: int | None) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "str", "slice", offset=offset, length=length + return self._compliant_expr._reuse_series_namespace_implementation( + "str", "slice", offset=offset, length=length ) def split(self: Self, by: str) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, "str", "split", by=by + return self._compliant_expr._reuse_series_namespace_implementation( + "str", "split", by=by ) def to_datetime(self: Self, format: str | None) -> PandasLikeExpr: # noqa: A002 - return reuse_series_namespace_implementation( - self._compliant_expr, - "str", - "to_datetime", - format=format, + return self._compliant_expr._reuse_series_namespace_implementation( + "str", "to_datetime", format=format ) def to_uppercase(self: Self) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, - "str", - "to_uppercase", + return self._compliant_expr._reuse_series_namespace_implementation( + "str", "to_uppercase" ) def to_lowercase(self: Self) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, - "str", - "to_lowercase", + return self._compliant_expr._reuse_series_namespace_implementation( + "str", "to_lowercase" ) From bf18326065b33459111fca5557c8592eaf8a68b6 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 7 Mar 2025 17:21:20 +0000 Subject: [PATCH 35/58] fix: dont `__bool__` a `pd.Series` --- narwhals/_pandas_like/series.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/narwhals/_pandas_like/series.py b/narwhals/_pandas_like/series.py index d7fe2fd4f0..65f5c67a17 100644 --- a/narwhals/_pandas_like/series.py +++ b/narwhals/_pandas_like/series.py @@ -171,7 +171,7 @@ def _from_iterable( native_series_from_iterable( data, name=name, - index=index or [], + index=[] if index is None else index, implementation=context._implementation, ), implementation=context._implementation, From 73656f1d0b17438c6ef9bbd18e86b37e7ecc37c4 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 7 Mar 2025 17:23:47 +0000 Subject: [PATCH 36/58] fix(DRAFT): extend `is_eager_namespace` --- narwhals/utils.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/narwhals/utils.py b/narwhals/utils.py index 462ca4d0b1..c311ce3389 100644 --- a/narwhals/utils.py +++ b/narwhals/utils.py @@ -1402,11 +1402,11 @@ def is_compliant_expr( return hasattr(obj, "__narwhals_expr__") -# NOTE: Temporary - just to introduce a path for the Arrow WIP +# NOTE: Temporary - just to introduce a path for the (Arrow|PandasLike) WIP def is_eager_namespace( obj: EagerNamespace[EagerDataFrameT, EagerSeriesT] | Any, ) -> TypeIs[EagerNamespace[EagerDataFrameT, EagerSeriesT]]: - return type(obj).__name__ == "ArrowNamespace" + return type(obj).__name__ in {"ArrowNamespace", "PandasLikeNamespace"} # return all(hasattr(obj, name) for name in ("selectors", "_expr", "_series")) # noqa: ERA001 From c606ea09b987373269ec2c2f2a5ff5756d293f6c Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 7 Mar 2025 17:29:12 +0000 Subject: [PATCH 37/58] refactor: drop all unused `_expression_parsing` Copied over the docs --- narwhals/_compliant/expr.py | 25 ++++++- narwhals/_expression_parsing.py | 121 -------------------------------- 2 files changed, 24 insertions(+), 122 deletions(-) diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index 3cda425e21..7bcda409dd 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -323,7 +323,6 @@ def from_column_indices( context: _FullContext, ) -> Self: ... - # https://github.com/narwhals-dev/narwhals/blob/35cef0b1e2c892fb24aa730902b08b6994008c18/narwhals/_protocols.py#L135 def _reuse_series_implementation( self: Self, attr: str, @@ -332,6 +331,20 @@ def _reuse_series_implementation( call_kwargs: dict[str, Any] | None = None, **expressifiable_args: Any, ) -> Self: + """Reuse Series implementation for expression. + + If Series.foo is already defined, and we'd like Expr.foo to be the same, we can + leverage this method to do that for us. + + Arguments: + attr: name of method. + returns_scalar: whether the Series version returns a scalar. In this case, + the expression version should return a 1-row Series. + call_kwargs: non-expressifiable args which we may need to reuse in `agg` or `over`, + such as `ddof` for `std` and `var`. + expressifiable_args: keyword arguments to pass to function, which may + be expressifiable (e.g. `nw.col('a').is_between(3, nw.col('b')))`). + """ func = partial( self._reuse_series_inner, method_name=attr, @@ -394,6 +407,16 @@ def _reuse_series_inner( def _reuse_series_namespace_implementation( self: Self, series_namespace: str, attr: str, **kwargs: Any ) -> Self: + """Reuse Series implementation for expression. + + Just like `_reuse_series_implementation`, but for e.g. `Expr.dt.foo` instead + of `Expr.foo`. + + Arguments: + series_namespace: The Series namespace (e.g. `dt`, `cat`, `str`, `list`, `name`) + attr: name of method. + kwargs: keyword arguments to pass to function. + """ return self._from_callable( lambda df: [ getattr(getattr(series, series_namespace), attr)(**kwargs) diff --git a/narwhals/_expression_parsing.py b/narwhals/_expression_parsing.py index 726589bc0f..45cd8d89c4 100644 --- a/narwhals/_expression_parsing.py +++ b/narwhals/_expression_parsing.py @@ -12,7 +12,6 @@ from typing import Literal from typing import Sequence from typing import TypeVar -from typing import overload from narwhals.dependencies import is_narwhals_series from narwhals.dependencies import is_numpy_array @@ -30,16 +29,12 @@ from narwhals._compliant import CompliantNamespace from narwhals._compliant import CompliantSeriesOrNativeExprT_co from narwhals._compliant import CompliantSeriesT_co - from narwhals._pandas_like.expr import PandasLikeExpr from narwhals.expr import Expr from narwhals.typing import CompliantDataFrame from narwhals.typing import CompliantLazyFrame - from narwhals.typing import CompliantSeries from narwhals.typing import IntoExpr from narwhals.typing import _1DArray - PandasLikeExprT = TypeVar("PandasLikeExprT", bound=PandasLikeExpr) - T = TypeVar("T") @@ -79,122 +74,6 @@ def evaluate_into_exprs( return list(chain.from_iterable(evaluate_into_expr(df, expr) for expr in exprs)) -@overload -def maybe_evaluate_expr( - df: CompliantFrameT, expr: CompliantExpr[CompliantFrameT, CompliantSeriesT_co] -) -> CompliantSeriesT_co: ... - - -@overload -def maybe_evaluate_expr(df: CompliantDataFrame[Any], expr: T) -> T: ... - - -def maybe_evaluate_expr( - df: Any, expr: CompliantExpr[Any, CompliantSeriesT_co] | T -) -> CompliantSeriesT_co | T: - """Evaluate `expr` if it's an expression, otherwise return it as is.""" - if is_compliant_expr(expr): - result: Sequence[CompliantSeriesT_co] = expr(df) - if len(result) > 1: - msg = "Multi-output expressions (e.g. `nw.all()` or `nw.col('a', 'b')`) are not supported in this context" - raise ValueError(msg) - return result[0] - return expr - - -def reuse_series_implementation( - expr: PandasLikeExprT, - attr: str, - *, - returns_scalar: bool = False, - call_kwargs: dict[str, Any] | None = None, - **expressifiable_args: Any, -) -> PandasLikeExprT: - """Reuse Series implementation for expression. - - If Series.foo is already defined, and we'd like Expr.foo to be the same, we can - leverage this method to do that for us. - - Arguments: - expr: expression object. - attr: name of method. - returns_scalar: whether the Series version returns a scalar. In this case, - the expression version should return a 1-row Series. - call_kwargs: non-expressifiable args which we may need to reuse in `agg` or `over`, - such as `ddof` for `std` and `var`. - expressifiable_args: keyword arguments to pass to function, which may - be expressifiable (e.g. `nw.col('a').is_between(3, nw.col('b')))`). - """ - plx = expr.__narwhals_namespace__() - - def func(df: CompliantDataFrame[Any]) -> Sequence[CompliantSeries]: - _kwargs = { - **(call_kwargs or {}), - **{ - arg_name: maybe_evaluate_expr(df, arg_value) - for arg_name, arg_value in expressifiable_args.items() - }, - } - - out: list[CompliantSeries] = [ - plx._create_series_from_scalar( # type: ignore # noqa: PGH003 - getattr(series, attr)(**_kwargs), - reference_series=series, - ) - if returns_scalar - else getattr(series, attr)(**_kwargs) - for series in expr(df) # type: ignore # noqa: PGH003 - ] - _, aliases = evaluate_output_names_and_aliases(expr, df, []) - if [s.name for s in out] != list(aliases): # pragma: no cover - msg = ( - f"Safety assertion failed, please report a bug to https://github.com/narwhals-dev/narwhals/issues\n" - f"Expression aliases: {aliases}\n" - f"Series names: {[s.name for s in out]}" - ) - raise AssertionError(msg) - return out - - return plx._create_expr_from_callable( # type: ignore # noqa: PGH003 - func, - depth=expr._depth + 1, - function_name=f"{expr._function_name}->{attr}", - evaluate_output_names=expr._evaluate_output_names, - alias_output_names=expr._alias_output_names, - call_kwargs=call_kwargs, - ) - - -def reuse_series_namespace_implementation( - expr: PandasLikeExprT, - series_namespace: str, - attr: str, - **kwargs: Any, -) -> PandasLikeExprT: - """Reuse Series implementation for expression. - - Just like `reuse_series_implementation`, but for e.g. `Expr.dt.foo` instead - of `Expr.foo`. - - Arguments: - expr: expression object. - series_namespace: The Series namespace (e.g. `dt`, `cat`, `str`, `list`, `name`) - attr: name of method. - kwargs: keyword arguments to pass to function. - """ - plx = expr.__narwhals_namespace__() - return plx._create_expr_from_callable( # type: ignore # noqa: PGH003 - lambda df: [ - getattr(getattr(series, series_namespace), attr)(**kwargs) - for series in expr(df) - ], - depth=expr._depth + 1, - function_name=f"{expr._function_name}->{series_namespace}.{attr}", - evaluate_output_names=expr._evaluate_output_names, - alias_output_names=expr._alias_output_names, - ) - - def is_elementary_expression(expr: CompliantExpr[Any, Any]) -> bool: """Check if expr is elementary. From 00a34423f1a30cb4de749113fcdb5ba1c0964f07 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 7 Mar 2025 17:43:59 +0000 Subject: [PATCH 38/58] refactor: remove `implementation` from method names The docs already provide a description --- narwhals/_arrow/expr.py | 16 ++-- narwhals/_arrow/expr_cat.py | 4 +- narwhals/_arrow/expr_dt.py | 62 +++++------- narwhals/_arrow/expr_list.py | 2 +- narwhals/_arrow/expr_str.py | 45 +++------ narwhals/_compliant/expr.py | 146 ++++++++++++++--------------- narwhals/_pandas_like/expr.py | 26 ++--- narwhals/_pandas_like/expr_cat.py | 4 +- narwhals/_pandas_like/expr_dt.py | 62 +++++------- narwhals/_pandas_like/expr_list.py | 2 +- narwhals/_pandas_like/expr_str.py | 32 +++---- 11 files changed, 155 insertions(+), 246 deletions(-) diff --git a/narwhals/_arrow/expr.py b/narwhals/_arrow/expr.py index 5538c1e725..8fff861171 100644 --- a/narwhals/_arrow/expr.py +++ b/narwhals/_arrow/expr.py @@ -138,10 +138,10 @@ def _reuse_series_extra_kwargs( return {"_return_py_scalar": False} if returns_scalar else {} def cum_sum(self: Self, *, reverse: bool) -> Self: - return self._reuse_series_implementation("cum_sum", reverse=reverse) + return self._reuse_series("cum_sum", reverse=reverse) def shift(self: Self, n: int) -> Self: - return self._reuse_series_implementation("shift", n=n) + return self._reuse_series("shift", n=n) def over(self: Self, keys: Sequence[str], kind: ExprKind) -> Self: if not is_scalar_like(kind): @@ -215,16 +215,16 @@ def func(df: ArrowDataFrame) -> list[ArrowSeries]: ) def cum_count(self: Self, *, reverse: bool) -> Self: - return self._reuse_series_implementation("cum_count", reverse=reverse) + return self._reuse_series("cum_count", reverse=reverse) def cum_min(self: Self, *, reverse: bool) -> Self: - return self._reuse_series_implementation("cum_min", reverse=reverse) + return self._reuse_series("cum_min", reverse=reverse) def cum_max(self: Self, *, reverse: bool) -> Self: - return self._reuse_series_implementation("cum_max", reverse=reverse) + return self._reuse_series("cum_max", reverse=reverse) def cum_prod(self: Self, *, reverse: bool) -> Self: - return self._reuse_series_implementation("cum_prod", reverse=reverse) + return self._reuse_series("cum_prod", reverse=reverse) def rank( self: Self, @@ -232,9 +232,7 @@ def rank( *, descending: bool, ) -> Self: - return self._reuse_series_implementation( - "rank", method=method, descending=descending - ) + return self._reuse_series("rank", method=method, descending=descending) ewm_mean = not_implemented() diff --git a/narwhals/_arrow/expr_cat.py b/narwhals/_arrow/expr_cat.py index 5c8861e3ba..c4b443ff03 100644 --- a/narwhals/_arrow/expr_cat.py +++ b/narwhals/_arrow/expr_cat.py @@ -13,6 +13,4 @@ def __init__(self: Self, expr: ArrowExpr) -> None: self._compliant_expr = expr def get_categories(self: Self) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "cat", "get_categories" - ) + return self._compliant_expr._reuse_series_namespace("cat", "get_categories") diff --git a/narwhals/_arrow/expr_dt.py b/narwhals/_arrow/expr_dt.py index 88d026ceb5..407184ceaa 100644 --- a/narwhals/_arrow/expr_dt.py +++ b/narwhals/_arrow/expr_dt.py @@ -14,92 +14,72 @@ def __init__(self: Self, expr: ArrowExpr) -> None: self._compliant_expr = expr def to_string(self: Self, format: str) -> ArrowExpr: # noqa: A002 - return self._compliant_expr._reuse_series_namespace_implementation( + return self._compliant_expr._reuse_series_namespace( "dt", "to_string", format=format ) def replace_time_zone(self: Self, time_zone: str | None) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( + return self._compliant_expr._reuse_series_namespace( "dt", "replace_time_zone", time_zone=time_zone ) def convert_time_zone(self: Self, time_zone: str) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( + return self._compliant_expr._reuse_series_namespace( "dt", "convert_time_zone", time_zone=time_zone ) def timestamp(self: Self, time_unit: TimeUnit) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( + return self._compliant_expr._reuse_series_namespace( "dt", "timestamp", time_unit=time_unit ) def date(self: Self) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation("dt", "date") + return self._compliant_expr._reuse_series_namespace("dt", "date") def year(self: Self) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation("dt", "year") + return self._compliant_expr._reuse_series_namespace("dt", "year") def month(self: Self) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation("dt", "month") + return self._compliant_expr._reuse_series_namespace("dt", "month") def day(self: Self) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation("dt", "day") + return self._compliant_expr._reuse_series_namespace("dt", "day") def hour(self: Self) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation("dt", "hour") + return self._compliant_expr._reuse_series_namespace("dt", "hour") def minute(self: Self) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation("dt", "minute") + return self._compliant_expr._reuse_series_namespace("dt", "minute") def second(self: Self) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation("dt", "second") + return self._compliant_expr._reuse_series_namespace("dt", "second") def millisecond(self: Self) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "dt", "millisecond" - ) + return self._compliant_expr._reuse_series_namespace("dt", "millisecond") def microsecond(self: Self) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "dt", "microsecond" - ) + return self._compliant_expr._reuse_series_namespace("dt", "microsecond") def nanosecond(self: Self) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "dt", "nanosecond" - ) + return self._compliant_expr._reuse_series_namespace("dt", "nanosecond") def ordinal_day(self: Self) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "dt", "ordinal_day" - ) + return self._compliant_expr._reuse_series_namespace("dt", "ordinal_day") def weekday(self: Self) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "dt", "weekday" - ) + return self._compliant_expr._reuse_series_namespace("dt", "weekday") def total_minutes(self: Self) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "dt", "total_minutes" - ) + return self._compliant_expr._reuse_series_namespace("dt", "total_minutes") def total_seconds(self: Self) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "dt", "total_seconds" - ) + return self._compliant_expr._reuse_series_namespace("dt", "total_seconds") def total_milliseconds(self: Self) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "dt", "total_milliseconds" - ) + return self._compliant_expr._reuse_series_namespace("dt", "total_milliseconds") def total_microseconds(self: Self) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "dt", "total_microseconds" - ) + return self._compliant_expr._reuse_series_namespace("dt", "total_microseconds") def total_nanoseconds(self: Self) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "dt", "total_nanoseconds" - ) + return self._compliant_expr._reuse_series_namespace("dt", "total_nanoseconds") diff --git a/narwhals/_arrow/expr_list.py b/narwhals/_arrow/expr_list.py index 90983c8b0a..b69d83ef44 100644 --- a/narwhals/_arrow/expr_list.py +++ b/narwhals/_arrow/expr_list.py @@ -13,4 +13,4 @@ def __init__(self: Self, expr: ArrowExpr) -> None: self._expr = expr def len(self: Self) -> ArrowExpr: - return self._expr._reuse_series_namespace_implementation("list", "len") + return self._expr._reuse_series_namespace("list", "len") diff --git a/narwhals/_arrow/expr_str.py b/narwhals/_arrow/expr_str.py index c3d71898e4..737959c69e 100644 --- a/narwhals/_arrow/expr_str.py +++ b/narwhals/_arrow/expr_str.py @@ -13,72 +13,55 @@ def __init__(self: Self, expr: ArrowExpr) -> None: self._compliant_expr = expr def len_chars(self: Self) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "str", "len_chars" - ) + return self._compliant_expr._reuse_series_namespace("str", "len_chars") def replace( self: Self, pattern: str, value: str, *, literal: bool, n: int ) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "str", - "replace", - pattern=pattern, - value=value, - literal=literal, - n=n, + return self._compliant_expr._reuse_series_namespace( + "str", "replace", pattern=pattern, value=value, literal=literal, n=n ) def replace_all(self: Self, pattern: str, value: str, *, literal: bool) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "str", - "replace_all", - pattern=pattern, - value=value, - literal=literal, + return self._compliant_expr._reuse_series_namespace( + "str", "replace_all", pattern=pattern, value=value, literal=literal ) def strip_chars(self: Self, characters: str | None) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( + return self._compliant_expr._reuse_series_namespace( "str", "strip_chars", characters=characters ) def starts_with(self: Self, prefix: str) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( + return self._compliant_expr._reuse_series_namespace( "str", "starts_with", prefix=prefix ) def ends_with(self: Self, suffix: str) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( + return self._compliant_expr._reuse_series_namespace( "str", "ends_with", suffix=suffix ) def contains(self: Self, pattern: str, *, literal: bool) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( + return self._compliant_expr._reuse_series_namespace( "str", "contains", pattern=pattern, literal=literal ) def slice(self: Self, offset: int, length: int | None) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( + return self._compliant_expr._reuse_series_namespace( "str", "slice", offset=offset, length=length ) def split(self: Self, by: str) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "str", "split", by=by - ) + return self._compliant_expr._reuse_series_namespace("str", "split", by=by) def to_datetime(self: Self, format: str | None) -> ArrowExpr: # noqa: A002 - return self._compliant_expr._reuse_series_namespace_implementation( + return self._compliant_expr._reuse_series_namespace( "str", "to_datetime", format=format ) def to_uppercase(self: Self) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "str", "to_uppercase" - ) + return self._compliant_expr._reuse_series_namespace("str", "to_uppercase") def to_lowercase(self: Self) -> ArrowExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "str", "to_lowercase" - ) + return self._compliant_expr._reuse_series_namespace("str", "to_lowercase") diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index 954a6a0e42..dbfdbadc44 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -323,7 +323,7 @@ def from_column_indices( context: _FullContext, ) -> Self: ... - def _reuse_series_implementation( + def _reuse_series( self: Self, attr: str, *, @@ -404,7 +404,7 @@ def _reuse_series_inner( raise AssertionError(msg) return out - def _reuse_series_namespace_implementation( + def _reuse_series_namespace( self: Self, series_namespace: str, attr: str, **kwargs: Any ) -> Self: """Reuse Series implementation for expression. @@ -454,138 +454,130 @@ def func(df: EagerDataFrameT) -> list[EagerSeriesT]: ) def cast(self, dtype: DType | type[DType]) -> Self: - return self._reuse_series_implementation("cast", dtype=dtype) + return self._reuse_series("cast", dtype=dtype) def __eq__(self, other: Self | Any) -> Self: # type: ignore[override] - return self._reuse_series_implementation("__eq__", other=other) + return self._reuse_series("__eq__", other=other) def __ne__(self, other: Self | Any) -> Self: # type: ignore[override] - return self._reuse_series_implementation("__ne__", other=other) + return self._reuse_series("__ne__", other=other) def __ge__(self, other: Self | Any) -> Self: - return self._reuse_series_implementation("__ge__", other=other) + return self._reuse_series("__ge__", other=other) def __gt__(self, other: Self | Any) -> Self: - return self._reuse_series_implementation("__gt__", other=other) + return self._reuse_series("__gt__", other=other) def __le__(self, other: Self | Any) -> Self: - return self._reuse_series_implementation("__le__", other=other) + return self._reuse_series("__le__", other=other) def __lt__(self, other: Self | Any) -> Self: - return self._reuse_series_implementation("__lt__", other=other) + return self._reuse_series("__lt__", other=other) def __and__(self, other: Self | bool | Any) -> Self: - return self._reuse_series_implementation("__and__", other=other) + return self._reuse_series("__and__", other=other) def __or__(self, other: Self | bool | Any) -> Self: - return self._reuse_series_implementation("__or__", other=other) + return self._reuse_series("__or__", other=other) def __add__(self, other: Self | Any) -> Self: - return self._reuse_series_implementation("__add__", other=other) + return self._reuse_series("__add__", other=other) def __sub__(self, other: Self | Any) -> Self: - return self._reuse_series_implementation("__sub__", other=other) + return self._reuse_series("__sub__", other=other) def __rsub__(self, other: Self | Any) -> Self: - return self.alias("literal")._reuse_series_implementation("__rsub__", other=other) + return self.alias("literal")._reuse_series("__rsub__", other=other) def __mul__(self, other: Self | Any) -> Self: - return self._reuse_series_implementation("__mul__", other=other) + return self._reuse_series("__mul__", other=other) def __truediv__(self, other: Self | Any) -> Self: - return self._reuse_series_implementation("__truediv__", other=other) + return self._reuse_series("__truediv__", other=other) def __rtruediv__(self, other: Self | Any) -> Self: - return self.alias("literal")._reuse_series_implementation( - "__rtruediv__", other=other - ) + return self.alias("literal")._reuse_series("__rtruediv__", other=other) def __floordiv__(self, other: Self | Any) -> Self: - return self._reuse_series_implementation("__floordiv__", other=other) + return self._reuse_series("__floordiv__", other=other) def __rfloordiv__(self, other: Self | Any) -> Self: - return self.alias("literal")._reuse_series_implementation( - "__rfloordiv__", other=other - ) + return self.alias("literal")._reuse_series("__rfloordiv__", other=other) def __pow__(self, other: Self | Any) -> Self: - return self._reuse_series_implementation("__pow__", other=other) + return self._reuse_series("__pow__", other=other) def __rpow__(self, other: Self | Any) -> Self: - return self.alias("literal")._reuse_series_implementation("__rpow__", other=other) + return self.alias("literal")._reuse_series("__rpow__", other=other) def __mod__(self, other: Self | Any) -> Self: - return self._reuse_series_implementation("__mod__", other=other) + return self._reuse_series("__mod__", other=other) def __rmod__(self, other: Self | Any) -> Self: - return self.alias("literal")._reuse_series_implementation("__rmod__", other=other) + return self.alias("literal")._reuse_series("__rmod__", other=other) # Unary def __invert__(self) -> Self: - return self._reuse_series_implementation("__invert__") + return self._reuse_series("__invert__") # Reductions def null_count(self) -> Self: - return self._reuse_series_implementation("null_count", returns_scalar=True) + return self._reuse_series("null_count", returns_scalar=True) def n_unique(self) -> Self: - return self._reuse_series_implementation("n_unique", returns_scalar=True) + return self._reuse_series("n_unique", returns_scalar=True) def sum(self) -> Self: - return self._reuse_series_implementation("sum", returns_scalar=True) + return self._reuse_series("sum", returns_scalar=True) def count(self) -> Self: - return self._reuse_series_implementation("count", returns_scalar=True) + return self._reuse_series("count", returns_scalar=True) def mean(self) -> Self: - return self._reuse_series_implementation("mean", returns_scalar=True) + return self._reuse_series("mean", returns_scalar=True) def median(self) -> Self: - return self._reuse_series_implementation("median", returns_scalar=True) + return self._reuse_series("median", returns_scalar=True) def std(self, *, ddof: int) -> Self: - return self._reuse_series_implementation( - "std", returns_scalar=True, call_kwargs={"ddof": ddof} - ) + return self._reuse_series("std", returns_scalar=True, call_kwargs={"ddof": ddof}) def var(self, *, ddof: int) -> Self: - return self._reuse_series_implementation( - "var", returns_scalar=True, call_kwargs={"ddof": ddof} - ) + return self._reuse_series("var", returns_scalar=True, call_kwargs={"ddof": ddof}) def skew(self) -> Self: - return self._reuse_series_implementation("skew", returns_scalar=True) + return self._reuse_series("skew", returns_scalar=True) def any(self) -> Self: - return self._reuse_series_implementation("any", returns_scalar=True) + return self._reuse_series("any", returns_scalar=True) def all(self) -> Self: - return self._reuse_series_implementation("all", returns_scalar=True) + return self._reuse_series("all", returns_scalar=True) def max(self) -> Self: - return self._reuse_series_implementation("max", returns_scalar=True) + return self._reuse_series("max", returns_scalar=True) def min(self) -> Self: - return self._reuse_series_implementation("min", returns_scalar=True) + return self._reuse_series("min", returns_scalar=True) def arg_min(self) -> Self: - return self._reuse_series_implementation("arg_min", returns_scalar=True) + return self._reuse_series("arg_min", returns_scalar=True) def arg_max(self) -> Self: - return self._reuse_series_implementation("arg_max", returns_scalar=True) + return self._reuse_series("arg_max", returns_scalar=True) # Other def clip(self, lower_bound: Any, upper_bound: Any) -> Self: - return self._reuse_series_implementation( + return self._reuse_series( "clip", lower_bound=lower_bound, upper_bound=upper_bound ) def is_null(self) -> Self: - return self._reuse_series_implementation("is_null") + return self._reuse_series("is_null") def is_nan(self) -> Self: - return self._reuse_series_implementation("is_nan") + return self._reuse_series("is_nan") def fill_null( self, @@ -593,25 +585,25 @@ def fill_null( strategy: Literal["forward", "backward"] | None, limit: int | None, ) -> Self: - return self._reuse_series_implementation( + return self._reuse_series( "fill_null", value=value, strategy=strategy, limit=limit ) def is_in(self, other: Any) -> Self: - return self._reuse_series_implementation("is_in", other=other) + return self._reuse_series("is_in", other=other) def arg_true(self) -> Self: - return self._reuse_series_implementation("arg_true") + return self._reuse_series("arg_true") # NOTE: `ewm_mean` not implemented `pyarrow` def filter(self, *predicates: Self) -> Self: plx = self.__narwhals_namespace__() predicate = plx.all_horizontal(*predicates) - return self._reuse_series_implementation("filter", predicate=predicate) + return self._reuse_series("filter", predicate=predicate) def drop_nulls(self) -> Self: - return self._reuse_series_implementation("drop_nulls") + return self._reuse_series("drop_nulls") def replace_strict( self, @@ -620,23 +612,21 @@ def replace_strict( *, return_dtype: DType | type[DType] | None, ) -> Self: - return self._reuse_series_implementation( + return self._reuse_series( "replace_strict", old=old, new=new, return_dtype=return_dtype ) def sort(self, *, descending: bool, nulls_last: bool) -> Self: - return self._reuse_series_implementation( - "sort", descending=descending, nulls_last=nulls_last - ) + return self._reuse_series("sort", descending=descending, nulls_last=nulls_last) def abs(self) -> Self: - return self._reuse_series_implementation("abs") + return self._reuse_series("abs") def unique(self) -> Self: - return self._reuse_series_implementation("unique", maintain_order=False) + return self._reuse_series("unique", maintain_order=False) def diff(self) -> Self: - return self._reuse_series_implementation("diff") + return self._reuse_series("diff") # NOTE: `shift` differs @@ -648,7 +638,7 @@ def sample( with_replacement: bool, seed: int | None, ) -> Self: - return self._reuse_series_implementation( + return self._reuse_series( "sample", n=n, fraction=fraction, with_replacement=with_replacement, seed=seed ) @@ -676,20 +666,20 @@ def alias_output_names(names: Sequence[str]) -> Sequence[str]: # NOTE: `over` differs def is_unique(self) -> Self: - return self._reuse_series_implementation("is_unique") + return self._reuse_series("is_unique") def is_first_distinct(self) -> Self: - return self._reuse_series_implementation("is_first_distinct") + return self._reuse_series("is_first_distinct") def is_last_distinct(self) -> Self: - return self._reuse_series_implementation("is_last_distinct") + return self._reuse_series("is_last_distinct") def quantile( self, quantile: float, interpolation: Literal["nearest", "higher", "lower", "midpoint", "linear"], ) -> Self: - return self._reuse_series_implementation( + return self._reuse_series( "quantile", quantile=quantile, interpolation=interpolation, @@ -697,34 +687,34 @@ def quantile( ) def head(self, n: int) -> Self: - return self._reuse_series_implementation("head", n=n) + return self._reuse_series("head", n=n) def tail(self, n: int) -> Self: - return self._reuse_series_implementation("tail", n=n) + return self._reuse_series("tail", n=n) def round(self, decimals: int) -> Self: - return self._reuse_series_implementation("round", decimals=decimals) + return self._reuse_series("round", decimals=decimals) def len(self) -> Self: - return self._reuse_series_implementation("len", returns_scalar=True) + return self._reuse_series("len", returns_scalar=True) def gather_every(self, n: int, offset: int) -> Self: - return self._reuse_series_implementation("gather_every", n=n, offset=offset) + return self._reuse_series("gather_every", n=n, offset=offset) def mode(self) -> Self: - return self._reuse_series_implementation("mode") + return self._reuse_series("mode") # NOTE: `map_batches` differs def is_finite(self) -> Self: - return self._reuse_series_implementation("is_finite") + return self._reuse_series("is_finite") # NOTE: `cum_(sum|count|min|max|prod)` differ def rolling_mean( self, window_size: int, *, min_samples: int | None, center: bool ) -> Self: - return self._reuse_series_implementation( + return self._reuse_series( "rolling_mean", window_size=window_size, min_samples=min_samples, @@ -734,7 +724,7 @@ def rolling_mean( def rolling_std( self, window_size: int, *, min_samples: int | None, center: bool, ddof: int ) -> Self: - return self._reuse_series_implementation( + return self._reuse_series( "rolling_std", window_size=window_size, min_samples=min_samples, @@ -745,14 +735,14 @@ def rolling_std( def rolling_sum( self, window_size: int, *, min_samples: int | None, center: bool ) -> Self: - return self._reuse_series_implementation( + return self._reuse_series( "rolling_sum", window_size=window_size, min_samples=min_samples, center=center ) def rolling_var( self, window_size: int, *, min_samples: int | None, center: bool, ddof: int ) -> Self: - return self._reuse_series_implementation( + return self._reuse_series( "rolling_var", window_size=window_size, min_samples=min_samples, diff --git a/narwhals/_pandas_like/expr.py b/narwhals/_pandas_like/expr.py index 2d605ba087..db40bacc7a 100644 --- a/narwhals/_pandas_like/expr.py +++ b/narwhals/_pandas_like/expr.py @@ -178,7 +178,7 @@ def ewm_mean( min_samples: int, ignore_nulls: bool, ) -> Self: - return self._reuse_series_implementation( + return self._reuse_series( "ewm_mean", com=com, span=span, @@ -190,12 +190,10 @@ def ewm_mean( ) def cum_sum(self: Self, *, reverse: bool) -> Self: - return self._reuse_series_implementation( - "cum_sum", call_kwargs={"reverse": reverse} - ) + return self._reuse_series("cum_sum", call_kwargs={"reverse": reverse}) def shift(self: Self, n: int) -> Self: - return self._reuse_series_implementation("shift", call_kwargs={"n": n}) + return self._reuse_series("shift", call_kwargs={"n": n}) def over(self: Self, partition_by: Sequence[str], kind: ExprKind) -> Self: if not is_elementary_expression(self): @@ -300,24 +298,16 @@ def func(df: PandasLikeDataFrame) -> list[PandasLikeSeries]: ) def cum_count(self: Self, *, reverse: bool) -> Self: - return self._reuse_series_implementation( - "cum_count", call_kwargs={"reverse": reverse} - ) + return self._reuse_series("cum_count", call_kwargs={"reverse": reverse}) def cum_min(self: Self, *, reverse: bool) -> Self: - return self._reuse_series_implementation( - "cum_min", call_kwargs={"reverse": reverse} - ) + return self._reuse_series("cum_min", call_kwargs={"reverse": reverse}) def cum_max(self: Self, *, reverse: bool) -> Self: - return self._reuse_series_implementation( - "cum_max", call_kwargs={"reverse": reverse} - ) + return self._reuse_series("cum_max", call_kwargs={"reverse": reverse}) def cum_prod(self: Self, *, reverse: bool) -> Self: - return self._reuse_series_implementation( - "cum_prod", call_kwargs={"reverse": reverse} - ) + return self._reuse_series("cum_prod", call_kwargs={"reverse": reverse}) def rank( self: Self, @@ -325,7 +315,7 @@ def rank( *, descending: bool, ) -> Self: - return self._reuse_series_implementation( + return self._reuse_series( "rank", call_kwargs={"method": method, "descending": descending} ) diff --git a/narwhals/_pandas_like/expr_cat.py b/narwhals/_pandas_like/expr_cat.py index 21fa2d7c0d..00a04bd3d9 100644 --- a/narwhals/_pandas_like/expr_cat.py +++ b/narwhals/_pandas_like/expr_cat.py @@ -13,6 +13,4 @@ def __init__(self: Self, expr: PandasLikeExpr) -> None: self._compliant_expr = expr def get_categories(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "cat", "get_categories" - ) + return self._compliant_expr._reuse_series_namespace("cat", "get_categories") diff --git a/narwhals/_pandas_like/expr_dt.py b/narwhals/_pandas_like/expr_dt.py index e37d575ac5..26b38394e5 100644 --- a/narwhals/_pandas_like/expr_dt.py +++ b/narwhals/_pandas_like/expr_dt.py @@ -14,92 +14,72 @@ def __init__(self: Self, expr: PandasLikeExpr) -> None: self._compliant_expr = expr def date(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation("dt", "date") + return self._compliant_expr._reuse_series_namespace("dt", "date") def year(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation("dt", "year") + return self._compliant_expr._reuse_series_namespace("dt", "year") def month(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation("dt", "month") + return self._compliant_expr._reuse_series_namespace("dt", "month") def day(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation("dt", "day") + return self._compliant_expr._reuse_series_namespace("dt", "day") def hour(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation("dt", "hour") + return self._compliant_expr._reuse_series_namespace("dt", "hour") def minute(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation("dt", "minute") + return self._compliant_expr._reuse_series_namespace("dt", "minute") def second(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation("dt", "second") + return self._compliant_expr._reuse_series_namespace("dt", "second") def millisecond(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "dt", "millisecond" - ) + return self._compliant_expr._reuse_series_namespace("dt", "millisecond") def microsecond(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "dt", "microsecond" - ) + return self._compliant_expr._reuse_series_namespace("dt", "microsecond") def nanosecond(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "dt", "nanosecond" - ) + return self._compliant_expr._reuse_series_namespace("dt", "nanosecond") def ordinal_day(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "dt", "ordinal_day" - ) + return self._compliant_expr._reuse_series_namespace("dt", "ordinal_day") def weekday(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "dt", "weekday" - ) + return self._compliant_expr._reuse_series_namespace("dt", "weekday") def total_minutes(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "dt", "total_minutes" - ) + return self._compliant_expr._reuse_series_namespace("dt", "total_minutes") def total_seconds(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "dt", "total_seconds" - ) + return self._compliant_expr._reuse_series_namespace("dt", "total_seconds") def total_milliseconds(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "dt", "total_milliseconds" - ) + return self._compliant_expr._reuse_series_namespace("dt", "total_milliseconds") def total_microseconds(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "dt", "total_microseconds" - ) + return self._compliant_expr._reuse_series_namespace("dt", "total_microseconds") def total_nanoseconds(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "dt", "total_nanoseconds" - ) + return self._compliant_expr._reuse_series_namespace("dt", "total_nanoseconds") def to_string(self: Self, format: str) -> PandasLikeExpr: # noqa: A002 - return self._compliant_expr._reuse_series_namespace_implementation( + return self._compliant_expr._reuse_series_namespace( "dt", "to_string", format=format ) def replace_time_zone(self: Self, time_zone: str | None) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( + return self._compliant_expr._reuse_series_namespace( "dt", "replace_time_zone", time_zone=time_zone ) def convert_time_zone(self: Self, time_zone: str) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( + return self._compliant_expr._reuse_series_namespace( "dt", "convert_time_zone", time_zone=time_zone ) def timestamp(self: Self, time_unit: TimeUnit) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( + return self._compliant_expr._reuse_series_namespace( "dt", "timestamp", time_unit=time_unit ) diff --git a/narwhals/_pandas_like/expr_list.py b/narwhals/_pandas_like/expr_list.py index 41ba1337fd..f8563c2553 100644 --- a/narwhals/_pandas_like/expr_list.py +++ b/narwhals/_pandas_like/expr_list.py @@ -13,4 +13,4 @@ def __init__(self: Self, expr: PandasLikeExpr) -> None: self._expr = expr def len(self: Self) -> PandasLikeExpr: - return self._expr._reuse_series_namespace_implementation("list", "len") + return self._expr._reuse_series_namespace("list", "len") diff --git a/narwhals/_pandas_like/expr_str.py b/narwhals/_pandas_like/expr_str.py index e349ffc757..c3ebbfcee1 100644 --- a/narwhals/_pandas_like/expr_str.py +++ b/narwhals/_pandas_like/expr_str.py @@ -13,65 +13,57 @@ def __init__(self: Self, expr: PandasLikeExpr) -> None: self._compliant_expr = expr def len_chars(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "str", "len_chars" - ) + return self._compliant_expr._reuse_series_namespace("str", "len_chars") def replace( self: Self, pattern: str, value: str, *, literal: bool, n: int ) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( + return self._compliant_expr._reuse_series_namespace( "str", "replace", pattern=pattern, value=value, literal=literal, n=n ) def replace_all( self: Self, pattern: str, value: str, *, literal: bool ) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( + return self._compliant_expr._reuse_series_namespace( "str", "replace_all", pattern=pattern, value=value, literal=literal ) def strip_chars(self: Self, characters: str | None) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( + return self._compliant_expr._reuse_series_namespace( "str", "strip_chars", characters=characters ) def starts_with(self: Self, prefix: str) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( + return self._compliant_expr._reuse_series_namespace( "str", "starts_with", prefix=prefix ) def ends_with(self: Self, suffix: str) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( + return self._compliant_expr._reuse_series_namespace( "str", "ends_with", suffix=suffix ) def contains(self: Self, pattern: str, *, literal: bool) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( + return self._compliant_expr._reuse_series_namespace( "str", "contains", pattern=pattern, literal=literal ) def slice(self: Self, offset: int, length: int | None) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( + return self._compliant_expr._reuse_series_namespace( "str", "slice", offset=offset, length=length ) def split(self: Self, by: str) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "str", "split", by=by - ) + return self._compliant_expr._reuse_series_namespace("str", "split", by=by) def to_datetime(self: Self, format: str | None) -> PandasLikeExpr: # noqa: A002 - return self._compliant_expr._reuse_series_namespace_implementation( + return self._compliant_expr._reuse_series_namespace( "str", "to_datetime", format=format ) def to_uppercase(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "str", "to_uppercase" - ) + return self._compliant_expr._reuse_series_namespace("str", "to_uppercase") def to_lowercase(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace_implementation( - "str", "to_lowercase" - ) + return self._compliant_expr._reuse_series_namespace("str", "to_lowercase") From 7cd3e2d96bb0549417f739bd82eab58371b3b811 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 7 Mar 2025 17:48:21 +0000 Subject: [PATCH 39/58] refactor: rename `attr` -> `method_name`, add annotation --- narwhals/_compliant/expr.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index dbfdbadc44..a28ac744ad 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -325,7 +325,7 @@ def from_column_indices( def _reuse_series( self: Self, - attr: str, + method_name: str, *, returns_scalar: bool = False, call_kwargs: dict[str, Any] | None = None, @@ -337,7 +337,7 @@ def _reuse_series( leverage this method to do that for us. Arguments: - attr: name of method. + method_name: name of method. returns_scalar: whether the Series version returns a scalar. In this case, the expression version should return a 1-row Series. call_kwargs: non-expressifiable args which we may need to reuse in `agg` or `over`, @@ -347,7 +347,7 @@ def _reuse_series( """ func = partial( self._reuse_series_inner, - method_name=attr, + method_name=method_name, returns_scalar=returns_scalar, call_kwargs=call_kwargs or {}, expressifiable_args=expressifiable_args, @@ -355,7 +355,7 @@ def _reuse_series( return self._from_callable( func, depth=self._depth + 1, - function_name=f"{self._function_name}->{attr}", + function_name=f"{self._function_name}->{method_name}", evaluate_output_names=self._evaluate_output_names, alias_output_names=self._alias_output_names, call_kwargs=call_kwargs, @@ -405,25 +405,28 @@ def _reuse_series_inner( return out def _reuse_series_namespace( - self: Self, series_namespace: str, attr: str, **kwargs: Any + self: Self, + series_namespace: Literal["cat", "dt", "list", "str", "name"], + method_name: str, + **kwargs: Any, ) -> Self: """Reuse Series implementation for expression. - Just like `_reuse_series_implementation`, but for e.g. `Expr.dt.foo` instead + Just like `_reuse_series`, but for e.g. `Expr.dt.foo` instead of `Expr.foo`. Arguments: series_namespace: The Series namespace (e.g. `dt`, `cat`, `str`, `list`, `name`) - attr: name of method. + method_name: name of method. kwargs: keyword arguments to pass to function. """ return self._from_callable( lambda df: [ - getattr(getattr(series, series_namespace), attr)(**kwargs) + getattr(getattr(series, series_namespace), method_name)(**kwargs) for series in self(df) ], depth=self._depth + 1, - function_name=f"{self._function_name}->{series_namespace}.{attr}", + function_name=f"{self._function_name}->{series_namespace}.{method_name}", evaluate_output_names=self._evaluate_output_names, alias_output_names=self._alias_output_names, call_kwargs={**self._call_kwargs, **kwargs}, From 035976bd718763f835121b301db350510d32a10b Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Sat, 8 Mar 2025 13:07:43 +0000 Subject: [PATCH 40/58] feat: Define `CompliantSeries._to_expr` Resolves: (https://github.com/narwhals-dev/narwhals/pull/2149#discussion_r1985465541) Doing this properly surfaced lots of issues - `Polars*` classes are *similar* to `Compliant*` - Not close enough to support typing - Lots of stuff is behind `__getattr__` - We don't need `PolarsExpr` to do a lot of the work `CompliantExpr` does - `nw.Series._compliant_series` is still a big PR away - Repeat of (#2119) - Last big hurdle to get (https://github.com/narwhals-dev/narwhals/pull/2104#discussion_r1971956027) --- narwhals/_compliant/series.py | 11 +++++++++++ narwhals/_expression_parsing.py | 10 ++-------- narwhals/_pandas_like/series.py | 8 ++++++++ narwhals/_polars/expr.py | 8 ++++++++ narwhals/_polars/namespace.py | 16 ++++++++-------- narwhals/_polars/series.py | 12 ++++++++++++ narwhals/dataframe.py | 9 ++------- narwhals/series.py | 2 ++ narwhals/utils.py | 11 ----------- 9 files changed, 53 insertions(+), 34 deletions(-) diff --git a/narwhals/_compliant/series.py b/narwhals/_compliant/series.py index bb5337d4e6..0d71c6ca31 100644 --- a/narwhals/_compliant/series.py +++ b/narwhals/_compliant/series.py @@ -9,6 +9,10 @@ if TYPE_CHECKING: from typing_extensions import Self + from narwhals._compliant.expr import CompliantExpr # noqa: F401 + from narwhals._compliant.expr import EagerExpr + from narwhals._compliant.namespace import CompliantNamespace # noqa: F401 + from narwhals._compliant.namespace import EagerNamespace from narwhals.dtypes import DType from narwhals.typing import NativeSeries from narwhals.utils import Implementation @@ -27,6 +31,8 @@ def dtype(self) -> DType: ... def name(self) -> str: ... def __narwhals_series__(self) -> CompliantSeries: ... def alias(self, name: str) -> Self: ... + def __narwhals_namespace__(self) -> Any: ... # CompliantNamespace[Any, Self]: ... + def _to_expr(self) -> Any: ... # CompliantExpr[Any, Self]: ... class EagerSeries(CompliantSeries, Protocol[NativeSeriesT_co]): @@ -50,6 +56,11 @@ def _from_iterable( cls: type[Self], data: Iterable[Any], name: str, *, context: _FullContext ) -> Self: ... + def __narwhals_namespace__(self) -> EagerNamespace[Any, Self]: ... + + def _to_expr(self) -> EagerExpr[Any, Self]: + return self.__narwhals_namespace__()._expr._from_series(self) + # TODO @dangotbanned: replacing `Namespace._create_compliant_series`` # - All usage within `*Expr.map_batches` # - `PandasLikeExpr` uses that **once** diff --git a/narwhals/_expression_parsing.py b/narwhals/_expression_parsing.py index 45cd8d89c4..b149aac8ca 100644 --- a/narwhals/_expression_parsing.py +++ b/narwhals/_expression_parsing.py @@ -18,7 +18,6 @@ from narwhals.exceptions import LengthChangingExprError from narwhals.exceptions import ShapeError from narwhals.utils import is_compliant_expr -from narwhals.utils import is_eager_namespace if TYPE_CHECKING: from typing_extensions import Never @@ -132,14 +131,9 @@ def extract_compliant( if isinstance(other, str) and not str_as_lit: return plx.col(other) if is_narwhals_series(other): - if is_eager_namespace(plx): - return plx._expr._from_series(other._compliant_series) - return plx._create_expr_from_series(other._compliant_series) # type: ignore[attr-defined] + return other._compliant_series._to_expr() if is_numpy_array(other): - if is_eager_namespace(plx): - return plx._expr._from_series(plx._create_compliant_series(other)) - series = plx._create_compliant_series(other) # type: ignore[attr-defined] - return plx._create_expr_from_series(series) # type: ignore[attr-defined] + return plx._create_compliant_series(other)._to_expr() # type: ignore[attr-defined] return other diff --git a/narwhals/_pandas_like/series.py b/narwhals/_pandas_like/series.py index f5772f6e3d..fc8a0112ce 100644 --- a/narwhals/_pandas_like/series.py +++ b/narwhals/_pandas_like/series.py @@ -39,6 +39,7 @@ from narwhals._arrow.typing import ArrowArray from narwhals._pandas_like.dataframe import PandasLikeDataFrame + from narwhals._pandas_like.namespace import PandasLikeNamespace from narwhals.dtypes import DType from narwhals.typing import _1DArray from narwhals.typing import _AnyDArray @@ -131,6 +132,13 @@ def __native_namespace__(self: Self) -> ModuleType: def __narwhals_series__(self: Self) -> Self: return self + def __narwhals_namespace__(self) -> PandasLikeNamespace: + from narwhals._pandas_like.namespace import PandasLikeNamespace + + return PandasLikeNamespace( + self._implementation, self._backend_version, self._version + ) + @overload def __getitem__(self: Self, idx: int) -> Any: ... diff --git a/narwhals/_polars/expr.py b/narwhals/_polars/expr.py index d39d2cdb31..f21361c57e 100644 --- a/narwhals/_polars/expr.py +++ b/narwhals/_polars/expr.py @@ -38,6 +38,14 @@ def _from_native_expr(self: Self, expr: pl.Expr) -> Self: expr, version=self._version, backend_version=self._backend_version ) + @classmethod + def _from_series(cls, series: Any) -> Self: + return cls( + series._native_series, + version=series._version, + backend_version=series._backend_version, + ) + def broadcast(self, kind: Literal[ExprKind.AGGREGATION, ExprKind.LITERAL]) -> Self: # Let Polars do its thing. return self diff --git a/narwhals/_polars/namespace.py b/narwhals/_polars/namespace.py index 6f1dc95daa..5e4302a5d0 100644 --- a/narwhals/_polars/namespace.py +++ b/narwhals/_polars/namespace.py @@ -49,19 +49,19 @@ def func(*args: Any, **kwargs: Any) -> Any: return func + @property + def _expr(self) -> type[PolarsExpr]: + return PolarsExpr + + @property + def _series(self) -> type[PolarsSeries]: + return PolarsSeries + def _create_compliant_series(self, value: Any) -> PolarsSeries: return PolarsSeries( pl.Series(value), backend_version=self._backend_version, version=self._version ) - def _create_expr_from_series(self, value: Any) -> PolarsExpr: - # Let Polars do its own thing. - return PolarsExpr( - value._native_series, - version=self._version, - backend_version=self._backend_version, - ) - def nth(self: Self, *indices: int) -> PolarsExpr: from narwhals._polars.expr import PolarsExpr diff --git a/narwhals/_polars/series.py b/narwhals/_polars/series.py index 73133a3bea..adb29eb800 100644 --- a/narwhals/_polars/series.py +++ b/narwhals/_polars/series.py @@ -23,6 +23,8 @@ from typing_extensions import Self from narwhals._polars.dataframe import PolarsDataFrame + from narwhals._polars.expr import PolarsExpr + from narwhals._polars.namespace import PolarsNamespace from narwhals.dtypes import DType from narwhals.typing import _1DArray from narwhals.utils import Version @@ -47,6 +49,13 @@ def __init__( def __repr__(self: Self) -> str: # pragma: no cover return "PolarsSeries" + def __narwhals_namespace__(self) -> PolarsNamespace: + from narwhals._polars.namespace import PolarsNamespace + + return PolarsNamespace( + backend_version=self._backend_version, version=self._version + ) + def __narwhals_series__(self: Self) -> Self: return self @@ -90,6 +99,9 @@ def _from_native_object( # scalar return series + def _to_expr(self) -> PolarsExpr: + return self.__narwhals_namespace__()._expr._from_series(self) + def __getattr__(self: Self, attr: str) -> Any: if attr == "as_py": # pragma: no cover raise AttributeError diff --git a/narwhals/dataframe.py b/narwhals/dataframe.py index d8f0fc09fd..2bbeb3918a 100644 --- a/narwhals/dataframe.py +++ b/narwhals/dataframe.py @@ -33,7 +33,6 @@ from narwhals.utils import find_stacklevel from narwhals.utils import flatten from narwhals.utils import generate_repr -from narwhals.utils import is_eager_namespace from narwhals.utils import is_sequence_but_not_str from narwhals.utils import issue_deprecation_warning from narwhals.utils import parse_version @@ -430,9 +429,7 @@ def _extract_compliant(self: Self, arg: Any) -> Any: if isinstance(arg, BaseFrame): return arg._compliant_frame if isinstance(arg, Series): - if is_eager_namespace(plx): - return plx._expr._from_series(arg._compliant_series) - return plx._create_expr_from_series(arg._compliant_series) + return arg._compliant_series._to_expr() if isinstance(arg, Expr): return arg._to_compliant_expr(self.__narwhals_namespace__()) if isinstance(arg, str): @@ -446,9 +443,7 @@ def _extract_compliant(self: Self, arg: Any) -> Any: ) raise TypeError(msg) if is_numpy_array(arg): - if is_eager_namespace(plx): - return plx._expr._from_series(plx._create_compliant_series(arg)) - return plx._create_expr_from_series(plx._create_compliant_series(arg)) + return plx._create_compliant_series(arg)._to_expr() raise InvalidIntoExprError.from_invalid_type(type(arg)) @property diff --git a/narwhals/series.py b/narwhals/series.py index 64c967d11a..a27f0de732 100644 --- a/narwhals/series.py +++ b/narwhals/series.py @@ -76,6 +76,8 @@ def __init__( ) -> None: self._level: Literal["full", "lazy", "interchange"] = level if hasattr(series, "__narwhals_series__"): + # TODO @dangotbanned: Repeat (#2119) for `CompliantSeries` to support typing + # morally: `CompliantSeries` self._compliant_series = series.__narwhals_series__() else: # pragma: no cover msg = f"Expected Polars Series or an object which implements `__narwhals_series__`, got: {type(series)}." diff --git a/narwhals/utils.py b/narwhals/utils.py index c311ce3389..798c6f5221 100644 --- a/narwhals/utils.py +++ b/narwhals/utils.py @@ -56,9 +56,6 @@ from narwhals._compliant import CompliantFrameT from narwhals._compliant import CompliantSeriesOrNativeExprT_co from narwhals._compliant import CompliantSeriesT_co - from narwhals._compliant import EagerDataFrameT - from narwhals._compliant import EagerNamespace - from narwhals._compliant import EagerSeriesT from narwhals.dataframe import DataFrame from narwhals.dataframe import LazyFrame from narwhals.dtypes import DType @@ -1402,14 +1399,6 @@ def is_compliant_expr( return hasattr(obj, "__narwhals_expr__") -# NOTE: Temporary - just to introduce a path for the (Arrow|PandasLike) WIP -def is_eager_namespace( - obj: EagerNamespace[EagerDataFrameT, EagerSeriesT] | Any, -) -> TypeIs[EagerNamespace[EagerDataFrameT, EagerSeriesT]]: - return type(obj).__name__ in {"ArrowNamespace", "PandasLikeNamespace"} - # return all(hasattr(obj, name) for name in ("selectors", "_expr", "_series")) # noqa: ERA001 - - def has_native_namespace(obj: Any) -> TypeIs[SupportsNativeNamespace]: return hasattr(obj, "__native_namespace__") From ba14a82f1beefcb45d157b0fd143a90555cf3d3e Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Sat, 8 Mar 2025 14:09:06 +0000 Subject: [PATCH 41/58] refactor: get coverage and simplify `_polars.namespace` Prompted by https://github.com/narwhals-dev/narwhals/actions/runs/13738142838/job/38424253024?pr=2149 --- narwhals/_polars/namespace.py | 54 +++++++---------------------------- 1 file changed, 11 insertions(+), 43 deletions(-) diff --git a/narwhals/_polars/namespace.py b/narwhals/_polars/namespace.py index 5e4302a5d0..2bcf62f949 100644 --- a/narwhals/_polars/namespace.py +++ b/narwhals/_polars/namespace.py @@ -37,11 +37,9 @@ def __init__( self._version = version def __getattr__(self: Self, attr: str) -> Any: - from narwhals._polars.expr import PolarsExpr - def func(*args: Any, **kwargs: Any) -> Any: args, kwargs = extract_args_kwargs(args, kwargs) # type: ignore[assignment] - return PolarsExpr( + return self._expr( getattr(pl, attr)(*args, **kwargs), version=self._version, backend_version=self._backend_version, @@ -58,30 +56,26 @@ def _series(self) -> type[PolarsSeries]: return PolarsSeries def _create_compliant_series(self, value: Any) -> PolarsSeries: - return PolarsSeries( + return self._series( pl.Series(value), backend_version=self._backend_version, version=self._version ) def nth(self: Self, *indices: int) -> PolarsExpr: - from narwhals._polars.expr import PolarsExpr - if self._backend_version < (1, 0, 0): msg = "`nth` is only supported for Polars>=1.0.0. Please use `col` for columns selection instead." raise AttributeError(msg) - return PolarsExpr( + return self._expr( pl.nth(*indices), version=self._version, backend_version=self._backend_version ) def len(self: Self) -> PolarsExpr: - from narwhals._polars.expr import PolarsExpr - if self._backend_version < (0, 20, 5): - return PolarsExpr( + return self._expr( pl.count().alias("len"), version=self._version, backend_version=self._backend_version, ) - return PolarsExpr( + return self._expr( pl.len(), version=self._version, backend_version=self._backend_version ) @@ -123,10 +117,8 @@ def concat( ) def lit(self: Self, value: Any, dtype: DType | None) -> PolarsExpr: - from narwhals._polars.expr import PolarsExpr - if dtype is not None: - return PolarsExpr( + return self._expr( pl.lit( value, dtype=narwhals_to_native_dtype( @@ -136,22 +128,20 @@ def lit(self: Self, value: Any, dtype: DType | None) -> PolarsExpr: version=self._version, backend_version=self._backend_version, ) - return PolarsExpr( + return self._expr( pl.lit(value), version=self._version, backend_version=self._backend_version ) def mean_horizontal(self: Self, *exprs: PolarsExpr) -> PolarsExpr: - from narwhals._polars.expr import PolarsExpr - if self._backend_version < (0, 20, 8): - return PolarsExpr( + return self._expr( pl.sum_horizontal(e._native_expr for e in exprs) / pl.sum_horizontal(1 - e.is_null()._native_expr for e in exprs), version=self._version, backend_version=self._backend_version, ) - return PolarsExpr( + return self._expr( pl.mean_horizontal(e._native_expr for e in exprs), version=self._version, backend_version=self._backend_version, @@ -163,8 +153,6 @@ def concat_str( separator: str, ignore_nulls: bool, ) -> PolarsExpr: - from narwhals._polars.expr import PolarsExpr - pl_exprs: list[pl.Expr] = [expr._native_expr for expr in exprs] if self._backend_version < (0, 20, 6): @@ -193,11 +181,11 @@ def concat_str( exprs=[s + v for s, v in zip(separators, values)], ) - return PolarsExpr( + return self._expr( result, version=self._version, backend_version=self._backend_version ) - return PolarsExpr( + return self._expr( pl.concat_str( pl_exprs, separator=separator, @@ -218,8 +206,6 @@ def __init__(self: Self, version: Version, backend_version: tuple[int, ...]) -> self._backend_version = backend_version def by_dtype(self: Self, dtypes: Iterable[DType]) -> PolarsExpr: - from narwhals._polars.expr import PolarsExpr - native_dtypes = [ narwhals_to_native_dtype( dtype, self._version, self._backend_version @@ -235,10 +221,6 @@ def by_dtype(self: Self, dtypes: Iterable[DType]) -> PolarsExpr: ) def matches(self: Self, pattern: str) -> PolarsExpr: - import polars as pl - - from narwhals._polars.expr import PolarsExpr - return PolarsExpr( pl.selectors.matches(pattern=pattern), version=self._version, @@ -246,8 +228,6 @@ def matches(self: Self, pattern: str) -> PolarsExpr: ) def numeric(self: Self) -> PolarsExpr: - from narwhals._polars.expr import PolarsExpr - return PolarsExpr( pl.selectors.numeric(), version=self._version, @@ -255,8 +235,6 @@ def numeric(self: Self) -> PolarsExpr: ) def boolean(self: Self) -> PolarsExpr: - from narwhals._polars.expr import PolarsExpr - return PolarsExpr( pl.selectors.boolean(), version=self._version, @@ -264,8 +242,6 @@ def boolean(self: Self) -> PolarsExpr: ) def string(self: Self) -> PolarsExpr: - from narwhals._polars.expr import PolarsExpr - return PolarsExpr( pl.selectors.string(), version=self._version, @@ -273,8 +249,6 @@ def string(self: Self) -> PolarsExpr: ) def categorical(self: Self) -> PolarsExpr: - from narwhals._polars.expr import PolarsExpr - return PolarsExpr( pl.selectors.categorical(), version=self._version, @@ -282,8 +256,6 @@ def categorical(self: Self) -> PolarsExpr: ) def all(self: Self) -> PolarsExpr: - from narwhals._polars.expr import PolarsExpr - return PolarsExpr( pl.selectors.all(), version=self._version, @@ -295,10 +267,6 @@ def datetime( time_unit: TimeUnit | Iterable[TimeUnit] | None, time_zone: str | timezone | Iterable[str | timezone | None] | None, ) -> PolarsExpr: - import polars as pl - - from narwhals._polars.expr import PolarsExpr - return PolarsExpr( pl.selectors.datetime(time_unit=time_unit, time_zone=time_zone), # type: ignore[arg-type] version=self._version, From 612df692b356f602c44738362174c2d3df1163c5 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Sat, 8 Mar 2025 16:15:00 +0000 Subject: [PATCH 42/58] refactor: avoid reimplementing `ArrowSeries._from_iterable` - Eventually want to get rid of `create_compliant_series` - `pandas` currently works differently in `create_compliant_series` and `native_series_from_iterable` --- narwhals/_arrow/namespace.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/narwhals/_arrow/namespace.py b/narwhals/_arrow/namespace.py index 946f5917d0..d1e95308cb 100644 --- a/narwhals/_arrow/namespace.py +++ b/narwhals/_arrow/namespace.py @@ -12,7 +12,6 @@ from typing import Literal from typing import Sequence -import pyarrow as pa import pyarrow.compute as pc from narwhals._arrow.dataframe import ArrowDataFrame @@ -58,12 +57,7 @@ def _series(self) -> type[ArrowSeries]: return ArrowSeries def _create_compliant_series(self: Self, value: Any) -> ArrowSeries: - return self._series( - native_series=pa.chunked_array([value]), - name="", - backend_version=self._backend_version, - version=self._version, - ) + return self._series._from_iterable(value, name="", context=self) # --- not in spec --- def __init__( From 5a13b63a12f211bbc58977b017bbf3783d61cf54 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Sat, 8 Mar 2025 18:43:17 +0000 Subject: [PATCH 43/58] =?UTF-8?q?feat:=20Super=20reusable=20namespaces=20?= =?UTF-8?q?=F0=9F=A4=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only `.name` differs between the two `EagerExpr` - so why not share? --- narwhals/_arrow/expr.py | 20 ---- narwhals/_arrow/expr_cat.py | 15 --- narwhals/_arrow/expr_dt.py | 82 ------------- narwhals/_arrow/expr_list.py | 15 --- narwhals/_arrow/expr_str.py | 60 ---------- narwhals/_compliant/any_namespace.py | 80 +++++++++++++ narwhals/_compliant/expr.py | 166 ++++++++++++++++++++++++++- narwhals/_compliant/typing.py | 2 + narwhals/_pandas_like/expr.py | 20 ---- narwhals/_pandas_like/expr_cat.py | 16 --- narwhals/_pandas_like/expr_dt.py | 85 -------------- narwhals/_pandas_like/expr_list.py | 16 --- narwhals/_pandas_like/expr_str.py | 69 ----------- 13 files changed, 247 insertions(+), 399 deletions(-) delete mode 100644 narwhals/_arrow/expr_cat.py delete mode 100644 narwhals/_arrow/expr_dt.py delete mode 100644 narwhals/_arrow/expr_list.py delete mode 100644 narwhals/_arrow/expr_str.py create mode 100644 narwhals/_compliant/any_namespace.py delete mode 100644 narwhals/_pandas_like/expr_cat.py delete mode 100644 narwhals/_pandas_like/expr_dt.py delete mode 100644 narwhals/_pandas_like/expr_list.py delete mode 100644 narwhals/_pandas_like/expr_str.py diff --git a/narwhals/_arrow/expr.py b/narwhals/_arrow/expr.py index e5b1cdf064..f0276e8a4b 100644 --- a/narwhals/_arrow/expr.py +++ b/narwhals/_arrow/expr.py @@ -8,11 +8,7 @@ import pyarrow.compute as pc -from narwhals._arrow.expr_cat import ArrowExprCatNamespace -from narwhals._arrow.expr_dt import ArrowExprDateTimeNamespace -from narwhals._arrow.expr_list import ArrowExprListNamespace from narwhals._arrow.expr_name import ArrowExprNameNamespace -from narwhals._arrow.expr_str import ArrowExprStringNamespace from narwhals._arrow.series import ArrowSeries from narwhals._compliant import EagerExpr from narwhals._expression_parsing import ExprKind @@ -268,22 +264,6 @@ def rank( ewm_mean = not_implemented() - @property - def dt(self: Self) -> ArrowExprDateTimeNamespace: - return ArrowExprDateTimeNamespace(self) - - @property - def str(self: Self) -> ArrowExprStringNamespace: - return ArrowExprStringNamespace(self) - - @property - def cat(self: Self) -> ArrowExprCatNamespace: - return ArrowExprCatNamespace(self) - @property def name(self: Self) -> ArrowExprNameNamespace: return ArrowExprNameNamespace(self) - - @property - def list(self: Self) -> ArrowExprListNamespace: - return ArrowExprListNamespace(self) diff --git a/narwhals/_arrow/expr_cat.py b/narwhals/_arrow/expr_cat.py deleted file mode 100644 index d7bf7891b6..0000000000 --- a/narwhals/_arrow/expr_cat.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -from narwhals._arrow.utils import ArrowExprNamespace - -if TYPE_CHECKING: - from typing_extensions import Self - - from narwhals._arrow.expr import ArrowExpr - - -class ArrowExprCatNamespace(ArrowExprNamespace): - def get_categories(self: Self) -> ArrowExpr: - return self.compliant._reuse_series_namespace("cat", "get_categories") diff --git a/narwhals/_arrow/expr_dt.py b/narwhals/_arrow/expr_dt.py deleted file mode 100644 index 38faaddf80..0000000000 --- a/narwhals/_arrow/expr_dt.py +++ /dev/null @@ -1,82 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -from narwhals._arrow.utils import ArrowExprNamespace - -if TYPE_CHECKING: - from typing_extensions import Self - - from narwhals._arrow.expr import ArrowExpr - from narwhals.typing import TimeUnit - - -class ArrowExprDateTimeNamespace(ArrowExprNamespace): - def to_string(self: Self, format: str) -> ArrowExpr: # noqa: A002 - return self.compliant._reuse_series_namespace("dt", "to_string", format=format) - - def replace_time_zone(self: Self, time_zone: str | None) -> ArrowExpr: - return self.compliant._reuse_series_namespace( - "dt", "replace_time_zone", time_zone=time_zone - ) - - def convert_time_zone(self: Self, time_zone: str) -> ArrowExpr: - return self.compliant._reuse_series_namespace( - "dt", "convert_time_zone", time_zone=time_zone - ) - - def timestamp(self: Self, time_unit: TimeUnit) -> ArrowExpr: - return self.compliant._reuse_series_namespace( - "dt", "timestamp", time_unit=time_unit - ) - - def date(self: Self) -> ArrowExpr: - return self.compliant._reuse_series_namespace("dt", "date") - - def year(self: Self) -> ArrowExpr: - return self.compliant._reuse_series_namespace("dt", "year") - - def month(self: Self) -> ArrowExpr: - return self.compliant._reuse_series_namespace("dt", "month") - - def day(self: Self) -> ArrowExpr: - return self.compliant._reuse_series_namespace("dt", "day") - - def hour(self: Self) -> ArrowExpr: - return self.compliant._reuse_series_namespace("dt", "hour") - - def minute(self: Self) -> ArrowExpr: - return self.compliant._reuse_series_namespace("dt", "minute") - - def second(self: Self) -> ArrowExpr: - return self.compliant._reuse_series_namespace("dt", "second") - - def millisecond(self: Self) -> ArrowExpr: - return self.compliant._reuse_series_namespace("dt", "millisecond") - - def microsecond(self: Self) -> ArrowExpr: - return self.compliant._reuse_series_namespace("dt", "microsecond") - - def nanosecond(self: Self) -> ArrowExpr: - return self.compliant._reuse_series_namespace("dt", "nanosecond") - - def ordinal_day(self: Self) -> ArrowExpr: - return self.compliant._reuse_series_namespace("dt", "ordinal_day") - - def weekday(self: Self) -> ArrowExpr: - return self.compliant._reuse_series_namespace("dt", "weekday") - - def total_minutes(self: Self) -> ArrowExpr: - return self.compliant._reuse_series_namespace("dt", "total_minutes") - - def total_seconds(self: Self) -> ArrowExpr: - return self.compliant._reuse_series_namespace("dt", "total_seconds") - - def total_milliseconds(self: Self) -> ArrowExpr: - return self.compliant._reuse_series_namespace("dt", "total_milliseconds") - - def total_microseconds(self: Self) -> ArrowExpr: - return self.compliant._reuse_series_namespace("dt", "total_microseconds") - - def total_nanoseconds(self: Self) -> ArrowExpr: - return self.compliant._reuse_series_namespace("dt", "total_nanoseconds") diff --git a/narwhals/_arrow/expr_list.py b/narwhals/_arrow/expr_list.py deleted file mode 100644 index fcfd29731b..0000000000 --- a/narwhals/_arrow/expr_list.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -from narwhals._arrow.utils import ArrowExprNamespace - -if TYPE_CHECKING: - from typing_extensions import Self - - from narwhals._arrow.expr import ArrowExpr - - -class ArrowExprListNamespace(ArrowExprNamespace): - def len(self: Self) -> ArrowExpr: - return self.compliant._reuse_series_namespace("list", "len") diff --git a/narwhals/_arrow/expr_str.py b/narwhals/_arrow/expr_str.py deleted file mode 100644 index 572eea69e0..0000000000 --- a/narwhals/_arrow/expr_str.py +++ /dev/null @@ -1,60 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -from narwhals._arrow.utils import ArrowExprNamespace - -if TYPE_CHECKING: - from typing_extensions import Self - - from narwhals._arrow.expr import ArrowExpr - - -class ArrowExprStringNamespace(ArrowExprNamespace): - def len_chars(self: Self) -> ArrowExpr: - return self.compliant._reuse_series_namespace("str", "len_chars") - - def replace( - self: Self, pattern: str, value: str, *, literal: bool, n: int - ) -> ArrowExpr: - return self.compliant._reuse_series_namespace( - "str", "replace", pattern=pattern, value=value, literal=literal, n=n - ) - - def replace_all(self: Self, pattern: str, value: str, *, literal: bool) -> ArrowExpr: - return self.compliant._reuse_series_namespace( - "str", "replace_all", pattern=pattern, value=value, literal=literal - ) - - def strip_chars(self: Self, characters: str | None) -> ArrowExpr: - return self.compliant._reuse_series_namespace( - "str", "strip_chars", characters=characters - ) - - def starts_with(self: Self, prefix: str) -> ArrowExpr: - return self.compliant._reuse_series_namespace("str", "starts_with", prefix=prefix) - - def ends_with(self: Self, suffix: str) -> ArrowExpr: - return self.compliant._reuse_series_namespace("str", "ends_with", suffix=suffix) - - def contains(self: Self, pattern: str, *, literal: bool) -> ArrowExpr: - return self.compliant._reuse_series_namespace( - "str", "contains", pattern=pattern, literal=literal - ) - - def slice(self: Self, offset: int, length: int | None) -> ArrowExpr: - return self.compliant._reuse_series_namespace( - "str", "slice", offset=offset, length=length - ) - - def split(self: Self, by: str) -> ArrowExpr: - return self.compliant._reuse_series_namespace("str", "split", by=by) - - def to_datetime(self: Self, format: str | None) -> ArrowExpr: # noqa: A002 - return self.compliant._reuse_series_namespace("str", "to_datetime", format=format) - - def to_uppercase(self: Self) -> ArrowExpr: - return self.compliant._reuse_series_namespace("str", "to_uppercase") - - def to_lowercase(self: Self) -> ArrowExpr: - return self.compliant._reuse_series_namespace("str", "to_lowercase") diff --git a/narwhals/_compliant/any_namespace.py b/narwhals/_compliant/any_namespace.py new file mode 100644 index 0000000000..c50b08eb22 --- /dev/null +++ b/narwhals/_compliant/any_namespace.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +from typing import TYPE_CHECKING +from typing import Protocol + +from narwhals.utils import CompliantT_co +from narwhals.utils import _StoresCompliant + +if TYPE_CHECKING: + from typing import Callable + + from narwhals.typing import TimeUnit + +__all__ = [ + "CatNamespace", + "DateTimeNamespace", + "ListNamespace", + "NameNamespace", + "StringNamespace", +] + + +class CatNamespace(_StoresCompliant[CompliantT_co], Protocol[CompliantT_co]): + def get_categories(self) -> CompliantT_co: ... + + +class DateTimeNamespace(_StoresCompliant[CompliantT_co], Protocol[CompliantT_co]): + def to_string(self, format: str) -> CompliantT_co: ... # noqa: A002 + def replace_time_zone(self, time_zone: str | None) -> CompliantT_co: ... + def convert_time_zone(self, time_zone: str) -> CompliantT_co: ... + def timestamp(self, time_unit: TimeUnit) -> CompliantT_co: ... + def date(self) -> CompliantT_co: ... + def year(self) -> CompliantT_co: ... + def month(self) -> CompliantT_co: ... + def day(self) -> CompliantT_co: ... + def hour(self) -> CompliantT_co: ... + def minute(self) -> CompliantT_co: ... + def second(self) -> CompliantT_co: ... + def millisecond(self) -> CompliantT_co: ... + def microsecond(self) -> CompliantT_co: ... + def nanosecond(self) -> CompliantT_co: ... + def ordinal_day(self) -> CompliantT_co: ... + def weekday(self) -> CompliantT_co: ... + def total_minutes(self) -> CompliantT_co: ... + def total_seconds(self) -> CompliantT_co: ... + def total_milliseconds(self) -> CompliantT_co: ... + def total_microseconds(self) -> CompliantT_co: ... + def total_nanoseconds(self) -> CompliantT_co: ... + + +class ListNamespace(_StoresCompliant[CompliantT_co], Protocol[CompliantT_co]): + def len(self) -> CompliantT_co: ... + + +class NameNamespace(_StoresCompliant[CompliantT_co], Protocol[CompliantT_co]): + def keep(self) -> CompliantT_co: ... + def map(self, function: Callable[[str], str]) -> CompliantT_co: ... + def prefix(self, prefix: str) -> CompliantT_co: ... + def suffix(self, suffix: str) -> CompliantT_co: ... + def to_lowercase(self) -> CompliantT_co: ... + def to_uppercase(self) -> CompliantT_co: ... + + +class StringNamespace(_StoresCompliant[CompliantT_co], Protocol[CompliantT_co]): + def len_chars(self) -> CompliantT_co: ... + def replace( + self, pattern: str, value: str, *, literal: bool, n: int + ) -> CompliantT_co: ... + def replace_all( + self, pattern: str, value: str, *, literal: bool + ) -> CompliantT_co: ... + def strip_chars(self, characters: str | None) -> CompliantT_co: ... + def starts_with(self, prefix: str) -> CompliantT_co: ... + def ends_with(self, suffix: str) -> CompliantT_co: ... + def contains(self, pattern: str, *, literal: bool) -> CompliantT_co: ... + def slice(self, offset: int, length: int | None) -> CompliantT_co: ... + def split(self, by: str) -> CompliantT_co: ... + def to_datetime(self, format: str | None) -> CompliantT_co: ... # noqa: A002 + def to_lowercase(self) -> CompliantT_co: ... + def to_uppercase(self) -> CompliantT_co: ... diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index b2cd8a64c2..42662cc685 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -6,20 +6,27 @@ from typing import TYPE_CHECKING from typing import Any from typing import Callable +from typing import Generic from typing import Literal from typing import Mapping from typing import Protocol from typing import Sequence +from narwhals._compliant.any_namespace import CatNamespace +from narwhals._compliant.any_namespace import DateTimeNamespace +from narwhals._compliant.any_namespace import ListNamespace +from narwhals._compliant.any_namespace import StringNamespace from narwhals._compliant.namespace import CompliantNamespace from narwhals._compliant.typing import CompliantFrameT from narwhals._compliant.typing import CompliantLazyFrameT from narwhals._compliant.typing import CompliantSeriesOrNativeExprT_co from narwhals._compliant.typing import EagerDataFrameT +from narwhals._compliant.typing import EagerExprT from narwhals._compliant.typing import EagerSeriesT from narwhals._compliant.typing import NativeExprT_co from narwhals._expression_parsing import evaluate_output_names_and_aliases from narwhals.dtypes import DType +from narwhals.utils import _ExprNamespace from narwhals.utils import deprecated from narwhals.utils import not_implemented from narwhals.utils import unstable @@ -44,6 +51,7 @@ from narwhals._compliant.series import CompliantSeries from narwhals._expression_parsing import ExprKind from narwhals.dtypes import DType + from narwhals.typing import TimeUnit from narwhals.utils import Implementation from narwhals.utils import Version from narwhals.utils import _FullContext @@ -757,7 +765,21 @@ def rolling_var( # NOTE: `rank` differs - # NOTE: All namespaces differ + @property + def cat(self) -> EagerExprCatNamespace[Self]: + return EagerExprCatNamespace(self) + + @property + def dt(self) -> EagerExprDateTimeNamespace[Self]: + return EagerExprDateTimeNamespace(self) + + @property + def list(self) -> EagerExprListNamespace[Self]: + return EagerExprListNamespace(self) + + @property + def str(self) -> EagerExprStringNamespace[Self]: + return EagerExprStringNamespace(self) # NOTE: See (https://github.com/narwhals-dev/narwhals/issues/2044#issuecomment-2674262833) @@ -783,3 +805,145 @@ class LazyExpr( gather_every: not_implemented = not_implemented() replace_strict: not_implemented = not_implemented() cat: not_implemented = not_implemented() # pyright: ignore[reportAssignmentType] + + +class EagerExprNamespace(_ExprNamespace[EagerExprT], Generic[EagerExprT]): + def __init__(self, expr: EagerExprT, /) -> None: + self._compliant_expr = expr + + +class EagerExprCatNamespace( + CatNamespace[EagerExprT], EagerExprNamespace[EagerExprT], Generic[EagerExprT] +): + def get_categories(self) -> EagerExprT: + return self.compliant._reuse_series_namespace("cat", "get_categories") + + +class EagerExprDateTimeNamespace( + DateTimeNamespace[EagerExprT], EagerExprNamespace[EagerExprT], Generic[EagerExprT] +): + def to_string(self, format: str) -> EagerExprT: # noqa: A002 + return self.compliant._reuse_series_namespace("dt", "to_string", format=format) + + def replace_time_zone(self, time_zone: str | None) -> EagerExprT: + return self.compliant._reuse_series_namespace( + "dt", "replace_time_zone", time_zone=time_zone + ) + + def convert_time_zone(self, time_zone: str) -> EagerExprT: + return self.compliant._reuse_series_namespace( + "dt", "convert_time_zone", time_zone=time_zone + ) + + def timestamp(self, time_unit: TimeUnit) -> EagerExprT: + return self.compliant._reuse_series_namespace( + "dt", "timestamp", time_unit=time_unit + ) + + def date(self) -> EagerExprT: + return self.compliant._reuse_series_namespace("dt", "date") + + def year(self) -> EagerExprT: + return self.compliant._reuse_series_namespace("dt", "year") + + def month(self) -> EagerExprT: + return self.compliant._reuse_series_namespace("dt", "month") + + def day(self) -> EagerExprT: + return self.compliant._reuse_series_namespace("dt", "day") + + def hour(self) -> EagerExprT: + return self.compliant._reuse_series_namespace("dt", "hour") + + def minute(self) -> EagerExprT: + return self.compliant._reuse_series_namespace("dt", "minute") + + def second(self) -> EagerExprT: + return self.compliant._reuse_series_namespace("dt", "second") + + def millisecond(self) -> EagerExprT: + return self.compliant._reuse_series_namespace("dt", "millisecond") + + def microsecond(self) -> EagerExprT: + return self.compliant._reuse_series_namespace("dt", "microsecond") + + def nanosecond(self) -> EagerExprT: + return self.compliant._reuse_series_namespace("dt", "nanosecond") + + def ordinal_day(self) -> EagerExprT: + return self.compliant._reuse_series_namespace("dt", "ordinal_day") + + def weekday(self) -> EagerExprT: + return self.compliant._reuse_series_namespace("dt", "weekday") + + def total_minutes(self) -> EagerExprT: + return self.compliant._reuse_series_namespace("dt", "total_minutes") + + def total_seconds(self) -> EagerExprT: + return self.compliant._reuse_series_namespace("dt", "total_seconds") + + def total_milliseconds(self) -> EagerExprT: + return self.compliant._reuse_series_namespace("dt", "total_milliseconds") + + def total_microseconds(self) -> EagerExprT: + return self.compliant._reuse_series_namespace("dt", "total_microseconds") + + def total_nanoseconds(self) -> EagerExprT: + return self.compliant._reuse_series_namespace("dt", "total_nanoseconds") + + +class EagerExprListNamespace( + ListNamespace[EagerExprT], EagerExprNamespace[EagerExprT], Generic[EagerExprT] +): + def len(self) -> EagerExprT: + return self.compliant._reuse_series_namespace("list", "len") + + +class EagerExprStringNamespace( + StringNamespace[EagerExprT], EagerExprNamespace[EagerExprT], Generic[EagerExprT] +): + def len_chars(self) -> EagerExprT: + return self.compliant._reuse_series_namespace("str", "len_chars") + + def replace(self, pattern: str, value: str, *, literal: bool, n: int) -> EagerExprT: + return self.compliant._reuse_series_namespace( + "str", "replace", pattern=pattern, value=value, literal=literal, n=n + ) + + def replace_all(self, pattern: str, value: str, *, literal: bool) -> EagerExprT: + return self.compliant._reuse_series_namespace( + "str", "replace_all", pattern=pattern, value=value, literal=literal + ) + + def strip_chars(self, characters: str | None) -> EagerExprT: + return self.compliant._reuse_series_namespace( + "str", "strip_chars", characters=characters + ) + + def starts_with(self, prefix: str) -> EagerExprT: + return self.compliant._reuse_series_namespace("str", "starts_with", prefix=prefix) + + def ends_with(self, suffix: str) -> EagerExprT: + return self.compliant._reuse_series_namespace("str", "ends_with", suffix=suffix) + + def contains(self, pattern: str, *, literal: bool) -> EagerExprT: + return self.compliant._reuse_series_namespace( + "str", "contains", pattern=pattern, literal=literal + ) + + def slice(self, offset: int, length: int | None) -> EagerExprT: + return self.compliant._reuse_series_namespace( + "str", "slice", offset=offset, length=length + ) + + def split(self, by: str) -> EagerExprT: + return self.compliant._reuse_series_namespace("str", "split", by=by) + + def to_datetime(self, format: str | None) -> EagerExprT: # noqa: A002 + return self.compliant._reuse_series_namespace("str", "to_datetime", format=format) + + def to_lowercase(self) -> EagerExprT: + return self.compliant._reuse_series_namespace("str", "to_lowercase") + + def to_uppercase(self) -> EagerExprT: + return self.compliant._reuse_series_namespace("str", "to_uppercase") diff --git a/narwhals/_compliant/typing.py b/narwhals/_compliant/typing.py index fb9aa1beb0..21d0cb1d61 100644 --- a/narwhals/_compliant/typing.py +++ b/narwhals/_compliant/typing.py @@ -11,6 +11,7 @@ from narwhals._compliant.dataframe import CompliantLazyFrame from narwhals._compliant.dataframe import EagerDataFrame from narwhals._compliant.expr import CompliantExpr + from narwhals._compliant.expr import EagerExpr from narwhals._compliant.expr import NativeExpr from narwhals._compliant.series import CompliantSeries from narwhals._compliant.series import EagerSeries @@ -41,3 +42,4 @@ EagerDataFrameT = TypeVar("EagerDataFrameT", bound="EagerDataFrame[Any]") EagerSeriesT = TypeVar("EagerSeriesT", bound="EagerSeries[Any]") EagerSeriesT_co = TypeVar("EagerSeriesT_co", bound="EagerSeries[Any]", covariant=True) +EagerExprT = TypeVar("EagerExprT", bound="EagerExpr[Any, Any]") diff --git a/narwhals/_pandas_like/expr.py b/narwhals/_pandas_like/expr.py index 20d0772f13..338b1528b1 100644 --- a/narwhals/_pandas_like/expr.py +++ b/narwhals/_pandas_like/expr.py @@ -11,11 +11,7 @@ from narwhals._expression_parsing import ExprKind from narwhals._expression_parsing import evaluate_output_names_and_aliases from narwhals._expression_parsing import is_elementary_expression -from narwhals._pandas_like.expr_cat import PandasLikeExprCatNamespace -from narwhals._pandas_like.expr_dt import PandasLikeExprDateTimeNamespace -from narwhals._pandas_like.expr_list import PandasLikeExprListNamespace from narwhals._pandas_like.expr_name import PandasLikeExprNameNamespace -from narwhals._pandas_like.expr_str import PandasLikeExprStringNamespace from narwhals._pandas_like.group_by import AGGREGATIONS_TO_PANDAS_EQUIVALENT from narwhals._pandas_like.series import PandasLikeSeries from narwhals.dependencies import get_numpy @@ -346,22 +342,6 @@ def rank( "rank", call_kwargs={"method": method, "descending": descending} ) - @property - def str(self: Self) -> PandasLikeExprStringNamespace: - return PandasLikeExprStringNamespace(self) - - @property - def dt(self: Self) -> PandasLikeExprDateTimeNamespace: - return PandasLikeExprDateTimeNamespace(self) - - @property - def cat(self: Self) -> PandasLikeExprCatNamespace: - return PandasLikeExprCatNamespace(self) - @property def name(self: Self) -> PandasLikeExprNameNamespace: return PandasLikeExprNameNamespace(self) - - @property - def list(self: Self) -> PandasLikeExprListNamespace: - return PandasLikeExprListNamespace(self) diff --git a/narwhals/_pandas_like/expr_cat.py b/narwhals/_pandas_like/expr_cat.py deleted file mode 100644 index 00a04bd3d9..0000000000 --- a/narwhals/_pandas_like/expr_cat.py +++ /dev/null @@ -1,16 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from typing_extensions import Self - - from narwhals._pandas_like.expr import PandasLikeExpr - - -class PandasLikeExprCatNamespace: - def __init__(self: Self, expr: PandasLikeExpr) -> None: - self._compliant_expr = expr - - def get_categories(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace("cat", "get_categories") diff --git a/narwhals/_pandas_like/expr_dt.py b/narwhals/_pandas_like/expr_dt.py deleted file mode 100644 index 26b38394e5..0000000000 --- a/narwhals/_pandas_like/expr_dt.py +++ /dev/null @@ -1,85 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from typing_extensions import Self - - from narwhals._pandas_like.expr import PandasLikeExpr - from narwhals.typing import TimeUnit - - -class PandasLikeExprDateTimeNamespace: - def __init__(self: Self, expr: PandasLikeExpr) -> None: - self._compliant_expr = expr - - def date(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace("dt", "date") - - def year(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace("dt", "year") - - def month(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace("dt", "month") - - def day(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace("dt", "day") - - def hour(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace("dt", "hour") - - def minute(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace("dt", "minute") - - def second(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace("dt", "second") - - def millisecond(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace("dt", "millisecond") - - def microsecond(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace("dt", "microsecond") - - def nanosecond(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace("dt", "nanosecond") - - def ordinal_day(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace("dt", "ordinal_day") - - def weekday(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace("dt", "weekday") - - def total_minutes(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace("dt", "total_minutes") - - def total_seconds(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace("dt", "total_seconds") - - def total_milliseconds(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace("dt", "total_milliseconds") - - def total_microseconds(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace("dt", "total_microseconds") - - def total_nanoseconds(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace("dt", "total_nanoseconds") - - def to_string(self: Self, format: str) -> PandasLikeExpr: # noqa: A002 - return self._compliant_expr._reuse_series_namespace( - "dt", "to_string", format=format - ) - - def replace_time_zone(self: Self, time_zone: str | None) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace( - "dt", "replace_time_zone", time_zone=time_zone - ) - - def convert_time_zone(self: Self, time_zone: str) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace( - "dt", "convert_time_zone", time_zone=time_zone - ) - - def timestamp(self: Self, time_unit: TimeUnit) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace( - "dt", "timestamp", time_unit=time_unit - ) diff --git a/narwhals/_pandas_like/expr_list.py b/narwhals/_pandas_like/expr_list.py deleted file mode 100644 index f8563c2553..0000000000 --- a/narwhals/_pandas_like/expr_list.py +++ /dev/null @@ -1,16 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from typing_extensions import Self - - from narwhals._pandas_like.expr import PandasLikeExpr - - -class PandasLikeExprListNamespace: - def __init__(self: Self, expr: PandasLikeExpr) -> None: - self._expr = expr - - def len(self: Self) -> PandasLikeExpr: - return self._expr._reuse_series_namespace("list", "len") diff --git a/narwhals/_pandas_like/expr_str.py b/narwhals/_pandas_like/expr_str.py deleted file mode 100644 index c3ebbfcee1..0000000000 --- a/narwhals/_pandas_like/expr_str.py +++ /dev/null @@ -1,69 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from typing_extensions import Self - - from narwhals._pandas_like.expr import PandasLikeExpr - - -class PandasLikeExprStringNamespace: - def __init__(self: Self, expr: PandasLikeExpr) -> None: - self._compliant_expr = expr - - def len_chars(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace("str", "len_chars") - - def replace( - self: Self, pattern: str, value: str, *, literal: bool, n: int - ) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace( - "str", "replace", pattern=pattern, value=value, literal=literal, n=n - ) - - def replace_all( - self: Self, pattern: str, value: str, *, literal: bool - ) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace( - "str", "replace_all", pattern=pattern, value=value, literal=literal - ) - - def strip_chars(self: Self, characters: str | None) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace( - "str", "strip_chars", characters=characters - ) - - def starts_with(self: Self, prefix: str) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace( - "str", "starts_with", prefix=prefix - ) - - def ends_with(self: Self, suffix: str) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace( - "str", "ends_with", suffix=suffix - ) - - def contains(self: Self, pattern: str, *, literal: bool) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace( - "str", "contains", pattern=pattern, literal=literal - ) - - def slice(self: Self, offset: int, length: int | None) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace( - "str", "slice", offset=offset, length=length - ) - - def split(self: Self, by: str) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace("str", "split", by=by) - - def to_datetime(self: Self, format: str | None) -> PandasLikeExpr: # noqa: A002 - return self._compliant_expr._reuse_series_namespace( - "str", "to_datetime", format=format - ) - - def to_uppercase(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace("str", "to_uppercase") - - def to_lowercase(self: Self) -> PandasLikeExpr: - return self._compliant_expr._reuse_series_namespace("str", "to_lowercase") From c662d85dd64dbd9406958ea2dbd5d09e123e651a Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Sat, 8 Mar 2025 19:28:58 +0000 Subject: [PATCH 44/58] maybe `3.8` compat? - Bit confused as what I've done is fairly similar to (#2130) - Getting the same issue as (#2084) https://github.com/narwhals-dev/narwhals/actions/runs/13740460535/job/38429062404?pr=2149 --- narwhals/_compliant/expr.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index 42662cc685..ec843811c0 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -813,14 +813,14 @@ def __init__(self, expr: EagerExprT, /) -> None: class EagerExprCatNamespace( - CatNamespace[EagerExprT], EagerExprNamespace[EagerExprT], Generic[EagerExprT] + EagerExprNamespace[EagerExprT], CatNamespace[EagerExprT], Generic[EagerExprT] ): def get_categories(self) -> EagerExprT: return self.compliant._reuse_series_namespace("cat", "get_categories") class EagerExprDateTimeNamespace( - DateTimeNamespace[EagerExprT], EagerExprNamespace[EagerExprT], Generic[EagerExprT] + EagerExprNamespace[EagerExprT], DateTimeNamespace[EagerExprT], Generic[EagerExprT] ): def to_string(self, format: str) -> EagerExprT: # noqa: A002 return self.compliant._reuse_series_namespace("dt", "to_string", format=format) @@ -893,14 +893,14 @@ def total_nanoseconds(self) -> EagerExprT: class EagerExprListNamespace( - ListNamespace[EagerExprT], EagerExprNamespace[EagerExprT], Generic[EagerExprT] + EagerExprNamespace[EagerExprT], ListNamespace[EagerExprT], Generic[EagerExprT] ): def len(self) -> EagerExprT: return self.compliant._reuse_series_namespace("list", "len") class EagerExprStringNamespace( - StringNamespace[EagerExprT], EagerExprNamespace[EagerExprT], Generic[EagerExprT] + EagerExprNamespace[EagerExprT], StringNamespace[EagerExprT], Generic[EagerExprT] ): def len_chars(self) -> EagerExprT: return self.compliant._reuse_series_namespace("str", "len_chars") From d53ce01d55f8f88e7042775900a20b27879d230c Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Sat, 8 Mar 2025 19:42:14 +0000 Subject: [PATCH 45/58] chore(typing): add some safety for `.name` `pandas` impl will fail now --- narwhals/_compliant/expr.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index ec843811c0..02ef8fbfac 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -46,6 +46,7 @@ from typing_extensions import Self + from narwhals._compliant.any_namespace import NameNamespace from narwhals._compliant.namespace import CompliantNamespace from narwhals._compliant.namespace import EagerNamespace from narwhals._compliant.series import CompliantSeries @@ -781,6 +782,9 @@ def list(self) -> EagerExprListNamespace[Self]: def str(self) -> EagerExprStringNamespace[Self]: return EagerExprStringNamespace(self) + @property + def name(self) -> NameNamespace[Self]: ... + # NOTE: See (https://github.com/narwhals-dev/narwhals/issues/2044#issuecomment-2674262833) class LazyExpr( From 277782219ea18a22fd5387ac03f9c6f658f1de3d Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Sat, 8 Mar 2025 20:28:13 +0000 Subject: [PATCH 46/58] feat: adds `EagerExprNameNamespace` Didn't realise until now that both backends did the same thing --- narwhals/_arrow/expr.py | 5 -- narwhals/_arrow/expr_name.py | 81 ---------------------------- narwhals/_arrow/utils.py | 7 --- narwhals/_compliant/expr.py | 76 ++++++++++++++++++++++++++- narwhals/_pandas_like/expr.py | 5 -- narwhals/_pandas_like/expr_name.py | 84 ------------------------------ 6 files changed, 74 insertions(+), 184 deletions(-) delete mode 100644 narwhals/_arrow/expr_name.py delete mode 100644 narwhals/_pandas_like/expr_name.py diff --git a/narwhals/_arrow/expr.py b/narwhals/_arrow/expr.py index f0276e8a4b..360132e80b 100644 --- a/narwhals/_arrow/expr.py +++ b/narwhals/_arrow/expr.py @@ -8,7 +8,6 @@ import pyarrow.compute as pc -from narwhals._arrow.expr_name import ArrowExprNameNamespace from narwhals._arrow.series import ArrowSeries from narwhals._compliant import EagerExpr from narwhals._expression_parsing import ExprKind @@ -263,7 +262,3 @@ def rank( return self._reuse_series("rank", method=method, descending=descending) ewm_mean = not_implemented() - - @property - def name(self: Self) -> ArrowExprNameNamespace: - return ArrowExprNameNamespace(self) diff --git a/narwhals/_arrow/expr_name.py b/narwhals/_arrow/expr_name.py deleted file mode 100644 index 2f8b368466..0000000000 --- a/narwhals/_arrow/expr_name.py +++ /dev/null @@ -1,81 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING -from typing import Callable -from typing import Sequence - -from narwhals._arrow.utils import ArrowExprNamespace - -if TYPE_CHECKING: - from typing_extensions import Self - - from narwhals._arrow.expr import ArrowExpr - - -class ArrowExprNameNamespace(ArrowExprNamespace): - def keep(self: Self) -> ArrowExpr: - return self._from_colname_func_and_alias_output_names( - name_mapping_func=lambda name: name, - alias_output_names=None, - ) - - def map(self: Self, function: Callable[[str], str]) -> ArrowExpr: - return self._from_colname_func_and_alias_output_names( - name_mapping_func=function, - alias_output_names=lambda output_names: [ - function(name) for name in output_names - ], - ) - - def prefix(self: Self, prefix: str) -> ArrowExpr: - return self._from_colname_func_and_alias_output_names( - name_mapping_func=lambda name: f"{prefix}{name}", - alias_output_names=lambda output_names: [ - f"{prefix}{output_name}" for output_name in output_names - ], - ) - - def suffix(self: Self, suffix: str) -> ArrowExpr: - return self._from_colname_func_and_alias_output_names( - name_mapping_func=lambda name: f"{name}{suffix}", - alias_output_names=lambda output_names: [ - f"{output_name}{suffix}" for output_name in output_names - ], - ) - - def to_lowercase(self: Self) -> ArrowExpr: - return self._from_colname_func_and_alias_output_names( - name_mapping_func=str.lower, - alias_output_names=lambda output_names: [ - name.lower() for name in output_names - ], - ) - - def to_uppercase(self: Self) -> ArrowExpr: - return self._from_colname_func_and_alias_output_names( - name_mapping_func=str.upper, - alias_output_names=lambda output_names: [ - name.upper() for name in output_names - ], - ) - - def _from_colname_func_and_alias_output_names( - self: Self, - name_mapping_func: Callable[[str], str], - alias_output_names: Callable[[Sequence[str]], Sequence[str]] | None, - ) -> ArrowExpr: - return self.compliant.__class__( - call=lambda df: [ - series.alias(name_mapping_func(name)) - for series, name in zip( - self.compliant._call(df), self.compliant._evaluate_output_names(df) - ) - ], - depth=self.compliant._depth, - function_name=self.compliant._function_name, - evaluate_output_names=self.compliant._evaluate_output_names, - alias_output_names=alias_output_names, - backend_version=self.compliant._backend_version, - version=self.compliant._version, - call_kwargs=self.compliant._call_kwargs, - ) diff --git a/narwhals/_arrow/utils.py b/narwhals/_arrow/utils.py index bfc617cee9..a1fd03f495 100644 --- a/narwhals/_arrow/utils.py +++ b/narwhals/_arrow/utils.py @@ -12,7 +12,6 @@ import pyarrow as pa import pyarrow.compute as pc -from narwhals.utils import _ExprNamespace from narwhals.utils import _SeriesNamespace from narwhals.utils import import_dtypes_module from narwhals.utils import isinstance_or_issubclass @@ -24,7 +23,6 @@ from typing_extensions import TypeAlias from typing_extensions import TypeIs - from narwhals._arrow.expr import ArrowExpr from narwhals._arrow.series import ArrowSeries from narwhals._arrow.typing import ArrowArray from narwhals._arrow.typing import ArrowChunkedArray @@ -548,8 +546,3 @@ def pad_series( class ArrowSeriesNamespace(_SeriesNamespace["ArrowSeries", "ArrowChunkedArray"]): def __init__(self: Self, series: ArrowSeries, /) -> None: self._compliant_series = series - - -class ArrowExprNamespace(_ExprNamespace["ArrowExpr"]): - def __init__(self: Self, expr: ArrowExpr, /) -> None: - self._compliant_expr = expr diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index 02ef8fbfac..af9dae4fe5 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -15,6 +15,7 @@ from narwhals._compliant.any_namespace import CatNamespace from narwhals._compliant.any_namespace import DateTimeNamespace from narwhals._compliant.any_namespace import ListNamespace +from narwhals._compliant.any_namespace import NameNamespace from narwhals._compliant.any_namespace import StringNamespace from narwhals._compliant.namespace import CompliantNamespace from narwhals._compliant.typing import CompliantFrameT @@ -46,7 +47,6 @@ from typing_extensions import Self - from narwhals._compliant.any_namespace import NameNamespace from narwhals._compliant.namespace import CompliantNamespace from narwhals._compliant.namespace import EagerNamespace from narwhals._compliant.series import CompliantSeries @@ -783,7 +783,8 @@ def str(self) -> EagerExprStringNamespace[Self]: return EagerExprStringNamespace(self) @property - def name(self) -> NameNamespace[Self]: ... + def name(self) -> EagerExprNameNamespace[Self]: + return EagerExprNameNamespace(self) # NOTE: See (https://github.com/narwhals-dev/narwhals/issues/2044#issuecomment-2674262833) @@ -903,6 +904,77 @@ def len(self) -> EagerExprT: return self.compliant._reuse_series_namespace("list", "len") +class EagerExprNameNamespace( + EagerExprNamespace[EagerExprT], NameNamespace[EagerExprT], Generic[EagerExprT] +): + def keep(self) -> EagerExprT: + return self._from_colname_func_and_alias_output_names( + name_mapping_func=lambda name: name + ) + + def map(self, function: Callable[[str], str]) -> EagerExprT: + return self._from_colname_func_and_alias_output_names( + name_mapping_func=function, + alias_output_names=lambda output_names: [ + function(name) for name in output_names + ], + ) + + def prefix(self, prefix: str) -> EagerExprT: + return self._from_colname_func_and_alias_output_names( + name_mapping_func=lambda name: f"{prefix}{name}", + alias_output_names=lambda output_names: [ + f"{prefix}{output_name}" for output_name in output_names + ], + ) + + def suffix(self, suffix: str) -> EagerExprT: + return self._from_colname_func_and_alias_output_names( + name_mapping_func=lambda name: f"{name}{suffix}", + alias_output_names=lambda output_names: [ + f"{output_name}{suffix}" for output_name in output_names + ], + ) + + def to_lowercase(self) -> EagerExprT: + return self._from_colname_func_and_alias_output_names( + name_mapping_func=str.lower, + alias_output_names=lambda output_names: [ + name.lower() for name in output_names + ], + ) + + def to_uppercase(self) -> EagerExprT: + return self._from_colname_func_and_alias_output_names( + name_mapping_func=str.upper, + alias_output_names=lambda output_names: [ + name.upper() for name in output_names + ], + ) + + def _from_colname_func_and_alias_output_names( + self, + name_mapping_func: Callable[[str], str], + alias_output_names: Callable[[Sequence[str]], Sequence[str]] | None = None, + ) -> EagerExprT: + return type(self.compliant)( + call=lambda df: [ + series.alias(name_mapping_func(name)) + for series, name in zip( + self.compliant._call(df), self.compliant._evaluate_output_names(df) + ) + ], + depth=self.compliant._depth, + function_name=self.compliant._function_name, + evaluate_output_names=self.compliant._evaluate_output_names, + alias_output_names=alias_output_names, + backend_version=self.compliant._backend_version, + implementation=self.compliant._implementation, + version=self.compliant._version, + call_kwargs=self.compliant._call_kwargs, + ) + + class EagerExprStringNamespace( EagerExprNamespace[EagerExprT], StringNamespace[EagerExprT], Generic[EagerExprT] ): diff --git a/narwhals/_pandas_like/expr.py b/narwhals/_pandas_like/expr.py index 338b1528b1..8507b3d55a 100644 --- a/narwhals/_pandas_like/expr.py +++ b/narwhals/_pandas_like/expr.py @@ -11,7 +11,6 @@ from narwhals._expression_parsing import ExprKind from narwhals._expression_parsing import evaluate_output_names_and_aliases from narwhals._expression_parsing import is_elementary_expression -from narwhals._pandas_like.expr_name import PandasLikeExprNameNamespace from narwhals._pandas_like.group_by import AGGREGATIONS_TO_PANDAS_EQUIVALENT from narwhals._pandas_like.series import PandasLikeSeries from narwhals.dependencies import get_numpy @@ -341,7 +340,3 @@ def rank( return self._reuse_series( "rank", call_kwargs={"method": method, "descending": descending} ) - - @property - def name(self: Self) -> PandasLikeExprNameNamespace: - return PandasLikeExprNameNamespace(self) diff --git a/narwhals/_pandas_like/expr_name.py b/narwhals/_pandas_like/expr_name.py deleted file mode 100644 index 2128b730a4..0000000000 --- a/narwhals/_pandas_like/expr_name.py +++ /dev/null @@ -1,84 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING -from typing import Callable -from typing import Sequence - -if TYPE_CHECKING: - from typing_extensions import Self - - from narwhals._pandas_like.expr import PandasLikeExpr - - -class PandasLikeExprNameNamespace: - def __init__(self: Self, expr: PandasLikeExpr) -> None: - self._compliant_expr = expr - - def keep(self: Self) -> PandasLikeExpr: - return self._from_colname_func_and_alias_output_names( - name_mapping_func=lambda name: name, - alias_output_names=None, - ) - - def map(self: Self, function: Callable[[str], str]) -> PandasLikeExpr: - return self._from_colname_func_and_alias_output_names( - name_mapping_func=lambda name: function(str(name)), - alias_output_names=lambda output_names: [ - function(str(name)) for name in output_names - ], - ) - - def prefix(self: Self, prefix: str) -> PandasLikeExpr: - return self._from_colname_func_and_alias_output_names( - name_mapping_func=lambda name: f"{prefix}{name}", - alias_output_names=lambda output_names: [ - f"{prefix}{output_name}" for output_name in output_names - ], - ) - - def suffix(self: Self, suffix: str) -> PandasLikeExpr: - return self._from_colname_func_and_alias_output_names( - name_mapping_func=lambda name: f"{name}{suffix}", - alias_output_names=lambda output_names: [ - f"{output_name}{suffix}" for output_name in output_names - ], - ) - - def to_lowercase(self: Self) -> PandasLikeExpr: - return self._from_colname_func_and_alias_output_names( - name_mapping_func=lambda name: str(name).lower(), - alias_output_names=lambda output_names: [ - str(name).lower() for name in output_names - ], - ) - - def to_uppercase(self: Self) -> PandasLikeExpr: - return self._from_colname_func_and_alias_output_names( - name_mapping_func=lambda name: str(name).upper(), - alias_output_names=lambda output_names: [ - str(name).upper() for name in output_names - ], - ) - - def _from_colname_func_and_alias_output_names( - self: Self, - name_mapping_func: Callable[[str], str], - alias_output_names: Callable[[Sequence[str]], Sequence[str]] | None, - ) -> PandasLikeExpr: - return self._compliant_expr.__class__( - call=lambda df: [ - series.alias(name_mapping_func(name)) - for series, name in zip( - self._compliant_expr._call(df), - self._compliant_expr._evaluate_output_names(df), - ) - ], - depth=self._compliant_expr._depth, - function_name=self._compliant_expr._function_name, - evaluate_output_names=self._compliant_expr._evaluate_output_names, - alias_output_names=alias_output_names, - backend_version=self._compliant_expr._backend_version, - implementation=self._compliant_expr._implementation, - version=self._compliant_expr._version, - call_kwargs=self._compliant_expr._call_kwargs, - ) From 067252bfc847f11f458cc12f20dcdd7c24daa4c7 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Sat, 8 Mar 2025 22:14:39 +0000 Subject: [PATCH 47/58] refactor: Simplify `EagerExprNameNamespace` --- narwhals/_compliant/expr.py | 83 +++++++++++++---------------------- narwhals/_compliant/typing.py | 6 +++ 2 files changed, 36 insertions(+), 53 deletions(-) diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index af9dae4fe5..8ee317a546 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -18,6 +18,8 @@ from narwhals._compliant.any_namespace import NameNamespace from narwhals._compliant.any_namespace import StringNamespace from narwhals._compliant.namespace import CompliantNamespace +from narwhals._compliant.typing import AliasName +from narwhals._compliant.typing import AliasNames from narwhals._compliant.typing import CompliantFrameT from narwhals._compliant.typing import CompliantLazyFrameT from narwhals._compliant.typing import CompliantSeriesOrNativeExprT_co @@ -908,70 +910,45 @@ class EagerExprNameNamespace( EagerExprNamespace[EagerExprT], NameNamespace[EagerExprT], Generic[EagerExprT] ): def keep(self) -> EagerExprT: - return self._from_colname_func_and_alias_output_names( - name_mapping_func=lambda name: name - ) + return self._from_callable(lambda name: name, alias=False) - def map(self, function: Callable[[str], str]) -> EagerExprT: - return self._from_colname_func_and_alias_output_names( - name_mapping_func=function, - alias_output_names=lambda output_names: [ - function(name) for name in output_names - ], - ) + def map(self, function: AliasName) -> EagerExprT: + return self._from_callable(function) def prefix(self, prefix: str) -> EagerExprT: - return self._from_colname_func_and_alias_output_names( - name_mapping_func=lambda name: f"{prefix}{name}", - alias_output_names=lambda output_names: [ - f"{prefix}{output_name}" for output_name in output_names - ], - ) + return self._from_callable(lambda name: f"{prefix}{name}") def suffix(self, suffix: str) -> EagerExprT: - return self._from_colname_func_and_alias_output_names( - name_mapping_func=lambda name: f"{name}{suffix}", - alias_output_names=lambda output_names: [ - f"{output_name}{suffix}" for output_name in output_names - ], - ) + return self._from_callable(lambda name: f"{name}{suffix}") def to_lowercase(self) -> EagerExprT: - return self._from_colname_func_and_alias_output_names( - name_mapping_func=str.lower, - alias_output_names=lambda output_names: [ - name.lower() for name in output_names - ], - ) + return self._from_callable(str.lower) def to_uppercase(self) -> EagerExprT: - return self._from_colname_func_and_alias_output_names( - name_mapping_func=str.upper, - alias_output_names=lambda output_names: [ - name.upper() for name in output_names - ], - ) + return self._from_callable(str.upper) - def _from_colname_func_and_alias_output_names( - self, - name_mapping_func: Callable[[str], str], - alias_output_names: Callable[[Sequence[str]], Sequence[str]] | None = None, - ) -> EagerExprT: - return type(self.compliant)( - call=lambda df: [ - series.alias(name_mapping_func(name)) - for series, name in zip( - self.compliant._call(df), self.compliant._evaluate_output_names(df) - ) + @staticmethod + def _alias_output_names(func: AliasName, /) -> AliasNames: + def fn(output_names: Sequence[str], /) -> Sequence[str]: + return [func(name) for name in output_names] + + return fn + + def _from_callable(self, func: AliasName, /, *, alias: bool = True) -> EagerExprT: + expr = self.compliant + return type(expr)( + lambda df: [ + series.alias(func(name)) + for series, name in zip(expr(df), expr._evaluate_output_names(df)) ], - depth=self.compliant._depth, - function_name=self.compliant._function_name, - evaluate_output_names=self.compliant._evaluate_output_names, - alias_output_names=alias_output_names, - backend_version=self.compliant._backend_version, - implementation=self.compliant._implementation, - version=self.compliant._version, - call_kwargs=self.compliant._call_kwargs, + depth=expr._depth, + function_name=expr._function_name, + evaluate_output_names=expr._evaluate_output_names, + alias_output_names=self._alias_output_names(func) if alias else None, + backend_version=expr._backend_version, + implementation=expr._implementation, + version=expr._version, + call_kwargs=expr._call_kwargs, ) diff --git a/narwhals/_compliant/typing.py b/narwhals/_compliant/typing.py index 21d0cb1d61..2513097a50 100644 --- a/narwhals/_compliant/typing.py +++ b/narwhals/_compliant/typing.py @@ -2,6 +2,8 @@ from typing import TYPE_CHECKING from typing import Any +from typing import Callable +from typing import Sequence from typing import TypeVar if TYPE_CHECKING: @@ -17,6 +19,8 @@ from narwhals._compliant.series import EagerSeries __all__ = [ + "AliasName", + "AliasNames", "CompliantDataFrameT", "CompliantFrameT", "CompliantLazyFrameT", @@ -43,3 +47,5 @@ EagerSeriesT = TypeVar("EagerSeriesT", bound="EagerSeries[Any]") EagerSeriesT_co = TypeVar("EagerSeriesT_co", bound="EagerSeries[Any]", covariant=True) EagerExprT = TypeVar("EagerExprT", bound="EagerExpr[Any, Any]") +AliasNames: TypeAlias = Callable[[Sequence[str]], Sequence[str]] +AliasName: TypeAlias = Callable[[str], str] From 309f910010cd9ea542739c51f182223cf0701af0 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Sun, 9 Mar 2025 14:39:35 +0000 Subject: [PATCH 48/58] chore: resolve more conflicts from (#2168) --- narwhals/_compliant/expr.py | 5 +---- narwhals/_duckdb/expr.py | 1 + 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index f67fedf9e5..0fd7f8e576 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -748,9 +748,7 @@ def rolling_std( ddof=ddof, ) - def rolling_sum( - self, window_size: int, *, min_samples: int | None, center: bool - ) -> Self: + def rolling_sum(self, window_size: int, *, min_samples: int, center: bool) -> Self: return self._reuse_series( "rolling_sum", window_size=window_size, min_samples=min_samples, center=center ) @@ -805,7 +803,6 @@ class LazyExpr( sample: not_implemented = not_implemented() map_batches: not_implemented = not_implemented() ewm_mean: not_implemented = not_implemented() - rolling_sum: not_implemented = not_implemented() rolling_mean: not_implemented = not_implemented() rolling_var: not_implemented = not_implemented() rolling_std: not_implemented = not_implemented() diff --git a/narwhals/_duckdb/expr.py b/narwhals/_duckdb/expr.py index 492c0f11ed..10c648c9b4 100644 --- a/narwhals/_duckdb/expr.py +++ b/narwhals/_duckdb/expr.py @@ -497,3 +497,4 @@ def list(self: Self) -> DuckDBExprListNamespace: cum_max = not_implemented() cum_prod = not_implemented() over = not_implemented() + rolling_sum = not_implemented() From 8ec9cd92fb37fabaf3d1e003d7d5e1fcb3fa60a0 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Sun, 9 Mar 2025 15:35:25 +0000 Subject: [PATCH 49/58] docs: add module-level doc Very open to alternatives on what the module should be called Was the *least-bad* one I thought of --- narwhals/_compliant/any_namespace.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/narwhals/_compliant/any_namespace.py b/narwhals/_compliant/any_namespace.py index c50b08eb22..5d46c8dedd 100644 --- a/narwhals/_compliant/any_namespace.py +++ b/narwhals/_compliant/any_namespace.py @@ -1,3 +1,5 @@ +"""`Expr` and `Series` namespace accessor protocols.""" + from __future__ import annotations from typing import TYPE_CHECKING From c368a602e992a2091dfe54b82f07bf05d4f04412 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Sun, 9 Mar 2025 16:02:35 +0000 Subject: [PATCH 50/58] docs(typing): Explain `is_eager_expr` Really hate this as a solution and keep trying to get `mypy` to let me do something else --- narwhals/_compliant/dataframe.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/narwhals/_compliant/dataframe.py b/narwhals/_compliant/dataframe.py index ba30f2515c..33ceacddaf 100644 --- a/narwhals/_compliant/dataframe.py +++ b/narwhals/_compliant/dataframe.py @@ -61,7 +61,7 @@ def _iter_columns(self) -> Iterator[Any]: ... class EagerDataFrame(CompliantDataFrame[EagerSeriesT_co], Protocol[EagerSeriesT_co]): def _maybe_evaluate_expr( - self, expr: EagerExpr[EagerDataFrame[EagerSeriesT_co], EagerSeriesT_co] | T, / + self, expr: EagerExpr[Self, EagerSeriesT_co] | T, / ) -> EagerSeriesT_co | T: if is_eager_expr(expr): result: Sequence[EagerSeriesT_co] = expr(self) @@ -75,7 +75,8 @@ def _maybe_evaluate_expr( return expr -# NOTE: DON'T CHANGE THIS EITHER +# NOTE: `mypy` is requiring the gymnastics here and is very fragile +# DON'T CHANGE THIS or `EagerDataFrame._maybe_evaluate_expr` def is_eager_expr( obj: EagerExpr[Any, EagerSeriesT] | Any, ) -> TypeIs[EagerExpr[Any, EagerSeriesT]]: From 7d3addcf6a7d7fee6f67bcd01a55d60c871c0a47 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Sun, 9 Mar 2025 16:11:08 +0000 Subject: [PATCH 51/58] docs: `NativeExpr` --- narwhals/_compliant/expr.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index 0fd7f8e576..96b6cefdbf 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -62,8 +62,13 @@ __all__ = ["CompliantExpr", "EagerExpr", "LazyExpr", "NativeExpr"] -# NOTE: Only common methods for lazy expr-like objects class NativeExpr(Protocol): + """An `Expr`-like object from a package with [Lazy-only support](https://narwhals-dev.github.io/narwhals/extending/#levels-of-support). + + Protocol members are chosen *purely* for matching statically - as they + are common to all currently supported packages. + """ + def between(self, *args: Any, **kwds: Any) -> Any: ... def isin(self, *args: Any, **kwds: Any) -> Any: ... From 66ecf843353d70e46fb1ecc570af533ce20891a9 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Sun, 9 Mar 2025 16:40:37 +0000 Subject: [PATCH 52/58] chore: remove outdated comments --- narwhals/_compliant/expr.py | 1 - narwhals/_compliant/series.py | 9 --------- 2 files changed, 10 deletions(-) diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index 96b6cefdbf..69ef737636 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -792,7 +792,6 @@ def name(self) -> EagerExprNameNamespace[Self]: return EagerExprNameNamespace(self) -# NOTE: See (https://github.com/narwhals-dev/narwhals/issues/2044#issuecomment-2674262833) class LazyExpr( CompliantExpr[CompliantLazyFrameT, NativeExprT_co], Protocol38[CompliantLazyFrameT, NativeExprT_co], diff --git a/narwhals/_compliant/series.py b/narwhals/_compliant/series.py index aede357152..d0f3e759f6 100644 --- a/narwhals/_compliant/series.py +++ b/narwhals/_compliant/series.py @@ -48,9 +48,6 @@ class EagerSeries(CompliantSeries, Protocol[NativeSeriesT_co]): @property def native(self) -> NativeSeriesT_co: ... - # NOTE: `ArrowSeries` needs to intercept `value` w/ - # if self._backend_version < (13,) and hasattr(value, "as_py"): - # value = value.as_py() # noqa: ERA001 def _from_scalar(self, value: Any) -> Self: return self._from_iterable([value], name=self.name, context=self) @@ -63,9 +60,3 @@ def __narwhals_namespace__(self) -> EagerNamespace[Any, Self]: ... def _to_expr(self) -> EagerExpr[Any, Self]: return self.__narwhals_namespace__()._expr._from_series(self) - - # TODO @dangotbanned: replacing `Namespace._create_compliant_series`` - # - All usage within `*Expr.map_batches` - # - `PandasLikeExpr` uses that **once** - # - `ArrowExpr` uses **twice** - # - `PandasLikeDataFrame.with_row_index` uses the wrapped `utils` function once From b3bdb3b3ba52dada9cadde3c50d68520f1270a09 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Sun, 9 Mar 2025 17:22:29 +0000 Subject: [PATCH 53/58] fix(typing): `EagerNamespace.all_horizontal` Was not easy getting that to please both `mypy` & `pyright` --- narwhals/_arrow/namespace.py | 6 ++---- narwhals/_compliant/expr.py | 4 +++- narwhals/_compliant/namespace.py | 18 ++++++++---------- narwhals/_compliant/series.py | 6 +++--- narwhals/_pandas_like/namespace.py | 8 ++++---- 5 files changed, 20 insertions(+), 22 deletions(-) diff --git a/narwhals/_arrow/namespace.py b/narwhals/_arrow/namespace.py index d1e95308cb..22e2e56184 100644 --- a/narwhals/_arrow/namespace.py +++ b/narwhals/_arrow/namespace.py @@ -47,7 +47,7 @@ _Scalar: TypeAlias = Any -class ArrowNamespace(EagerNamespace[ArrowDataFrame, ArrowSeries]): +class ArrowNamespace(EagerNamespace[ArrowDataFrame, ArrowSeries, ArrowExpr]): @property def _expr(self) -> type[ArrowExpr]: return ArrowExpr @@ -123,9 +123,7 @@ def _lit_arrow_series(_: ArrowDataFrame) -> ArrowSeries: version=self._version, ) - # NOTE: Needs to be resolved in `EagerNamespace` - # Probably, by adding an `EagerExprT` typevar - def all_horizontal(self: Self, *exprs: ArrowExpr) -> ArrowExpr: # type: ignore[override] + def all_horizontal(self: Self, *exprs: ArrowExpr) -> ArrowExpr: def func(df: ArrowDataFrame) -> list[ArrowSeries]: series = chain.from_iterable(expr(df) for expr in exprs) return [reduce(operator.and_, align_series_full_broadcast(*series))] diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index 69ef737636..2ed2726074 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -285,7 +285,9 @@ def __call__(self, df: EagerDataFrameT) -> Sequence[EagerSeriesT]: def __repr__(self) -> str: # pragma: no cover return f"{type(self).__name__}(depth={self._depth}, function_name={self._function_name})" - def __narwhals_namespace__(self) -> EagerNamespace[EagerDataFrameT, EagerSeriesT]: ... + def __narwhals_namespace__( + self, + ) -> EagerNamespace[EagerDataFrameT, EagerSeriesT, Self]: ... def __narwhals_expr__(self) -> None: ... @classmethod diff --git a/narwhals/_compliant/namespace.py b/narwhals/_compliant/namespace.py index 49fb4915d2..91f79dcb2b 100644 --- a/narwhals/_compliant/namespace.py +++ b/narwhals/_compliant/namespace.py @@ -7,12 +7,12 @@ from narwhals._compliant.typing import CompliantFrameT from narwhals._compliant.typing import CompliantSeriesOrNativeExprT_co from narwhals._compliant.typing import EagerDataFrameT -from narwhals._compliant.typing import EagerSeriesT +from narwhals._compliant.typing import EagerExprT +from narwhals._compliant.typing import EagerSeriesT_co from narwhals.utils import deprecated if TYPE_CHECKING: from narwhals._compliant.expr import CompliantExpr - from narwhals._compliant.expr import EagerExpr from narwhals._compliant.selectors import CompliantSelectorNamespace from narwhals.dtypes import DType @@ -31,25 +31,23 @@ def selectors(self) -> CompliantSelectorNamespace[Any, Any]: ... class EagerNamespace( - CompliantNamespace[EagerDataFrameT, EagerSeriesT], - Protocol[EagerDataFrameT, EagerSeriesT], + CompliantNamespace[EagerDataFrameT, EagerSeriesT_co], + Protocol[EagerDataFrameT, EagerSeriesT_co, EagerExprT], ): # NOTE: Supporting moved ops # - `self_create_expr_from_callable` -> `self._expr._from_callable` # - `self_create_expr_from_series` -> `self._expr._from_series` @property - def _expr(self) -> type[EagerExpr[EagerDataFrameT, EagerSeriesT]]: ... + def _expr(self) -> type[EagerExprT]: ... # NOTE: Supporting moved ops # - `self._create_series_from_scalar` -> `EagerSeries()._from_scalar` # - Was dependent on a `reference_series`, so is now an instance method # - `._from_iterable` -> `self._series._from_iterable` @property - def _series(self) -> type[EagerSeriesT]: ... + def _series(self) -> type[EagerSeriesT_co]: ... - def all_horizontal( - self, *exprs: EagerExpr[EagerDataFrameT, EagerSeriesT] - ) -> EagerExpr[EagerDataFrameT, EagerSeriesT]: ... + def all_horizontal(self, *exprs: EagerExprT) -> EagerExprT: ... @deprecated("ref'd in untyped code") - def _create_compliant_series(self, value: Any) -> EagerSeriesT: ... + def _create_compliant_series(self, value: Any) -> EagerSeriesT_co: ... diff --git a/narwhals/_compliant/series.py b/narwhals/_compliant/series.py index d0f3e759f6..e6be377046 100644 --- a/narwhals/_compliant/series.py +++ b/narwhals/_compliant/series.py @@ -56,7 +56,7 @@ def _from_iterable( cls: type[Self], data: Iterable[Any], name: str, *, context: _FullContext ) -> Self: ... - def __narwhals_namespace__(self) -> EagerNamespace[Any, Self]: ... + def __narwhals_namespace__(self) -> EagerNamespace[Any, Self, Any]: ... - def _to_expr(self) -> EagerExpr[Any, Self]: - return self.__narwhals_namespace__()._expr._from_series(self) + def _to_expr(self) -> EagerExpr[Any, Any]: + return self.__narwhals_namespace__()._expr._from_series(self) # type: ignore[no-any-return] diff --git a/narwhals/_pandas_like/namespace.py b/narwhals/_pandas_like/namespace.py index 1af7ee17a9..e9902e6f8b 100644 --- a/narwhals/_pandas_like/namespace.py +++ b/narwhals/_pandas_like/namespace.py @@ -40,7 +40,9 @@ _Scalar: TypeAlias = Any -class PandasLikeNamespace(EagerNamespace[PandasLikeDataFrame, PandasLikeSeries]): +class PandasLikeNamespace( + EagerNamespace[PandasLikeDataFrame, PandasLikeSeries, PandasLikeExpr] +): @property def _expr(self) -> type[PandasLikeExpr]: return PandasLikeExpr @@ -149,9 +151,7 @@ def func(df: PandasLikeDataFrame) -> list[PandasLikeSeries]: context=self, ) - # NOTE: Needs to be resolved in `EagerNamespace` - # Probably, by adding an `EagerExprT` typevar - def all_horizontal(self: Self, *exprs: PandasLikeExpr) -> PandasLikeExpr: # type: ignore[override] + def all_horizontal(self: Self, *exprs: PandasLikeExpr) -> PandasLikeExpr: def func(df: PandasLikeDataFrame) -> list[PandasLikeSeries]: series = align_series_full_broadcast( *(s for _expr in exprs for s in _expr(df)) From 40d718f5834c2bc3ec452d41e2a60b2e272e7050 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Mon, 10 Mar 2025 09:43:31 +0000 Subject: [PATCH 54/58] refactor: Add `EagerExprStructNamespace` Integrates #2146 --- narwhals/_arrow/expr.py | 5 ----- narwhals/_arrow/expr_struct.py | 23 ----------------------- narwhals/_compliant/any_namespace.py | 5 +++++ narwhals/_compliant/expr.py | 24 +++++++++++++++++++----- narwhals/_pandas_like/expr.py | 5 ----- narwhals/_pandas_like/expr_struct.py | 23 ----------------------- 6 files changed, 24 insertions(+), 61 deletions(-) delete mode 100644 narwhals/_arrow/expr_struct.py delete mode 100644 narwhals/_pandas_like/expr_struct.py diff --git a/narwhals/_arrow/expr.py b/narwhals/_arrow/expr.py index e858edfadc..360132e80b 100644 --- a/narwhals/_arrow/expr.py +++ b/narwhals/_arrow/expr.py @@ -8,7 +8,6 @@ import pyarrow.compute as pc -from narwhals._arrow.expr_struct import ArrowExprStructNamespace from narwhals._arrow.series import ArrowSeries from narwhals._compliant import EagerExpr from narwhals._expression_parsing import ExprKind @@ -263,7 +262,3 @@ def rank( return self._reuse_series("rank", method=method, descending=descending) ewm_mean = not_implemented() - - @property - def struct(self: Self) -> ArrowExprStructNamespace: - return ArrowExprStructNamespace(self) diff --git a/narwhals/_arrow/expr_struct.py b/narwhals/_arrow/expr_struct.py deleted file mode 100644 index 4d4f1863d2..0000000000 --- a/narwhals/_arrow/expr_struct.py +++ /dev/null @@ -1,23 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -from narwhals._expression_parsing import reuse_series_namespace_implementation - -if TYPE_CHECKING: - from typing_extensions import Self - - from narwhals._arrow.expr import ArrowExpr - - -class ArrowExprStructNamespace: - def __init__(self: Self, expr: ArrowExpr) -> None: - self._compliant_expr = expr - - def field(self: Self, name: str) -> ArrowExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, - "struct", - "field", - name=name, - ).alias(name) diff --git a/narwhals/_compliant/any_namespace.py b/narwhals/_compliant/any_namespace.py index 5d46c8dedd..d67b0a434c 100644 --- a/narwhals/_compliant/any_namespace.py +++ b/narwhals/_compliant/any_namespace.py @@ -19,6 +19,7 @@ "ListNamespace", "NameNamespace", "StringNamespace", + "StructNamespace", ] @@ -80,3 +81,7 @@ def split(self, by: str) -> CompliantT_co: ... def to_datetime(self, format: str | None) -> CompliantT_co: ... # noqa: A002 def to_lowercase(self) -> CompliantT_co: ... def to_uppercase(self) -> CompliantT_co: ... + + +class StructNamespace(_StoresCompliant[CompliantT_co], Protocol[CompliantT_co]): + def field(self, name: str) -> CompliantT_co: ... diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index 76f9c681e6..01719c64af 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -17,6 +17,7 @@ from narwhals._compliant.any_namespace import ListNamespace from narwhals._compliant.any_namespace import NameNamespace from narwhals._compliant.any_namespace import StringNamespace +from narwhals._compliant.any_namespace import StructNamespace from narwhals._compliant.namespace import CompliantNamespace from narwhals._compliant.typing import AliasName from narwhals._compliant.typing import AliasNames @@ -428,7 +429,7 @@ def _reuse_series_inner( def _reuse_series_namespace( self: Self, - series_namespace: Literal["cat", "dt", "list", "str", "name"], + series_namespace: Literal["cat", "dt", "list", "name", "str", "struct"], method_name: str, **kwargs: Any, ) -> Self: @@ -438,8 +439,8 @@ def _reuse_series_namespace( of `Expr.foo`. Arguments: - series_namespace: The Series namespace (e.g. `dt`, `cat`, `str`, `list`, `name`) - method_name: name of method. + series_namespace: The Series namespace. + method_name: name of method, within `series_namespace`. kwargs: keyword arguments to pass to function. """ return self._from_callable( @@ -787,13 +788,17 @@ def dt(self) -> EagerExprDateTimeNamespace[Self]: def list(self) -> EagerExprListNamespace[Self]: return EagerExprListNamespace(self) + @property + def name(self) -> EagerExprNameNamespace[Self]: + return EagerExprNameNamespace(self) + @property def str(self) -> EagerExprStringNamespace[Self]: return EagerExprStringNamespace(self) @property - def name(self) -> EagerExprNameNamespace[Self]: - return EagerExprNameNamespace(self) + def struct(self) -> EagerExprStructNamespace[Self]: + return EagerExprStructNamespace(self) class LazyExpr( @@ -1005,3 +1010,12 @@ def to_lowercase(self) -> EagerExprT: def to_uppercase(self) -> EagerExprT: return self.compliant._reuse_series_namespace("str", "to_uppercase") + + +class EagerExprStructNamespace( + EagerExprNamespace[EagerExprT], StructNamespace[EagerExprT], Generic[EagerExprT] +): + def field(self, name: str) -> EagerExprT: + return self.compliant._reuse_series_namespace("struct", "field", name=name).alias( + name + ) diff --git a/narwhals/_pandas_like/expr.py b/narwhals/_pandas_like/expr.py index 72ccd0f11f..bc18626778 100644 --- a/narwhals/_pandas_like/expr.py +++ b/narwhals/_pandas_like/expr.py @@ -11,7 +11,6 @@ from narwhals._expression_parsing import ExprKind from narwhals._expression_parsing import evaluate_output_names_and_aliases from narwhals._expression_parsing import is_elementary_expression -from narwhals._pandas_like.expr_struct import PandasLikeExprStructNamespace from narwhals._pandas_like.group_by import AGGREGATIONS_TO_PANDAS_EQUIVALENT from narwhals._pandas_like.series import PandasLikeSeries from narwhals.dependencies import get_numpy @@ -366,7 +365,3 @@ def rank( return self._reuse_series( "rank", call_kwargs={"method": method, "descending": descending} ) - - @property - def struct(self: Self) -> PandasLikeExprStructNamespace: - return PandasLikeExprStructNamespace(self) diff --git a/narwhals/_pandas_like/expr_struct.py b/narwhals/_pandas_like/expr_struct.py deleted file mode 100644 index 997ce1dab7..0000000000 --- a/narwhals/_pandas_like/expr_struct.py +++ /dev/null @@ -1,23 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -from narwhals._expression_parsing import reuse_series_namespace_implementation - -if TYPE_CHECKING: - from typing_extensions import Self - - from narwhals._pandas_like.expr import PandasLikeExpr - - -class PandasLikeExprStructNamespace: - def __init__(self: Self, expr: PandasLikeExpr) -> None: - self._compliant_expr = expr - - def field(self, name: str) -> PandasLikeExpr: - return reuse_series_namespace_implementation( - self._compliant_expr, - "struct", - "field", - name=name, - ).alias(name) From 42dd2c4d1bac8afb997dd6fbc8f57722f883aa5b Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Mon, 10 Mar 2025 12:26:39 +0000 Subject: [PATCH 55/58] refactor: add `EagerDataFrame._evaluate_into_expr(s)` - Forgot that I did that in the other PR - Important as it changes the variance of `EagerSeries` https://github.com/narwhals-dev/narwhals/blob/35cef0b1e2c892fb24aa730902b08b6994008c18/narwhals/_protocols.py#L50-L64 --- narwhals/_arrow/dataframe.py | 7 +++--- narwhals/_compliant/dataframe.py | 35 +++++++++++++++++++++++++----- narwhals/_expression_parsing.py | 30 ------------------------- narwhals/_pandas_like/dataframe.py | 7 +++--- 4 files changed, 36 insertions(+), 43 deletions(-) diff --git a/narwhals/_arrow/dataframe.py b/narwhals/_arrow/dataframe.py index 4b7766c487..1b885631fe 100644 --- a/narwhals/_arrow/dataframe.py +++ b/narwhals/_arrow/dataframe.py @@ -19,7 +19,6 @@ from narwhals._arrow.utils import select_rows from narwhals._compliant import EagerDataFrame from narwhals._expression_parsing import ExprKind -from narwhals._expression_parsing import evaluate_into_exprs from narwhals.dependencies import is_numpy_array_1d from narwhals.utils import Implementation from narwhals.utils import Version @@ -361,7 +360,7 @@ def aggregate(self: ArrowDataFrame, *exprs: ArrowExpr) -> ArrowDataFrame: return self.select(*exprs) def select(self: ArrowDataFrame, *exprs: ArrowExpr) -> ArrowDataFrame: - new_series = evaluate_into_exprs(self, *exprs) + new_series = self._evaluate_into_exprs(*exprs) if not new_series: # return empty dataframe, like Polars does return self._from_native_frame( @@ -374,7 +373,7 @@ def select(self: ArrowDataFrame, *exprs: ArrowExpr) -> ArrowDataFrame: def with_columns(self: ArrowDataFrame, *exprs: ArrowExpr) -> ArrowDataFrame: native_frame = self._native_frame - new_columns = evaluate_into_exprs(self, *exprs) + new_columns = self._evaluate_into_exprs(*exprs) length = len(self) columns = self.columns @@ -561,7 +560,7 @@ def filter( mask_native: Mask | ArrowChunkedArray = predicate else: # `[0]` is safe as the predicate's expression only returns a single column - mask_native = evaluate_into_exprs(self, predicate)[0]._native_series + mask_native = self._evaluate_into_exprs(predicate)[0]._native_series return self._from_native_frame( self._native_frame.filter(mask_native), # pyright: ignore[reportArgumentType] validate_column_names=False, diff --git a/narwhals/_compliant/dataframe.py b/narwhals/_compliant/dataframe.py index 33ceacddaf..ed1d83b998 100644 --- a/narwhals/_compliant/dataframe.py +++ b/narwhals/_compliant/dataframe.py @@ -1,5 +1,6 @@ from __future__ import annotations +from itertools import chain from typing import TYPE_CHECKING from typing import Any from typing import Iterator @@ -10,7 +11,7 @@ from narwhals._compliant.typing import CompliantSeriesT_co from narwhals._compliant.typing import EagerSeriesT -from narwhals._compliant.typing import EagerSeriesT_co +from narwhals._expression_parsing import evaluate_output_names_and_aliases if TYPE_CHECKING: from typing_extensions import Self @@ -59,12 +60,12 @@ def schema(self) -> Mapping[str, DType]: ... def _iter_columns(self) -> Iterator[Any]: ... -class EagerDataFrame(CompliantDataFrame[EagerSeriesT_co], Protocol[EagerSeriesT_co]): +class EagerDataFrame(CompliantDataFrame[EagerSeriesT], Protocol[EagerSeriesT]): def _maybe_evaluate_expr( - self, expr: EagerExpr[Self, EagerSeriesT_co] | T, / - ) -> EagerSeriesT_co | T: + self, expr: EagerExpr[Self, EagerSeriesT] | T, / + ) -> EagerSeriesT | T: if is_eager_expr(expr): - result: Sequence[EagerSeriesT_co] = expr(self) + result: Sequence[EagerSeriesT] = expr(self) if len(result) > 1: msg = ( "Multi-output expressions (e.g. `nw.all()` or `nw.col('a', 'b')`) " @@ -74,6 +75,30 @@ def _maybe_evaluate_expr( return result[0] return expr + def _evaluate_into_exprs( + self, *exprs: EagerExpr[Self, EagerSeriesT] + ) -> Sequence[EagerSeriesT]: + return list(chain.from_iterable(self._evaluate_into_expr(expr) for expr in exprs)) + + def _evaluate_into_expr( + self, expr: EagerExpr[Self, EagerSeriesT], / + ) -> Sequence[EagerSeriesT]: + """Return list of raw columns. + + For eager backends we alias operations at each step. + + As a safety precaution, here we can check that the expected result names match those + we were expecting from the various `evaluate_output_names` / `alias_output_names` calls. + + Note that for PySpark / DuckDB, we are less free to liberally set aliases whenever we want. + """ + _, aliases = evaluate_output_names_and_aliases(expr, self, []) + result = expr(self) + if list(aliases) != [s.name for s in result]: + msg = f"Safety assertion failed, expected {aliases}, got {result}" + raise AssertionError(msg) + return result + # NOTE: `mypy` is requiring the gymnastics here and is very fragile # DON'T CHANGE THIS or `EagerDataFrame._maybe_evaluate_expr` diff --git a/narwhals/_expression_parsing.py b/narwhals/_expression_parsing.py index b149aac8ca..f5d091c4eb 100644 --- a/narwhals/_expression_parsing.py +++ b/narwhals/_expression_parsing.py @@ -27,7 +27,6 @@ from narwhals._compliant import CompliantFrameT from narwhals._compliant import CompliantNamespace from narwhals._compliant import CompliantSeriesOrNativeExprT_co - from narwhals._compliant import CompliantSeriesT_co from narwhals.expr import Expr from narwhals.typing import CompliantDataFrame from narwhals.typing import CompliantLazyFrame @@ -44,35 +43,6 @@ def is_expr(obj: Any) -> TypeIs[Expr]: return isinstance(obj, Expr) -def evaluate_into_expr( - df: CompliantFrameT, expr: CompliantExpr[CompliantFrameT, CompliantSeriesT_co] -) -> Sequence[CompliantSeriesT_co]: - """Return list of raw columns. - - This is only use for eager backends (pandas, PyArrow), where we - alias operations at each step. As a safety precaution, here we - can check that the expected result names match those we were - expecting from the various `evaluate_output_names` / `alias_output_names` - calls. Note that for PySpark / DuckDB, we are less free to liberally - set aliases whenever we want. - """ - _, aliases = evaluate_output_names_and_aliases(expr, df, []) - result = expr(df) - if list(aliases) != [s.name for s in result]: # pragma: no cover - msg = f"Safety assertion failed, expected {aliases}, got {result}" - raise AssertionError(msg) - return result - - -def evaluate_into_exprs( - df: CompliantFrameT, - /, - *exprs: CompliantExpr[CompliantFrameT, CompliantSeriesT_co], -) -> list[CompliantSeriesT_co]: - """Evaluate each expr into Series.""" - return list(chain.from_iterable(evaluate_into_expr(df, expr) for expr in exprs)) - - def is_elementary_expression(expr: CompliantExpr[Any, Any]) -> bool: """Check if expr is elementary. diff --git a/narwhals/_pandas_like/dataframe.py b/narwhals/_pandas_like/dataframe.py index 6684a97dc7..d3e57bae89 100644 --- a/narwhals/_pandas_like/dataframe.py +++ b/narwhals/_pandas_like/dataframe.py @@ -11,7 +11,6 @@ import numpy as np from narwhals._compliant import EagerDataFrame -from narwhals._expression_parsing import evaluate_into_exprs from narwhals._pandas_like.series import PANDAS_TO_NUMPY_DTYPE_MISSING from narwhals._pandas_like.series import PandasLikeSeries from narwhals._pandas_like.utils import align_series_full_broadcast @@ -405,7 +404,7 @@ def aggregate( return self.select(*exprs) def select(self: PandasLikeDataFrame, *exprs: PandasLikeExpr) -> PandasLikeDataFrame: - new_series = evaluate_into_exprs(self, *exprs) + new_series = self._evaluate_into_exprs(*exprs) if not new_series: # return empty dataframe, like Polars does return self._from_native_frame( @@ -459,7 +458,7 @@ def filter( mask_native: pd.Series[Any] | list[bool] = predicate else: # `[0]` is safe as the predicate's expression only returns a single column - mask = evaluate_into_exprs(self, predicate)[0] + mask = self._evaluate_into_exprs(predicate)[0] mask_native = extract_dataframe_comparand(self._native_frame.index, mask) return self._from_native_frame( @@ -470,7 +469,7 @@ def with_columns( self: PandasLikeDataFrame, *exprs: PandasLikeExpr ) -> PandasLikeDataFrame: index = self._native_frame.index - new_columns = evaluate_into_exprs(self, *exprs) + new_columns = self._evaluate_into_exprs(*exprs) if not new_columns and len(self) == 0: return self From 12da35bebf62ad4b847af725b2595bf3c72f66cd Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Tue, 11 Mar 2025 08:54:17 +0000 Subject: [PATCH 56/58] chore: remove `expr.py` notes https://github.com/narwhals-dev/narwhals/pull/2149#discussion_r1988032209 --- narwhals/_compliant/expr.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/narwhals/_compliant/expr.py b/narwhals/_compliant/expr.py index 01719c64af..30e9bfa0a7 100644 --- a/narwhals/_compliant/expr.py +++ b/narwhals/_compliant/expr.py @@ -621,8 +621,6 @@ def is_in(self, other: Any) -> Self: def arg_true(self) -> Self: return self._reuse_series("arg_true") - # NOTE: `ewm_mean` not implemented `pyarrow` - def filter(self, *predicates: Self) -> Self: plx = self.__narwhals_namespace__() predicate = plx.all_horizontal(*predicates) @@ -654,8 +652,6 @@ def unique(self) -> Self: def diff(self) -> Self: return self._reuse_series("diff") - # NOTE: `shift` differs - def sample( self, n: int | None, @@ -689,8 +685,6 @@ def alias_output_names(names: Sequence[str]) -> Sequence[str]: call_kwargs=self._call_kwargs, ) - # NOTE: `over` differs - def is_unique(self) -> Self: return self._reuse_series("is_unique") @@ -730,13 +724,9 @@ def gather_every(self, n: int, offset: int) -> Self: def mode(self) -> Self: return self._reuse_series("mode") - # NOTE: `map_batches` differs - def is_finite(self) -> Self: return self._reuse_series("is_finite") - # NOTE: `cum_(sum|count|min|max|prod)` differ - def rolling_mean( self, window_size: int, *, min_samples: int | None, center: bool ) -> Self: @@ -774,8 +764,6 @@ def rolling_var( ddof=ddof, ) - # NOTE: `rank` differs - @property def cat(self) -> EagerExprCatNamespace[Self]: return EagerExprCatNamespace(self) From 79c7c8d0ada2fb16a88daa6fa2a8c3b8e302d809 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Tue, 11 Mar 2025 08:55:31 +0000 Subject: [PATCH 57/58] chore: remove `namespace.py` notes https://github.com/narwhals-dev/narwhals/pull/2149#discussion_r1988103046 --- narwhals/_compliant/namespace.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/narwhals/_compliant/namespace.py b/narwhals/_compliant/namespace.py index 91f79dcb2b..158a6ecca6 100644 --- a/narwhals/_compliant/namespace.py +++ b/narwhals/_compliant/namespace.py @@ -34,19 +34,10 @@ class EagerNamespace( CompliantNamespace[EagerDataFrameT, EagerSeriesT_co], Protocol[EagerDataFrameT, EagerSeriesT_co, EagerExprT], ): - # NOTE: Supporting moved ops - # - `self_create_expr_from_callable` -> `self._expr._from_callable` - # - `self_create_expr_from_series` -> `self._expr._from_series` @property def _expr(self) -> type[EagerExprT]: ... - - # NOTE: Supporting moved ops - # - `self._create_series_from_scalar` -> `EagerSeries()._from_scalar` - # - Was dependent on a `reference_series`, so is now an instance method - # - `._from_iterable` -> `self._series._from_iterable` @property def _series(self) -> type[EagerSeriesT_co]: ... - def all_horizontal(self, *exprs: EagerExprT) -> EagerExprT: ... @deprecated("ref'd in untyped code") From 34dd94a8496f5d4fd0b2955ee45ce5eb629c0bf6 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Tue, 11 Mar 2025 09:06:17 +0000 Subject: [PATCH 58/58] docs: Use more helpful deprecation message The points I'm trying to make are: - the method isn't self-explanatory from its name - `Find All References` won't show every use - This will be replaced soon - but still gathering the details on how/with what --- narwhals/_compliant/namespace.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/narwhals/_compliant/namespace.py b/narwhals/_compliant/namespace.py index 158a6ecca6..688f2770c2 100644 --- a/narwhals/_compliant/namespace.py +++ b/narwhals/_compliant/namespace.py @@ -40,5 +40,12 @@ def _expr(self) -> type[EagerExprT]: ... def _series(self) -> type[EagerSeriesT_co]: ... def all_horizontal(self, *exprs: EagerExprT) -> EagerExprT: ... - @deprecated("ref'd in untyped code") + @deprecated( + "Internally used for `numpy.ndarray` -> `CompliantSeries`\n" + "Also referenced in untyped `nw.dataframe.DataFrame._extract_compliant`\n" + "See Also:\n" + " - https://github.com/narwhals-dev/narwhals/pull/2149#discussion_r1986283345\n" + " - https://github.com/narwhals-dev/narwhals/issues/2116\n" + " - https://github.com/narwhals-dev/narwhals/pull/2169" + ) def _create_compliant_series(self, value: Any) -> EagerSeriesT_co: ...