Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
38 commits
Select commit Hold shift + click to select a range
4c22a90
fix everything except `join`, `v1`, `v2`
dangotbanned Aug 4, 2025
b5f76f0
fix(typing): Get `mypy` happier
dangotbanned Aug 4, 2025
2b3d2bf
fix(typing): Finish `v1`
dangotbanned Aug 4, 2025
a57fb05
ignore everything in `join_test`
dangotbanned Aug 4, 2025
6f764a6
fix(typing): `v2` happy?
dangotbanned Aug 5, 2025
c102992
fix(typing): avoid `[assignment]`
dangotbanned Aug 5, 2025
c0e08a9
fix(typing): Avoid confusing mypy
dangotbanned Aug 5, 2025
2cae9bd
Merge remote-tracking branch 'upstream/main' into native-dataframe-sized
dangotbanned Aug 5, 2025
89095ce
oooooh i think we're close
dangotbanned Aug 5, 2025
789a6da
Merge remote-tracking branch 'upstream/main' into native-dataframe-sized
dangotbanned Aug 6, 2025
13ab677
Merge branch 'main' into native-dataframe-sized
dangotbanned Aug 7, 2025
0d4b6f7
Merge branch 'main' into native-dataframe-sized
dangotbanned Aug 7, 2025
9cd60d1
Merge remote-tracking branch 'upstream/main' into native-dataframe-sized
dangotbanned Aug 8, 2025
bb34017
Merge branch 'main' into native-dataframe-sized
dangotbanned Aug 12, 2025
7329992
Merge remote-tracking branch 'upstream/main' into native-dataframe-sized
dangotbanned Aug 13, 2025
31d785b
Merge branch 'main' into native-dataframe-sized
dangotbanned Aug 14, 2025
6574c57
Merge branch 'main' into native-dataframe-sized
dangotbanned Aug 15, 2025
f5ae9fb
Merge branch 'main' into native-dataframe-sized
dangotbanned Aug 15, 2025
ac662ea
Merge branch 'main' into native-dataframe-sized
dangotbanned Aug 18, 2025
56102a9
fix: avoid introducing `__all__` extension
dangotbanned Aug 18, 2025
3e14692
refactor: rename external import guard
dangotbanned Aug 18, 2025
eb7929a
refactor: Reuse `v1.typing.DataFrameLike` def
dangotbanned Aug 18, 2025
ea96b84
refactor(typing): Realign, simplify `get_native_namespace`
dangotbanned Aug 18, 2025
33178dc
test(typing): Ensure type ignore only on lazy branch
dangotbanned Aug 18, 2025
7b329c5
test(typing): Fix dask `union-attr`
dangotbanned Aug 18, 2025
9a7bf8f
test(typing): repeat pandas assign cast fix
dangotbanned Aug 18, 2025
e9b2435
Merge branch 'main' into native-dataframe-sized
dangotbanned Aug 18, 2025
974c4a5
Merge branch 'main' into native-dataframe-sized
dangotbanned Aug 18, 2025
cd54eaf
Merge branch 'main' into native-dataframe-sized
dangotbanned Aug 19, 2025
99d19e4
Merge remote-tracking branch 'upstream/main' into native-dataframe-sized
dangotbanned Aug 19, 2025
482eca8
Merge branch 'main' into native-dataframe-sized
dangotbanned Aug 19, 2025
dbe4432
test: Call `.lazy()` before every join test
dangotbanned Aug 20, 2025
3c7e1cc
test: Move into a documented helper
dangotbanned Aug 20, 2025
2426e36
Merge branch 'main' into native-dataframe-sized
dangotbanned Aug 20, 2025
dcd67f5
Merge branch 'main' into native-dataframe-sized
dangotbanned Aug 20, 2025
a1002f1
Merge branch 'main' into native-dataframe-sized
dangotbanned Aug 20, 2025
4eadd63
test: `PolarsDataFrame.join` coverage
dangotbanned Aug 21, 2025
d3b8e13
test: `DataFrame.join_asof` coverage
dangotbanned Aug 21, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 20 additions & 12 deletions narwhals/_compliant/dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@
CompliantSeriesT,
EagerExprT,
EagerSeriesT,
NativeDataFrameT,
NativeFrameT,
NativeLazyFrameT,
NativeSeriesT,
)
from narwhals._translate import (
Expand Down Expand Up @@ -261,20 +263,22 @@ def write_parquet(self, file: str | Path | BytesIO) -> None: ...


class CompliantLazyFrame(
_StoresNative[NativeFrameT],
FromNative[NativeFrameT],
_StoresNative[NativeLazyFrameT],
FromNative[NativeLazyFrameT],
ToNarwhals[ToNarwhalsT_co],
Protocol[CompliantExprT_contra, NativeFrameT, ToNarwhalsT_co],
Protocol[CompliantExprT_contra, NativeLazyFrameT, ToNarwhalsT_co],
):
_native_frame: NativeFrameT
_native_frame: NativeLazyFrameT
_implementation: Implementation
_version: Version

def __narwhals_lazyframe__(self) -> Self: ...
def __narwhals_namespace__(self) -> Any: ...

@classmethod
def from_native(cls, data: NativeFrameT, /, *, context: _LimitedContext) -> Self: ...
def from_native(
cls, data: NativeLazyFrameT, /, *, context: _LimitedContext
) -> Self: ...

def simple_select(self, *column_names: str) -> Self:
"""`select` where all args are column names."""
Expand All @@ -290,7 +294,7 @@ def aggregate(self, *exprs: CompliantExprT_contra) -> Self:
def _with_version(self, version: Version) -> Self: ...

@property
def native(self) -> NativeFrameT:
def native(self) -> NativeLazyFrameT:
return self._native_frame

@property
Expand Down Expand Up @@ -354,24 +358,28 @@ def with_row_index(self, name: str, order_by: Sequence[str]) -> Self: ...


class EagerDataFrame(
CompliantDataFrame[EagerSeriesT, EagerExprT, NativeFrameT, "DataFrame[NativeFrameT]"],
CompliantLazyFrame[EagerExprT, NativeFrameT, "DataFrame[NativeFrameT]"],
CompliantDataFrame[
EagerSeriesT, EagerExprT, NativeDataFrameT, "DataFrame[NativeDataFrameT]"
],
CompliantLazyFrame[EagerExprT, "Incomplete", "DataFrame[NativeDataFrameT]"],
Copy link
Member Author

@dangotbanned dangotbanned Aug 5, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Pretty sure this Incomplete is unavoidable, just like the [type-var] warnings on v1.DataFrameLike

# NOTE legit
class DataFrame(NwDataFrame[IntoDataFrameT]): # type: ignore[type-var]
_version = Version.V1

I'm much happier to eat those two, when it fixes sooooooooo much else πŸ₯³

ValidateBackendVersion,
Protocol[EagerSeriesT, EagerExprT, NativeFrameT, NativeSeriesT],
Protocol[EagerSeriesT, EagerExprT, NativeDataFrameT, NativeSeriesT],
):
@property
def _backend_version(self) -> tuple[int, ...]:
return self._implementation._backend_version()

def __narwhals_namespace__(
self,
) -> EagerNamespace[Self, EagerSeriesT, EagerExprT, NativeFrameT, NativeSeriesT]: ...
) -> EagerNamespace[
Self, EagerSeriesT, EagerExprT, NativeDataFrameT, NativeSeriesT
]: ...

def to_narwhals(self) -> DataFrame[NativeFrameT]:
def to_narwhals(self) -> DataFrame[NativeDataFrameT]:
return self._version.dataframe(self, level="full")

def _with_native(
self, df: NativeFrameT, *, validate_column_names: bool = True
self, df: NativeDataFrameT, *, validate_column_names: bool = True
) -> Self: ...

def _check_columns_exist(self, subset: Sequence[str]) -> ColumnNotFoundError | None:
Expand Down
4 changes: 4 additions & 0 deletions narwhals/_compliant/typing.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@
from narwhals._compliant.window import WindowInputs
from narwhals.typing import (
FillNullStrategy,
IntoLazyFrame,
NativeDataFrame,
NativeFrame,
NativeSeries,
RankMethod,
Expand Down Expand Up @@ -90,6 +92,8 @@ class ScalarKwargs(TypedDict, total=False):
NativeSeriesT_contra = TypeVar(
"NativeSeriesT_contra", bound="NativeSeries", contravariant=True
)
NativeDataFrameT = TypeVar("NativeDataFrameT", bound="NativeDataFrame")
NativeLazyFrameT = TypeVar("NativeLazyFrameT", bound="IntoLazyFrame")
NativeFrameT = TypeVar("NativeFrameT", bound="NativeFrame")
NativeFrameT_co = TypeVar("NativeFrameT_co", bound="NativeFrame", covariant=True)
NativeFrameT_contra = TypeVar(
Expand Down
8 changes: 6 additions & 2 deletions narwhals/_interchange/dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,11 @@
if TYPE_CHECKING:
import pandas as pd
import pyarrow as pa
from typing_extensions import Self
from typing_extensions import Self, TypeIs

from narwhals._interchange.series import InterchangeSeries
from narwhals.dtypes import DType
from narwhals.typing import DataFrameLike
from narwhals.stable.v1.typing import DataFrameLike


class DtypeKind(enum.IntEnum):
Expand Down Expand Up @@ -153,3 +153,7 @@ def select(self, *exprs: str) -> Self: # pragma: no cover
"at https://github.com/narwhals-dev/narwhals/issues."
)
raise NotImplementedError(msg)


def supports_dataframe_interchange(obj: Any) -> TypeIs[DataFrameLike]:
return hasattr(obj, "__dataframe__")
8 changes: 3 additions & 5 deletions narwhals/_namespace.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@
Polars,
SparkLike,
)
from narwhals.typing import DataFrameLike, NativeFrame, NativeLazyFrame, NativeSeries
from narwhals.typing import NativeDataFrame, NativeLazyFrame, NativeSeries

T = TypeVar("T")

Expand All @@ -86,7 +86,7 @@ def rename(self, *args: Any, inplace: Literal[False], **kwds: Any) -> Self:
"""`inplace=False` is required to avoid (incorrect?) default overloads."""
...

class _BasePandasLikeFrame(NativeFrame, _BasePandasLike, Protocol): ...
class _BasePandasLikeFrame(NativeDataFrame, _BasePandasLike, Protocol): ...

class _BasePandasLikeSeries(NativeSeries, _BasePandasLike, Protocol):
def where(self, cond: Any, other: Any = ..., **kwds: Any) -> Any: ...
Expand Down Expand Up @@ -131,9 +131,7 @@ class _ModinSeries(_BasePandasLikeSeries, Protocol):
)

NativeKnown: TypeAlias = "_NativePolars | _NativeArrow | _NativePandasLike | _NativeSparkLike | _NativeDuckDB | _NativeDask | _NativeIbis"
NativeUnknown: TypeAlias = (
"NativeFrame | NativeSeries | NativeLazyFrame | DataFrameLike"
)
NativeUnknown: TypeAlias = "NativeDataFrame | NativeSeries | NativeLazyFrame"
NativeAny: TypeAlias = "NativeKnown | NativeUnknown"

__all__ = ["Namespace"]
Expand Down
10 changes: 7 additions & 3 deletions narwhals/_sql/dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,11 @@
from typing import TYPE_CHECKING, Any, Protocol

from narwhals._compliant.dataframe import CompliantLazyFrame
from narwhals._compliant.typing import CompliantExprT_contra, NativeExprT, NativeFrameT
from narwhals._compliant.typing import (
CompliantExprT_contra,
NativeExprT,
NativeLazyFrameT,
)
from narwhals._translate import ToNarwhalsT_co
from narwhals._utils import check_columns_exist

Expand All @@ -20,8 +24,8 @@


class SQLLazyFrame(
CompliantLazyFrame[CompliantExprT_contra, NativeFrameT, ToNarwhalsT_co],
Protocol[CompliantExprT_contra, NativeFrameT, ToNarwhalsT_co],
CompliantLazyFrame[CompliantExprT_contra, NativeLazyFrameT, ToNarwhalsT_co],
Protocol[CompliantExprT_contra, NativeLazyFrameT, ToNarwhalsT_co],
):
def _evaluate_window_expr(
self,
Expand Down
11 changes: 3 additions & 8 deletions narwhals/_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@
NativeFrameT_co,
NativeSeriesT_co,
)
from narwhals._compliant.typing import EvalNames
from narwhals._compliant.typing import EvalNames, NativeLazyFrameT
from narwhals._namespace import Namespace
from narwhals._translate import ArrowStreamExportable, IntoArrowTable, ToNarwhalsT_co
from narwhals._typing import (
Expand All @@ -90,7 +90,6 @@
CompliantDataFrame,
CompliantLazyFrame,
CompliantSeries,
DataFrameLike,
DTypes,
IntoSeriesT,
MultiIndexSelector,
Expand Down Expand Up @@ -1623,8 +1622,8 @@ def is_compliant_dataframe(


def is_compliant_lazyframe(
obj: CompliantLazyFrame[CompliantExprT, NativeFrameT_co, ToNarwhalsT_co] | Any,
) -> TypeIs[CompliantLazyFrame[CompliantExprT, NativeFrameT_co, ToNarwhalsT_co]]:
obj: CompliantLazyFrame[CompliantExprT, NativeLazyFrameT, ToNarwhalsT_co] | Any,
) -> TypeIs[CompliantLazyFrame[CompliantExprT, NativeLazyFrameT, ToNarwhalsT_co]]:
return _hasattr_static(obj, "__narwhals_lazyframe__")


Expand Down Expand Up @@ -1676,10 +1675,6 @@ def has_native_namespace(obj: Any) -> TypeIs[SupportsNativeNamespace]:
return _hasattr_static(obj, "__native_namespace__")


def _supports_dataframe_interchange(obj: Any) -> TypeIs[DataFrameLike]:
return hasattr(obj, "__dataframe__")


def supports_arrow_c_stream(obj: Any) -> TypeIs[ArrowStreamExportable]:
return _hasattr_static(obj, "__arrow_c_stream__")

Expand Down
12 changes: 6 additions & 6 deletions narwhals/dataframe.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@
IntoDataFrame,
IntoExpr,
IntoFrame,
IntoLazyFrame,
IntoSchema,
JoinStrategy,
LazyUniqueKeepStrategy,
Expand All @@ -95,7 +96,7 @@
PS = ParamSpec("PS")

_FrameT = TypeVar("_FrameT", bound="IntoFrame")
FrameT = TypeVar("FrameT", bound="IntoFrame")
LazyFrameT = TypeVar("LazyFrameT", bound="IntoLazyFrame")
DataFrameT = TypeVar("DataFrameT", bound="IntoDataFrame")
R = TypeVar("R")

Expand Down Expand Up @@ -469,8 +470,7 @@ def _lazyframe(self) -> type[LazyFrame[Any]]:

def __init__(self, df: Any, *, level: Literal["full", "lazy", "interchange"]) -> None:
self._level: Literal["full", "lazy", "interchange"] = level
# NOTE: Interchange support (`DataFrameLike`) is the source of the error
self._compliant_frame: CompliantDataFrame[Any, Any, DataFrameT, Self] # type: ignore[type-var]
self._compliant_frame: CompliantDataFrame[Any, Any, DataFrameT, Self]
if is_compliant_dataframe(df):
self._compliant_frame = df.__narwhals_dataframe__()
else: # pragma: no cover
Expand Down Expand Up @@ -2219,7 +2219,7 @@ def explode(self, columns: str | Sequence[str], *more_columns: str) -> Self:
return super().explode(columns, *more_columns)


class LazyFrame(BaseFrame[FrameT]):
class LazyFrame(BaseFrame[LazyFrameT]):
"""Narwhals LazyFrame, backed by a native lazyframe.

Warning:
Expand Down Expand Up @@ -2285,7 +2285,7 @@ def _dataframe(self) -> type[DataFrame[Any]]:

def __init__(self, df: Any, *, level: Literal["full", "lazy", "interchange"]) -> None:
self._level = level
self._compliant_frame: CompliantLazyFrame[Any, FrameT, Self] # type: ignore[type-var]
self._compliant_frame: CompliantLazyFrame[Any, LazyFrameT, Self]
if is_compliant_lazyframe(df):
self._compliant_frame = df.__narwhals_lazyframe__()
else: # pragma: no cover
Expand Down Expand Up @@ -2384,7 +2384,7 @@ def collect(
msg = f"Unsupported `backend` value.\nExpected one of {get_args(_LazyFrameCollectImpl)} or None, got: {eager_backend}."
raise ValueError(msg)

def to_native(self) -> FrameT:
def to_native(self) -> LazyFrameT:
"""Convert Narwhals LazyFrame to native one.

Examples:
Expand Down
20 changes: 12 additions & 8 deletions narwhals/functions.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
IntoDType,
IntoExpr,
IntoSchema,
NativeFrame,
NativeDataFrame,
NativeLazyFrame,
NativeSeries,
NonNestedLiteral,
Expand Down Expand Up @@ -301,7 +301,9 @@ def from_dict(
try:
# implementation is UNKNOWN, Narwhals extension using this feature should
# implement `from_dict` function in the top-level namespace.
native_frame: NativeFrame = _native_namespace.from_dict(data, schema=schema)
native_frame: NativeDataFrame = _native_namespace.from_dict(
data, schema=schema
)
except AttributeError as e:
msg = "Unknown namespace is expected to implement `from_dict` function."
raise AttributeError(msg) from e
Expand Down Expand Up @@ -397,7 +399,9 @@ def from_numpy(
try:
# implementation is UNKNOWN, Narwhals extension using this feature should
# implement `from_numpy` function in the top-level namespace.
native_frame: NativeFrame = _native_namespace.from_numpy(data, schema=schema)
native_frame: NativeDataFrame = _native_namespace.from_numpy(
data, schema=schema
)
except AttributeError as e:
msg = "Unknown namespace is expected to implement `from_numpy` function."
raise AttributeError(msg) from e
Expand Down Expand Up @@ -470,7 +474,7 @@ def from_arrow(
try:
# implementation is UNKNOWN, Narwhals extension using this feature should
# implement PyCapsule support
native: NativeFrame = _native_namespace.DataFrame(native_frame)
native: NativeDataFrame = _native_namespace.DataFrame(native_frame)
except AttributeError as e:
msg = "Unknown namespace is expected to implement `DataFrame` class which accepts object which supports PyCapsule Interface."
raise AttributeError(msg) from e
Expand Down Expand Up @@ -594,7 +598,7 @@ def read_csv(
"""
impl = Implementation.from_backend(backend)
native_namespace = impl.to_native_namespace()
native_frame: NativeFrame
native_frame: NativeDataFrame
if impl in {
Implementation.POLARS,
Implementation.PANDAS,
Expand Down Expand Up @@ -670,7 +674,7 @@ def scan_csv(
"""
implementation = Implementation.from_backend(backend)
native_namespace = implementation.to_native_namespace()
native_frame: NativeFrame | NativeLazyFrame
native_frame: NativeDataFrame | NativeLazyFrame
if implementation is Implementation.POLARS:
native_frame = native_namespace.scan_csv(source, **kwargs)
elif implementation in {
Expand Down Expand Up @@ -750,7 +754,7 @@ def read_parquet(
"""
impl = Implementation.from_backend(backend)
native_namespace = impl.to_native_namespace()
native_frame: NativeFrame
native_frame: NativeDataFrame
if impl in {
Implementation.POLARS,
Implementation.PANDAS,
Expand Down Expand Up @@ -853,7 +857,7 @@ def scan_parquet(
"""
implementation = Implementation.from_backend(backend)
native_namespace = implementation.to_native_namespace()
native_frame: NativeFrame | NativeLazyFrame
native_frame: NativeDataFrame | NativeLazyFrame
if implementation is Implementation.POLARS:
native_frame = native_namespace.scan_parquet(source, **kwargs)
elif implementation in {
Expand Down
Loading
Loading