From ff8e647a1ba717925a35abdc9418310e0fc57d35 Mon Sep 17 00:00:00 2001 From: FBruzzesi Date: Tue, 27 Jan 2026 15:55:39 +0100 Subject: [PATCH 01/13] WIP: all but pandas and ibis? --- narwhals/_arrow/namespace.py | 4 +-- narwhals/_duckdb/namespace.py | 8 ++++-- narwhals/_spark_like/namespace.py | 19 ++++++++++-- narwhals/functions.py | 43 +++++++++++++++++++++++---- narwhals/typing.py | 4 ++- tests/expr_and_series/lit_test.py | 48 +++++++++++++++++++++++++------ 6 files changed, 103 insertions(+), 23 deletions(-) diff --git a/narwhals/_arrow/namespace.py b/narwhals/_arrow/namespace.py index 98282a575e..83bb1c2bce 100644 --- a/narwhals/_arrow/namespace.py +++ b/narwhals/_arrow/namespace.py @@ -25,7 +25,7 @@ from narwhals._arrow.typing import ChunkedArrayAny, Incomplete, ScalarAny from narwhals._utils import Version - from narwhals.typing import IntoDType, NonNestedLiteral + from narwhals.typing import IntoDType, PythonLiteral class ArrowNamespace( @@ -64,7 +64,7 @@ def len(self) -> ArrowExpr: version=self._version, ) - def lit(self, value: NonNestedLiteral, dtype: IntoDType | None) -> ArrowExpr: + def lit(self, value: PythonLiteral, dtype: IntoDType | None) -> ArrowExpr: def _lit_arrow_series(_: ArrowDataFrame) -> ArrowSeries: arrow_series = ArrowSeries.from_iterable( data=[value], name="literal", context=self diff --git a/narwhals/_duckdb/namespace.py b/narwhals/_duckdb/namespace.py index ebc5041e68..476820feed 100644 --- a/narwhals/_duckdb/namespace.py +++ b/narwhals/_duckdb/namespace.py @@ -35,7 +35,7 @@ from narwhals._compliant.window import WindowInputs from narwhals._utils import Version - from narwhals.typing import ConcatMethod, IntoDType, NonNestedLiteral + from narwhals.typing import ConcatMethod, IntoDType, PythonLiteral VARCHAR = duckdb_dtypes.VARCHAR @@ -130,8 +130,12 @@ def func(cols: Iterable[Expression]) -> Expression: return self._expr._from_elementwise_horizontal_op(func, *exprs) - def lit(self, value: NonNestedLiteral, dtype: IntoDType | None) -> DuckDBExpr: + def lit(self, value: PythonLiteral, dtype: IntoDType | None) -> DuckDBExpr: def func(df: DuckDBLazyFrame) -> list[Expression]: + if isinstance(value, dict) and len(value) == 0: + msg = "Cannot create an empty struct type for DuckDB backend" + raise NotImplementedError(msg) + tz = DeferredTimeZone(df.native) if dtype is not None: target = narwhals_to_native_dtype(dtype, self._version, tz) diff --git a/narwhals/_spark_like/namespace.py b/narwhals/_spark_like/namespace.py index c660b67298..ecbecd1a52 100644 --- a/narwhals/_spark_like/namespace.py +++ b/narwhals/_spark_like/namespace.py @@ -28,7 +28,7 @@ from narwhals._compliant.window import WindowInputs from narwhals._spark_like.dataframe import SQLFrameDataFrame # noqa: F401 from narwhals._utils import Implementation, Version - from narwhals.typing import ConcatMethod, IntoDType, NonNestedLiteral, PythonLiteral + from narwhals.typing import ConcatMethod, IntoDType, PythonLiteral # Adjust slight SQL vs PySpark differences FUNCTION_REMAPPINGS = { @@ -91,9 +91,22 @@ def _when( def _coalesce(self, *exprs: Column) -> Column: return self._F.coalesce(*exprs) - def lit(self, value: NonNestedLiteral, dtype: IntoDType | None) -> SparkLikeExpr: + def lit(self, value: PythonLiteral, dtype: IntoDType | None) -> SparkLikeExpr: def func(df: SparkLikeLazyFrame) -> list[Column]: - column = df._F.lit(value) + F = df._F + + if isinstance(value, (list, tuple)): + lit_values = [F.lit(v) for v in value] + column = F.lit(F.array(lit_values)) + elif isinstance(value, dict): + if self._implementation.is_sqlframe() and len(value) == 0: + msg = "Cannot create an empty struct type for SQLFrame backend" + raise NotImplementedError(msg) + lit_values = [F.lit(v).alias(k) for k, v in value.items()] + column = F.struct(*lit_values) + else: + column = F.lit(value) + if dtype: native_dtype = narwhals_to_native_dtype( dtype, self._version, df._native_dtypes, df.native.sparkSession diff --git a/narwhals/functions.py b/narwhals/functions.py index 322da7019e..6a909d951e 100644 --- a/narwhals/functions.py +++ b/narwhals/functions.py @@ -46,6 +46,7 @@ IntoExpr, IntoSchema, NonNestedLiteral, + PythonLiteral, _2DArray, ) @@ -1422,15 +1423,19 @@ def all_horizontal(*exprs: IntoExpr | Iterable[IntoExpr], ignore_nulls: bool) -> ) -def lit(value: NonNestedLiteral, dtype: IntoDType | None = None) -> Expr: +def lit(value: PythonLiteral, dtype: IntoDType | None = None) -> Expr: """Return an expression representing a literal value. Arguments: - value: The value to use as literal. + value: The value to use as literal. Can be a scalar value, list, tuple, or dict. + Lists and tuples are converted to `List` dtype, dicts to `Struct` dtype. dtype: The data type of the literal value. If not provided, the data type will - be inferred by the native library. + be inferred by the native library. For empty lists/dicts, dtype must be + specified explicitly. Examples: + Scalar literals: + >>> import pandas as pd >>> import narwhals as nw >>> @@ -1443,6 +1448,32 @@ def lit(value: NonNestedLiteral, dtype: IntoDType | None = None) -> Expr: | 0 1 3 | | 1 2 3 | └──────────────────┘ + + List literals (creates a List column): + + >>> df_native = pd.DataFrame({"a": [1, 2]}) + >>> nw.from_native(df_native).with_columns(nw.lit([1, 2, 3]).alias("list_col")) + ┌──────────────────────┐ + |Narwhals DataFrame | + |----------------------| + | a list_col | + | 0 1 [1, 2, 3] | + | 1 2 [1, 2, 3] | + └──────────────────────┘ + + Dict literals (creates a Struct column): + + >>> df_native = pd.DataFrame({"a": [1, 2]}) + >>> nw.from_native(df_native).with_columns( + ... nw.lit({"x": 1, "y": 2}).alias("struct_col") + ... ) + ┌───────────────────────┐ + |Narwhals DataFrame | + |-----------------------| + | a struct_col | + | 0 1 {1, 2} | + | 1 2 {1, 2} | + └───────────────────────┘ """ if is_numpy_array(value): msg = ( @@ -1451,9 +1482,9 @@ def lit(value: NonNestedLiteral, dtype: IntoDType | None = None) -> Expr: ) raise ValueError(msg) - if isinstance(value, (list, tuple)): - msg = f"Nested datatypes are not supported yet. Got {value}" - raise NotImplementedError(msg) + if isinstance(value, (list, tuple, dict)) and len(value) == 0 and dtype is None: + msg = "Cannot infer dtype for empty nested structure. Please provide an explicit dtype parameter." + raise ValueError(msg) return Expr(ExprNode(ExprKind.LITERAL, "lit", value=value, dtype=dtype)) diff --git a/narwhals/typing.py b/narwhals/typing.py index cf2852fc83..bffe3c7e9f 100644 --- a/narwhals/typing.py +++ b/narwhals/typing.py @@ -270,7 +270,9 @@ def Binary(self) -> type[dtypes.Binary]: ... NonNestedLiteral: TypeAlias = ( "NumericLiteral | TemporalLiteral | str | bool | bytes | None" ) -PythonLiteral: TypeAlias = "NonNestedLiteral | list[Any] | tuple[Any, ...]" +PythonLiteral: TypeAlias = ( + "NonNestedLiteral | list[Any] | tuple[Any, ...] | dict[str, Any]" +) NonNestedDType: TypeAlias = "dtypes.NumericType | dtypes.TemporalType | dtypes.String | dtypes.Boolean | dtypes.Binary | dtypes.Categorical | dtypes.Unknown | dtypes.Object" """Any Narwhals DType that does not have required arguments.""" diff --git a/tests/expr_and_series/lit_test.py b/tests/expr_and_series/lit_test.py index 4a23ab9629..d667f43962 100644 --- a/tests/expr_and_series/lit_test.py +++ b/tests/expr_and_series/lit_test.py @@ -1,6 +1,5 @@ from __future__ import annotations -import re from datetime import date from typing import TYPE_CHECKING, Any @@ -17,6 +16,7 @@ if TYPE_CHECKING: from narwhals.dtypes import DType + from narwhals.typing import IntoDType, PythonLiteral @pytest.mark.parametrize( @@ -45,14 +45,6 @@ def test_lit_error(constructor: Constructor) -> None: ValueError, match="numpy arrays are not supported as literal values" ): _ = df.with_columns(nw.lit(np.array([1, 2])).alias("lit")) # pyright: ignore[reportArgumentType] - with pytest.raises( - NotImplementedError, match=re.escape("Nested datatypes are not supported yet.") - ): - _ = df.with_columns(nw.lit((1, 2)).alias("lit")) # type: ignore[arg-type] - with pytest.raises( - NotImplementedError, match=re.escape("Nested datatypes are not supported yet.") - ): - _ = df.with_columns(nw.lit([1, 2]).alias("lit")) # type: ignore[arg-type] def test_lit_out_name(constructor: Constructor) -> None: @@ -143,3 +135,41 @@ def test_pyarrow_lit_string() -> None: assert pa.types.is_string(result.type) result = df.select(nw.lit("foo", dtype=nw.String)).to_native().schema.field("literal") assert pa.types.is_string(result.type) + + +@pytest.mark.parametrize( + ("value", "dtype"), + [ + ((), nw.List(nw.Int32())), + ([], nw.List(nw.Int32())), + ({}, nw.Struct({})), + (("foo", "bar"), None), + (["orca", "narwhal"], None), + ({"field_1": 42}, None), + ], +) +def test_nested_structures( + request: pytest.FixtureRequest, + constructor: Constructor, + value: PythonLiteral, + dtype: IntoDType | None, +) -> None: + is_empty_dict = isinstance(value, dict) and len(value) == 0 + if any(x in str(constructor) for x in ("duckdb", "sqlframe")) and is_empty_dict: + reason = "Cannot create an empty struct type for backend" + request.applymarker(pytest.mark.xfail(reason=reason, raises=NotImplementedError)) + + size = 3 + data = {"a": list(range(size))} + frame = nw.from_native(constructor(data)) + result = frame.with_columns(nested=nw.lit(value, dtype=dtype)) + + value = list(value) if isinstance(value, tuple) else value + assert_equal_data(result, {"a": data["a"], "nested": [value] * size}) + + +@pytest.mark.parametrize("value", [[], (), {}]) +def test_raise_empty_nested_structures(value: PythonLiteral) -> None: + msg = "Cannot infer dtype for empty nested structure. Please provide an explicit dtype parameter." + with pytest.raises(ValueError, match=msg): + nw.lit(value=value) From db185900c2ca42b84858c82adb01a4809abdc2e8 Mon Sep 17 00:00:00 2001 From: FBruzzesi Date: Tue, 27 Jan 2026 16:46:55 +0100 Subject: [PATCH 02/13] add ibis to the mix, xfail pandas-like --- narwhals/_ibis/namespace.py | 8 +++++++- narwhals/_pandas_like/namespace.py | 19 +++++++++++++------ tests/expr_and_series/lit_test.py | 19 ++++++++++++++++++- 3 files changed, 38 insertions(+), 8 deletions(-) diff --git a/narwhals/_ibis/namespace.py b/narwhals/_ibis/namespace.py index ddda316742..092b0a2593 100644 --- a/narwhals/_ibis/namespace.py +++ b/narwhals/_ibis/namespace.py @@ -114,9 +114,15 @@ def func(cols: Iterable[ir.Value]) -> ir.Value: return self._expr._from_elementwise_horizontal_op(func, *exprs) - def lit(self, value: Any, dtype: IntoDType | None) -> IbisExpr: + def lit(self, value: PythonLiteral, dtype: IntoDType | None) -> IbisExpr: def func(_df: IbisLazyFrame) -> Sequence[ir.Value]: + if isinstance(value, dict) and len(value) == 0: + msg = "Cannot create an empty struct type for Ibis backend" + raise NotImplementedError(msg) + ibis_dtype = narwhals_to_native_dtype(dtype, self._version) if dtype else None + if isinstance(value, dict): + return [ibis.struct(value, type=ibis_dtype)] return [lit(value, ibis_dtype)] return self._expr( diff --git a/narwhals/_pandas_like/namespace.py b/narwhals/_pandas_like/namespace.py index 96b7a22290..81296ce483 100644 --- a/narwhals/_pandas_like/namespace.py +++ b/narwhals/_pandas_like/namespace.py @@ -17,7 +17,7 @@ from narwhals._pandas_like.series import PandasLikeSeries from narwhals._pandas_like.typing import NativeDataFrameT, NativeSeriesT from narwhals._pandas_like.utils import is_non_nullable_boolean -from narwhals._utils import zip_strict +from narwhals._utils import qualified_type_name, zip_strict if TYPE_CHECKING: from collections.abc import Iterable, Sequence @@ -25,7 +25,7 @@ from typing_extensions import TypeAlias from narwhals._utils import Implementation, Version - from narwhals.typing import IntoDType, NonNestedLiteral + from narwhals.typing import IntoDType, PythonLiteral Incomplete: TypeAlias = Any @@ -83,17 +83,24 @@ def func(df: PandasLikeDataFrame) -> list[PandasLikeSeries]: context=self, ) - def lit(self, value: NonNestedLiteral, dtype: IntoDType | None) -> PandasLikeExpr: + def lit(self, value: PythonLiteral, dtype: IntoDType | None) -> PandasLikeExpr: def _lit_pandas_series(df: PandasLikeDataFrame) -> PandasLikeSeries: - pandas_series = self._series.from_iterable( + if isinstance(value, (list, tuple, dict)): + msg = ( + "Nested structures are not support for Pandas-like backend, " + f" found {qualified_type_name(value)}" + ) + raise NotImplementedError(msg) + + pandas_like_series = self._series.from_iterable( data=[value], name="literal", index=df._native_frame.index[0:1], context=self, ) if dtype: - return pandas_series.cast(dtype) - return pandas_series + return pandas_like_series.cast(dtype) + return pandas_like_series return PandasLikeExpr( lambda df: [_lit_pandas_series(df)], diff --git a/tests/expr_and_series/lit_test.py b/tests/expr_and_series/lit_test.py index d667f43962..6dfefe4e28 100644 --- a/tests/expr_and_series/lit_test.py +++ b/tests/expr_and_series/lit_test.py @@ -140,12 +140,24 @@ def test_pyarrow_lit_string() -> None: @pytest.mark.parametrize( ("value", "dtype"), [ + # Empty nested structures ((), nw.List(nw.Int32())), ([], nw.List(nw.Int32())), ({}, nw.Struct({})), + # Nested structures with different size from dataframe (("foo", "bar"), None), (["orca", "narwhal"], None), ({"field_1": 42}, None), + # Nested structures with same size as dataframe + (("foo", "bar", "baz"), None), + (["orca", "narwhal", "penguin"], None), + ( + {"field_1": 42, "field_2": 1.2, "field_3": True}, + nw.Struct( + {"field_1": nw.Int32(), "field_2": nw.Float64(), "field_3": nw.Boolean()} + ), + ), + # TODO(FBruzzesi): double nested? ], ) def test_nested_structures( @@ -155,10 +167,15 @@ def test_nested_structures( dtype: IntoDType | None, ) -> None: is_empty_dict = isinstance(value, dict) and len(value) == 0 - if any(x in str(constructor) for x in ("duckdb", "sqlframe")) and is_empty_dict: + non_pyspark_sql_like = ("duckdb", "sqlframe", "ibis") + if any(x in str(constructor) for x in non_pyspark_sql_like) and is_empty_dict: reason = "Cannot create an empty struct type for backend" request.applymarker(pytest.mark.xfail(reason=reason, raises=NotImplementedError)) + if any(x in str(constructor) for x in ("cudf", "modin", "pandas")): + reason = "Nested structures are not support for backend" + request.applymarker(pytest.mark.xfail(reason=reason, raises=NotImplementedError)) + size = 3 data = {"a": list(range(size))} frame = nw.from_native(constructor(data)) From c49ccbbea31aba4663023323f119a1a7b5a81f47 Mon Sep 17 00:00:00 2001 From: FBruzzesi Date: Tue, 27 Jan 2026 17:01:54 +0100 Subject: [PATCH 03/13] fix doctest, test select --- narwhals/functions.py | 65 ++++++++++++++++++------------- tests/expr_and_series/lit_test.py | 15 +++++-- 2 files changed, 50 insertions(+), 30 deletions(-) diff --git a/narwhals/functions.py b/narwhals/functions.py index 6a909d951e..7dc452451c 100644 --- a/narwhals/functions.py +++ b/narwhals/functions.py @@ -1436,44 +1436,57 @@ def lit(value: PythonLiteral, dtype: IntoDType | None = None) -> Expr: Examples: Scalar literals: - >>> import pandas as pd + >>> import pyarrow as pa >>> import narwhals as nw >>> - >>> df_native = pd.DataFrame({"a": [1, 2]}) - >>> nw.from_native(df_native).with_columns(nw.lit(3)) + >>> df_nw = nw.from_native(pa.table({"a": [1, 2]})) + >>> df_nw.with_columns(nw.lit(3)) ┌──────────────────┐ |Narwhals DataFrame| |------------------| - | a literal | - | 0 1 3 | - | 1 2 3 | + | pyarrow.Table | + | a: int64 | + | literal: int64 | + | ---- | + | a: [[1,2]] | + | literal: [[3,3]] | └──────────────────┘ List literals (creates a List column): - >>> df_native = pd.DataFrame({"a": [1, 2]}) - >>> nw.from_native(df_native).with_columns(nw.lit([1, 2, 3]).alias("list_col")) - ┌──────────────────────┐ - |Narwhals DataFrame | - |----------------------| - | a list_col | - | 0 1 [1, 2, 3] | - | 1 2 [1, 2, 3] | - └──────────────────────┘ + >>> df_nw.with_columns(nw.lit([1, 2, 3]).alias("list_col")) + ┌─────────────────────────────┐ + | Narwhals DataFrame | + |-----------------------------| + |pyarrow.Table | + |a: int64 | + |list_col: list | + | child 0, item: int64 | + |---- | + |a: [[1,2]] | + |list_col: [[[1,2,3],[1,2,3]]]| + └─────────────────────────────┘ Dict literals (creates a Struct column): - >>> df_native = pd.DataFrame({"a": [1, 2]}) - >>> nw.from_native(df_native).with_columns( - ... nw.lit({"x": 1, "y": 2}).alias("struct_col") - ... ) - ┌───────────────────────┐ - |Narwhals DataFrame | - |-----------------------| - | a struct_col | - | 0 1 {1, 2} | - | 1 2 {1, 2} | - └───────────────────────┘ + >>> df_nw.with_columns(nw.lit({"x": 1, "y": 2}).alias("struct_col")) + ┌──────────────────────────────────────┐ + | Narwhals DataFrame | + |--------------------------------------| + |pyarrow.Table | + |a: int64 | + |struct_col: struct| + | child 0, x: int64 | + | child 1, y: int64 | + |---- | + |a: [[1,2]] | + |struct_col: [ | + | -- is_valid: all not null | + | -- child 0 type: int64 | + |[1,1] | + | -- child 1 type: int64 | + |[2,2]] | + └──────────────────────────────────────┘ """ if is_numpy_array(value): msg = ( diff --git a/tests/expr_and_series/lit_test.py b/tests/expr_and_series/lit_test.py index 6dfefe4e28..cbbd92f299 100644 --- a/tests/expr_and_series/lit_test.py +++ b/tests/expr_and_series/lit_test.py @@ -157,7 +157,7 @@ def test_pyarrow_lit_string() -> None: {"field_1": nw.Int32(), "field_2": nw.Float64(), "field_3": nw.Boolean()} ), ), - # TODO(FBruzzesi): double nested? + # TODO(FBruzzesi): deeper nesting? ], ) def test_nested_structures( @@ -178,11 +178,18 @@ def test_nested_structures( size = 3 data = {"a": list(range(size))} + expr = nw.lit(value, dtype=dtype).alias("nested") + + value_ = list(value) if isinstance(value, tuple) else value + expected_nested = {"nested": [value_] * size} + frame = nw.from_native(constructor(data)) - result = frame.with_columns(nested=nw.lit(value, dtype=dtype)) - value = list(value) if isinstance(value, tuple) else value - assert_equal_data(result, {"a": data["a"], "nested": [value] * size}) + result_with_cols = frame.with_columns(expr) + assert_equal_data(result_with_cols, {**data, **expected_nested}) + + result_select = frame.select(expr, nw.col("a")) + assert_equal_data(result_select, {**expected_nested, **data}) @pytest.mark.parametrize("value", [[], (), {}]) From a897470e3e73964cfb026a1865c785c7eae5cbf3 Mon Sep 17 00:00:00 2001 From: FBruzzesi Date: Tue, 27 Jan 2026 23:45:20 +0100 Subject: [PATCH 04/13] pandas support! --- narwhals/_dask/namespace.py | 9 ++++++++- narwhals/_pandas_like/dataframe.py | 14 ++++++++++++- narwhals/_pandas_like/namespace.py | 32 +++++++++++++++++++++++++----- narwhals/_pandas_like/series.py | 16 ++++++++++++--- tests/expr_and_series/lit_test.py | 3 ++- 5 files changed, 63 insertions(+), 11 deletions(-) diff --git a/narwhals/_dask/namespace.py b/narwhals/_dask/namespace.py index c4791a7e0d..b75c8d2cde 100644 --- a/narwhals/_dask/namespace.py +++ b/narwhals/_dask/namespace.py @@ -22,7 +22,7 @@ combine_alias_output_names, combine_evaluate_output_names, ) -from narwhals._utils import Implementation, zip_strict +from narwhals._utils import Implementation, qualified_type_name, zip_strict if TYPE_CHECKING: from collections.abc import Iterable, Iterator @@ -55,6 +55,13 @@ def __init__(self, *, version: Version) -> None: self._version = version def lit(self, value: NonNestedLiteral, dtype: IntoDType | None) -> DaskExpr: + if isinstance(value, (list, tuple, dict)): + msg = ( + "Nested structures are not support for Dask backend, " + f" found {qualified_type_name(value)}" + ) + raise NotImplementedError(msg) + def func(df: DaskLazyFrame) -> list[dx.Series]: if dtype is not None: native_dtype = narwhals_to_native_dtype(dtype, self._version) diff --git a/narwhals/_pandas_like/dataframe.py b/narwhals/_pandas_like/dataframe.py index c822ecec04..184f9b1292 100644 --- a/narwhals/_pandas_like/dataframe.py +++ b/narwhals/_pandas_like/dataframe.py @@ -308,7 +308,19 @@ def _extract_comparand(self, other: PandasLikeSeries) -> pd.Series[Any]: index = self.native.index if other._broadcast: s = other.native - return type(s)(s.iloc[0], index=index, dtype=s.dtype, name=s.name) + value = s.iloc[0] + if other.dtype.is_nested(): + import pandas as pd # ignore-banned-import + import pyarrow as pa # ignore-banned-import + + pa_type = s.dtype.pyarrow_dtype + pa_array = pd.arrays.ArrowExtensionArray( # type: ignore[attr-defined] + pa.repeat(pa.scalar(value, pa_type), len(index)) # type: ignore[arg-type] + ) + return type(s)(pa_array, index=index, name=s.name) + + return type(s)(value, index=index, dtype=s.dtype, name=s.name) + if (len_other := len(other)) != (len_idx := len(index)): msg = f"Expected object of length {len_idx}, got: {len_other}." raise ShapeError(msg) diff --git a/narwhals/_pandas_like/namespace.py b/narwhals/_pandas_like/namespace.py index 81296ce483..33815a58ff 100644 --- a/narwhals/_pandas_like/namespace.py +++ b/narwhals/_pandas_like/namespace.py @@ -17,7 +17,7 @@ from narwhals._pandas_like.series import PandasLikeSeries from narwhals._pandas_like.typing import NativeDataFrameT, NativeSeriesT from narwhals._pandas_like.utils import is_non_nullable_boolean -from narwhals._utils import qualified_type_name, zip_strict +from narwhals._utils import zip_strict if TYPE_CHECKING: from collections.abc import Iterable, Sequence @@ -86,11 +86,33 @@ def func(df: PandasLikeDataFrame) -> list[PandasLikeSeries]: def lit(self, value: PythonLiteral, dtype: IntoDType | None) -> PandasLikeExpr: def _lit_pandas_series(df: PandasLikeDataFrame) -> PandasLikeSeries: if isinstance(value, (list, tuple, dict)): - msg = ( - "Nested structures are not support for Pandas-like backend, " - f" found {qualified_type_name(value)}" + try: + import pandas as pd # ignore-banned-import + import pyarrow as pa # ignore-banned-import + except ImportError as exc: + msg = ( + "Nested structures require pyarrow to be installed for pandas backend. " + "Please install pyarrow: pip install pyarrow" + ) + raise ImportError(msg) from exc + + from narwhals._arrow.utils import ( + narwhals_to_native_dtype as _to_arrow_dtype, ) - raise NotImplementedError(msg) + + array_value = list(value) if isinstance(value, tuple) else value + pa_dtype = _to_arrow_dtype(dtype, self._version) if dtype else None + pa_array = pa.array([array_value], type=pa_dtype) # type: ignore[arg-type] + + # Use ArrowExtensionArray to avoid pandas unpacking the nested structure + ns = self._implementation.to_native_namespace() + pandas_series_native = ns.Series( + pd.arrays.ArrowExtensionArray(pa_array), # type: ignore[attr-defined] + name="literal", + index=df._native_frame.index[0:1], + ) + + return self._series.from_native(pandas_series_native, context=self) pandas_like_series = self._series.from_iterable( data=[value], diff --git a/narwhals/_pandas_like/series.py b/narwhals/_pandas_like/series.py index 09cae7120e..092c4eb990 100644 --- a/narwhals/_pandas_like/series.py +++ b/narwhals/_pandas_like/series.py @@ -211,9 +211,19 @@ def _align_full_broadcast(cls, *series: Self) -> Sequence[Self]: reindexed = [] for s in series: if s._broadcast: - native = Series( - s.native.iloc[0], index=idx, name=s.name, dtype=s.native.dtype - ) + value = s.native.iloc[0] + if s.dtype.is_nested(): + import pandas as pd # ignore-banned-import + import pyarrow as pa # ignore-banned-import + + pa_type = s.native.dtype.pyarrow_dtype + pa_array = pd.arrays.ArrowExtensionArray( # type: ignore[attr-defined] + pa.repeat(pa.scalar(value, pa_type), len(idx)) # type: ignore[arg-type] + ) + native = Series(pa_array, index=idx, name=s.name) + else: + native = Series(value, index=idx, name=s.name, dtype=s.native.dtype) + compliant = s._with_native(native) elif s.native.index is not idx: native = set_index(s.native, idx, implementation=s._implementation) diff --git a/tests/expr_and_series/lit_test.py b/tests/expr_and_series/lit_test.py index cbbd92f299..cd768d97f2 100644 --- a/tests/expr_and_series/lit_test.py +++ b/tests/expr_and_series/lit_test.py @@ -172,7 +172,8 @@ def test_nested_structures( reason = "Cannot create an empty struct type for backend" request.applymarker(pytest.mark.xfail(reason=reason, raises=NotImplementedError)) - if any(x in str(constructor) for x in ("cudf", "modin", "pandas")): + # TODO(FBruzzesi): Check cudf + if any(x in str(constructor) for x in ("cudf", "dask")): reason = "Nested structures are not support for backend" request.applymarker(pytest.mark.xfail(reason=reason, raises=NotImplementedError)) From 3697d12cb27c7f73d159269afefc9989b4855274 Mon Sep 17 00:00:00 2001 From: FBruzzesi Date: Wed, 28 Jan 2026 00:12:18 +0100 Subject: [PATCH 05/13] factor out broadcasting, ignore typing, try to skip old pandas cases --- narwhals/_pandas_like/dataframe.py | 19 +++++---------- narwhals/_pandas_like/namespace.py | 2 +- narwhals/_pandas_like/series.py | 17 ++++---------- narwhals/_pandas_like/utils.py | 37 +++++++++++++++++++++++++++++- tests/expr_and_series/lit_test.py | 6 +++++ 5 files changed, 53 insertions(+), 28 deletions(-) diff --git a/narwhals/_pandas_like/dataframe.py b/narwhals/_pandas_like/dataframe.py index 184f9b1292..257f52dcaa 100644 --- a/narwhals/_pandas_like/dataframe.py +++ b/narwhals/_pandas_like/dataframe.py @@ -10,6 +10,7 @@ from narwhals._pandas_like.series import PANDAS_TO_NUMPY_DTYPE_MISSING, PandasLikeSeries from narwhals._pandas_like.utils import ( align_and_extract_native, + broadcast_series_to_index, get_dtype_backend, import_array_module, iter_dtype_backends, @@ -307,19 +308,11 @@ def _with_native(self, df: Any, *, validate_column_names: bool = True) -> Self: def _extract_comparand(self, other: PandasLikeSeries) -> pd.Series[Any]: index = self.native.index if other._broadcast: - s = other.native - value = s.iloc[0] - if other.dtype.is_nested(): - import pandas as pd # ignore-banned-import - import pyarrow as pa # ignore-banned-import - - pa_type = s.dtype.pyarrow_dtype - pa_array = pd.arrays.ArrowExtensionArray( # type: ignore[attr-defined] - pa.repeat(pa.scalar(value, pa_type), len(index)) # type: ignore[arg-type] - ) - return type(s)(pa_array, index=index, name=s.name) - - return type(s)(value, index=index, dtype=s.dtype, name=s.name) + native = other.native + is_nested = other.dtype.is_nested() + return broadcast_series_to_index( + native, index, is_nested=is_nested, series_class=type(native) + ) if (len_other := len(other)) != (len_idx := len(index)): msg = f"Expected object of length {len_idx}, got: {len_other}." diff --git a/narwhals/_pandas_like/namespace.py b/narwhals/_pandas_like/namespace.py index 33815a58ff..2bbc73874d 100644 --- a/narwhals/_pandas_like/namespace.py +++ b/narwhals/_pandas_like/namespace.py @@ -102,7 +102,7 @@ def _lit_pandas_series(df: PandasLikeDataFrame) -> PandasLikeSeries: array_value = list(value) if isinstance(value, tuple) else value pa_dtype = _to_arrow_dtype(dtype, self._version) if dtype else None - pa_array = pa.array([array_value], type=pa_dtype) # type: ignore[arg-type] + pa_array = pa.array([array_value], type=pa_dtype) # type: ignore[arg-type, list-item] # Use ArrowExtensionArray to avoid pandas unpacking the nested structure ns = self._implementation.to_native_namespace() diff --git a/narwhals/_pandas_like/series.py b/narwhals/_pandas_like/series.py index 092c4eb990..187a5c6c42 100644 --- a/narwhals/_pandas_like/series.py +++ b/narwhals/_pandas_like/series.py @@ -14,6 +14,7 @@ from narwhals._pandas_like.series_struct import PandasLikeSeriesStructNamespace from narwhals._pandas_like.utils import ( align_and_extract_native, + broadcast_series_to_index, get_dtype_backend, import_array_module, narwhals_to_native_dtype, @@ -211,19 +212,9 @@ def _align_full_broadcast(cls, *series: Self) -> Sequence[Self]: reindexed = [] for s in series: if s._broadcast: - value = s.native.iloc[0] - if s.dtype.is_nested(): - import pandas as pd # ignore-banned-import - import pyarrow as pa # ignore-banned-import - - pa_type = s.native.dtype.pyarrow_dtype - pa_array = pd.arrays.ArrowExtensionArray( # type: ignore[attr-defined] - pa.repeat(pa.scalar(value, pa_type), len(idx)) # type: ignore[arg-type] - ) - native = Series(pa_array, index=idx, name=s.name) - else: - native = Series(value, index=idx, name=s.name, dtype=s.native.dtype) - + native = broadcast_series_to_index( + s.native, idx, is_nested=s.dtype.is_nested(), series_class=Series + ) compliant = s._with_native(native) elif s.native.index is not idx: native = set_index(s.native, idx, implementation=s._implementation) diff --git a/narwhals/_pandas_like/utils.py b/narwhals/_pandas_like/utils.py index 2de5813b81..0fc0382d79 100644 --- a/narwhals/_pandas_like/utils.py +++ b/narwhals/_pandas_like/utils.py @@ -3,7 +3,7 @@ import functools import operator import re -from typing import TYPE_CHECKING, Any, Callable, Literal, TypeVar, cast +from typing import TYPE_CHECKING, Any, Callable, Literal, TypeVar, cast, no_type_check import pandas as pd @@ -663,3 +663,38 @@ class PandasLikeSeriesNamespace(EagerSeriesNamespace["PandasLikeSeries", Any]): def make_group_by_kwargs(*, drop_null_keys: bool) -> dict[str, bool]: return {"sort": False, "as_index": True, "dropna": drop_null_keys, "observed": True} + + +@no_type_check +def broadcast_series_to_index( + native: NativeSeriesT, + index: Any, + *, + is_nested: bool, + series_class: type[NativeSeriesT], +) -> NativeSeriesT: + """Broadcast a scalar value from a (one element) Series to match a target index. + + For nested (arrow-backed) types, we rely on + [pandas.arrays.ArrowExtensionArray](https://pandas.pydata.org/docs/reference/api/pandas.arrays.ArrowExtensionArray.html). + + Arguments: + native: The native pandas-like Series containing the scalar value to broadcast. + index: The target index to broadcast to. + is_nested: Whether the Series has a nested (arrow-backed) dtype. + series_class: Series class to use for constructing the result. + + Returns: + A new Series with the scalar value broadcast to match the target index. + """ + value = native.iloc[0] + if is_nested: + import pyarrow as pa # ignore-banned-import + + pa_type = native.dtype.pyarrow_dtype + pa_array = pd.arrays.ArrowExtensionArray( + pa.repeat(pa.scalar(value, pa_type), len(index)) + ) + return series_class(pa_array, index=index, name=native.name) + + return series_class(value, index=index, dtype=native.dtype, name=native.name) diff --git a/tests/expr_and_series/lit_test.py b/tests/expr_and_series/lit_test.py index cd768d97f2..9b1f5d8439 100644 --- a/tests/expr_and_series/lit_test.py +++ b/tests/expr_and_series/lit_test.py @@ -177,6 +177,12 @@ def test_nested_structures( reason = "Nested structures are not support for backend" request.applymarker(pytest.mark.xfail(reason=reason, raises=NotImplementedError)) + if any(x in str(constructor) for x in ("pandas", "modin")): + pytest.importorskip("pyarrow") + + if PANDAS_VERSION < (2, 0): + pytest.skip() + size = 3 data = {"a": list(range(size))} expr = nw.lit(value, dtype=dtype).alias("nested") From d66d4e62397d3ea03058dbb54e118f53a2331e1f Mon Sep 17 00:00:00 2001 From: FBruzzesi Date: Wed, 28 Jan 2026 00:23:33 +0100 Subject: [PATCH 06/13] port docstrings --- narwhals/stable/v2/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/narwhals/stable/v2/__init__.py b/narwhals/stable/v2/__init__.py index 80599a2728..76b0e06f86 100644 --- a/narwhals/stable/v2/__init__.py +++ b/narwhals/stable/v2/__init__.py @@ -653,9 +653,11 @@ def lit(value: NonNestedLiteral, dtype: IntoDType | None = None) -> Expr: """Return an expression representing a literal value. Arguments: - value: The value to use as literal. + value: The value to use as literal. Can be a scalar value, list, tuple, or dict. + Lists and tuples are converted to `List` dtype, dicts to `Struct` dtype. dtype: The data type of the literal value. If not provided, the data type will - be inferred by the native library. + be inferred by the native library. For empty lists/dicts, dtype must be + specified explicitly. """ return _stableify(nw.lit(value, dtype)) From e3cee739712545dc7ad04ac67d153065394cb50c Mon Sep 17 00:00:00 2001 From: FBruzzesi Date: Wed, 28 Jan 2026 09:57:21 +0100 Subject: [PATCH 07/13] skip polars pre 1.0 --- tests/expr_and_series/lit_test.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/expr_and_series/lit_test.py b/tests/expr_and_series/lit_test.py index 9b1f5d8439..800dc31db4 100644 --- a/tests/expr_and_series/lit_test.py +++ b/tests/expr_and_series/lit_test.py @@ -10,6 +10,8 @@ CUDF_VERSION, DASK_VERSION, PANDAS_VERSION, + POLARS_VERSION, + PYARROW_VERSION, Constructor, assert_equal_data, ) @@ -177,11 +179,19 @@ def test_nested_structures( reason = "Nested structures are not support for backend" request.applymarker(pytest.mark.xfail(reason=reason, raises=NotImplementedError)) - if any(x in str(constructor) for x in ("pandas", "modin")): - pytest.importorskip("pyarrow") + if any(x in str(constructor) for x in ("pandas", "modin")) and ( + PYARROW_VERSION == (0, 0, 0) or PANDAS_VERSION < (2, 0) + ): + reason = "Requires pyarrow and pandas 2.0+" + pytest.skip(reason=reason) - if PANDAS_VERSION < (2, 0): - pytest.skip() + if ( + "polars" in str(constructor) + and isinstance(value, dict) + and POLARS_VERSION < (1, 0, 0) + ): + reason = "polars<1.0 does not support dict to struct in lit" + pytest.skip(reason=reason) size = 3 data = {"a": list(range(size))} From 9b615c54f3627dca1db1f4b4253a8cdca14cb596 Mon Sep 17 00:00:00 2001 From: FBruzzesi Date: Wed, 28 Jan 2026 11:33:09 +0100 Subject: [PATCH 08/13] pragma no cover, xfail pyspark connect for empty dict --- narwhals/_pandas_like/namespace.py | 2 +- narwhals/_spark_like/namespace.py | 4 ++-- tests/expr_and_series/lit_test.py | 8 ++++--- tests/expr_and_series/replace_strict_test.py | 24 ++++++++++---------- tests/utils.py | 15 +++++++++++- 5 files changed, 34 insertions(+), 19 deletions(-) diff --git a/narwhals/_pandas_like/namespace.py b/narwhals/_pandas_like/namespace.py index 2bbc73874d..604d06bd25 100644 --- a/narwhals/_pandas_like/namespace.py +++ b/narwhals/_pandas_like/namespace.py @@ -89,7 +89,7 @@ def _lit_pandas_series(df: PandasLikeDataFrame) -> PandasLikeSeries: try: import pandas as pd # ignore-banned-import import pyarrow as pa # ignore-banned-import - except ImportError as exc: + except ImportError as exc: # pragma: no cover msg = ( "Nested structures require pyarrow to be installed for pandas backend. " "Please install pyarrow: pip install pyarrow" diff --git a/narwhals/_spark_like/namespace.py b/narwhals/_spark_like/namespace.py index ecbecd1a52..b96b807575 100644 --- a/narwhals/_spark_like/namespace.py +++ b/narwhals/_spark_like/namespace.py @@ -99,8 +99,8 @@ def func(df: SparkLikeLazyFrame) -> list[Column]: lit_values = [F.lit(v) for v in value] column = F.lit(F.array(lit_values)) elif isinstance(value, dict): - if self._implementation.is_sqlframe() and len(value) == 0: - msg = "Cannot create an empty struct type for SQLFrame backend" + if (not self._implementation.is_pyspark()) and (len(value) == 0): + msg = f"Cannot create an empty struct type for {self._implementation} backend" raise NotImplementedError(msg) lit_values = [F.lit(v).alias(k) for k, v in value.items()] column = F.struct(*lit_values) diff --git a/tests/expr_and_series/lit_test.py b/tests/expr_and_series/lit_test.py index 800dc31db4..a1369dfb3e 100644 --- a/tests/expr_and_series/lit_test.py +++ b/tests/expr_and_series/lit_test.py @@ -14,6 +14,7 @@ PYARROW_VERSION, Constructor, assert_equal_data, + is_pyspark_connect, ) if TYPE_CHECKING: @@ -170,7 +171,8 @@ def test_nested_structures( ) -> None: is_empty_dict = isinstance(value, dict) and len(value) == 0 non_pyspark_sql_like = ("duckdb", "sqlframe", "ibis") - if any(x in str(constructor) for x in non_pyspark_sql_like) and is_empty_dict: + is_non_pyspark_sql_like = any(x in str(constructor) for x in non_pyspark_sql_like) + if (is_non_pyspark_sql_like or is_pyspark_connect(constructor)) and is_empty_dict: reason = "Cannot create an empty struct type for backend" request.applymarker(pytest.mark.xfail(reason=reason, raises=NotImplementedError)) @@ -181,7 +183,7 @@ def test_nested_structures( if any(x in str(constructor) for x in ("pandas", "modin")) and ( PYARROW_VERSION == (0, 0, 0) or PANDAS_VERSION < (2, 0) - ): + ): # pragma: no cover reason = "Requires pyarrow and pandas 2.0+" pytest.skip(reason=reason) @@ -189,7 +191,7 @@ def test_nested_structures( "polars" in str(constructor) and isinstance(value, dict) and POLARS_VERSION < (1, 0, 0) - ): + ): # pragma: no cover reason = "polars<1.0 does not support dict to struct in lit" pytest.skip(reason=reason) diff --git a/tests/expr_and_series/replace_strict_test.py b/tests/expr_and_series/replace_strict_test.py index 7d4b5f4bf6..14bce078e1 100644 --- a/tests/expr_and_series/replace_strict_test.py +++ b/tests/expr_and_series/replace_strict_test.py @@ -1,13 +1,18 @@ from __future__ import annotations -import os from typing import TYPE_CHECKING, Any import pytest import narwhals as nw from narwhals.exceptions import InvalidOperationError -from tests.utils import POLARS_VERSION, Constructor, ConstructorEager, assert_equal_data +from tests.utils import ( + POLARS_VERSION, + Constructor, + ConstructorEager, + assert_equal_data, + xfail_if_pyspark_connect, +) if TYPE_CHECKING: from collections.abc import Mapping, Sequence @@ -30,15 +35,6 @@ def xfail_if_no_default(constructor: Constructor, request: pytest.FixtureRequest request.applymarker(pytest.mark.xfail(reason=reason)) -def xfail_if_pyspark_connect( # pragma: no cover - constructor: Constructor, request: pytest.FixtureRequest -) -> None: - is_spark_connect = os.environ.get("SPARK_CONNECT", None) - if is_spark_connect and "pyspark" in str(constructor): - reason = "`mapping_expr[expr]` raises: pyspark.errors.exceptions.base.PySparkTypeError: [UNSUPPORTED_DATA_TYPE] Unsupported DataType `Column`." - request.applymarker(pytest.mark.xfail(reason=reason)) - - @pytest.mark.parametrize( ("old", "new", "return_dtype"), [ @@ -139,7 +135,11 @@ def test_replace_strict_pandas_unnamed_series() -> None: def test_replace_strict_expr_with_default( constructor: Constructor, request: pytest.FixtureRequest, return_dtype: DType | None ) -> None: - xfail_if_pyspark_connect(constructor, request) + spark_connect_reason = ( + "`mapping_expr[expr]` raises: pyspark.errors.exceptions.base.PySparkTypeError: " + "[UNSUPPORTED_DATA_TYPE] Unsupported DataType `Column`." + ) + xfail_if_pyspark_connect(constructor, request, reason=spark_connect_reason) if "polars" in str(constructor) and polars_lt_v1: pytest.skip(reason=pl_skip_reason) diff --git a/tests/utils.py b/tests/utils.py index 693569feb6..47bcd34884 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -8,6 +8,8 @@ from pathlib import Path from typing import TYPE_CHECKING, Any, Callable, cast +import pytest + import narwhals as nw from narwhals._utils import Implementation, parse_version, zip_strict from narwhals.dependencies import get_pandas @@ -17,7 +19,6 @@ from collections.abc import Mapping, Sequence import pandas as pd - import pytest from pyspark.sql import SparkSession from sqlframe.duckdb import DuckDBSession from typing_extensions import TypeAlias @@ -252,3 +253,15 @@ def time_unit_compat(time_unit: TimeUnit, request: pytest.FixtureRequest, /) -> if PANDAS_VERSION < (2,) and any(name in request_id for name in pandas_like): return "ns" return time_unit + + +def is_pyspark_connect(constructor: Constructor) -> bool: + is_spark_connect = bool(os.environ.get("SPARK_CONNECT", None)) + return is_spark_connect and ("pyspark" in str(constructor)) + + +def xfail_if_pyspark_connect( # pragma: no cover + constructor: Constructor, request: pytest.FixtureRequest, reason: str = "" +) -> None: + if is_pyspark_connect(constructor): + request.applymarker(pytest.mark.xfail(reason=reason)) From 8cf634b217e1675ddb6e6fc650fa215569ca879f Mon Sep 17 00:00:00 2001 From: FBruzzesi Date: Fri, 30 Jan 2026 10:19:49 +0100 Subject: [PATCH 09/13] apply Dan's suggestions --- narwhals/_pandas_like/utils.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/narwhals/_pandas_like/utils.py b/narwhals/_pandas_like/utils.py index 0fc0382d79..1b5b60e71b 100644 --- a/narwhals/_pandas_like/utils.py +++ b/narwhals/_pandas_like/utils.py @@ -3,7 +3,7 @@ import functools import operator import re -from typing import TYPE_CHECKING, Any, Callable, Literal, TypeVar, cast, no_type_check +from typing import TYPE_CHECKING, Any, Callable, Literal, TypeVar, cast import pandas as pd @@ -665,18 +665,17 @@ def make_group_by_kwargs(*, drop_null_keys: bool) -> dict[str, bool]: return {"sort": False, "as_index": True, "dropna": drop_null_keys, "observed": True} -@no_type_check def broadcast_series_to_index( - native: NativeSeriesT, + native: pd.Series[Any], index: Any, *, is_nested: bool, - series_class: type[NativeSeriesT], -) -> NativeSeriesT: + series_class: type[pd.Series[Any]], +) -> pd.Series[Any]: """Broadcast a scalar value from a (one element) Series to match a target index. For nested (arrow-backed) types, we rely on - [pandas.arrays.ArrowExtensionArray](https://pandas.pydata.org/docs/reference/api/pandas.arrays.ArrowExtensionArray.html). + [`pandas.array`](https://pandas.pydata.org/docs/reference/api/pandas.array.html). Arguments: native: The native pandas-like Series containing the scalar value to broadcast. @@ -689,12 +688,12 @@ def broadcast_series_to_index( """ value = native.iloc[0] if is_nested: - import pyarrow as pa # ignore-banned-import + from narwhals._arrow.utils import repeat + + # NOTE: Ignore typing because `pandas-stubs` are wrong + # TODO(FBruzzesi): Should we pass the `copy=False` flag? + pa_array = pd.array(repeat(value, len(index)), dtype=native.dtype) # type: ignore[arg-type] - pa_type = native.dtype.pyarrow_dtype - pa_array = pd.arrays.ArrowExtensionArray( - pa.repeat(pa.scalar(value, pa_type), len(index)) - ) return series_class(pa_array, index=index, name=native.name) return series_class(value, index=index, dtype=native.dtype, name=native.name) From 1eb8451979d730e5d8f9cd0f4cd54e771c03963e Mon Sep 17 00:00:00 2001 From: FBruzzesi Date: Fri, 30 Jan 2026 16:19:57 +0100 Subject: [PATCH 10/13] explicitly raise NotImplementedError for multi-nested values --- narwhals/functions.py | 17 ++++++++++++++--- tests/expr_and_series/lit_test.py | 24 +++++++++++++++++++++++- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/narwhals/functions.py b/narwhals/functions.py index 7dc452451c..543303e64e 100644 --- a/narwhals/functions.py +++ b/narwhals/functions.py @@ -1495,9 +1495,20 @@ def lit(value: PythonLiteral, dtype: IntoDType | None = None) -> Expr: ) raise ValueError(msg) - if isinstance(value, (list, tuple, dict)) and len(value) == 0 and dtype is None: - msg = "Cannot infer dtype for empty nested structure. Please provide an explicit dtype parameter." - raise ValueError(msg) + is_list_or_tuple = isinstance(value, (list, tuple)) + is_dict = isinstance(value, dict) + if is_list_or_tuple or is_dict: + if (length := len(value)) == 0 and dtype is None: + msg = "Cannot infer dtype for empty nested structure. Please provide an explicit dtype parameter." + raise ValueError(msg) + + nested_types = (list, tuple, dict) + if length > 0 and ( + (is_list_or_tuple and isinstance(value[0], nested_types)) + or (is_dict and any(isinstance(v, nested_types) for v in value.values())) + ): + msg = "Nested structures with nested values are not supported." + raise NotImplementedError(msg) return Expr(ExprNode(ExprKind.LITERAL, "lit", value=value, dtype=dtype)) diff --git a/tests/expr_and_series/lit_test.py b/tests/expr_and_series/lit_test.py index a1369dfb3e..f0106f76e2 100644 --- a/tests/expr_and_series/lit_test.py +++ b/tests/expr_and_series/lit_test.py @@ -160,7 +160,6 @@ def test_pyarrow_lit_string() -> None: {"field_1": nw.Int32(), "field_2": nw.Float64(), "field_3": nw.Boolean()} ), ), - # TODO(FBruzzesi): deeper nesting? ], ) def test_nested_structures( @@ -216,3 +215,26 @@ def test_raise_empty_nested_structures(value: PythonLiteral) -> None: msg = "Cannot infer dtype for empty nested structure. Please provide an explicit dtype parameter." with pytest.raises(ValueError, match=msg): nw.lit(value=value) + + +@pytest.mark.parametrize( + "value", + [ + # List containing nested structures + [[1, 2], [3, 4]], + [(1, 2), (3, 4)], + [{"a": 1}, {"a": 2}], + # Tuple containing nested structures + ([1, 2], [3, 4]), + ((1, 2), (3, 4)), + ({"a": 1}, {"a": 2}), + # Dict containing nested structures + {"a": [1, 2], "b": [3, 4]}, + {"a": (1, 2), "b": (3, 4)}, + {"a": {"x": 1}, "b": {"y": 2}}, + ], +) +def test_raise_nested_structures_with_nested_values(value: Any) -> None: + msg = "Nested structures with nested values are not supported." + with pytest.raises(NotImplementedError, match=msg): + nw.lit(value=value) From 69abb919c4baabd799fe9b428133de1b32eff735 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 30 Jan 2026 20:18:21 +0000 Subject: [PATCH 11/13] fix(typing): Add `is_nested_literal`, resolve `functions.lit` --- narwhals/_utils.py | 5 +++++ narwhals/functions.py | 25 +++++++++++-------------- narwhals/typing.py | 5 ++--- 3 files changed, 18 insertions(+), 17 deletions(-) diff --git a/narwhals/_utils.py b/narwhals/_utils.py index a02552e90a..bc1795fb59 100644 --- a/narwhals/_utils.py +++ b/narwhals/_utils.py @@ -122,6 +122,7 @@ FileSource, IntoSeriesT, MultiIndexSelector, + NestedLiteral, SingleIndexSelector, SizedMultiBoolSelector, SizedMultiIndexSelector, @@ -1371,6 +1372,10 @@ def is_sequence_of(obj: Any, tp: type[_T]) -> TypeIs[Sequence[_T]]: ) +def is_nested_literal(obj: Any) -> TypeIs[NestedLiteral]: + return isinstance(obj, (list, tuple, dict)) + + def validate_strict_and_pass_though( strict: bool | None, # noqa: FBT001 pass_through: bool | None, # noqa: FBT001 diff --git a/narwhals/functions.py b/narwhals/functions.py index 543303e64e..a19478eb21 100644 --- a/narwhals/functions.py +++ b/narwhals/functions.py @@ -12,6 +12,7 @@ deprecate_native_namespace, flatten, is_eager_allowed, + is_nested_literal, is_sequence_but_not_str, normalize_path, supports_arrow_c_stream, @@ -1494,22 +1495,18 @@ def lit(value: PythonLiteral, dtype: IntoDType | None = None) -> Expr: "Consider using `with_columns` to create a new column from the array." ) raise ValueError(msg) - - is_list_or_tuple = isinstance(value, (list, tuple)) - is_dict = isinstance(value, dict) - if is_list_or_tuple or is_dict: - if (length := len(value)) == 0 and dtype is None: - msg = "Cannot infer dtype for empty nested structure. Please provide an explicit dtype parameter." - raise ValueError(msg) - - nested_types = (list, tuple, dict) - if length > 0 and ( - (is_list_or_tuple and isinstance(value[0], nested_types)) - or (is_dict and any(isinstance(v, nested_types) for v in value.values())) - ): + if is_nested_literal(value): + if not value: + if not dtype: + msg = "Cannot infer dtype for empty nested structure. Please provide an explicit dtype parameter." + raise ValueError(msg) + elif isinstance(value, dict): + if any(is_nested_literal(v) for v in value.values()): + msg = "Nested structures with nested values are not supported." + raise NotImplementedError(msg) + elif is_nested_literal(value[0]): msg = "Nested structures with nested values are not supported." raise NotImplementedError(msg) - return Expr(ExprNode(ExprKind.LITERAL, "lit", value=value, dtype=dtype)) diff --git a/narwhals/typing.py b/narwhals/typing.py index bffe3c7e9f..9454186722 100644 --- a/narwhals/typing.py +++ b/narwhals/typing.py @@ -270,9 +270,8 @@ def Binary(self) -> type[dtypes.Binary]: ... NonNestedLiteral: TypeAlias = ( "NumericLiteral | TemporalLiteral | str | bool | bytes | None" ) -PythonLiteral: TypeAlias = ( - "NonNestedLiteral | list[Any] | tuple[Any, ...] | dict[str, Any]" -) +NestedLiteral: TypeAlias = "list[Any] | tuple[Any, ...] | dict[str, Any]" +PythonLiteral: TypeAlias = "NonNestedLiteral | NestedLiteral" NonNestedDType: TypeAlias = "dtypes.NumericType | dtypes.TemporalType | dtypes.String | dtypes.Boolean | dtypes.Binary | dtypes.Categorical | dtypes.Unknown | dtypes.Object" """Any Narwhals DType that does not have required arguments.""" From dd5c6236be8e7736e621fbe7ddb1a826f2cdece4 Mon Sep 17 00:00:00 2001 From: dangotbanned <125183946+dangotbanned@users.noreply.github.com> Date: Fri, 30 Jan 2026 20:23:53 +0000 Subject: [PATCH 12/13] refactor: Reuse `is_nested_literal` + dask drive-by - Fixed typo - Skip using `qualified_type_name` when we know it would be `builtins` (and get stripped anyway) --- narwhals/_dask/namespace.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/narwhals/_dask/namespace.py b/narwhals/_dask/namespace.py index b75c8d2cde..af3a9bc2f9 100644 --- a/narwhals/_dask/namespace.py +++ b/narwhals/_dask/namespace.py @@ -22,7 +22,7 @@ combine_alias_output_names, combine_evaluate_output_names, ) -from narwhals._utils import Implementation, qualified_type_name, zip_strict +from narwhals._utils import Implementation, is_nested_literal, zip_strict if TYPE_CHECKING: from collections.abc import Iterable, Iterator @@ -55,11 +55,8 @@ def __init__(self, *, version: Version) -> None: self._version = version def lit(self, value: NonNestedLiteral, dtype: IntoDType | None) -> DaskExpr: - if isinstance(value, (list, tuple, dict)): - msg = ( - "Nested structures are not support for Dask backend, " - f" found {qualified_type_name(value)}" - ) + if is_nested_literal(value): + msg = f"Nested structures are not supported for Dask backend, found {type(value).__name__}" raise NotImplementedError(msg) def func(df: DaskLazyFrame) -> list[dx.Series]: From 9f65f3a1bae61b9d0d2ec3c79552ed39257326c1 Mon Sep 17 00:00:00 2001 From: FBruzzesi Date: Fri, 30 Jan 2026 22:39:17 +0100 Subject: [PATCH 13/13] apply Dan's suggestions --- narwhals/_duckdb/namespace.py | 2 +- narwhals/_ibis/namespace.py | 11 +++++------ 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/narwhals/_duckdb/namespace.py b/narwhals/_duckdb/namespace.py index 476820feed..71a94e4a95 100644 --- a/narwhals/_duckdb/namespace.py +++ b/narwhals/_duckdb/namespace.py @@ -132,7 +132,7 @@ def func(cols: Iterable[Expression]) -> Expression: def lit(self, value: PythonLiteral, dtype: IntoDType | None) -> DuckDBExpr: def func(df: DuckDBLazyFrame) -> list[Expression]: - if isinstance(value, dict) and len(value) == 0: + if isinstance(value, dict) and not value: msg = "Cannot create an empty struct type for DuckDB backend" raise NotImplementedError(msg) diff --git a/narwhals/_ibis/namespace.py b/narwhals/_ibis/namespace.py index 092b0a2593..801a04dbd4 100644 --- a/narwhals/_ibis/namespace.py +++ b/narwhals/_ibis/namespace.py @@ -116,14 +116,13 @@ def func(cols: Iterable[ir.Value]) -> ir.Value: def lit(self, value: PythonLiteral, dtype: IntoDType | None) -> IbisExpr: def func(_df: IbisLazyFrame) -> Sequence[ir.Value]: - if isinstance(value, dict) and len(value) == 0: - msg = "Cannot create an empty struct type for Ibis backend" - raise NotImplementedError(msg) - ibis_dtype = narwhals_to_native_dtype(dtype, self._version) if dtype else None - if isinstance(value, dict): + if not isinstance(value, dict): + return [lit(value, ibis_dtype)] + if value: return [ibis.struct(value, type=ibis_dtype)] - return [lit(value, ibis_dtype)] + msg = "Cannot create an empty struct type for Ibis backend" + raise NotImplementedError(msg) return self._expr( func,