BUG: interchange protocol with nullable datatypes a non-null validity provides nonsense results

Author: MarcoGorelli

pandas/core/interchange/column.py

@@ -190,6 +190,10 @@ def describe_categorical(self):
 
     @property
     def describe_null(self):
+        if isinstance(self._col.dtype, BaseMaskedDtype):
+            column_null_dtype = ColumnNullType.USE_BYTEMASK
+            null_value = 1
+            return column_null_dtype, null_value
         kind = self.dtype[0]
         try:
             null, value = _NULL_DESCRIPTION[kind]
@@ -298,7 +302,13 @@ def _get_data_buffer(
             DtypeKind.FLOAT,
             DtypeKind.BOOL,
         ):
-            np_arr = self._col.to_numpy()
+            arr = self._col.array
+            if isinstance(self._col.dtype, BaseMaskedDtype):
+                np_arr = arr._data
+            elif isinstance(self._col.dtype, ArrowDtype):
+                raise NotImplementedError("ArrowDtype not handled yet")
+            else:
+                np_arr = arr._ndarray
             buffer = PandasBuffer(np_arr, allow_copy=self._allow_copy)
             dtype = self.dtype
         elif self.dtype[0] == DtypeKind.CATEGORICAL:
@@ -341,6 +351,20 @@ def _get_validity_buffer(self) -> tuple[PandasBuffer, Any]:
         """
         null, invalid = self.describe_null
 
+        if isinstance(self._col.dtype, BaseMaskedDtype):
+            buf = self._col.array._data
+
+            mask = self._col.array._mask
+
+            # Convert the mask array to a Pandas "buffer" using
+            # a NumPy array as the backing store
+            buffer = PandasBuffer(mask)
+
+            # Define the dtype of the returned buffer
+            dtype = (DtypeKind.BOOL, 8, ArrowCTypes.BOOL, Endianness.NATIVE)
+
+            return buffer, dtype
+
         if self.dtype[0] == DtypeKind.STRING:
             # For now, use byte array as the mask.
             # TODO: maybe store as bit array to save space?..

pandas/tests/interchange/test_impl.py

@@ -8,7 +8,6 @@
     is_ci_environment,
     is_platform_windows,
 )
-import pandas.util._test_decorators as td
 
 import pandas as pd
 import pandas._testing as tm
@@ -417,17 +416,35 @@ def test_non_str_names_w_duplicates():
         pd.api.interchange.from_dataframe(dfi, allow_copy=False)
 
 
-@pytest.mark.parametrize(
-    "dtype", ["Int8", pytest.param("Int8[pyarrow]", marks=td.skip_if_no("pyarrow"))]
-)
-def test_nullable_integers(dtype: str) -> None:
+def test_nullable_integers() -> None:
+    # https://github.com/pandas-dev/pandas/issues/55069
+    df = pd.DataFrame({"a": [1]}, dtype="Int8")
+    expected = pd.DataFrame({"a": [1]}, dtype="int8")
+    result = pd.api.interchange.from_dataframe(df.__dataframe__())
+    tm.assert_frame_equal(result, expected)
+
+
+@pytest.mark.xfail(reason="https://github.com/pandas-dev/pandas/issues/57664")
+def test_nullable_integers_pyarrow() -> None:
     # https://github.com/pandas-dev/pandas/issues/55069
-    df = pd.DataFrame({"a": [1]}, dtype=dtype)
+    df = pd.DataFrame({"a": [1]}, dtype="Int8[pyarrow]")
     expected = pd.DataFrame({"a": [1]}, dtype="int8")
     result = pd.api.interchange.from_dataframe(df.__dataframe__())
     tm.assert_frame_equal(result, expected)
 
 
+def test_nullable_integers_w_missing_values() -> None:
+    # https://github.com/pandas-dev/pandas/issues/57643
+    pytest.importorskip("pyarrow", "11.0.0")
+    import pyarrow.interchange as pai
+
+    df = pd.DataFrame({"a": [1, 2, None]}, dtype="Int64")
+    result = pai.from_dataframe(df.__dataframe__())["a"]
+    assert result[0].as_py() == 1
+    assert result[1].as_py() == 2
+    assert result[2].as_py() is None
+
+
 def test_empty_dataframe():
     # https://github.com/pandas-dev/pandas/issues/56700
     df = pd.DataFrame({"a": []}, dtype="int8")