Upgrade pyarrow minimum version to 7.0 (#51151)

* Upgrade ci minimum version to 7.0

* Fix

* Remove

* Add cond back in

* Remove super

* Remove if

* Fix merge

Author: phofl

ci/deps/actions-38-minimum_versions.yaml

@@ -43,7 +43,7 @@ dependencies:
   - openpyxl=3.0.7
   - pandas-gbq=0.15.0
   - psycopg2=2.8.6
-  - pyarrow=6.0.0
+  - pyarrow=7.0.0
   - pymysql=1.0.2
   - pyreadstat=1.1.2
   - pytables=3.6.1

doc/source/getting_started/install.rst

@@ -441,7 +441,7 @@ PyTables                  3.6.1              hdf5             HDF5-based reading
 blosc                     1.21.0             hdf5             Compression for HDF5; only available on ``conda``
 zlib                                         hdf5             Compression for HDF5
 fastparquet               0.6.3              -                Parquet reading / writing (pyarrow is default)
-pyarrow                   6.0.0              parquet, feather Parquet, ORC, and feather reading / writing
+pyarrow                   7.0.0              parquet, feather Parquet, ORC, and feather reading / writing
 pyreadstat                1.1.2              spss             SPSS files (.sav) reading
 odfpy                     1.4.1              excel            Open document format (.odf, .ods, .odt) reading / writing
 ========================= ================== ================ =============================================================

doc/source/whatsnew/v2.0.0.rst

@@ -668,7 +668,7 @@ Optional libraries below the lowest tested version may still work, but are not c
 +-----------------+-----------------+---------+
 | Package         | Minimum Version | Changed |
 +=================+=================+=========+
-| pyarrow         | 6.0.0           |    X    |
+| pyarrow         | 7.0.0           |    X    |
 +-----------------+-----------------+---------+
 | matplotlib      | 3.6.1           |    X    |
 +-----------------+-----------------+---------+

pandas/_testing/__init__.py

@@ -29,7 +29,7 @@
     Frequency,
     NpDtype,
 )
-from pandas.compat import pa_version_under6p0
+from pandas.compat import pa_version_under7p0
 
 from pandas.core.dtypes.common import (
     is_float_dtype,
@@ -191,7 +191,7 @@
     ]
 ]
 
-if not pa_version_under6p0:
+if not pa_version_under7p0:
     import pyarrow as pa
 
     UNSIGNED_INT_PYARROW_DTYPES = [pa.uint8(), pa.uint16(), pa.uint32(), pa.uint64()]

pandas/compat/__init__.py

@@ -27,7 +27,6 @@
     np_version_under1p21,
 )
 from pandas.compat.pyarrow import (
-    pa_version_under6p0,
     pa_version_under7p0,
     pa_version_under8p0,
     pa_version_under9p0,
@@ -157,7 +156,6 @@ def get_lzma_file() -> type[pandas.compat.compressors.LZMAFile]:
 __all__ = [
     "is_numpy_dev",
     "np_version_under1p21",
-    "pa_version_under6p0",
     "pa_version_under7p0",
     "pa_version_under8p0",
     "pa_version_under9p0",

pandas/compat/_optional.py

@@ -31,7 +31,7 @@
     "pandas_gbq": "0.15.0",
     "psycopg2": "2.8.6",  # (dt dec pq3 ext lo64)
     "pymysql": "1.0.2",
-    "pyarrow": "6.0.0",
+    "pyarrow": "7.0.0",
     "pyreadstat": "1.1.2",
     "pytest": "7.0.0",
     "pyxlsb": "1.0.8",

pandas/compat/pyarrow.py

@@ -9,13 +9,11 @@
 
     _pa_version = pa.__version__
     _palv = Version(_pa_version)
-    pa_version_under6p0 = _palv < Version("6.0.0")
     pa_version_under7p0 = _palv < Version("7.0.0")
     pa_version_under8p0 = _palv < Version("8.0.0")
     pa_version_under9p0 = _palv < Version("9.0.0")
     pa_version_under10p0 = _palv < Version("10.0.0")
 except ImportError:
-    pa_version_under6p0 = True
     pa_version_under7p0 = True
     pa_version_under8p0 = True
     pa_version_under9p0 = True

pandas/core/arrays/arrow/array.py

@@ -26,7 +26,6 @@
     npt,
 )
 from pandas.compat import (
-    pa_version_under6p0,
     pa_version_under7p0,
     pa_version_under8p0,
     pa_version_under9p0,
@@ -54,7 +53,7 @@
     validate_indices,
 )
 
-if not pa_version_under6p0:
+if not pa_version_under7p0:
     import pyarrow as pa
     import pyarrow.compute as pc
 
@@ -199,8 +198,8 @@ class ArrowExtensionArray(OpsMixin, ExtensionArray):
     _dtype: ArrowDtype
 
     def __init__(self, values: pa.Array | pa.ChunkedArray) -> None:
-        if pa_version_under6p0:
-            msg = "pyarrow>=6.0.0 is required for PyArrow backed ArrowExtensionArray."
+        if pa_version_under7p0:
+            msg = "pyarrow>=7.0.0 is required for PyArrow backed ArrowExtensionArray."
             raise ImportError(msg)
         if isinstance(values, pa.Array):
             self._data = pa.chunked_array([values])
@@ -529,11 +528,6 @@ def _argmin_max(self, skipna: bool, method: str) -> int:
             # let ExtensionArray.arg{max|min} raise
             return getattr(super(), f"arg{method}")(skipna=skipna)
 
-        if pa_version_under6p0:
-            raise NotImplementedError(
-                f"arg{method} only implemented for pyarrow version >= 6.0"
-            )
-
         data = self._data
         if pa.types.is_duration(data.type):
             data = data.cast(pa.int64())
@@ -567,11 +561,7 @@ def dropna(self: ArrowExtensionArrayT) -> ArrowExtensionArrayT:
         -------
         ArrowExtensionArray
         """
-        if pa_version_under6p0:
-            fallback_performancewarning(version="6")
-            return super().dropna()
-        else:
-            return type(self)(pc.drop_null(self._data))
+        return type(self)(pc.drop_null(self._data))
 
     @doc(ExtensionArray.fillna)
     def fillna(
@@ -1293,9 +1283,6 @@ def _mode(self: ArrowExtensionArrayT, dropna: bool = True) -> ArrowExtensionArra
         same type as self
             Sorted, if possible.
         """
-        if pa_version_under6p0:
-            raise NotImplementedError("mode only supported for pyarrow version >= 6.0")
-
         pa_type = self._data.type
         if pa.types.is_temporal(pa_type):
             nbits = pa_type.bit_width

pandas/core/arrays/arrow/dtype.py

@@ -5,15 +5,15 @@
 import numpy as np
 
 from pandas._typing import DtypeObj
-from pandas.compat import pa_version_under6p0
+from pandas.compat import pa_version_under7p0
 from pandas.util._decorators import cache_readonly
 
 from pandas.core.dtypes.base import (
     StorageExtensionDtype,
     register_extension_dtype,
 )
 
-if not pa_version_under6p0:
+if not pa_version_under7p0:
     import pyarrow as pa
 
 
@@ -66,8 +66,8 @@ class ArrowDtype(StorageExtensionDtype):
 
     def __init__(self, pyarrow_dtype: pa.DataType) -> None:
         super().__init__("pyarrow")
-        if pa_version_under6p0:
-            raise ImportError("pyarrow>=6.0.0 is required for ArrowDtype")
+        if pa_version_under7p0:
+            raise ImportError("pyarrow>=7.0.0 is required for ArrowDtype")
         if not isinstance(pyarrow_dtype, pa.DataType):
             raise ValueError(
                 f"pyarrow_dtype ({pyarrow_dtype}) must be an instance "

pandas/core/arrays/string_.py

@@ -21,7 +21,7 @@
     npt,
     type_t,
 )
-from pandas.compat import pa_version_under6p0
+from pandas.compat import pa_version_under7p0
 from pandas.compat.numpy import function as nv
 from pandas.util._decorators import doc
 
@@ -115,9 +115,9 @@ def __init__(self, storage=None) -> None:
             raise ValueError(
                 f"Storage must be 'python' or 'pyarrow'. Got {storage} instead."
             )
-        if storage == "pyarrow" and pa_version_under6p0:
+        if storage == "pyarrow" and pa_version_under7p0:
             raise ImportError(
-                "pyarrow>=6.0.0 is required for PyArrow backed StringArray."
+                "pyarrow>=7.0.0 is required for PyArrow backed StringArray."
             )
         self.storage = storage