ENH: Add pre-commit check for setup.cfg options.extras_require (#49261)

* Add dependency check for setup.cfg

* improve diff alert, add install mapping, remove test deps

* align versions across min version files

* fix version issues for boto3, s3fs and pyqt5. Small change to script

* fix version equality, modify script for CI check

* fix version typo in min version yaml and update min version script

* bump fsspec, gcsfs to match in whats new

* Revert "bump fsspec, gcsfs to match in whats new"

This reverts commit 52efecb.

* align fastparquet, lowercase PyQt5

* ENH: align matplotlib versions

* ENH: align matplotlib in setup.cfg

* bump s3fs across min version files

* Update scripts/validate_min_versions_in_sync.py

Co-authored-by: Marco Edward Gorelli <[email protected]>

Co-authored-by: Marco Edward Gorelli <[email protected]>

Author: kostyafarber

ci/deps/actions-38-minimum_versions.yaml

@@ -36,6 +36,7 @@ dependencies:
   - numba=0.53.1
   - numexpr=2.7.3
   - odfpy=1.4.1
+  - qtpy=2.2.0
   - openpyxl=3.0.7
   - pandas-gbq=0.15.0
   - psycopg2=2.8.6
@@ -54,3 +55,6 @@ dependencies:
   - xlrd=2.0.1
   - xlsxwriter=1.4.3
   - zstandard=0.15.2
+
+  - pip:
+    - pyqt5==5.15.1

pandas/compat/_optional.py

@@ -46,6 +46,8 @@
     "xlsxwriter": "1.4.3",
     "zstandard": "0.15.2",
     "tzdata": "2022.1",
+    "qtpy": "2.2.0",
+    "pyqt5": "5.15.1",
 }
 
 # A mapping from import name to package name (on PyPI) for packages where

scripts/validate_min_versions_in_sync.py

@@ -4,6 +4,7 @@
 
 ci/deps/actions-.*-minimum_versions.yaml
 pandas/compat/_optional.py
+setup.cfg
 
 TODO: doc/source/getting_started/install.rst
 
@@ -13,6 +14,7 @@
 """
 from __future__ import annotations
 
+import configparser
 import pathlib
 import sys
 
@@ -21,6 +23,7 @@
     pathlib.Path("ci/deps").absolute().glob("actions-*-minimum_versions.yaml")
 )
 CODE_PATH = pathlib.Path("pandas/compat/_optional.py").resolve()
+SETUP_PATH = pathlib.Path("setup.cfg").resolve()
 EXCLUDE_DEPS = {"tzdata"}
 # pandas package is not available
 # in pre-commit environment
@@ -57,29 +60,76 @@ def get_versions_from_ci(content: list[str]) -> tuple[dict[str, str], dict[str,
             seen_required = True
         elif "# optional dependencies" in line:
             seen_optional = True
+        elif "- pip:" in line:
+            continue
         elif seen_required and line.strip():
-            package, version = line.strip().split("=")
+            if "==" in line:
+                package, version = line.strip().split("==")
+
+            else:
+                package, version = line.strip().split("=")
             package = package[2:]
             if package in EXCLUDE_DEPS:
                 continue
             if not seen_optional:
-                required_deps[package] = version
+                required_deps[package.casefold()] = version
             else:
-                optional_deps[package] = version
+                optional_deps[package.casefold()] = version
     return required_deps, optional_deps
 
 
+def get_versions_from_setup() -> dict[str, str]:
+    install_map = _optional.INSTALL_MAPPING
+    optional_dependencies = {}
+
+    parser = configparser.ConfigParser()
+    parser.read(SETUP_PATH)
+    setup_optional = parser["options.extras_require"]["all"]
+    dependencies = setup_optional[1:].split("\n")
+
+    # remove test dependencies
+    test = parser["options.extras_require"]["test"]
+    test_dependencies = set(test[1:].split("\n"))
+    dependencies = [
+        package for package in dependencies if package not in test_dependencies
+    ]
+
+    for dependency in dependencies:
+        package, version = dependency.strip().split(">=")
+        optional_dependencies[install_map.get(package, package).casefold()] = version
+
+    for item in EXCLUDE_DEPS:
+        optional_dependencies.pop(item)
+
+    return optional_dependencies
+
+
 def main():
     with open(CI_PATH, encoding="utf-8") as f:
         _, ci_optional = get_versions_from_ci(f.readlines())
     code_optional = get_versions_from_code()
-    diff = set(ci_optional.items()).symmetric_difference(code_optional.items())
+    setup_optional = get_versions_from_setup()
+
+    diff = (ci_optional.items() | code_optional.items() | setup_optional.items()) - (
+        ci_optional.items() & code_optional.items() & setup_optional.items()
+    )
+
     if diff:
-        sys.stdout.write(
+        packages = {package for package, _ in diff}
+        out = sys.stdout
+        out.write(
             f"The follow minimum version differences were found between  "
-            f"{CI_PATH} and {CODE_PATH}. Please ensure these are aligned: "
-            f"{diff}\n"
+            f"{CI_PATH}, {CODE_PATH} AND {SETUP_PATH}. "
+            f"Please ensure these are aligned: \n\n"
         )
+
+        for package in packages:
+            out.write(
+                f"{package}\n"
+                f"{CI_PATH}: {ci_optional.get(package, 'Not specified')}\n"
+                f"{CODE_PATH}: {code_optional.get(package, 'Not specified')}\n"
+                f"{SETUP_PATH}: {setup_optional.get(package, 'Not specified')}\n\n"
+            )
         sys.exit(1)
     sys.exit(0)
 

setup.cfg

@@ -60,20 +60,19 @@ test =
 # see: doc/source/getting_started/install.rst
 performance =
     bottleneck>=1.3.2
-    numba>=0.53.0
+    numba>=0.53.1
     numexpr>=2.7.1
 timezone =
     tzdata>=2022.1
 computation =
     scipy>=1.7.1
     xarray>=0.19.0
 fss =
-    fsspec>=2021.7.0
+    fsspec>=2021.07.0
 aws =
-    boto3>=1.22.7
-    s3fs>=0.4.0
+    s3fs>=2021.08.0
 gcp =
-    gcsfs>=2021.05.0
+    gcsfs>=2021.07.0
     pandas-gbq>=0.15.0
 excel =
     odfpy>=1.4.1
@@ -105,7 +104,7 @@ html =
 xml =
     lxml>=4.6.3
 plot =
-    matplotlib>=3.3.2
+    matplotlib>=3.6.1
 output_formatting =
     jinja2>=3.0.0
     tabulate>=0.8.9
@@ -123,19 +122,18 @@ compression =
 all =
     beautifulsoup4>=4.9.3
     blosc>=1.21.0
-    bottleneck>=1.3.1
-    boto3>=1.22.7
+    bottleneck>=1.3.2
     brotlipy>=0.7.0
-    fastparquet>=0.4.0
-    fsspec>=2021.7.0
-    gcsfs>=2021.05.0
+    fastparquet>=0.6.3
+    fsspec>=2021.07.0
+    gcsfs>=2021.07.0
     html5lib>=1.1
-    hypothesis>=5.5.3
+    hypothesis>=6.13.0
     jinja2>=3.0.0
     lxml>=4.6.3
-    matplotlib>=3.3.2
-    numba>=0.53.0
-    numexpr>=2.7.1
+    matplotlib>=3.6.1
+    numba>=0.53.1
+    numexpr>=2.7.3
     odfpy>=1.4.1
     openpyxl>=3.0.7
     pandas-gbq>=0.15.0
@@ -151,7 +149,7 @@ all =
     pyxlsb>=1.0.8
     qtpy>=2.2.0
     scipy>=1.7.1
-    s3fs>=0.4.0
+    s3fs>=2021.08.0
     SQLAlchemy>=1.4.16
     tables>=3.6.1
     tabulate>=0.8.9