From b3e7809951ab4cafb5cf641250f895b953c91160 Mon Sep 17 00:00:00 2001 From: panbingkun Date: Thu, 24 Nov 2022 15:29:26 +0800 Subject: [PATCH 1/2] [SPARK-41251][PS][INFRA] Upgrade pandas from 1.5.1 to 1.5.2 --- dev/infra/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dev/infra/Dockerfile b/dev/infra/Dockerfile index 26c4e13035e2..1c326d437c79 100644 --- a/dev/infra/Dockerfile +++ b/dev/infra/Dockerfile @@ -64,8 +64,8 @@ RUN Rscript -e "devtools::install_version('roxygen2', version='7.2.0', repos='ht # See more in SPARK-39735 ENV R_LIBS_SITE "/usr/local/lib/R/site-library:${R_LIBS_SITE}:/usr/lib/R/library" -RUN pypy3 -m pip install numpy 'pandas<=1.5.1' scipy coverage matplotlib -RUN python3.9 -m pip install numpy pyarrow 'pandas<=1.5.1' scipy unittest-xml-reporting plotly>=4.8 sklearn 'mlflow>=1.0' coverage matplotlib openpyxl 'memory-profiler==0.60.0' +RUN pypy3 -m pip install numpy 'pandas<=1.5.2' scipy coverage matplotlib +RUN python3.9 -m pip install numpy pyarrow 'pandas<=1.5.2' scipy unittest-xml-reporting plotly>=4.8 sklearn 'mlflow>=1.0' coverage matplotlib openpyxl 'memory-profiler==0.60.0' # Add Python deps for Spark Connect. RUN python3.9 -m pip install grpcio protobuf From ab214a60b36577cd0c6566d3fe1f9d1627b4d01d Mon Sep 17 00:00:00 2001 From: panbingkun Date: Thu, 24 Nov 2022 15:33:23 +0800 Subject: [PATCH 2/2] [SPARK-41251][PS][INFRA] Upgrade pandas from 1.5.1 to 1.5.2 --- python/pyspark/pandas/supported_api_gen.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/pyspark/pandas/supported_api_gen.py b/python/pyspark/pandas/supported_api_gen.py index 9b25f6145359..e60cba204f93 100644 --- a/python/pyspark/pandas/supported_api_gen.py +++ b/python/pyspark/pandas/supported_api_gen.py @@ -98,7 +98,7 @@ def generate_supported_api(output_rst_file_path: str) -> None: Write supported APIs documentation. """ - pandas_latest_version = "1.5.1" + pandas_latest_version = "1.5.2" if LooseVersion(pd.__version__) != LooseVersion(pandas_latest_version): msg = ( "Warning: Latest version of pandas (%s) is required to generate the documentation; "