From e5dc6ba0e99a8e37c7ff331b3e01672707c2f224 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Tue, 15 Oct 2024 14:46:35 +0800 Subject: [PATCH 1/3] fix --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index a3ce1c6f70eb..8c5aa9f29f05 100644 --- a/pom.xml +++ b/pom.xml @@ -341,7 +341,7 @@ 3.5 spark35 spark-sql-columnar-shims-spark35 - 3.5.2 + 3.5.3 1.5.0 delta-spark 3.2.0 From 38a1b762ae734f7c921ee03854f0dc1db4247c89 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Wed, 16 Oct 2024 10:09:44 +0800 Subject: [PATCH 2/3] do rebase --- .../workflows/util/install_spark_resources.sh | 20 +++++++++---------- .github/workflows/velox_backend.yml | 10 +++++----- .../sql/shims/spark35/SparkShimProvider.scala | 2 +- tools/gluten-it/pom.xml | 2 +- 4 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/util/install_spark_resources.sh b/.github/workflows/util/install_spark_resources.sh index dd2afec821d4..d5f413ee2116 100755 --- a/.github/workflows/util/install_spark_resources.sh +++ b/.github/workflows/util/install_spark_resources.sh @@ -63,26 +63,26 @@ case "$1" in 3.5) # Spark-3.5 cd ${INSTALL_DIR} && \ - wget -nv https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz && \ - tar --strip-components=1 -xf spark-3.5.2-bin-hadoop3.tgz spark-3.5.2-bin-hadoop3/jars/ && \ - rm -rf spark-3.5.2-bin-hadoop3.tgz && \ + wget -nv https://archive.apache.org/dist/spark/spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz && \ + tar --strip-components=1 -xf spark-3.5.3-bin-hadoop3.tgz spark-3.5.3-bin-hadoop3/jars/ && \ + rm -rf spark-3.5.3-bin-hadoop3.tgz && \ mkdir -p ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.12 && \ mv jars ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.12 && \ - wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.2.tar.gz && \ - tar --strip-components=1 -xf v3.5.2.tar.gz spark-3.5.2/sql/core/src/test/resources/ && \ + wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.3.tar.gz && \ + tar --strip-components=1 -xf v3.5.3.tar.gz spark-3.5.3/sql/core/src/test/resources/ && \ mkdir -p shims/spark35/spark_home/ && \ mv sql shims/spark35/spark_home/ ;; 3.5-scala2.13) # Spark-3.5, scala 2.13 cd ${INSTALL_DIR} && \ - wget -nv https://archive.apache.org/dist/spark/spark-3.5.2/spark-3.5.2-bin-hadoop3.tgz && \ - tar --strip-components=1 -xf spark-3.5.2-bin-hadoop3.tgz spark-3.5.2-bin-hadoop3/jars/ && \ - rm -rf spark-3.5.2-bin-hadoop3.tgz && \ + wget -nv https://archive.apache.org/dist/spark/spark-3.5.3/spark-3.5.3-bin-hadoop3.tgz && \ + tar --strip-components=1 -xf spark-3.5.3-bin-hadoop3.tgz spark-3.5.3-bin-hadoop3/jars/ && \ + rm -rf spark-3.5.3-bin-hadoop3.tgz && \ mkdir -p ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.13 && \ mv jars ${INSTALL_DIR}/shims/spark35/spark_home/assembly/target/scala-2.13 && \ - wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.2.tar.gz && \ - tar --strip-components=1 -xf v3.5.2.tar.gz spark-3.5.2/sql/core/src/test/resources/ && \ + wget -nv https://github.com/apache/spark/archive/refs/tags/v3.5.3.tar.gz && \ + tar --strip-components=1 -xf v3.5.3.tar.gz spark-3.5.3/sql/core/src/test/resources/ && \ mkdir -p shims/spark35/spark_home/ && \ mv sql shims/spark35/spark_home/ ;; diff --git a/.github/workflows/velox_backend.yml b/.github/workflows/velox_backend.yml index 8637c92ffe7c..740748af1875 100644 --- a/.github/workflows/velox_backend.yml +++ b/.github/workflows/velox_backend.yml @@ -860,9 +860,9 @@ jobs: dnf module -y install python39 && \ alternatives --set python3 /usr/bin/python3.9 && \ pip3 install setuptools && \ - pip3 install pyspark==3.5.2 cython && \ + pip3 install pyspark==3.5.3 cython && \ pip3 install pandas pyarrow - - name: Build and Run unit test for Spark 3.5.2 (other tests) + - name: Build and Run unit test for Spark 3.5.3 (other tests) run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.12 @@ -906,9 +906,9 @@ jobs: dnf module -y install python39 && \ alternatives --set python3 /usr/bin/python3.9 && \ pip3 install setuptools && \ - pip3 install pyspark==3.5.2 cython && \ + pip3 install pyspark==3.5.3 cython && \ pip3 install pandas pyarrow - - name: Build and Run unit test for Spark 3.5.2 with scala-2.13 (other tests) + - name: Build and Run unit test for Spark 3.5.3 with scala-2.13 (other tests) run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.13 @@ -949,7 +949,7 @@ jobs: - name: Prepare spark.test.home for Spark 3.5.2 (other tests) run: | bash .github/workflows/util/install_spark_resources.sh 3.5 - - name: Build and Run unit test for Spark 3.5.2 (slow tests) + - name: Build and Run unit test for Spark 3.5.3 (slow tests) run: | cd $GITHUB_WORKSPACE/ $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Phudi -Pspark-ut \ diff --git a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/SparkShimProvider.scala b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/SparkShimProvider.scala index eab32ab9d0b9..ad99f7be2547 100644 --- a/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/SparkShimProvider.scala +++ b/shims/spark35/src/main/scala/org/apache/gluten/sql/shims/spark35/SparkShimProvider.scala @@ -20,7 +20,7 @@ import org.apache.gluten.sql.shims.{SparkShimDescriptor, SparkShims} import org.apache.gluten.sql.shims.spark35.SparkShimProvider.DESCRIPTOR object SparkShimProvider { - val DESCRIPTOR = SparkShimDescriptor(3, 5, 2) + val DESCRIPTOR = SparkShimDescriptor(3, 5, 3) } class SparkShimProvider extends org.apache.gluten.sql.shims.SparkShimProvider { diff --git a/tools/gluten-it/pom.xml b/tools/gluten-it/pom.xml index bad4d6087f11..70b8d20c5ba4 100644 --- a/tools/gluten-it/pom.xml +++ b/tools/gluten-it/pom.xml @@ -164,7 +164,7 @@ spark-3.5 - 3.5.2 + 3.5.3 2.12.18 From 3eb4411b2292824ef0ce15b7bb81987c52c88882 Mon Sep 17 00:00:00 2001 From: Yang Zhang Date: Thu, 17 Oct 2024 12:34:48 +0800 Subject: [PATCH 3/3] disable hudi suite and bump delta version --- .github/workflows/velox_backend.yml | 4 ++-- pom.xml | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/velox_backend.yml b/.github/workflows/velox_backend.yml index 740748af1875..dd31b8e77bb2 100644 --- a/.github/workflows/velox_backend.yml +++ b/.github/workflows/velox_backend.yml @@ -866,7 +866,7 @@ jobs: run: | cd $GITHUB_WORKSPACE/ export SPARK_SCALA_VERSION=2.12 - $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Phudi -Pspark-ut \ + $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut \ -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" \ -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.SkipTestTags - name: Upload test report @@ -952,7 +952,7 @@ jobs: - name: Build and Run unit test for Spark 3.5.3 (slow tests) run: | cd $GITHUB_WORKSPACE/ - $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Phudi -Pspark-ut \ + $MVN_CMD clean test -Pspark-3.5 -Pbackends-velox -Pceleborn -Piceberg -Pdelta -Pspark-ut \ -DargLine="-Dspark.test.home=$GITHUB_WORKSPACE//shims/spark35/spark_home/" \ -DtagsToInclude=org.apache.spark.tags.ExtendedSQLTest - name: Upload test report diff --git a/pom.xml b/pom.xml index 8c5aa9f29f05..ea8a9f980854 100644 --- a/pom.xml +++ b/pom.xml @@ -344,9 +344,9 @@ 3.5.3 1.5.0 delta-spark - 3.2.0 - 32 - 0.15.0 + 3.2.1 + 32 + 0.15.0 2.15.1 3.3.4 4.9.3