From af2d6b7f7f82267d64ec4c46d05823b0fb14effd Mon Sep 17 00:00:00 2001
From: Raymond Xu <2701446+xushiyan@users.noreply.github.com>
Date: Thu, 27 Jan 2022 23:01:39 -0800
Subject: [PATCH] [HUDI-3088] Use Spark 3.2 as default Spark version

---
 azure-pipelines.yml                           |  2 +-
 hudi-spark-datasource/hudi-spark/pom.xml      |  2 +-
 .../hudi-spark2-common/pom.xml                | 11 ++-
 .../hudi-spark3-common/pom.xml                | 12 ++-
 hudi-spark-datasource/hudi-spark3.1.x/pom.xml |  8 +-
 hudi-spark-datasource/hudi-spark3/pom.xml     |  8 +-
 hudi-utilities/pom.xml                        |  2 +-
 packaging/hudi-integ-test-bundle/pom.xml      |  4 +-
 packaging/hudi-spark-bundle/pom.xml           |  4 +-
 packaging/hudi-utilities-bundle/pom.xml       |  4 +-
 pom.xml                                       | 97 ++++++++++---------
 11 files changed, 80 insertions(+), 74 deletions(-)

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index cd75e28dae24e..2d6d766051942 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -24,7 +24,7 @@ pool:
 variables:
   MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository
   MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) -Dcheckstyle.skip=true -Drat.skip=true -Djacoco.skip=true'
-  SPARK_VERSION: '2.4.4'
+  SPARK_VERSION: '3.2.0'
   HADOOP_VERSION: '2.7'
   SPARK_ARCHIVE: spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)
 
diff --git a/hudi-spark-datasource/hudi-spark/pom.xml b/hudi-spark-datasource/hudi-spark/pom.xml
index 606f6fa894d72..e5026ebac3e48 100644
--- a/hudi-spark-datasource/hudi-spark/pom.xml
+++ b/hudi-spark-datasource/hudi-spark/pom.xml
@@ -232,7 +232,7 @@
 
     <dependency>
       <groupId>org.apache.hudi</groupId>
-      <artifactId>${hudi.spark.module}_${scala.binary.version}</artifactId>
+      <artifactId>${hudi.spark.module}</artifactId>
       <version>${project.version}</version>
       <exclusions>
         <exclusion>
diff --git a/hudi-spark-datasource/hudi-spark2-common/pom.xml b/hudi-spark-datasource/hudi-spark2-common/pom.xml
index 403c2fe1e9db2..6af76c0b4d628 100644
--- a/hudi-spark-datasource/hudi-spark2-common/pom.xml
+++ b/hudi-spark-datasource/hudi-spark2-common/pom.xml
@@ -9,11 +9,14 @@
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
-    <artifactId>hudi-spark2-common</artifactId>
+    <artifactId>hudi-spark2-common_${scala.binary.version}</artifactId>
+    <version>0.11.0-SNAPSHOT</version>
+
+    <name>hudi-spark2-common_${scala.binary.version}</name>
+    <packaging>jar</packaging>
 
     <properties>
-        <maven.compiler.source>8</maven.compiler.source>
-        <maven.compiler.target>8</maven.compiler.target>
+        <main.basedir>${project.parent.parent.basedir}</main.basedir>
     </properties>
 
-</project>
\ No newline at end of file
+</project>
diff --git a/hudi-spark-datasource/hudi-spark3-common/pom.xml b/hudi-spark-datasource/hudi-spark3-common/pom.xml
index affa987372963..d44687715f437 100644
--- a/hudi-spark-datasource/hudi-spark3-common/pom.xml
+++ b/hudi-spark-datasource/hudi-spark3-common/pom.xml
@@ -25,12 +25,14 @@
     </parent>
     <modelVersion>4.0.0</modelVersion>
 
-    <artifactId>hudi-spark3-common</artifactId>
+    <artifactId>hudi-spark3-common_${spark3.scala.binary.version}</artifactId>
+    <version>0.11.0-SNAPSHOT</version>
+
+    <name>hudi-spark3-common_${spark3.scala.binary.version}</name>
+    <packaging>jar</packaging>
 
     <properties>
         <main.basedir>${project.parent.parent.basedir}</main.basedir>
-        <maven.compiler.source>8</maven.compiler.source>
-        <maven.compiler.target>8</maven.compiler.target>
     </properties>
 
     <build>
@@ -166,7 +168,7 @@
 
         <dependency>
             <groupId>org.apache.spark</groupId>
-            <artifactId>spark-sql_2.12</artifactId>
+            <artifactId>spark-sql_${spark3.scala.binary.version}</artifactId>
             <version>${spark3.version}</version>
             <optional>true</optional>
         </dependency>
@@ -244,4 +246,4 @@
         </dependency>
     </dependencies>
 
-</project>
\ No newline at end of file
+</project>
diff --git a/hudi-spark-datasource/hudi-spark3.1.x/pom.xml b/hudi-spark-datasource/hudi-spark3.1.x/pom.xml
index f6d9f7d557216..091434eb3eac5 100644
--- a/hudi-spark-datasource/hudi-spark3.1.x/pom.xml
+++ b/hudi-spark-datasource/hudi-spark3.1.x/pom.xml
@@ -21,10 +21,10 @@
   </parent>
   <modelVersion>4.0.0</modelVersion>
 
-  <artifactId>hudi-spark3.1.x_2.12</artifactId>
+  <artifactId>hudi-spark3.1.x_${spark3.scala.binary.version}</artifactId>
   <version>0.11.0-SNAPSHOT</version>
 
-  <name>hudi-spark3.1.x_2.12</name>
+  <name>hudi-spark3.1.x_${spark3.scala.binary.version}</name>
   <packaging>jar</packaging>
 
   <properties>
@@ -156,7 +156,7 @@
 
     <dependency>
       <groupId>org.apache.spark</groupId>
-      <artifactId>spark-sql_2.12</artifactId>
+      <artifactId>spark-sql_${spark3.scala.binary.version}</artifactId>
       <version>${spark3.version}</version>
       <optional>true</optional>
     </dependency>
@@ -189,7 +189,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hudi</groupId>
-      <artifactId>hudi-spark3-common</artifactId>
+      <artifactId>hudi-spark3-common_${spark3.scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
 
diff --git a/hudi-spark-datasource/hudi-spark3/pom.xml b/hudi-spark-datasource/hudi-spark3/pom.xml
index d8dba8384886c..5cbd88d858a47 100644
--- a/hudi-spark-datasource/hudi-spark3/pom.xml
+++ b/hudi-spark-datasource/hudi-spark3/pom.xml
@@ -21,10 +21,10 @@
   </parent>
   <modelVersion>4.0.0</modelVersion>
 
-  <artifactId>hudi-spark3_2.12</artifactId>
+  <artifactId>hudi-spark3_${spark3.scala.binary.version}</artifactId>
   <version>0.11.0-SNAPSHOT</version>
 
-  <name>hudi-spark3_2.12</name>
+  <name>hudi-spark3_${spark3.scala.binary.version}</name>
   <packaging>jar</packaging>
 
   <properties>
@@ -156,7 +156,7 @@
 
     <dependency>
       <groupId>org.apache.spark</groupId>
-      <artifactId>spark-sql_2.12</artifactId>
+      <artifactId>spark-sql_${spark3.scala.binary.version}</artifactId>
       <version>${spark3.version}</version>
       <optional>true</optional>
     </dependency>
@@ -189,7 +189,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hudi</groupId>
-      <artifactId>hudi-spark3-common</artifactId>
+      <artifactId>hudi-spark3-common_${spark3.scala.binary.version}</artifactId>
       <version>${project.version}</version>
     </dependency>
 
diff --git a/hudi-utilities/pom.xml b/hudi-utilities/pom.xml
index 39510537ba2fe..05793bc8df78e 100644
--- a/hudi-utilities/pom.xml
+++ b/hudi-utilities/pom.xml
@@ -137,7 +137,7 @@
 
     <dependency>
       <groupId>org.apache.hudi</groupId>
-      <artifactId>${hudi.spark.module}_${scala.binary.version}</artifactId>
+      <artifactId>${hudi.spark.module}</artifactId>
       <version>${project.version}</version>
     </dependency>
 
diff --git a/packaging/hudi-integ-test-bundle/pom.xml b/packaging/hudi-integ-test-bundle/pom.xml
index b53e02aaf7768..e0a478b2c7b44 100644
--- a/packaging/hudi-integ-test-bundle/pom.xml
+++ b/packaging/hudi-integ-test-bundle/pom.xml
@@ -76,7 +76,7 @@
                   <include>org.apache.hudi:hudi-spark-common_${scala.binary.version}</include>
                   <include>org.apache.hudi:hudi-utilities_${scala.binary.version}</include>
                   <include>org.apache.hudi:hudi-spark_${scala.binary.version}</include>
-                  <include>org.apache.hudi:${hudi.spark.module}_${scala.binary.version}</include>
+                  <include>org.apache.hudi:${hudi.spark.module}</include>
                   <include>org.apache.hudi:${hudi.spark.common.module}</include>
                   <include>org.apache.hudi:hudi-hive-sync</include>
                   <include>org.apache.hudi:hudi-sync-common</include>
@@ -350,7 +350,7 @@
 
     <dependency>
       <groupId>org.apache.hudi</groupId>
-      <artifactId>${hudi.spark.module}_${scala.binary.version}</artifactId>
+      <artifactId>${hudi.spark.module}</artifactId>
       <version>${project.version}</version>
     </dependency>
 
diff --git a/packaging/hudi-spark-bundle/pom.xml b/packaging/hudi-spark-bundle/pom.xml
index a877d10a586a8..1ec383613e7f3 100644
--- a/packaging/hudi-spark-bundle/pom.xml
+++ b/packaging/hudi-spark-bundle/pom.xml
@@ -71,7 +71,7 @@
                   <include>org.apache.hudi:hudi-spark-client</include>
                   <include>org.apache.hudi:hudi-spark-common_${scala.binary.version}</include>
                   <include>org.apache.hudi:hudi-spark_${scala.binary.version}</include>
-                  <include>org.apache.hudi:${hudi.spark.module}_${scala.binary.version}</include>
+                  <include>org.apache.hudi:${hudi.spark.module}</include>
                   <include>org.apache.hudi:${hudi.spark.common.module}</include>
                   <include>org.apache.hudi:hudi-hive-sync</include>
                   <include>org.apache.hudi:hudi-sync-common</include>
@@ -250,7 +250,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hudi</groupId>
-      <artifactId>${hudi.spark.module}_${scala.binary.version}</artifactId>
+      <artifactId>${hudi.spark.module}</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>
diff --git a/packaging/hudi-utilities-bundle/pom.xml b/packaging/hudi-utilities-bundle/pom.xml
index 1ffca7634a1ff..b7aa9902317e5 100644
--- a/packaging/hudi-utilities-bundle/pom.xml
+++ b/packaging/hudi-utilities-bundle/pom.xml
@@ -95,7 +95,7 @@
                   <include>org.apache.hudi:hudi-utilities_${scala.binary.version}</include>
                   <include>org.apache.hudi:hudi-spark-common_${scala.binary.version}</include>
                   <include>org.apache.hudi:hudi-spark_${scala.binary.version}</include>
-                  <include>org.apache.hudi:${hudi.spark.module}_${scala.binary.version}</include>
+                  <include>org.apache.hudi:${hudi.spark.module}</include>
                   <include>org.apache.hudi:${hudi.spark.common.module}</include>
                   <include>org.apache.hudi:hudi-hive-sync</include>
                   <include>org.apache.hudi:hudi-sync-common</include>
@@ -276,7 +276,7 @@
     </dependency>
     <dependency>
       <groupId>org.apache.hudi</groupId>
-      <artifactId>${hudi.spark.module}_${scala.binary.version}</artifactId>
+      <artifactId>${hudi.spark.module}</artifactId>
       <version>${project.version}</version>
     </dependency>
     <dependency>
diff --git a/pom.xml b/pom.xml
index 1d0e21d83d7c1..d8be55257df04 100644
--- a/pom.xml
+++ b/pom.xml
@@ -88,16 +88,16 @@
     <maven-docker-plugin.version>0.37.0</maven-docker-plugin.version>
 
     <java.version>1.8</java.version>
-    <fasterxml.version>2.6.7</fasterxml.version>
-    <fasterxml.jackson.databind.version>2.6.7.3</fasterxml.jackson.databind.version>
-    <fasterxml.jackson.module.scala.version>2.6.7.1</fasterxml.jackson.module.scala.version>
-    <fasterxml.jackson.dataformat.yaml.version>2.7.4</fasterxml.jackson.dataformat.yaml.version>
+    <fasterxml.version>${fasterxml.spark3.version}</fasterxml.version>
+    <fasterxml.jackson.databind.version>${fasterxml.spark3.version}</fasterxml.jackson.databind.version>
+    <fasterxml.jackson.module.scala.version>${fasterxml.spark3.version}</fasterxml.jackson.module.scala.version>
+    <fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}</fasterxml.jackson.dataformat.yaml.version>
     <fasterxml.spark3.version>2.10.0</fasterxml.spark3.version>
-    <kafka.version>2.0.0</kafka.version>
+    <kafka.version>2.4.1</kafka.version>
     <pulsar.version>2.8.1</pulsar.version>
     <confluent.version>5.3.4</confluent.version>
     <glassfish.version>2.17</glassfish.version>
-    <parquet.version>1.10.1</parquet.version>
+    <parquet.version>1.12.1</parquet.version>
     <junit.jupiter.version>5.7.0-M1</junit.jupiter.version>
     <junit.vintage.version>5.7.0-M1</junit.vintage.version>
     <junit.platform.version>1.7.0-M1</junit.platform.version>
@@ -115,21 +115,25 @@
     <airlift.version>0.16</airlift.version>
     <prometheus.version>0.8.0</prometheus.version>
     <http.version>4.4.1</http.version>
-    <spark.version>${spark2.version}</spark.version>
-    <sparkbundle.version></sparkbundle.version>
+    <spark.version>${spark3.version}</spark.version>
     <flink.version>1.13.1</flink.version>
     <spark2.version>2.4.4</spark2.version>
     <spark3.version>3.2.0</spark3.version>
-    <hudi.spark.module>hudi-spark2</hudi.spark.module>
-    <hudi.spark.common.module>hudi-spark2-common</hudi.spark.common.module>
+    <spark2.bundle.version>2.4</spark2.bundle.version>
+    <spark3.bundle.version>3.2</spark3.bundle.version>
+    <sparkbundle.version>${spark3.bundle.version}</sparkbundle.version>
+    <hudi.spark.module>hudi-spark3_${spark3.scala.binary.version}</hudi.spark.module>
+    <hudi.spark.common.module>hudi-spark3-common_${spark3.scala.binary.version}</hudi.spark.common.module>
     <avro.version>1.8.2</avro.version>
     <scala11.version>2.11.12</scala11.version>
     <scala12.version>2.12.10</scala12.version>
-    <scala.version>${scala11.version}</scala.version>
-    <scala.binary.version>2.11</scala.binary.version>
+    <scala.version>${scala12.version}</scala.version>
+    <spark2.scala.binary.version>2.11</spark2.scala.binary.version>
+    <spark3.scala.binary.version>2.12</spark3.scala.binary.version>
+    <scala.binary.version>${spark3.scala.binary.version}</scala.binary.version>
     <apache-rat-plugin.version>0.12</apache-rat-plugin.version>
     <scala-maven-plugin.version>3.3.1</scala-maven-plugin.version>
-    <scalatest.version>3.0.1</scalatest.version>
+    <scalatest.version>3.1.0</scalatest.version>
     <surefire-log4j.file>file://${project.basedir}/src/test/resources/log4j-surefire.properties</surefire-log4j.file>
     <thrift.version>0.12.0</thrift.version>
     <jetty.version>9.4.15.v20190215</jetty.version>
@@ -1526,23 +1530,22 @@
         </plugins>
       </build>
     </profile>
-    <!-- Exists for backwards compatibility; profile doesn't do anything -->
     <profile>
       <id>scala-2.11</id>
-    </profile>
-    <profile>
-      <id>scala-2.12</id>
       <properties>
-        <scala.version>${scala12.version}</scala.version>
-        <scala.binary.version>2.12</scala.binary.version>
+        <scala.version>${scala11.version}</scala.version>
+        <scala.binary.version>2.11</scala.binary.version>
         <skip.hudi-spark3.unit.tests>true</skip.hudi-spark3.unit.tests>
         <skipITs>true</skipITs>
       </properties>
       <activation>
         <property>
-          <name>scala-2.12</name>
+          <name>scala-2.11</name>
         </property>
       </activation>
+    </profile>
+    <profile>
+      <id>scala-2.12</id>
       <build>
         <plugins>
           <plugin>
@@ -1571,28 +1574,36 @@
       </build>
     </profile>
 
-    <!-- Exists for backwards compatibility; profile doesn't do anything -->
     <profile>
       <id>spark2</id>
+      <properties>
+        <spark.version>${spark2.version}</spark.version>
+        <sparkbundle.version>${spark2.bundle.version}</sparkbundle.version>
+        <scala.version>${scala11.version}</scala.version>
+        <scala.binary.version>${spark2.scala.binary.version}</scala.binary.version>
+        <hudi.spark.module>hudi-spark2_${scala.binary.version}</hudi.spark.module>
+        <hudi.spark.common.module>hudi-spark2-common_${scala.binary.version}</hudi.spark.common.module>
+        <scalatest.version>3.0.1</scalatest.version>
+        <kafka.version>2.0.0</kafka.version>
+        <parquet.version>1.10.1</parquet.version>
+        <fasterxml.version>2.6.7</fasterxml.version>
+        <fasterxml.jackson.databind.version>2.6.7.3</fasterxml.jackson.databind.version>
+        <fasterxml.jackson.module.scala.version>2.6.7.1</fasterxml.jackson.module.scala.version>
+        <fasterxml.jackson.dataformat.yaml.version>2.7.4</fasterxml.jackson.dataformat.yaml.version>
+        <skip.hudi-spark2.unit.tests>false</skip.hudi-spark2.unit.tests>
+        <skipITs>true</skipITs>
+      </properties>
+      <activation>
+        <property>
+          <name>spark2</name>
+        </property>
+      </activation>
     </profile>
     <profile>
       <id>spark3</id>
       <properties>
-        <spark.version>${spark3.version}</spark.version>
-        <sparkbundle.version>${spark3.version}</sparkbundle.version>
-        <scala.version>${scala12.version}</scala.version>
-        <scala.binary.version>2.12</scala.binary.version>
-        <hudi.spark.module>hudi-spark3</hudi.spark.module>
-        <hudi.spark.common.module>hudi-spark3-common</hudi.spark.common.module>
-        <scalatest.version>3.1.0</scalatest.version>
-        <kafka.version>2.4.1</kafka.version>
-        <parquet.version>1.12.1</parquet.version>
-        <fasterxml.version>${fasterxml.spark3.version}</fasterxml.version>
-        <fasterxml.jackson.databind.version>${fasterxml.spark3.version}</fasterxml.jackson.databind.version>
-        <fasterxml.jackson.module.scala.version>${fasterxml.spark3.version}</fasterxml.jackson.module.scala.version>
-        <fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}</fasterxml.jackson.dataformat.yaml.version>
+        <sparkbundle.version>${spark3.bundle.version}</sparkbundle.version>
         <skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
-        <skipITs>true</skipITs>
       </properties>
       <activation>
         <property>
@@ -1605,20 +1616,10 @@
       <id>spark3.1.x</id>
       <properties>
         <spark3.version>3.1.2</spark3.version>
-        <spark.version>${spark3.version}</spark.version>
-        <sparkbundle.version>${spark3.version}</sparkbundle.version>
-        <scala.version>${scala12.version}</scala.version>
-        <scala.binary.version>2.12</scala.binary.version>
-        <hudi.spark.module>hudi-spark3.1.x</hudi.spark.module>
-        <hudi.spark.common.module>hudi-spark3-common</hudi.spark.common.module>
-        <scalatest.version>3.1.0</scalatest.version>
-        <kafka.version>2.4.1</kafka.version>
-        <fasterxml.version>${fasterxml.spark3.version}</fasterxml.version>
-        <fasterxml.jackson.databind.version>${fasterxml.spark3.version}</fasterxml.jackson.databind.version>
-        <fasterxml.jackson.module.scala.version>${fasterxml.spark3.version}</fasterxml.jackson.module.scala.version>
-        <fasterxml.jackson.dataformat.yaml.version>${fasterxml.spark3.version}</fasterxml.jackson.dataformat.yaml.version>
+        <hudi.spark.module>hudi-spark3.1.x_${spark3.scala.binary.version}</hudi.spark.module>
+        <parquet.version>1.10.1</parquet.version>
+        <sparkbundle.version>3.1</sparkbundle.version>
         <skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
-        <skipITs>true</skipITs>
       </properties>
       <activation>
         <property>