merge from master

.github/workflows/bot.yml

@@ -14,20 +14,53 @@ jobs:
   build:
     runs-on: ubuntu-latest
     strategy:
+      max-parallel: 8
       matrix:
         include:
-          - scala: "scala-2.11"
-            spark: "spark2"
-          - scala: "scala-2.11"
-            spark: "spark2,spark-shade-unbundle-avro"
-          - scala: "scala-2.12"
-            spark: "spark3.1.x"
-          - scala: "scala-2.12"
-            spark: "spark3.1.x,spark-shade-unbundle-avro"
-          - scala: "scala-2.12"
-            spark: "spark3"
-          - scala: "scala-2.12"
-            spark: "spark3,spark-shade-unbundle-avro"
+          # Spark 2.4.4, scala 2.11
+          - scalaProfile: "scala-2.11"
+            sparkProfile: "spark2.4"
+            sparkVersion: "2.4.4"
+            flinkProfile: "flink1.13"
+
+          # Spark 2.4.4, scala 2.12
+          - scalaProfile: "scala-2.12"
+            sparkProfile: "spark2.4"
+            sparkVersion: "2.4.4"
+            flinkProfile: "flink1.14"
+
+          # Spark 3.1.x
+          - scalaProfile: "scala-2.12"
+            sparkProfile: "spark3.1"
+            sparkVersion: "3.1.0"
+            flinkProfile: "flink1.13"
+
+          - scalaProfile: "scala-2.12"
+            sparkProfile: "spark3.1"
+            sparkVersion: "3.1.1"
+            flinkProfile: "flink1.13"
+
+          - scalaProfile: "scala-2.12"
+            sparkProfile: "spark3.1"
+            sparkVersion: "3.1.2"
+            flinkProfile: "flink1.14"
+
+          - scalaProfile: "scala-2.12"
+            sparkProfile: "spark3.1"
+            sparkVersion: "3.1.3"
+            flinkProfile: "flink1.14"
+
+          # Spark 3.2.x
+          - scalaProfile: "scala-2.12"
+            sparkProfile: "spark3.2"
+            sparkVersion: "3.2.0"
+            flinkProfile: "flink1.13"
+
+          - scalaProfile: "scala-2.12"
+            sparkProfile: "spark3.2"
+            sparkVersion: "3.2.1"
+            flinkProfile: "flink1.14"
+
     steps:
       - uses: actions/checkout@v2
       - name: Set up JDK 8
@@ -38,6 +71,18 @@ jobs:
           architecture: x64
       - name: Build Project
         env:
-          SCALA_PROFILE: ${{ matrix.scala }}
-          SPARK_PROFILE: ${{ matrix.spark }}
-        run: mvn install -P "$SCALA_PROFILE,$SPARK_PROFILE" -DskipTests=true -Dmaven.javadoc.skip=true -B -V
+          SCALA_PROFILE: ${{ matrix.scalaProfile }}
+          SPARK_PROFILE: ${{ matrix.sparkProfile }}
+          SPARK_VERSION: ${{ matrix.sparkVersion }}
+          FLINK_PROFILE: ${{ matrix.flinkProfile }}
+        run:
+          mvn clean install -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -Dspark.version="$SPARK_VERSION" -Pintegration-tests -DskipTests=true -B -V
+      - name: Quickstart Test
+        env:
+          SCALA_PROFILE: ${{ matrix.scalaProfile }}
+          SPARK_PROFILE: ${{ matrix.sparkProfile }}
+          SPARK_VERSION: ${{ matrix.sparkVersion }}
+          FLINK_PROFILE: ${{ matrix.flinkProfile }}
+        if: ${{ !startsWith(env.SPARK_VERSION, '3.2.') }} # skip test spark 3.2 before hadoop upgrade to 3.x
+        run:
+          mvn test -P "unit-tests" -D"$SCALA_PROFILE" -D"$SPARK_PROFILE" -D"$FLINK_PROFILE" -Dspark.version="$SPARK_VERSION" -DfailIfNoTests=false -pl hudi-examples/hudi-examples-flink,hudi-examples/hudi-examples-java,hudi-examples/hudi-examples-spark

README.md

@@ -70,41 +70,36 @@ To build the Javadoc for all Java and Scala classes:
 mvn clean javadoc:aggregate -Pjavadocs
 ```
 
-### Build with Scala 2.12
+### Build with different Spark versions
 
-The default Scala version supported is 2.11. To build for Scala 2.12 version, build using `scala-2.12` profile
+The default Spark version supported is 2.4.4. To build for different Spark versions and Scala 2.12, use the
+corresponding profile
 
-```
-mvn clean package -DskipTests -Dscala-2.12
-```
-
-### Build with Spark 3
-
-The default Spark version supported is 2.4.4. To build for different Spark 3 versions, use the corresponding profile
+| Label | Artifact Name for Spark Bundle | Maven Profile Option | Notes |
+|--|--|--|--|
+| Spark 2.4, Scala 2.11  | hudi-spark2.4-bundle_2.11 | `-Pspark2.4` | For Spark 2.4.4, which is the same as the default  |
+| Spark 2.4, Scala 2.12 | hudi-spark2.4-bundle_2.12 | `-Pspark2.4,scala-2.12` | For Spark 2.4.4, which is the same as the default and Scala 2.12 |
+| Spark 3.1, Scala 2.12 | hudi-spark3.1-bundle_2.12 | `-Pspark3.1` | For Spark 3.1.x |
+| Spark 3.2, Scala 2.12 | hudi-spark3.2-bundle_2.12 | `-Pspark3.2` | For Spark 3.2.x |
+| Spark 3, Scala 2.12 | hudi-spark3-bundle_2.12 | `-Pspark3` | This is the same as `Spark 3.2, Scala 2.12` |
+| Spark, Scala 2.11 | hudi-spark-bundle_2.11 | Default | The default profile, supporting Spark 2.4.4 |
+| Spark, Scala 2.12 | hudi-spark-bundle_2.12 | `-Pscala-2.12` | The default profile (for Spark 2.4.4) with Scala 2.12 |
 
+For example,
 ```
-# Build against Spark 3.2.1 (the default build shipped with the public Spark 3 bundle)
-mvn clean package -DskipTests -Dspark3
+# Build against Spark 3.2.x (the default build shipped with the public Spark 3 bundle)
+mvn clean package -DskipTests -Pspark3.2
 
-# Build against Spark 3.1.2
-mvn clean package -DskipTests -Dspark3.1.x
-```
-
-### Build without spark-avro module
-
-The default hudi-jar bundles spark-avro module. To build without spark-avro module, build using `spark-shade-unbundle-avro` profile
+# Build against Spark 3.1.x
+mvn clean package -DskipTests -Pspark3.1
 
+# Build against Spark 2.4.4 and Scala 2.12
+mvn clean package -DskipTests -Pspark2.4,scala-2.12
 ```
-# Checkout code and build
-git clone https://github.com/apache/hudi.git && cd hudi
-mvn clean package -DskipTests -Pspark-shade-unbundle-avro
 
-# Start command
-spark-2.4.4-bin-hadoop2.7/bin/spark-shell \
-  --packages org.apache.spark:spark-avro_2.11:2.4.4 \
-  --jars `ls packaging/hudi-spark-bundle/target/hudi-spark-bundle_2.11-*.*.*-SNAPSHOT.jar` \
-  --conf 'spark.serializer=org.apache.spark.serializer.KryoSerializer'
-```
+### What about "spark-avro" module? 
+
+Starting from versions 0.11, Hudi no longer requires `spark-avro` to be specified using `--packages`
 
 ## Running Tests
 

azure-pipelines.yml

@@ -22,11 +22,11 @@ pool:
   vmImage: 'ubuntu-18.04'
 
 variables:
-  MAVEN_CACHE_FOLDER: $(Pipeline.Workspace)/.m2/repository
-  MAVEN_OPTS: '-Dmaven.repo.local=$(MAVEN_CACHE_FOLDER) -Dcheckstyle.skip=true -Drat.skip=true -Djacoco.skip=true'
+  MAVEN_OPTS: '-Dcheckstyle.skip=true -Drat.skip=true -Djacoco.skip=true'
   SPARK_VERSION: '2.4.4'
   HADOOP_VERSION: '2.7'
   SPARK_ARCHIVE: spark-$(SPARK_VERSION)-bin-hadoop$(HADOOP_VERSION)
+  EXCLUDE_TESTED_MODULES: '!hudi-examples/hudi-examples-common,!hudi-examples/hudi-examples-flink,!hudi-examples/hudi-examples-java,!hudi-examples/hudi-examples-spark,!hudi-common,!hudi-flink-datasource/hudi-flink,!hudi-client/hudi-spark-client,!hudi-client/hudi-client-common,!hudi-client/hudi-flink-client,!hudi-client/hudi-java-client,!hudi-cli,!hudi-utilities,!hudi-sync/hudi-hive-sync'
 
 stages:
   - stage: test
@@ -35,23 +35,15 @@ stages:
         displayName: UT FT common & flink & UT client/spark-client
         timeoutInMinutes: '90'
         steps:
-          - task: Cache@2
-            displayName: set cache
-            inputs:
-              key: 'maven | "$(Agent.OS)" | **/pom.xml'
-              restoreKeys: |
-                maven | "$(Agent.OS)"
-                maven
-              path: $(MAVEN_CACHE_FOLDER)
           - task: Maven@3
             displayName: maven install
             inputs:
               mavenPomFile: 'pom.xml'
-              goals: 'install'
+              goals: 'clean install'
               options: -T 2.5C -DskipTests
               publishJUnitResults: false
               jdkVersionOption: '1.8'
-              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
+              mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
           - task: Maven@3
             displayName: UT common flink client/spark-client
             inputs:
@@ -60,7 +52,7 @@ stages:
               options: -Punit-tests -pl hudi-common,hudi-flink-datasource/hudi-flink,hudi-client/hudi-spark-client
               publishJUnitResults: false
               jdkVersionOption: '1.8'
-              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
+              mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
           - task: Maven@3
             displayName: FT common flink
             inputs:
@@ -69,28 +61,20 @@ stages:
               options: -Pfunctional-tests -pl hudi-common,hudi-flink-datasource/hudi-flink
               publishJUnitResults: false
               jdkVersionOption: '1.8'
-              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
+              mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
       - job: UT_FT_2
         displayName: FT client/spark-client
         timeoutInMinutes: '90'
         steps:
-          - task: Cache@2
-            displayName: set cache
-            inputs:
-              key: 'maven | "$(Agent.OS)" | **/pom.xml'
-              restoreKeys: |
-                maven | "$(Agent.OS)"
-                maven
-              path: $(MAVEN_CACHE_FOLDER)
           - task: Maven@3
             displayName: maven install
             inputs:
               mavenPomFile: 'pom.xml'
-              goals: 'install'
+              goals: 'clean install'
               options: -T 2.5C -DskipTests
               publishJUnitResults: false
               jdkVersionOption: '1.8'
-              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
+              mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
           - task: Maven@3
             displayName: FT client/spark-client
             inputs:
@@ -99,28 +83,20 @@ stages:
               options: -Pfunctional-tests -pl hudi-client/hudi-spark-client
               publishJUnitResults: false
               jdkVersionOption: '1.8'
-              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
+              mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
       - job: UT_FT_3
         displayName: UT FT clients & cli & utilities & sync/hive-sync
         timeoutInMinutes: '90'
         steps:
-          - task: Cache@2
-            displayName: set cache
-            inputs:
-              key: 'maven | "$(Agent.OS)" | **/pom.xml'
-              restoreKeys: |
-                maven | "$(Agent.OS)"
-                maven
-              path: $(MAVEN_CACHE_FOLDER)
           - task: Maven@3
             displayName: maven install
             inputs:
               mavenPomFile: 'pom.xml'
-              goals: 'install'
+              goals: 'clean install'
               options: -T 2.5C -DskipTests
               publishJUnitResults: false
               jdkVersionOption: '1.8'
-              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
+              mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
           - task: Maven@3
             displayName: UT clients & cli & utilities & sync/hive-sync
             inputs:
@@ -129,7 +105,7 @@ stages:
               options: -Punit-tests -pl hudi-client/hudi-client-common,hudi-client/hudi-flink-client,hudi-client/hudi-java-client,hudi-cli,hudi-utilities,hudi-sync/hudi-hive-sync
               publishJUnitResults: false
               jdkVersionOption: '1.8'
-              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
+              mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
           - task: Maven@3
             displayName: FT clients & cli & utilities & sync/hive-sync
             inputs:
@@ -138,46 +114,38 @@ stages:
               options: -Pfunctional-tests -pl hudi-client/hudi-client-common,hudi-client/hudi-flink-client,hudi-client/hudi-java-client,hudi-cli,hudi-utilities,hudi-sync/hudi-hive-sync
               publishJUnitResults: false
               jdkVersionOption: '1.8'
-              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
+              mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
       - job: UT_FT_4
         displayName: UT FT other modules
         timeoutInMinutes: '90'
         steps:
-          - task: Cache@2
-            displayName: set cache
-            inputs:
-              key: 'maven | "$(Agent.OS)" | **/pom.xml'
-              restoreKeys: |
-                maven | "$(Agent.OS)"
-                maven
-              path: $(MAVEN_CACHE_FOLDER)
           - task: Maven@3
             displayName: maven install
             inputs:
               mavenPomFile: 'pom.xml'
-              goals: 'install'
+              goals: 'clean install'
               options: -T 2.5C -DskipTests
               publishJUnitResults: false
               jdkVersionOption: '1.8'
-              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
+              mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
           - task: Maven@3
             displayName: UT other modules
             inputs:
               mavenPomFile: 'pom.xml'
               goals: 'test'
-              options: -Punit-tests -pl !hudi-common,!hudi-flink-datasource/hudi-flink,!hudi-client/hudi-spark-client,!hudi-client/hudi-client-common,!hudi-client/hudi-flink-client,!hudi-client/hudi-java-client,!hudi-cli,!hudi-utilities,!hudi-sync/hudi-hive-sync
+              options: -Punit-tests -pl $(EXCLUDE_TESTED_MODULES)
               publishJUnitResults: false
               jdkVersionOption: '1.8'
-              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
+              mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
           - task: Maven@3
             displayName: FT other modules
             inputs:
               mavenPomFile: 'pom.xml'
               goals: 'test'
-              options: -Pfunctional-tests -pl !hudi-common,!hudi-flink-datasource/hudi-flink,!hudi-client/hudi-spark-client,!hudi-client/hudi-client-common,!hudi-client/hudi-flink-client,!hudi-client/hudi-java-client,!hudi-cli,!hudi-utilities,!hudi-sync/hudi-hive-sync
+              options: -Pfunctional-tests -pl $(EXCLUDE_TESTED_MODULES)
               publishJUnitResults: false
               jdkVersionOption: '1.8'
-              mavenOptions: '-Xmx2g $(MAVEN_OPTS)'
+              mavenOptions: '-Xmx4g $(MAVEN_OPTS)'
       - job: IT
         displayName: IT modules
         timeoutInMinutes: '90'

docker/README.md

@@ -51,9 +51,19 @@ mvn clean pre-integration-test -DskipTests -Ddocker.compose.skip=true -Ddocker.b
 mvn clean pre-integration-test -DskipTests -Ddocker.compose.skip=true -Ddocker.build.skip=false -pl :hudi-hadoop-trinobase-docker -am
 ```
 
-Alternatively, you can use `docker` cli directly under `hoodie/hadoop`. Note that, you need to manually name your local
-image by using `-t` option to match the naming in the `pom.xml`, so that you can update the corresponding image
-repository in Docker Hub (detailed steps in the next section).
+Alternatively, you can use `docker` cli directly under `hoodie/hadoop` to build images in a faster way. If you use this
+approach, make sure you first build Hudi modules with `integration-tests` profile as below so that the latest Hudi jars
+built are copied to the corresponding Hudi docker folder, e.g., `$HUDI_DIR/docker/hoodie/hadoop/hive_base/target`, which
+is required to build each docker image. Otherwise, the `target/` folder can be missing and `docker` cli complains about
+that: `failed to compute cache key: "/target" not found: not found`.
+
+```shell
+mvn -Pintegration-tests clean package -DskipTests
+```
+
+Note that, to build the image with `docker` cli, you need to manually name your local image by using `-t` option to
+match the naming in the `pom.xml`, so that you can update the corresponding image repository in Docker Hub (detailed
+steps in the next section).
 
 ```shell
 # Run under hoodie/hadoop, the <tag> is optional, "latest" by default
@@ -82,7 +92,7 @@ docker push apachehudi/hudi-hadoop_2.8.4-trinobase_368
 You can also easily push the image to the Docker Hub using Docker Desktop app: go to `Images`, search for the image by
 the name, and then click on the three dots and `Push to Hub`.
 
-![Push to Docker Hub](push_to_docker_hub.png)
+![Push to Docker Hub](images/push_to_docker_hub.png)
 
 Note that you need to ask for permission to upload the Hudi Docker Demo images to the repositories.
 

docker/demo/config/test-suite/cow-spark-long-running.yaml

@@ -38,7 +38,7 @@ dag_content:
   first_delete:
     config:
       num_partitions_delete: 50
-      num_records_delete: 8000
+      num_records_delete: 4000
     type: SparkDeleteNode
     deps: first_upsert
   second_validate:

docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions-hive.yaml

@@ -60,7 +60,7 @@ dag_content:
   first_delete:
     config:
       num_partitions_delete: 50
-      num_records_delete: 8000
+      num_records_delete: 4000
     type: DeleteNode
     deps: first_upsert
   second_hive_sync:

docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions-metadata.yaml

@@ -54,7 +54,7 @@ dag_content:
   first_delete:
     config:
       num_partitions_delete: 50
-      num_records_delete: 8000
+      num_records_delete: 4000
     type: DeleteNode
     deps: first_upsert
   second_validate:

docker/demo/config/test-suite/deltastreamer-long-running-multi-partitions.yaml

@@ -54,7 +54,7 @@ dag_content:
   first_delete:
     config:
       num_partitions_delete: 50
-      num_records_delete: 8000
+      num_records_delete: 4000
     type: DeleteNode
     deps: first_upsert
   second_validate: