[HUDI-5975] Release 0.12.3 prep triage flaky test

.github/workflows/bot.yml

@@ -6,6 +6,15 @@ on:
       - master
       - 'release-*'
   pull_request:
+    paths-ignore:
+      - '**.bmp'
+      - '**.gif'
+      - '**.jpg'
+      - '**.jpeg'
+      - '**.md'
+      - '**.pdf'
+      - '**.png'
+      - '**.svg'
     branches:
       - master
       - 'release-*'

.gitignore

@@ -1,6 +1,7 @@
 # Directories #
 /build/
 target/
+metastore_db/
 .mvn/
 
 # OS Files #

azure-pipelines.yml

@@ -19,7 +19,7 @@ trigger:
       - '*'  # must quote since "*" is a YAML reserved character; we want a string
 
 pool:
-  vmImage: 'ubuntu-18.04'
+  vmImage: 'ubuntu-22.04'
 
 parameters:
   - name: job1Modules
@@ -46,23 +46,7 @@ parameters:
   - name: job4Modules
     type: object
     default:
-      - '!hudi-client/hudi-spark-client'
-      - '!hudi-common'
-      - '!hudi-examples'
-      - '!hudi-examples/hudi-examples-common'
-      - '!hudi-examples/hudi-examples-flink'
-      - '!hudi-examples/hudi-examples-java'
-      - '!hudi-examples/hudi-examples-spark'
-      - '!hudi-flink-datasource'
-      - '!hudi-flink-datasource/hudi-flink'
-      - '!hudi-flink-datasource/hudi-flink1.13.x'
-      - '!hudi-flink-datasource/hudi-flink1.14.x'
-      - '!hudi-flink-datasource/hudi-flink1.15.x'
-      - '!hudi-spark-datasource'
-      - '!hudi-spark-datasource/hudi-spark'
-      - '!hudi-spark-datasource/hudi-spark2'
-      - '!hudi-spark-datasource/hudi-spark2-common'
-      - '!hudi-spark-datasource/hudi-spark-common'
+      - 'hudi-utilities'
 
 variables:
   BUILD_PROFILES: '-Dscala-2.11 -Dspark2.4 -Dflink1.14'
@@ -80,96 +64,6 @@ variables:
 stages:
   - stage: test
     jobs:
-      - job: UT_FT_1
-        displayName: UT FT common & flink & UT client/spark-client
-        timeoutInMinutes: '150'
-        steps:
-          - task: Maven@4
-            displayName: maven install
-            inputs:
-              mavenPomFile: 'pom.xml'
-              goals: 'clean install'
-              options: $(MVN_OPTS_INSTALL)
-              publishJUnitResults: false
-              jdkVersionOption: '1.8'
-          - task: Maven@4
-            displayName: UT common flink client/spark-client
-            inputs:
-              mavenPomFile: 'pom.xml'
-              goals: 'test'
-              options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB1_MODULES),hudi-client/hudi-spark-client
-              publishJUnitResults: false
-              jdkVersionOption: '1.8'
-              mavenOptions: '-Xmx4g'
-          - task: Maven@4
-            displayName: FT common flink
-            inputs:
-              mavenPomFile: 'pom.xml'
-              goals: 'test'
-              options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB1_MODULES)
-              publishJUnitResults: false
-              jdkVersionOption: '1.8'
-              mavenOptions: '-Xmx4g'
-          - script: |
-              grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100
-            displayName: Top 100 long-running testcases
-      - job: UT_FT_2
-        displayName: FT client/spark-client
-        timeoutInMinutes: '150'
-        steps:
-          - task: Maven@4
-            displayName: maven install
-            inputs:
-              mavenPomFile: 'pom.xml'
-              goals: 'clean install'
-              options: $(MVN_OPTS_INSTALL)
-              publishJUnitResults: false
-              jdkVersionOption: '1.8'
-          - task: Maven@4
-            displayName: FT client/spark-client
-            inputs:
-              mavenPomFile: 'pom.xml'
-              goals: 'test'
-              options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB2_MODULES)
-              publishJUnitResults: false
-              jdkVersionOption: '1.8'
-              mavenOptions: '-Xmx4g'
-          - script: |
-              grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100
-            displayName: Top 100 long-running testcases
-      - job: UT_FT_3
-        displayName: UT FT spark-datasource
-        timeoutInMinutes: '150'
-        steps:
-          - task: Maven@4
-            displayName: maven install
-            inputs:
-              mavenPomFile: 'pom.xml'
-              goals: 'clean install'
-              options: $(MVN_OPTS_INSTALL)
-              publishJUnitResults: false
-              jdkVersionOption: '1.8'
-          - task: Maven@4
-            displayName: UT spark-datasource
-            inputs:
-              mavenPomFile: 'pom.xml'
-              goals: 'test'
-              options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB3_MODULES)
-              publishJUnitResults: false
-              jdkVersionOption: '1.8'
-              mavenOptions: '-Xmx4g'
-          - task: Maven@4
-            displayName: FT spark-datasource
-            inputs:
-              mavenPomFile: 'pom.xml'
-              goals: 'test'
-              options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB3_MODULES)
-              publishJUnitResults: false
-              jdkVersionOption: '1.8'
-              mavenOptions: '-Xmx4g'
-          - script: |
-              grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100
-            displayName: Top 100 long-running testcases
       - job: UT_FT_4
         displayName: UT FT other modules
         timeoutInMinutes: '150'
@@ -191,51 +85,6 @@ stages:
               publishJUnitResults: false
               jdkVersionOption: '1.8'
               mavenOptions: '-Xmx4g'
-          - task: Maven@4
-            displayName: FT other modules
-            inputs:
-              mavenPomFile: 'pom.xml'
-              goals: 'test'
-              options: $(MVN_OPTS_TEST) -Pfunctional-tests -pl $(JOB4_MODULES)
-              publishJUnitResults: false
-              jdkVersionOption: '1.8'
-              mavenOptions: '-Xmx4g'
           - script: |
               grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100
-            displayName: Top 100 long-running testcases
-      - job: IT
-        displayName: IT modules
-        timeoutInMinutes: '150'
-        steps:
-          - task: Maven@4
-            displayName: maven install
-            inputs:
-              mavenPomFile: 'pom.xml'
-              goals: 'clean install'
-              options: $(MVN_OPTS_INSTALL) -Pintegration-tests
-              publishJUnitResults: false
-              jdkVersionOption: '1.8'
-          - task: Maven@4
-            displayName: UT integ-test
-            inputs:
-              mavenPomFile: 'pom.xml'
-              goals: 'test'
-              options: $(MVN_OPTS_TEST) -Pintegration-tests -DskipUTs=false -DskipITs=true -pl hudi-integ-test
-              publishJUnitResults: false
-              jdkVersionOption: '1.8'
-              mavenOptions: '-Xmx4g'
-          - task: AzureCLI@2
-            displayName: Prepare for IT
-            inputs:
-              azureSubscription: apachehudici-service-connection
-              scriptType: bash
-              scriptLocation: inlineScript
-              inlineScript: |
-                echo 'Downloading $(SPARK_ARCHIVE)'
-                az storage blob download -c ci-caches -n $(SPARK_ARCHIVE).tgz -f $(Pipeline.Workspace)/$(SPARK_ARCHIVE).tgz --account-name apachehudici
-                tar -xvf $(Pipeline.Workspace)/$(SPARK_ARCHIVE).tgz -C $(Pipeline.Workspace)/
-                mkdir /tmp/spark-events/
-          - script: |
-              export SPARK_HOME=$(Pipeline.Workspace)/$(SPARK_ARCHIVE)
-              mvn $(MVN_OPTS_TEST) -Pintegration-tests verify
-            displayName: IT
+            displayName: Top 100 long-running testcases

doap_HUDI.rdf

@@ -101,6 +101,16 @@
         <created>2022-08-16</created>
         <revision>0.12.0</revision>
       </Version>
+      <Version>
+        <name>Apache Hudi 0.12.1</name>
+        <created>2022-10-18</created>
+        <revision>0.12.1</revision>
+      </Version>
+      <Version>
+        <name>Apache Hudi 0.12.2</name>
+        <created>2022-12-28</created>
+        <revision>0.12.2</revision>
+      </Version>
     </release>
     <repository>
       <GitRepository>

docker/demo/config/test-suite/compact-test.properties

@@ -22,7 +22,7 @@ hoodie.bulkinsert.shuffle.parallelism=100
 hoodie.deltastreamer.source.test.num_partitions=100
 hoodie.deltastreamer.source.test.datagen.use_rocksdb_for_storing_existing_keys=false
 hoodie.deltastreamer.source.test.max_unique_records=100000000
-hoodie.embed.timeline.server=false
+
 hoodie.deltastreamer.source.input.selector=org.apache.hudi.integ.testsuite.helpers.DFSTestSuitePathSelector
 
 hoodie.deltastreamer.source.input.selector=org.apache.hudi.integ.testsuite.helpers.DFSTestSuitePathSelector

docker/demo/config/test-suite/multi-writer-1.properties

@@ -25,7 +25,7 @@ hoodie.metadata.enable=false
 hoodie.deltastreamer.source.test.num_partitions=100
 hoodie.deltastreamer.source.test.datagen.use_rocksdb_for_storing_existing_keys=false
 hoodie.deltastreamer.source.test.max_unique_records=100000000
-hoodie.embed.timeline.server=false
+
 hoodie.deltastreamer.source.input.selector=org.apache.hudi.integ.testsuite.helpers.DFSTestSuitePathSelector
 
 hoodie.deltastreamer.source.input.selector=org.apache.hudi.integ.testsuite.helpers.DFSTestSuitePathSelector

docker/demo/config/test-suite/multi-writer-2.properties

@@ -25,7 +25,7 @@ hoodie.metadata.enable=false
 hoodie.deltastreamer.source.test.num_partitions=100
 hoodie.deltastreamer.source.test.datagen.use_rocksdb_for_storing_existing_keys=false
 hoodie.deltastreamer.source.test.max_unique_records=100000000
-hoodie.embed.timeline.server=false
+
 hoodie.deltastreamer.source.input.selector=org.apache.hudi.integ.testsuite.helpers.DFSTestSuitePathSelector
 
 hoodie.deltastreamer.source.input.selector=org.apache.hudi.integ.testsuite.helpers.DFSTestSuitePathSelector

docker/demo/config/test-suite/multi-writer-local-1.properties

@@ -28,7 +28,7 @@ hoodie.metadata.enable=false
 hoodie.deltastreamer.source.test.num_partitions=100
 hoodie.deltastreamer.source.test.datagen.use_rocksdb_for_storing_existing_keys=false
 hoodie.deltastreamer.source.test.max_unique_records=100000000
-hoodie.embed.timeline.server=false
+
 hoodie.deltastreamer.source.input.selector=org.apache.hudi.integ.testsuite.helpers.DFSTestSuitePathSelector
 
 hoodie.deltastreamer.source.input.selector=org.apache.hudi.integ.testsuite.helpers.DFSTestSuitePathSelector

docker/demo/config/test-suite/multi-writer-local-2.properties

@@ -28,7 +28,7 @@ hoodie.metadata.enable=false
 hoodie.deltastreamer.source.test.num_partitions=100
 hoodie.deltastreamer.source.test.datagen.use_rocksdb_for_storing_existing_keys=false
 hoodie.deltastreamer.source.test.max_unique_records=100000000
-hoodie.embed.timeline.server=false
+
 hoodie.deltastreamer.source.input.selector=org.apache.hudi.integ.testsuite.helpers.DFSTestSuitePathSelector
 
 hoodie.deltastreamer.source.input.selector=org.apache.hudi.integ.testsuite.helpers.DFSTestSuitePathSelector

docker/demo/config/test-suite/multi-writer-local-3.properties

@@ -28,7 +28,7 @@ hoodie.metadata.enable=false
 hoodie.deltastreamer.source.test.num_partitions=100
 hoodie.deltastreamer.source.test.datagen.use_rocksdb_for_storing_existing_keys=false
 hoodie.deltastreamer.source.test.max_unique_records=100000000
-hoodie.embed.timeline.server=false
+
 hoodie.deltastreamer.source.input.selector=org.apache.hudi.integ.testsuite.helpers.DFSTestSuitePathSelector
 
 hoodie.deltastreamer.source.input.selector=org.apache.hudi.integ.testsuite.helpers.DFSTestSuitePathSelector

docker/demo/config/test-suite/multi-writer-local-4.properties

@@ -28,7 +28,7 @@ hoodie.metadata.enable=false
 hoodie.deltastreamer.source.test.num_partitions=100
 hoodie.deltastreamer.source.test.datagen.use_rocksdb_for_storing_existing_keys=false
 hoodie.deltastreamer.source.test.max_unique_records=100000000
-hoodie.embed.timeline.server=false
+
 hoodie.deltastreamer.source.input.selector=org.apache.hudi.integ.testsuite.helpers.DFSTestSuitePathSelector
 
 hoodie.deltastreamer.source.input.selector=org.apache.hudi.integ.testsuite.helpers.DFSTestSuitePathSelector

docker/demo/config/test-suite/templates/test.properties.template

@@ -22,7 +22,7 @@ hoodie.bulkinsert.shuffle.parallelism=100
 hoodie.deltastreamer.source.test.num_partitions=100
 hoodie.deltastreamer.source.test.datagen.use_rocksdb_for_storing_existing_keys=false
 hoodie.deltastreamer.source.test.max_unique_records=100000000
-hoodie.embed.timeline.server=false
+
 hoodie.deltastreamer.source.input.selector=org.apache.hudi.integ.testsuite.helpers.DFSTestSuitePathSelector
 
 hoodie.datasource.hive_sync.skip_ro_suffix=true

docker/demo/config/test-suite/test-aggressive-clean-archival-inline-compact.properties

@@ -28,7 +28,7 @@ hoodie.keep.min.commits=12
 hoodie.keep.max.commits=14
 
 hoodie.compact.inline=true
-hoodie.embed.timeline.server=false
+
 
 hoodie.deltastreamer.source.test.num_partitions=100
 hoodie.deltastreamer.source.test.datagen.use_rocksdb_for_storing_existing_keys=false

docker/demo/config/test-suite/test-aggressive-clean-archival.properties

@@ -26,7 +26,7 @@ hoodie.delete.shuffle.parallelism=25
 hoodie.cleaner.commits.retained=8
 hoodie.keep.min.commits=12
 hoodie.keep.max.commits=14
-hoodie.embed.timeline.server=false
+
 
 hoodie.deltastreamer.source.test.num_partitions=100
 hoodie.deltastreamer.source.test.datagen.use_rocksdb_for_storing_existing_keys=false

docker/demo/config/test-suite/test-clustering-aggressive-clean-archival-inline-compact.properties

@@ -24,7 +24,7 @@ hoodie.bulkinsert.shuffle.parallelism=25
 hoodie.delete.shuffle.parallelism=25
 
 hoodie.compact.inline=true
-hoodie.embed.timeline.server=false
+
 
 hoodie.cleaner.commits.retained=8
 hoodie.keep.min.commits=12

docker/demo/config/test-suite/test-clustering-aggressive-clean-archival.properties

@@ -26,7 +26,7 @@ hoodie.delete.shuffle.parallelism=25
 hoodie.cleaner.commits.retained=8
 hoodie.keep.min.commits=12
 hoodie.keep.max.commits=14
-hoodie.embed.timeline.server=false
+
 
 hoodie.deltastreamer.source.test.num_partitions=100
 hoodie.deltastreamer.source.test.datagen.use_rocksdb_for_storing_existing_keys=false

docker/demo/config/test-suite/test-clustering-inline-compact.properties

@@ -23,7 +23,7 @@ hoodie.upsert.shuffle.parallelism=25
 hoodie.bulkinsert.shuffle.parallelism=25
 hoodie.delete.shuffle.parallelism=25
 
-hoodie.embed.timeline.server=false
+
 
 hoodie.compact.inline=true
 hoodie.deltastreamer.source.test.num_partitions=100

docker/demo/config/test-suite/test-clustering-metadata-aggressive-clean-archival-inline-compact.properties

@@ -29,7 +29,7 @@ hoodie.keep.max.commits=14
 
 hoodie.compact.inline=true
 hoodie.metadata.enable=true
-hoodie.embed.timeline.server=false
+
 
 hoodie.deltastreamer.source.test.num_partitions=100
 hoodie.deltastreamer.source.test.datagen.use_rocksdb_for_storing_existing_keys=false

docker/demo/config/test-suite/test-clustering-metadata-aggressive-clean-archival.properties

@@ -27,7 +27,7 @@ hoodie.cleaner.commits.retained=8
 hoodie.keep.min.commits=12
 hoodie.keep.max.commits=14
 
-hoodie.embed.timeline.server=false
+
 hoodie.metadata.enable=true
 
 hoodie.deltastreamer.source.test.num_partitions=100

docker/demo/config/test-suite/test-clustering.properties

@@ -23,7 +23,7 @@ hoodie.upsert.shuffle.parallelism=25
 hoodie.bulkinsert.shuffle.parallelism=25
 hoodie.delete.shuffle.parallelism=25
 
-hoodie.embed.timeline.server=false
+
 
 hoodie.deltastreamer.source.test.num_partitions=100
 hoodie.deltastreamer.source.test.datagen.use_rocksdb_for_storing_existing_keys=false

docker/demo/config/test-suite/test-inline-compact.properties

@@ -25,7 +25,7 @@ hoodie.delete.shuffle.parallelism=25
 
 hoodie.metadata.enable=false
 hoodie.compact.inline=true
-hoodie.embed.timeline.server=false
+
 
 hoodie.deltastreamer.source.test.num_partitions=100
 hoodie.deltastreamer.source.test.datagen.use_rocksdb_for_storing_existing_keys=false

docker/demo/config/test-suite/test-metadata-aggressive-clean-archival-inline-compact.properties

@@ -27,7 +27,7 @@ hoodie.cleaner.commits.retained=8
 hoodie.keep.min.commits=12
 hoodie.keep.max.commits=14
 
-hoodie.embed.timeline.server=false
+
 hoodie.metadata.enable=true
 hoodie.compact.inline=true
 

docker/demo/config/test-suite/test-metadata-aggressive-clean-archival.properties

@@ -28,7 +28,7 @@ hoodie.keep.min.commits=12
 hoodie.keep.max.commits=14
 
 hoodie.metadata.enable=true
-hoodie.embed.timeline.server=false
+
 
 hoodie.deltastreamer.source.test.num_partitions=100
 hoodie.deltastreamer.source.test.datagen.use_rocksdb_for_storing_existing_keys=false