From 1efaafcea3cac4875cfa071f8db432f3aa2da890 Mon Sep 17 00:00:00 2001 From: Lokesh Jain Date: Mon, 5 Dec 2022 10:53:47 +0530 Subject: [PATCH 1/8] [HUDI-4827] Rebase Azure Image on Ubuntu 22.04 --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 93c5c8db0fe8b..71387e7f0c8db 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,7 +19,7 @@ trigger: - '*' # must quote since "*" is a YAML reserved character; we want a string pool: - vmImage: 'ubuntu-18.04' + vmImage: 'ubuntu-20.04' parameters: - name: job1Modules From 607224088aba2a29ff7ab51d0ad4de968bb712a6 Mon Sep 17 00:00:00 2001 From: Lokesh Jain Date: Mon, 5 Dec 2022 16:02:55 +0530 Subject: [PATCH 2/8] Remove target arg from scala-maven-plugin --- hudi-spark-datasource/hudi-spark/pom.xml | 1 - 1 file changed, 1 deletion(-) diff --git a/hudi-spark-datasource/hudi-spark/pom.xml b/hudi-spark-datasource/hudi-spark/pom.xml index 91e06f0a922fd..ddf700e64a4ec 100644 --- a/hudi-spark-datasource/hudi-spark/pom.xml +++ b/hudi-spark-datasource/hudi-spark/pom.xml @@ -48,7 +48,6 @@ -nobootcp - -target:jvm-1.8 false From 399aec43d22aba091df5b360a7912ce0a3b9033c Mon Sep 17 00:00:00 2001 From: Lokesh Jain Date: Mon, 5 Dec 2022 16:19:02 +0530 Subject: [PATCH 3/8] maven compiler plugin set source target --- azure-pipelines.yml | 3 +++ hudi-spark-datasource/hudi-spark/pom.xml | 5 +++++ 2 files changed, 8 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 71387e7f0c8db..220fde0834e75 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -97,6 +97,7 @@ stages: - stage: test jobs: - job: UT_FT_1 + condition: false displayName: UT FT common & flink & UT client/spark-client timeoutInMinutes: '150' steps: @@ -130,6 +131,7 @@ stages: grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 displayName: Top 100 long-running testcases - job: UT_FT_2 + condition: false displayName: FT client/spark-client timeoutInMinutes: '150' steps: @@ -178,6 +180,7 @@ stages: grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 displayName: Top 100 long-running testcases - job: UT_FT_4 + condition: false displayName: UT FT other modules timeoutInMinutes: '180' steps: diff --git a/hudi-spark-datasource/hudi-spark/pom.xml b/hudi-spark-datasource/hudi-spark/pom.xml index ddf700e64a4ec..26f5fd1330095 100644 --- a/hudi-spark-datasource/hudi-spark/pom.xml +++ b/hudi-spark-datasource/hudi-spark/pom.xml @@ -48,6 +48,7 @@ -nobootcp + -target:jvm-1.8 false @@ -55,6 +56,10 @@ org.apache.maven.plugins maven-compiler-plugin + + 1.8 + 1.8 + From 1484956a2e5c949cb6df99932d048aec151397a7 Mon Sep 17 00:00:00 2001 From: Lokesh Jain Date: Thu, 15 Dec 2022 11:50:01 +0530 Subject: [PATCH 4/8] use javaHomeOption --- azure-pipelines.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 220fde0834e75..b004d1d52f1da 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -167,6 +167,7 @@ stages: options: $(MVN_OPTS_INSTALL) publishJUnitResults: false jdkVersionOption: '1.8' + javaHomeOption: 'JDKVersion' - task: Maven@4 displayName: UT spark-datasource inputs: @@ -175,6 +176,7 @@ stages: options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB3_MODULES) publishJUnitResults: false jdkVersionOption: '1.8' + javaHomeOption: 'JDKVersion' mavenOptions: '-Xmx4g' - script: | grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 @@ -214,6 +216,7 @@ stages: grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 displayName: Top 100 long-running testcases - job: IT + condition: false displayName: IT modules timeoutInMinutes: '150' steps: From 3b6872b0db6909c043358460e078f2d1dedfb5b3 Mon Sep 17 00:00:00 2001 From: Lokesh Jain Date: Tue, 20 Dec 2022 16:20:09 +0530 Subject: [PATCH 5/8] Update scalatest-maven-plugin version --- azure-pipelines.yml | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b004d1d52f1da..341e48e671d37 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,7 +19,7 @@ trigger: - '*' # must quote since "*" is a YAML reserved character; we want a string pool: - vmImage: 'ubuntu-20.04' + vmImage: 'ubuntu-22.04' parameters: - name: job1Modules diff --git a/pom.xml b/pom.xml index 16e2108ce543b..5ff9995297534 100644 --- a/pom.xml +++ b/pom.xml @@ -495,7 +495,7 @@ org.scalatest scalatest-maven-plugin - 1.0 + 2.2.0 ${skipUTs} ${project.build.directory}/surefire-reports From 69813b14590139be134289204e7ad17f1d4e63c8 Mon Sep 17 00:00:00 2001 From: Lokesh Jain Date: Wed, 21 Dec 2022 12:45:36 +0530 Subject: [PATCH 6/8] remove redundant changes --- azure-pipelines.yml | 6 ------ hudi-spark-datasource/hudi-spark/pom.xml | 4 ---- 2 files changed, 10 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 341e48e671d37..07330cca44300 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -97,7 +97,6 @@ stages: - stage: test jobs: - job: UT_FT_1 - condition: false displayName: UT FT common & flink & UT client/spark-client timeoutInMinutes: '150' steps: @@ -131,7 +130,6 @@ stages: grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 displayName: Top 100 long-running testcases - job: UT_FT_2 - condition: false displayName: FT client/spark-client timeoutInMinutes: '150' steps: @@ -167,7 +165,6 @@ stages: options: $(MVN_OPTS_INSTALL) publishJUnitResults: false jdkVersionOption: '1.8' - javaHomeOption: 'JDKVersion' - task: Maven@4 displayName: UT spark-datasource inputs: @@ -176,13 +173,11 @@ stages: options: $(MVN_OPTS_TEST) -Punit-tests -pl $(JOB3_MODULES) publishJUnitResults: false jdkVersionOption: '1.8' - javaHomeOption: 'JDKVersion' mavenOptions: '-Xmx4g' - script: | grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 displayName: Top 100 long-running testcases - job: UT_FT_4 - condition: false displayName: UT FT other modules timeoutInMinutes: '180' steps: @@ -216,7 +211,6 @@ stages: grep "testcase" */target/surefire-reports/*.xml */*/target/surefire-reports/*.xml | awk -F'"' ' { print $6,$4,$2 } ' | sort -nr | head -n 100 displayName: Top 100 long-running testcases - job: IT - condition: false displayName: IT modules timeoutInMinutes: '150' steps: diff --git a/hudi-spark-datasource/hudi-spark/pom.xml b/hudi-spark-datasource/hudi-spark/pom.xml index 26f5fd1330095..91e06f0a922fd 100644 --- a/hudi-spark-datasource/hudi-spark/pom.xml +++ b/hudi-spark-datasource/hudi-spark/pom.xml @@ -56,10 +56,6 @@ org.apache.maven.plugins maven-compiler-plugin - - 1.8 - 1.8 - From b47cb3602fd327ca2fadac185ec1be200a5ce671 Mon Sep 17 00:00:00 2001 From: Lokesh Jain Date: Thu, 22 Dec 2022 13:35:26 +0530 Subject: [PATCH 7/8] Trigger CI From 91fbb11275f860d9575a6e0f476d8df527a62c1d Mon Sep 17 00:00:00 2001 From: Lokesh Jain Date: Thu, 22 Dec 2022 15:48:57 +0530 Subject: [PATCH 8/8] Trigger CI