diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml index 384c39eebadb7..59eada9cc647f 100644 --- a/.github/workflows/bot.yml +++ b/.github/workflows/bot.yml @@ -146,14 +146,19 @@ jobs: include: - flinkProfile: 'flink1.17' sparkProfile: 'spark3.3' + sparkRuntime: 'spark3.3.2' - flinkProfile: 'flink1.16' sparkProfile: 'spark3.3' + sparkRuntime: 'spark3.3.2' - flinkProfile: 'flink1.15' sparkProfile: 'spark3.3' + sparkRuntime: 'spark3.3.1' - flinkProfile: 'flink1.14' sparkProfile: 'spark3.2' + sparkRuntime: 'spark3.2.3' - flinkProfile: 'flink1.13' sparkProfile: 'spark3.1' + sparkRuntime: 'spark3.1.3' steps: - uses: actions/checkout@v2 - name: Set up JDK 8 @@ -175,7 +180,7 @@ jobs: - name: IT - Bundle Validation - OpenJDK 8 env: FLINK_PROFILE: ${{ matrix.flinkProfile }} - SPARK_PROFILE: ${{ matrix.sparkProfile }} + SPARK_RUNTIME: ${{ matrix.sparkRuntime }} SCALA_PROFILE: 'scala-2.12' run: | HUDI_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) @@ -183,7 +188,7 @@ jobs: - name: IT - Bundle Validation - OpenJDK 11 env: FLINK_PROFILE: ${{ matrix.flinkProfile }} - SPARK_PROFILE: ${{ matrix.sparkProfile }} + SPARK_RUNTIME: ${{ matrix.sparkRuntime }} SCALA_PROFILE: 'scala-2.12' run: | HUDI_VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) @@ -192,6 +197,7 @@ jobs: env: FLINK_PROFILE: ${{ matrix.flinkProfile }} SPARK_PROFILE: ${{ matrix.sparkProfile }} + SPARK_RUNTIME: ${{ matrix.sparkRuntime }} SCALA_PROFILE: 'scala-2.12' if: ${{ endsWith(env.SPARK_PROFILE, '3.3') }} # Only Spark 3.3 supports Java 17 as of now run: | diff --git a/packaging/bundle-validation/ci_run.sh b/packaging/bundle-validation/ci_run.sh index 70faebe563d74..bef2ab3282c63 100755 --- a/packaging/bundle-validation/ci_run.sh +++ b/packaging/bundle-validation/ci_run.sh @@ -31,7 +31,7 @@ HUDI_VERSION=$1 JAVA_RUNTIME_VERSION=$2 # choose versions based on build profiles -if [[ ${SPARK_PROFILE} == 'spark2.4' ]]; then +if [[ ${SPARK_RUNTIME} == 'spark2.4' ]]; then HADOOP_VERSION=2.7.7 HIVE_VERSION=2.3.9 DERBY_VERSION=10.10.2.0 @@ -41,7 +41,7 @@ if [[ ${SPARK_PROFILE} == 'spark2.4' ]]; then CONFLUENT_VERSION=5.5.12 KAFKA_CONNECT_HDFS_VERSION=10.1.13 IMAGE_TAG=flink1136hive239spark248 -elif [[ ${SPARK_PROFILE} == 'spark3.1' ]]; then +elif [[ ${SPARK_RUNTIME} == 'spark3.1.3' ]]; then HADOOP_VERSION=2.7.7 HIVE_VERSION=3.1.3 DERBY_VERSION=10.14.1.0 @@ -51,7 +51,7 @@ elif [[ ${SPARK_PROFILE} == 'spark3.1' ]]; then CONFLUENT_VERSION=5.5.12 KAFKA_CONNECT_HDFS_VERSION=10.1.13 IMAGE_TAG=flink1136hive313spark313 -elif [[ ${SPARK_PROFILE} == 'spark3.2' ]]; then +elif [[ ${SPARK_RUNTIME} == 'spark3.2.3' ]]; then HADOOP_VERSION=2.7.7 HIVE_VERSION=3.1.3 DERBY_VERSION=10.14.1.0 @@ -61,7 +61,7 @@ elif [[ ${SPARK_PROFILE} == 'spark3.2' ]]; then CONFLUENT_VERSION=5.5.12 KAFKA_CONNECT_HDFS_VERSION=10.1.13 IMAGE_TAG=flink1146hive313spark323 -elif [[ ${SPARK_PROFILE} == 'spark3.3' ]]; then +elif [[ ${SPARK_RUNTIME} == 'spark3.3.1' ]]; then HADOOP_VERSION=2.7.7 HIVE_VERSION=3.1.3 DERBY_VERSION=10.14.1.0 @@ -71,6 +71,16 @@ elif [[ ${SPARK_PROFILE} == 'spark3.3' ]]; then CONFLUENT_VERSION=5.5.12 KAFKA_CONNECT_HDFS_VERSION=10.1.13 IMAGE_TAG=flink1153hive313spark331 +elif [[ ${SPARK_RUNTIME} == 'spark3.3.2' ]]; then + HADOOP_VERSION=2.7.7 + HIVE_VERSION=3.1.3 + DERBY_VERSION=10.14.1.0 + FLINK_VERSION=1.15.3 + SPARK_VERSION=3.3.2 + SPARK_HADOOP_VERSION=2 + CONFLUENT_VERSION=5.5.12 + KAFKA_CONNECT_HDFS_VERSION=10.1.13 + IMAGE_TAG=flink1153hive313spark332 fi # Copy bundle jars to temp dir for mounting