diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 5eb00c4aba0f..5ddd34893b50 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -16,10 +16,15 @@ jobs: matrix: java: [ '1.8', '11' ] hadoop: [ 'hadoop-2.7', 'hadoop-3.2' ] + hive: [ 'hive-1.2', 'hive-2.3' ] exclude: - java: '11' hadoop: 'hadoop-2.7' - name: Build Spark with JDK ${{ matrix.java }} and ${{ matrix.hadoop }} + - java: '11' + hive: 'hive-1.2' + - hadoop: 'hadoop-3.2' + hive: 'hive-1.2' + name: Build Spark - JDK${{ matrix.java }}/${{ matrix.hadoop }}/${{ matrix.hive }} steps: - uses: actions/checkout@master @@ -44,7 +49,7 @@ jobs: run: | export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN" export MAVEN_CLI_OPTS="--no-transfer-progress" - ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -Phive-thriftserver -P${{ matrix.hadoop }} -Phadoop-cloud -Djava.version=${{ matrix.java }} install + ./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -P${{ matrix.hive }} -Phive-thriftserver -P${{ matrix.hadoop }} -Phadoop-cloud -Djava.version=${{ matrix.java }} install rm -rf ~/.m2/repository/org/apache/spark diff --git a/appveyor.yml b/appveyor.yml index c3baa1f82a39..1c8ecede4c1d 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -54,7 +54,7 @@ install: build_script: # '-Djna.nosys=true' is required to avoid kernel32.dll load failure. # See SPARK-28759. - - cmd: mvn -DskipTests -Psparkr -Phive -Djna.nosys=true package + - cmd: mvn -DskipTests -Psparkr -Phive -Phive-1.2 -Djna.nosys=true package environment: NOT_CRAN: true diff --git a/dev/run-tests.py b/dev/run-tests.py index fc8b7251a85f..2d52ead06a04 100755 --- a/dev/run-tests.py +++ b/dev/run-tests.py @@ -283,8 +283,8 @@ def get_hadoop_profiles(hadoop_version): """ sbt_maven_hadoop_profiles = { - "hadoop2.7": ["-Phadoop-2.7"], - "hadoop3.2": ["-Phadoop-3.2"], + "hadoop2.7": ["-Phadoop-2.7", "-Phive-1.2"], + "hadoop3.2": ["-Phadoop-3.2", "-Phive-2.3"], } if hadoop_version in sbt_maven_hadoop_profiles: diff --git a/dev/test-dependencies.sh b/dev/test-dependencies.sh index cc0292e9c2ea..7d5725aaf137 100755 --- a/dev/test-dependencies.sh +++ b/dev/test-dependencies.sh @@ -67,15 +67,20 @@ $MVN -q versions:set -DnewVersion=$TEMP_VERSION -DgenerateBackupPoms=false > /de # Generate manifests for each Hadoop profile: for HADOOP_PROFILE in "${HADOOP_PROFILES[@]}"; do + if [[ $HADOOP_PROFILE == **hadoop-3** ]]; then + HIVE_PROFILE=hive-2.3 + else + HIVE_PROFILE=hive-1.2 + fi echo "Performing Maven install for $HADOOP_PROFILE" - $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE jar:jar jar:test-jar install:install clean -q + $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE jar:jar jar:test-jar install:install clean -q echo "Performing Maven validate for $HADOOP_PROFILE" - $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE validate -q + $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE validate -q echo "Generating dependency manifest for $HADOOP_PROFILE" mkdir -p dev/pr-deps - $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE dependency:build-classpath -pl assembly -am \ + $MVN $HADOOP2_MODULE_PROFILES -P$HADOOP_PROFILE -P$HIVE_PROFILE dependency:build-classpath -pl assembly -am \ | grep "Dependencies classpath:" -A 1 \ | tail -n 1 | tr ":" "\n" | rev | cut -d "/" -f 1 | rev | sort \ | grep -v spark > dev/pr-deps/spark-deps-$HADOOP_PROFILE diff --git a/pom.xml b/pom.xml index 44593b78c9a0..a9b544bba158 100644 --- a/pom.xml +++ b/pom.xml @@ -128,19 +128,19 @@ 3.4.14 2.7.1 0.4.2 - org.spark-project.hive - + org.apache.hive + core - 1.2.1.spark2 + 2.3.6 2.3.6 - 1.2.1 + 2.3.5 2.3.1 10.12.1.1 1.10.1 1.5.7 - nohive + com.twitter 1.6.0 9.4.18.v20190429 @@ -181,7 +181,7 @@ 3.8.1 2.6.2 - 3.2.10 + 4.1.17 3.0.15 2.29 2.10.5 @@ -228,7 +228,7 @@ --> compile compile - ${hive.deps.scope} + provided compile compile test @@ -2921,16 +2921,27 @@ 3.2.0 2.13.0 - org.apache.hive - core - ${hive23.version} - 2.3.5 - - provided - - 4.1.17 + + + + hive-1.2 + + org.spark-project.hive + + + 1.2.1.spark2 + + 1.2.1 + ${hive.deps.scope} + nohive + 3.2.10 + + + + + hive-2.3 + diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml index 882cabd34707..69961acd510d 100644 --- a/sql/hive/pom.xml +++ b/sql/hive/pom.xml @@ -209,7 +209,7 @@ - hadoop-3.2 + hive-2.3 ${hive.group}