From a6e5c0424478c716bcc8ca33591bc777f483d827 Mon Sep 17 00:00:00 2001 From: Lorenzo Martini Date: Mon, 19 Apr 2021 09:17:00 +0100 Subject: [PATCH] Hive-2.3.8 --- dev/deps/spark-deps-hadoop-palantir | 16 ++++++++++++++- docs/building-spark.md | 4 ++-- docs/sql-data-sources-hive-tables.md | 8 ++++---- docs/sql-migration-guide.md | 2 +- pom.xml | 20 +++++++++++++++++-- .../HiveThriftServer2Suites.scala | 4 ++-- .../org/apache/spark/sql/hive/HiveUtils.scala | 2 +- .../hive/client/IsolatedClientLoader.scala | 4 ++-- .../spark/sql/hive/client/package.scala | 10 +++++----- .../HiveExternalCatalogVersionsSuite.scala | 5 +++++ 10 files changed, 55 insertions(+), 20 deletions(-) diff --git a/dev/deps/spark-deps-hadoop-palantir b/dev/deps/spark-deps-hadoop-palantir index d26545e74fd19..cab174afa5e04 100644 --- a/dev/deps/spark-deps-hadoop-palantir +++ b/dev/deps/spark-deps-hadoop-palantir @@ -65,7 +65,21 @@ hadoop-yarn-client/2.9.2-palantir.15//hadoop-yarn-client-2.9.2-palantir.15.jar hadoop-yarn-common/2.9.2-palantir.15//hadoop-yarn-common-2.9.2-palantir.15.jar hadoop-yarn-registry/2.9.2-palantir.15//hadoop-yarn-registry-2.9.2-palantir.15.jar hadoop-yarn-server-common/2.9.2-palantir.15//hadoop-yarn-server-common-2.9.2-palantir.15.jar -hive-storage-api/2.7.1//hive-storage-api-2.7.1.jar +hive-beeline/2.3.8//hive-beeline-2.3.8.jar +hive-cli/2.3.8//hive-cli-2.3.8.jar +hive-common/2.3.8//hive-common-2.3.8.jar +hive-exec/2.3.8/core/hive-exec-2.3.8-core.jar +hive-jdbc/2.3.8//hive-jdbc-2.3.8.jar +hive-llap-common/2.3.8//hive-llap-common-2.3.8.jar +hive-metastore/2.3.8//hive-metastore-2.3.8.jar +hive-serde/2.3.8//hive-serde-2.3.8.jar +hive-service-rpc/3.1.2//hive-service-rpc-3.1.2.jar +hive-shims-0.23/2.3.8//hive-shims-0.23-2.3.8.jar +hive-shims-common/2.3.8//hive-shims-common-2.3.8.jar +hive-shims-scheduler/2.3.8//hive-shims-scheduler-2.3.8.jar +hive-shims/2.3.8//hive-shims-2.3.8.jar +hive-storage-api/2.7.2//hive-storage-api-2.7.2.jar +hive-vector-code-gen/2.3.8//hive-vector-code-gen-2.3.8.jar hk2-api/2.6.1//hk2-api-2.6.1.jar hk2-locator/2.6.1//hk2-locator-2.6.1.jar hk2-utils/2.6.1//hk2-utils-2.6.1.jar diff --git a/docs/building-spark.md b/docs/building-spark.md index 3d12a60e2b974..8384126778b19 100644 --- a/docs/building-spark.md +++ b/docs/building-spark.md @@ -83,9 +83,9 @@ Example: To enable Hive integration for Spark SQL along with its JDBC server and CLI, add the `-Phive` and `-Phive-thriftserver` profiles to your existing build options. -By default Spark will build with Hive 2.3.7. +By default Spark will build with Hive 2.3.8. - # With Hive 2.3.7 support + # With Hive 2.3.8 support ./build/mvn -Pyarn -Phive -Phive-thriftserver -DskipTests clean package ## Packaging without Hadoop Dependencies for YARN diff --git a/docs/sql-data-sources-hive-tables.md b/docs/sql-data-sources-hive-tables.md index ae3572c474e1a..9a1dcc1f9e48c 100644 --- a/docs/sql-data-sources-hive-tables.md +++ b/docs/sql-data-sources-hive-tables.md @@ -127,10 +127,10 @@ The following options can be used to configure the version of Hive that is used Property NameDefaultMeaningSince Version spark.sql.hive.metastore.version - 2.3.7 + 2.3.8 Version of the Hive metastore. Available - options are 0.12.0 through 2.3.7 and 3.0.0 through 3.1.2. + options are 0.12.0 through 2.3.8 and 3.0.0 through 3.1.2. 1.4.0 @@ -142,9 +142,9 @@ The following options can be used to configure the version of Hive that is used property can be one of three options:
  1. builtin
    2. - Use Hive 2.3.7, which is bundled with the Spark assembly when -Phive is + Use Hive 2.3.8, which is bundled with the Spark assembly when -Phive is enabled. When this option is chosen, spark.sql.hive.metastore.version must be - either 2.3.7 or not defined. + either 2.3.8 or not defined.
  2. maven
    3. Use Hive jars of specified version downloaded from Maven repositories. This configuration is not generally recommended for production deployments. diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md index 742a05b425ee2..515b606492950 100644 --- a/docs/sql-migration-guide.md +++ b/docs/sql-migration-guide.md @@ -810,7 +810,7 @@ Python UDF registration is unchanged. Spark SQL is designed to be compatible with the Hive Metastore, SerDes and UDFs. Currently, Hive SerDes and UDFs are based on built-in Hive, and Spark SQL can be connected to different versions of Hive Metastore -(from 0.12.0 to 2.3.7 and 3.0.0 to 3.1.2. Also see [Interacting with Different Versions of Hive Metastore](sql-data-sources-hive-tables.html#interacting-with-different-versions-of-hive-metastore)). +(from 0.12.0 to 2.3.8 and 3.0.0 to 3.1.2. Also see [Interacting with Different Versions of Hive Metastore](sql-data-sources-hive-tables.html#interacting-with-different-versions-of-hive-metastore)). #### Deploying in Existing Hive Warehouses {:.no_toc} diff --git a/pom.xml b/pom.xml index 445aadd1500a9..a1675d8d74dc5 100644 --- a/pom.xml +++ b/pom.xml @@ -131,8 +131,8 @@ org.apache.hive core - 2.3.7 - 2.3.7 + 2.3.8 + 2.3.8 2.3 @@ -1846,6 +1846,22 @@ org.apache.logging.log4j * + + net.hydromatic + eigenbase-properties + + + org.codehaus.janino + commons-compiler + + + org.codehaus.janino + janino + + + org.pentaho + pentaho-aggdesigner-algorithm + diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala index 396e7f9d397db..45a0f1b310dc9 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/HiveThriftServer2Suites.scala @@ -545,7 +545,7 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest { } if (HiveUtils.isHive23) { - assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("2.3.7")) + assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("2.3.8")) } else { assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("1.2.1")) } @@ -562,7 +562,7 @@ class HiveThriftBinaryServerSuite extends HiveThriftJdbcTest { } if (HiveUtils.isHive23) { - assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("2.3.7")) + assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("2.3.8")) } else { assert(conf.get(HiveUtils.FAKE_HIVE_VERSION.key) === Some("1.2.1")) } diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala index 04caf57efdc74..9abc538cd315e 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveUtils.scala @@ -63,7 +63,7 @@ private[spark] object HiveUtils extends Logging { val HIVE_METASTORE_VERSION = buildStaticConf("spark.sql.hive.metastore.version") .doc("Version of the Hive metastore. Available options are " + - "0.12.0 through 2.3.7 and " + + "0.12.0 through 2.3.8 and " + "3.0.0 through 3.1.2.") .version("1.4.0") .stringConf diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala index 42a0ec0253b85..8907332c9a452 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala @@ -104,8 +104,8 @@ private[hive] object IsolatedClientLoader extends Logging { case "2.0" | "2.0.0" | "2.0.1" => hive.v2_0 case "2.1" | "2.1.0" | "2.1.1" => hive.v2_1 case "2.2" | "2.2.0" => hive.v2_2 - case "2.3" | "2.3.0" | "2.3.1" | "2.3.2" | "2.3.3" | "2.3.4" | "2.3.5" | "2.3.6" | "2.3.7" => - hive.v2_3 + case "2.3" | "2.3.0" | "2.3.1" | "2.3.2" | "2.3.3" | "2.3.4" | "2.3.5" | "2.3.6" | "2.3.7" | + "2.3.8" => hive.v2_3 case "3.0" | "3.0.0" => hive.v3_0 case "3.1" | "3.1.0" | "3.1.1" | "3.1.2" => hive.v3_1 case version => diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala index 8526d86454604..7fb15469caa5f 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/package.scala @@ -87,11 +87,13 @@ package object client { "org.apache.curator:*", "org.pentaho:pentaho-aggdesigner-algorithm")) - // Since HIVE-14496, Hive materialized view need calcite-core. + // Since HIVE-23980, calcite-core included in Hive package jar. // For spark, only VersionsSuite currently creates a hive materialized view for testing. - case object v2_3 extends HiveVersion("2.3.7", - exclusions = Seq("org.apache.calcite:calcite-druid", + case object v2_3 extends HiveVersion("2.3.8", + exclusions = Seq("org.apache.calcite:calcite-core", + "org.apache.calcite:calcite-druid", "org.apache.calcite.avatica:avatica", + "com.fasterxml.jackson.core:*", "org.apache.curator:*", "org.pentaho:pentaho-aggdesigner-algorithm")) @@ -101,7 +103,6 @@ package object client { extraDeps = Seq("org.apache.logging.log4j:log4j-api:2.10.0", "org.apache.derby:derby:10.14.1.0"), exclusions = Seq("org.apache.calcite:calcite-druid", - "org.apache.calcite.avatica:avatica", "org.apache.curator:*", "org.pentaho:pentaho-aggdesigner-algorithm")) @@ -111,7 +112,6 @@ package object client { extraDeps = Seq("org.apache.logging.log4j:log4j-api:2.10.0", "org.apache.derby:derby:10.14.1.0"), exclusions = Seq("org.apache.calcite:calcite-druid", - "org.apache.calcite.avatica:avatica", "org.apache.curator:*", "org.pentaho:pentaho-aggdesigner-algorithm")) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index 52fd5d77e5bb6..d5b365c79c09a 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -58,6 +58,11 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { // avoid downloading Spark of different versions in each run. private val sparkTestingDir = new File("/tmp/test-spark") private val unusedJar = TestUtils.createJarWithClasses(Seq.empty) + val hiveVersion = if (SystemUtils.isJavaVersionAtLeast(JavaVersion.JAVA_9)) { + "2.3.8" + } else { + "1.2.1" + } override def afterAll(): Unit = { try {