[DO-NOT-MERGE] Run tests against hadoop-3.1 to see the test failures

dev/deps/spark-deps-hadoop-3.1

@@ -93,8 +93,6 @@ jackson-core-2.6.7.jar
 jackson-core-asl-1.9.13.jar
 jackson-databind-2.6.7.1.jar
 jackson-dataformat-yaml-2.6.7.jar
-jackson-jaxrs-base-2.7.8.jar
-jackson-jaxrs-json-provider-2.7.8.jar
 jackson-mapper-asl-1.9.13.jar
 jackson-module-jaxb-annotations-2.6.7.jar
 jackson-module-paranamer-2.7.9.jar

dev/run-tests.py

@@ -305,8 +305,8 @@ def get_hadoop_profiles(hadoop_version):
     """
 
     sbt_maven_hadoop_profiles = {
-        "hadoop2.6": ["-Phadoop-2.6"],
-        "hadoop2.7": ["-Phadoop-2.7"],
+        "hadoop2.6": ["-Phadoop-3.1"],
+        "hadoop2.7": ["-Phadoop-3.1"],
     }
 
     if hadoop_version in sbt_maven_hadoop_profiles:

pom.xml

@@ -123,7 +123,7 @@
     <flume.version>1.6.0</flume.version>
     <zookeeper.version>3.4.6</zookeeper.version>
     <curator.version>2.6.0</curator.version>
-    <hive.group>org.spark-project.hive</hive.group>
+    <hive.group>com.github.hyukjinkwon</hive.group>
     <!-- Version used in Maven Hive dependency -->
     <hive.version>1.2.1.spark2</hive.version>
     <!-- Version used for internal directory structure -->
@@ -1080,6 +1080,10 @@
             <groupId>com.sun.jersey.contribs</groupId>
             <artifactId>*</artifactId>
           </exclusion>
+          <exclusion>
+            <groupId>com.fasterxml.jackson.jaxrs</groupId>
+            <artifactId>jackson-jaxrs-json-provider</artifactId>
+          </exclusion>
         </exclusions>
       </dependency>
       <dependency>

project/SparkBuild.scala

@@ -317,7 +317,7 @@ object SparkBuild extends PomBuild {
   // Note ordering of these settings matter.
   /* Enable shared settings on all projects */
   (allProjects ++ optionallyEnabledProjects ++ assemblyProjects ++ copyJarsProjects ++ Seq(spark, tools))
-    .foreach(enable(sharedSettings ++ DependencyOverrides.settings ++
+    .foreach(enable(sharedSettings ++ DependencyOverrides.settings ++ ExcludeDependencies.settings ++
       ExcludedDependencies.settings ++ Checkstyle.settings))
 
   /* Enable tests settings for all projects except examples, assembly and tools */
@@ -464,7 +464,20 @@ object DockerIntegrationTests {
  */
 object DependencyOverrides {
   lazy val settings = Seq(
-    dependencyOverrides += "com.google.guava" % "guava" % "14.0.1")
+    dependencyOverrides += "com.google.guava" % "guava" % "14.0.1",
+    dependencyOverrides += "com.fasterxml.jackson.core" % "jackson-annotations" % "2.6.7",
+    dependencyOverrides += "com.fasterxml.jackson.core" % "jackson-core" % "2.6.7",
+    dependencyOverrides += "com.fasterxml.jackson.core" % "jackson-module-jaxb-annotations" % "2.6.7",
+    dependencyOverrides += "com.fasterxml.jackson.core" % "jackson-databind" % "2.6.7")
+}
+
+/**
+  * Exclusions to work around sbt's dependency resolution being different from Maven's.
+  */
+object ExcludeDependencies {
+  lazy val settings = Seq(
+    excludeDependencies += "com.fasterxml.jackson.jaxrs" % "jackson-jaxrs-json-provider",
+    excludeDependencies += "javax.ws.rs" % "jsr311-api")
 }
 
 /**

sql/hive/pom.xml

@@ -121,7 +121,7 @@
       <artifactId>avro</artifactId>
     </dependency>
     <!-- use the build matching the hadoop api of avro-mapred (i.e. no classifier for hadoop 1 API,
-    hadoop2 classifier for hadoop 2 API. avro-mapred is a dependency of org.spark-project.hive:hive-serde -->
+    hadoop2 classifier for hadoop 2 API. avro-mapred is a dependency of com.github.hyukjinkwon:hive-serde -->
     <dependency>
       <groupId>org.apache.avro</groupId>
       <artifactId>avro-mapred</artifactId>

sql/hive/src/main/scala/org/apache/spark/sql/hive/client/IsolatedClientLoader.scala

@@ -177,7 +177,10 @@ private[hive] class IsolatedClientLoader(
 
   protected def isSharedClass(name: String): Boolean = {
     val isHadoopClass =
-      name.startsWith("org.apache.hadoop.") && !name.startsWith("org.apache.hadoop.hive.")
+      name.startsWith("org.apache.hadoop.") && !name.startsWith("org.apache.hadoop.hive.") ||
+      // Also, includes configuration2 as a min fix for Hadoop 3+ for now. This is failed
+      // during class resolution. It is fine when 'sharesHadoopClasses' is disabled.
+      name.startsWith("org.apache.commons.configuration2.")
 
     name.startsWith("org.slf4j") ||
     name.startsWith("org.apache.log4j") || // log4j1.x

sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveSparkSubmitSuite.scala

@@ -22,6 +22,7 @@ import java.io.{BufferedWriter, File, FileWriter}
 import scala.tools.nsc.Properties
 
 import org.apache.hadoop.fs.Path
+import org.apache.hadoop.util.VersionInfo
 import org.scalatest.{BeforeAndAfterEach, Matchers}
 
 import org.apache.spark._
@@ -123,6 +124,7 @@ class HiveSparkSubmitSuite
   }
 
   test("SPARK-8020: set sql conf in spark conf") {
+    assume(VersionInfo.getVersion < "3.0.0", "Only Hive 2.3+ supports Hadoop 3+. See HIVE-16081.")
     val unusedJar = TestUtils.createJarWithClasses(Seq.empty)
     val args = Seq(
       "--class", SparkSQLConfTest.getClass.getName.stripSuffix("$"),

sql/hive/src/test/scala/org/apache/spark/sql/hive/client/HiveVersionSuite.scala

@@ -18,6 +18,7 @@
 package org.apache.spark.sql.hive.client
 
 import org.apache.hadoop.conf.Configuration
+import org.apache.hadoop.util.VersionInfo
 import org.scalactic.source.Position
 import org.scalatest.Tag
 
@@ -49,6 +50,9 @@ private[client] abstract class HiveVersionSuite(version: String) extends SparkFu
 
   override protected def test(testName: String, testTags: Tag*)(testFun: => Any)
       (implicit pos: Position): Unit = {
+    assume(
+      VersionInfo.getVersion < "3.0.0" || version >= "2.3",
+      "Hive 2.3+ supports Hadoop 3+. See HIVE-16081.")
     super.test(s"$version: $testName", testTags: _*)(testFun)
   }
 }