From 8d06b98eeed459991937b66a607d6dca885b14c7 Mon Sep 17 00:00:00 2001 From: Cheng Lian Date: Mon, 9 Nov 2015 22:35:21 +0800 Subject: [PATCH 1/2] Fixes SPARK-11595 for branch-1.5 --- .../spark/sql/hive/execution/commands.scala | 15 ++++++++++++++- .../spark/sql/hive/execution/HiveQuerySuite.scala | 12 ++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala index 9f654eed5761..1615803d692f 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/execution/commands.scala @@ -17,6 +17,9 @@ package org.apache.spark.sql.hive.execution +import java.io.File + +import org.apache.hadoop.fs.Path import org.apache.hadoop.hive.metastore.MetaStoreUtils import org.apache.spark.sql._ import org.apache.spark.sql.catalyst.{TableIdentifier, SqlParser} @@ -90,7 +93,17 @@ case class AddJar(path: String) extends RunnableCommand { val currentClassLoader = Utils.getContextOrSparkClassLoader // Add jar to current context - val jarURL = new java.io.File(path).toURI.toURL + val jarURL = { + val uri = new Path(path).toUri + if (uri.getScheme == null) { + // `path` is a local file path without a URL scheme + new File(path).toURI.toURL + } else { + // `path` is a URL with a scheme + uri.toURL + } + } + val newClassLoader = new java.net.URLClassLoader(Array(jarURL), currentClassLoader) Thread.currentThread.setContextClassLoader(newClassLoader) // We need to explicitly set the class loader associated with the conf in executionHive's diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index 83f9f3eaa3a5..e1d53e296f63 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -898,6 +898,18 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { sql("DROP TABLE t1") } + test("CREATE TEMPORARY FUNCTION") { + val funcJar = TestHive.getHiveFile("TestUDTF.jar").getCanonicalPath + val jarURL = s"file://$funcJar" + sql(s"ADD JAR $jarURL") + sql( + """CREATE TEMPORARY FUNCTION udtf_count2 AS + |'org.apache.spark.sql.hive.execution.GenericUDTFCount2' + """.stripMargin) + assert(sql("DESCRIBE FUNCTION udtf_count2").count > 1) + sql("DROP TEMPORARY FUNCTION udtf_count2") + } + test("ADD FILE command") { val testFile = TestHive.getHiveFile("data/files/v1.txt").getCanonicalFile sql(s"ADD FILE $testFile") From 3c0f368c6c2f1f71f3b5a5690c4ba4fe9fe47b6a Mon Sep 17 00:00:00 2001 From: Cheng Lian Date: Wed, 11 Nov 2015 22:25:19 +0800 Subject: [PATCH 2/2] Fixes test failure --- .../spark/sql/hive/execution/HiveQuerySuite.scala | 12 ------------ .../spark/sql/hive/execution/SQLQuerySuite.scala | 7 ++++++- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala index e1d53e296f63..83f9f3eaa3a5 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala @@ -898,18 +898,6 @@ class HiveQuerySuite extends HiveComparisonTest with BeforeAndAfter { sql("DROP TABLE t1") } - test("CREATE TEMPORARY FUNCTION") { - val funcJar = TestHive.getHiveFile("TestUDTF.jar").getCanonicalPath - val jarURL = s"file://$funcJar" - sql(s"ADD JAR $jarURL") - sql( - """CREATE TEMPORARY FUNCTION udtf_count2 AS - |'org.apache.spark.sql.hive.execution.GenericUDTFCount2' - """.stripMargin) - assert(sql("DESCRIBE FUNCTION udtf_count2").count > 1) - sql("DROP TEMPORARY FUNCTION udtf_count2") - } - test("ADD FILE command") { val testFile = TestHive.getHiveFile("data/files/v1.txt").getCanonicalFile sql(s"ADD FILE $testFile") diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala index c98fddb3cb47..05504f134afb 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala @@ -70,7 +70,12 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils { private val sqlContext = _sqlContext test("UDTF") { - sql(s"ADD JAR ${TestHive.getHiveFile("TestUDTF.jar").getCanonicalPath()}") + val jarPath = TestHive.getHiveFile("TestUDTF.jar").getCanonicalPath + + // SPARK-11595 Fixes ADD JAR when input path contains URL scheme + val jarURL = s"file://$jarPath" + + sql(s"ADD JAR $jarURL") // The function source code can be found at: // https://cwiki.apache.org/confluence/display/Hive/DeveloperGuide+UDTF sql(