From 86d8325ed10444074ccfbcb145f511492db5cf44 Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Fri, 18 Oct 2019 17:19:39 +0800 Subject: [PATCH 1/2] CatalogTable to HiveTable should not change the table's ownership --- .../sql/hive/client/HiveClientImpl.scala | 5 ++--- .../sql/hive/HiveExternalCatalogSuite.scala | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala index 783bc5b562ff4..12c9a972c1aff 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/client/HiveClientImpl.scala @@ -573,9 +573,8 @@ private[hive] class HiveClientImpl( // If users explicitly alter these Hive-specific properties through ALTER TABLE DDL, we respect // these user-specified values. verifyColumnDataType(table.dataSchema) - val owner = Option(table.owner).filter(_.nonEmpty).getOrElse(userName) val hiveTable = toHiveTable( - table.copy(properties = table.ignoredProperties ++ table.properties), Some(owner)) + table.copy(properties = table.ignoredProperties ++ table.properties), Some(userName)) // Do not use `table.qualifiedName` here because this may be a rename val qualifiedTableName = s"$dbName.$tableName" shim.alterTable(client, qualifiedTableName, hiveTable) @@ -1039,7 +1038,7 @@ private[hive] object HiveClientImpl { } hiveTable.setFields(schema.asJava) hiveTable.setPartCols(partCols.asJava) - userName.foreach(hiveTable.setOwner) + Option(table.owner).filter(_.nonEmpty).orElse(userName).foreach(hiveTable.setOwner) hiveTable.setCreateTime(MILLISECONDS.toSeconds(table.createTime).toInt) hiveTable.setLastAccessTime(MILLISECONDS.toSeconds(table.lastAccessTime).toInt) table.storage.locationUri.map(CatalogUtils.URIToString).foreach { loc => diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala index 0a522b6a11c80..eace443eff384 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala @@ -113,4 +113,26 @@ class HiveExternalCatalogSuite extends ExternalCatalogSuite { catalog.createDatabase(newDb("dbWithNullDesc").copy(description = null), ignoreIfExists = false) assert(catalog.getDatabase("dbWithNullDesc").description == "") } + + test("SPARK-29498 CatalogTable to HiveTable should not change the table's ownership") { + val catalog = newBasicCatalog() + val identifier = TableIdentifier("spark_29498", Some("default")) + val owner = "SPARK-29498" + val newTable = CatalogTable( + identifier, + tableType = CatalogTableType.MANAGED, + storage = CatalogStorageFormat( + locationUri = None, + inputFormat = None, + outputFormat = None, + serde = None, + compressed = false, + properties = Map.empty), + owner = owner, + schema = new StructType().add("i", "int"), + provider = Some("hive")) + + catalog.createTable(newTable, false) + assert(catalog.getTable("default", "spark_29498").owner === owner) + } } From 507fceda6f0dc15e2f0e5adbe3c1b368344ed90f Mon Sep 17 00:00:00 2001 From: Yuming Wang Date: Fri, 18 Oct 2019 20:21:09 +0800 Subject: [PATCH 2/2] Fix test error --- .../sql/hive/HiveExternalCatalogSuite.scala | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala index eace443eff384..46623000405cb 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogSuite.scala @@ -116,23 +116,16 @@ class HiveExternalCatalogSuite extends ExternalCatalogSuite { test("SPARK-29498 CatalogTable to HiveTable should not change the table's ownership") { val catalog = newBasicCatalog() - val identifier = TableIdentifier("spark_29498", Some("default")) val owner = "SPARK-29498" - val newTable = CatalogTable( - identifier, + val hiveTable = CatalogTable( + identifier = TableIdentifier("spark_29498", Some("db1")), tableType = CatalogTableType.MANAGED, - storage = CatalogStorageFormat( - locationUri = None, - inputFormat = None, - outputFormat = None, - serde = None, - compressed = false, - properties = Map.empty), + storage = storageFormat, owner = owner, schema = new StructType().add("i", "int"), provider = Some("hive")) - catalog.createTable(newTable, false) - assert(catalog.getTable("default", "spark_29498").owner === owner) + catalog.createTable(hiveTable, ignoreIfExists = false) + assert(catalog.getTable("db1", "spark_29498").owner === owner) } }