From 53a0d48be313a4f6b43491f2d378e69fe084ae87 Mon Sep 17 00:00:00 2001 From: PengLei Date: Thu, 25 Nov 2021 11:14:50 +0800 Subject: [PATCH] add draft --- .../sql/catalyst/parser/DDLParserSuite.scala | 13 - .../spark/sql/execution/command/tables.scala | 2 +- .../datasources/v2/ShowCreateTableExec.scala | 2 +- .../sql-tests/results/group-by.sql.out | 2 +- .../results/show-create-table.sql.out | 6 +- .../spark/sql/ShowCreateTableSuite.scala | 267 ------------------ .../sql/connector/DataSourceV2SQLSuite.scala | 107 ------- .../sql/execution/SQLViewTestSuite.scala | 64 ++++- .../command/ShowCreateTableParserSuite.scala | 37 +++ .../command/ShowCreateTableSuiteBase.scala | 194 +++++++++++++ .../command/v1/ShowCreateTableSuite.scala | 141 +++++++++ .../command/v2/ShowCreateTableSuite.scala | 139 +++++++++ .../sql/hive/execution/HiveSQLViewSuite.scala | 41 ++- .../command/ShowCreateTableSuite.scala} | 249 ++++++---------- 14 files changed, 709 insertions(+), 555 deletions(-) delete mode 100644 sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowCreateTableParserSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowCreateTableSuiteBase.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala rename sql/hive/src/test/scala/org/apache/spark/sql/hive/{HiveShowCreateTableSuite.scala => execution/command/ShowCreateTableSuite.scala} (63%) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala index 04309297bb4f..56fdffdd82bc 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala @@ -1839,19 +1839,6 @@ class DDLParserSuite extends AnalysisTest { Some(Map("ds" -> "2017-06-10")))) } - test("SHOW CREATE table") { - comparePlans( - parsePlan("SHOW CREATE TABLE a.b.c"), - ShowCreateTable( - UnresolvedTableOrView(Seq("a", "b", "c"), "SHOW CREATE TABLE", allowTempView = false))) - - comparePlans( - parsePlan("SHOW CREATE TABLE a.b.c AS SERDE"), - ShowCreateTable( - UnresolvedTableOrView(Seq("a", "b", "c"), "SHOW CREATE TABLE", allowTempView = false), - asSerde = true)) - } - test("CACHE TABLE") { comparePlans( parsePlan("CACHE TABLE a.b.c"), diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala index 761a0d508e87..b989224d4e0f 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala @@ -1247,7 +1247,7 @@ case class ShowCreateTableAsSerdeCommand( s"Unknown table type is found at showCreateHiveTable: $t") } - builder ++= s"CREATE$tableTypeString ${table.quotedString}" + builder ++= s"CREATE$tableTypeString ${table.quotedString} " if (metadata.tableType == VIEW) { showCreateView(metadata, builder) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateTableExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateTableExec.scala index 5eaa16961886..f21b9a5095a3 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateTableExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateTableExec.scala @@ -74,7 +74,7 @@ case class ShowCreateTableExec( val props = tableOptions.toSeq.sortBy(_._1).map { case (key, value) => s"'${escapeSingleQuotedString(key)}' = '${escapeSingleQuotedString(value)}'" } - builder ++= "OPTIONS" + builder ++= "OPTIONS " builder ++= concatByMultiLines(props) } } diff --git a/sql/core/src/test/resources/sql-tests/results/group-by.sql.out b/sql/core/src/test/resources/sql-tests/results/group-by.sql.out index dce5fd02ba71..cd0fa486cdb6 100644 --- a/sql/core/src/test/resources/sql-tests/results/group-by.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/group-by.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 74 +-- Number of queries: 76 -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out b/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out index e7399e45c357..49c27a2229c5 100644 --- a/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/show-create-table.sql.out @@ -295,7 +295,7 @@ SHOW CREATE TABLE view_SPARK_30302 AS SERDE -- !query schema struct -- !query output -CREATE VIEW `default`.`view_SPARK_30302`( +CREATE VIEW `default`.`view_SPARK_30302` ( `aaa`, `bbb`) AS SELECT a, b FROM tbl @@ -335,7 +335,7 @@ SHOW CREATE TABLE view_SPARK_30302 AS SERDE -- !query schema struct -- !query output -CREATE VIEW `default`.`view_SPARK_30302`( +CREATE VIEW `default`.`view_SPARK_30302` ( `aaa` COMMENT 'comment with \'quoted text\' for aaa', `bbb`) COMMENT 'This is a comment with \'quoted text\' for view' @@ -377,7 +377,7 @@ SHOW CREATE TABLE view_SPARK_30302 AS SERDE -- !query schema struct -- !query output -CREATE VIEW `default`.`view_SPARK_30302`( +CREATE VIEW `default`.`view_SPARK_30302` ( `aaa`, `bbb`) TBLPROPERTIES ( diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala deleted file mode 100644 index 13983120955f..000000000000 --- a/sql/core/src/test/scala/org/apache/spark/sql/ShowCreateTableSuite.scala +++ /dev/null @@ -1,267 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql - -import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.catalog.CatalogTable -import org.apache.spark.sql.sources.SimpleInsertSource -import org.apache.spark.sql.test.{SharedSparkSession, SQLTestUtils} -import org.apache.spark.util.Utils - -class SimpleShowCreateTableSuite extends ShowCreateTableSuite with SharedSparkSession - -abstract class ShowCreateTableSuite extends QueryTest with SQLTestUtils { - import testImplicits._ - - test("data source table with user specified schema") { - withTable("ddl_test") { - val jsonFilePath = Utils.getSparkClassLoader.getResource("sample.json").getFile - - sql( - s"""CREATE TABLE ddl_test ( - | a STRING, - | b STRING, - | `extra col` ARRAY, - | `` STRUCT> - |) - |USING json - |OPTIONS ( - | PATH '$jsonFilePath' - |) - """.stripMargin - ) - - checkCreateTable("ddl_test") - } - } - - test("data source table CTAS") { - withTable("ddl_test") { - sql( - s"""CREATE TABLE ddl_test - |USING json - |AS SELECT 1 AS a, "foo" AS b - """.stripMargin - ) - - checkCreateTable("ddl_test") - } - } - - test("partitioned data source table") { - withTable("ddl_test") { - sql( - s"""CREATE TABLE ddl_test - |USING json - |PARTITIONED BY (b) - |AS SELECT 1 AS a, "foo" AS b - """.stripMargin - ) - - checkCreateTable("ddl_test") - } - } - - test("bucketed data source table") { - withTable("ddl_test") { - sql( - s"""CREATE TABLE ddl_test - |USING json - |CLUSTERED BY (a) SORTED BY (b) INTO 2 BUCKETS - |AS SELECT 1 AS a, "foo" AS b - """.stripMargin - ) - - checkCreateTable("ddl_test") - } - } - - test("partitioned bucketed data source table") { - withTable("ddl_test") { - sql( - s"""CREATE TABLE ddl_test - |USING json - |PARTITIONED BY (c) - |CLUSTERED BY (a) SORTED BY (b) INTO 2 BUCKETS - |AS SELECT 1 AS a, "foo" AS b, 2.5 AS c - """.stripMargin - ) - - checkCreateTable("ddl_test") - } - } - - test("data source table with a comment") { - withTable("ddl_test") { - sql( - s"""CREATE TABLE ddl_test - |USING json - |COMMENT 'This is a comment' - |AS SELECT 1 AS a, "foo" AS b, 2.5 AS c - """.stripMargin - ) - - checkCreateTable("ddl_test") - } - } - - test("data source table with table properties") { - withTable("ddl_test") { - sql( - s"""CREATE TABLE ddl_test - |USING json - |TBLPROPERTIES ('a' = '1') - |AS SELECT 1 AS a, "foo" AS b, 2.5 AS c - """.stripMargin - ) - - checkCreateTable("ddl_test") - } - } - - test("data source table using Dataset API") { - withTable("ddl_test") { - spark - .range(3) - .select('id as 'a, 'id as 'b, 'id as 'c, 'id as 'd, 'id as 'e) - .write - .mode("overwrite") - .partitionBy("a", "b") - .bucketBy(2, "c", "d") - .saveAsTable("ddl_test") - - checkCreateTable("ddl_test") - } - } - - test("temp view") { - val viewName = "spark_28383" - withTempView(viewName) { - sql(s"CREATE TEMPORARY VIEW $viewName AS SELECT 1 AS a") - val ex = intercept[AnalysisException] { - sql(s"SHOW CREATE TABLE $viewName") - } - assert(ex.getMessage.contains( - s"$viewName is a temp view. 'SHOW CREATE TABLE' expects a table or permanent view.")) - } - - withGlobalTempView(viewName) { - sql(s"CREATE GLOBAL TEMPORARY VIEW $viewName AS SELECT 1 AS a") - val globalTempViewDb = spark.sessionState.catalog.globalTempViewManager.database - val ex = intercept[AnalysisException] { - sql(s"SHOW CREATE TABLE $globalTempViewDb.$viewName") - } - assert(ex.getMessage.contains( - s"$globalTempViewDb.$viewName is a temp view. " + - "'SHOW CREATE TABLE' expects a table or permanent view.")) - } - } - - test("SPARK-24911: keep quotes for nested fields") { - withTable("t1") { - val createTable = "CREATE TABLE `t1` (`a` STRUCT<`b`: STRING>)" - sql(s"$createTable USING json") - val shownDDL = getShowDDL("SHOW CREATE TABLE t1") - assert(shownDDL == "CREATE TABLE `default`.`t1` ( `a` STRUCT<`b`: STRING>) USING json") - - checkCreateTable("t1") - } - } - - test("SPARK-36012: Add NULL flag when SHOW CREATE TABLE") { - val t = "SPARK_36012" - withTable(t) { - sql( - s""" - |CREATE TABLE $t ( - | a bigint NOT NULL, - | b bigint - |) - |USING ${classOf[SimpleInsertSource].getName} - """.stripMargin) - val showDDL = getShowDDL(s"SHOW CREATE TABLE $t") - assert(showDDL == s"CREATE TABLE `default`.`$t` ( `a` BIGINT NOT NULL," + - s" `b` BIGINT) USING ${classOf[SimpleInsertSource].getName}") - } - } - - test("SPARK-37494: Unify v1 and v2 option output") { - withTable("ddl_test") { - sql( - s"""CREATE TABLE ddl_test ( - | a STRING - |) - |USING json - |TBLPROPERTIES ( - | 'b' = '1', - | 'a' = '2') - |OPTIONS ( - | k4 'v4', - | `k3` 'v3', - | 'k5' 'v5', - | 'k1' = 'v1', - | k2 = 'v2' - |) - """.stripMargin - ) - val expected = "CREATE TABLE `default`.`ddl_test` ( `a` STRING) USING json" + - " OPTIONS ( 'k1' = 'v1', 'k2' = 'v2', 'k3' = 'v3', 'k4' = 'v4', 'k5' = 'v5')" + - " TBLPROPERTIES ( 'a' = '2', 'b' = '1')" - assert(getShowDDL("SHOW CREATE TABLE ddl_test") == expected) - } - } - - protected def getShowDDL(showCreateTableSql: String): String = { - sql(showCreateTableSql).head().getString(0).split("\n").map(_.trim).mkString(" ") - } - - protected def checkCreateTable(table: String, serde: Boolean = false): Unit = { - checkCreateTableOrView(TableIdentifier(table, Some("default")), "TABLE", serde) - } - - protected def checkCreateView(table: String, serde: Boolean = false): Unit = { - checkCreateTableOrView(TableIdentifier(table, Some("default")), "VIEW", serde) - } - - protected def checkCreateTableOrView( - table: TableIdentifier, - checkType: String, - serde: Boolean): Unit = { - val db = table.database.getOrElse("default") - val expected = spark.sharedState.externalCatalog.getTable(db, table.table) - val shownDDL = if (serde) { - sql(s"SHOW CREATE TABLE ${table.quotedString} AS SERDE").head().getString(0) - } else { - sql(s"SHOW CREATE TABLE ${table.quotedString}").head().getString(0) - } - - sql(s"DROP $checkType ${table.quotedString}") - - try { - sql(shownDDL) - val actual = spark.sharedState.externalCatalog.getTable(db, table.table) - checkCatalogTables(expected, actual) - } finally { - sql(s"DROP $checkType IF EXISTS ${table.table}") - } - } - - protected def checkCatalogTables(expected: CatalogTable, actual: CatalogTable): Unit = { - assert(CatalogTable.normalize(actual) == CatalogTable.normalize(expected)) - } -} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala index 1fcfd94263c5..b1ec8d20a91e 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala @@ -1854,109 +1854,6 @@ class DataSourceV2SQLSuite } } - test("SPARK-33898: SHOW CREATE TABLE AS SERDE") { - val t = "testcat.ns1.ns2.tbl" - withTable(t) { - spark.sql(s"CREATE TABLE $t (id bigint, data string) USING foo") - val e = intercept[AnalysisException] { - sql(s"SHOW CREATE TABLE $t AS SERDE") - } - assert(e.message.contains(s"SHOW CREATE TABLE AS SERDE is not supported for v2 tables.")) - } - } - - test("SPARK-33898: SHOW CREATE TABLE") { - val t = "testcat.ns1.ns2.tbl" - withTable(t) { - sql( - s""" - |CREATE TABLE $t ( - | a bigint NOT NULL, - | b bigint, - | c bigint, - | `extra col` ARRAY, - | `` STRUCT> - |) - |USING foo - |OPTIONS ( - | from = 0, - | to = 1, - | via = 2) - |COMMENT 'This is a comment' - |TBLPROPERTIES ('prop1' = '1', 'prop2' = '2', 'prop3' = 3, 'prop4' = 4) - |PARTITIONED BY (a) - |LOCATION 'file:/tmp' - """.stripMargin) - val showDDL = getShowCreateDDL(s"SHOW CREATE TABLE $t") - assert(showDDL === Array( - "CREATE TABLE testcat.ns1.ns2.tbl (", - "`a` BIGINT NOT NULL,", - "`b` BIGINT,", - "`c` BIGINT,", - "`extra col` ARRAY,", - "`` STRUCT<`x`: INT, `y`: ARRAY>)", - "USING foo", - "OPTIONS(", - "'from' = '0',", - "'to' = '1',", - "'via' = '2')", - "PARTITIONED BY (a)", - "COMMENT 'This is a comment'", - "LOCATION 'file:/tmp'", - "TBLPROPERTIES (", - "'prop1' = '1',", - "'prop2' = '2',", - "'prop3' = '3',", - "'prop4' = '4')" - )) - } - } - - test("SPARK-33898: SHOW CREATE TABLE WITH AS SELECT") { - val t = "testcat.ns1.ns2.tbl" - withTable(t) { - sql( - s""" - |CREATE TABLE $t - |USING foo - |AS SELECT 1 AS a, "foo" AS b - """.stripMargin) - val showDDL = getShowCreateDDL(s"SHOW CREATE TABLE $t") - assert(showDDL === Array( - "CREATE TABLE testcat.ns1.ns2.tbl (", - "`a` INT,", - "`b` STRING)", - "USING foo" - )) - } - } - - test("SPARK-33898: SHOW CREATE TABLE PARTITIONED BY Transforms") { - val t = "testcat.ns1.ns2.tbl" - withTable(t) { - sql( - s""" - |CREATE TABLE $t (a INT, b STRING, ts TIMESTAMP) USING foo - |PARTITIONED BY ( - | a, - | bucket(16, b), - | years(ts), - | months(ts), - | days(ts), - | hours(ts)) - """.stripMargin) - val showDDL = getShowCreateDDL(s"SHOW CREATE TABLE $t") - assert(showDDL === Array( - "CREATE TABLE testcat.ns1.ns2.tbl (", - "`a` INT,", - "`b` STRING,", - "`ts` TIMESTAMP)", - "USING foo", - "PARTITIONED BY (a, bucket(16, b), years(ts), months(ts), days(ts), hours(ts))" - )) - } - } - test("CACHE/UNCACHE TABLE") { val t = "testcat.ns1.ns2.tbl" withTable(t) { @@ -2868,10 +2765,6 @@ class DataSourceV2SQLSuite assert(ex.getErrorClass == expectedErrorClass) assert(ex.messageParameters.sameElements(expectedErrorMessageParameters)) } - - private def getShowCreateDDL(showCreateTableSql: String): Array[String] = { - sql(showCreateTableSql).head().getString(0).split("\n").map(_.trim) - } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala index 730299c2f2c9..433264951ace 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala @@ -45,10 +45,12 @@ abstract class SQLViewTestSuite extends QueryTest with SQLTestUtils { viewName: String, sqlText: String, columnNames: Seq[String] = Seq.empty, + others: Seq[String] = Seq.empty, replace: Boolean = false): String = { val replaceString = if (replace) "OR REPLACE" else "" val columnString = if (columnNames.nonEmpty) columnNames.mkString("(", ",", ")") else "" - sql(s"CREATE $replaceString $viewTypeString $viewName $columnString AS $sqlText") + val othersString = if (others.nonEmpty) others.mkString(" ") else "" + sql(s"CREATE $replaceString $viewTypeString $viewName $columnString $othersString AS $sqlText") formattedViewName(viewName) } @@ -421,6 +423,18 @@ abstract class TempViewTestSuite extends SQLViewTestSuite { } } } + + test("show create table does not support temp view") { + val viewName = "spark_28383" + withView(viewName) { + createView(viewName, "SELECT 1 AS a") + val ex = intercept[AnalysisException] { + sql(s"SHOW CREATE TABLE ${formattedViewName(viewName)}") + } + assert(ex.getMessage.contains( + s"$viewName is a temp view. 'SHOW CREATE TABLE' expects a table or permanent view.")) + } + } } class LocalTempViewTestSuite extends TempViewTestSuite with SharedSparkSession { @@ -591,4 +605,52 @@ class PersistedViewTestSuite extends SQLViewTestSuite with SharedSparkSession { s" The view ${table.qualifiedName} may have been tampered with")) } } + + test("show create table for persisted simple view") { + val viewName = "v1" + Seq(true, false).foreach { serde => + withView(viewName) { + createView(viewName, "SELECT 1 AS a") + val expected = "CREATE VIEW `default`.`v1` ( `a`) AS SELECT 1 AS a" + assert(getShowCreateDDL(formattedViewName(viewName), serde) == expected) + } + } + } + + test("show create table for persisted view with output columns") { + val viewName = "v1" + Seq(true, false).foreach { serde => + withView(viewName) { + createView(viewName, "SELECT 1 AS a, 2 AS b", Seq("a", "b COMMENT 'b column'")) + val expected = "CREATE VIEW `default`.`v1` ( `a`, `b` COMMENT 'b column')" + + " AS SELECT 1 AS a, 2 AS b" + assert(getShowCreateDDL(formattedViewName(viewName), serde) == expected) + } + } + } + + test("show create table for persisted simple view with table comment and properties") { + val viewName = "v1" + Seq(true, false).foreach { serde => + withView(viewName) { + createView(viewName, "SELECT 1 AS c1, '2' AS c2", Seq("c1 COMMENT 'bla'", "c2"), + Seq("COMMENT 'table comment'", "TBLPROPERTIES ( 'prop1' = 'value1', 'prop2' = 'value2')")) + + val expected = "CREATE VIEW `default`.`v1` ( `c1` COMMENT 'bla', `c2`)" + + " COMMENT 'table comment'" + + " TBLPROPERTIES ( 'prop1' = 'value1', 'prop2' = 'value2')" + + " AS SELECT 1 AS c1, '2' AS c2" + assert(getShowCreateDDL(formattedViewName(viewName), serde) == expected) + } + } + } + + def getShowCreateDDL(view: String, serde: Boolean = false): String = { + val result = if (serde) { + sql(s"SHOW CREATE TABLE $view AS SERDE") + } else { + sql(s"SHOW CREATE TABLE $view") + } + result.head().getString(0).split("\n").map(_.trim).mkString(" ") + } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowCreateTableParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowCreateTableParserSuite.scala new file mode 100644 index 000000000000..ab7c6e4dec56 --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowCreateTableParserSuite.scala @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.command + +import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedTableOrView} +import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan +import org.apache.spark.sql.catalyst.plans.logical.ShowCreateTable + +class ShowCreateTableParserSuite extends AnalysisTest { + test("show create table") { + comparePlans( + parsePlan("SHOW CREATE TABLE a.b.c"), + ShowCreateTable( + UnresolvedTableOrView(Seq("a", "b", "c"), "SHOW CREATE TABLE", allowTempView = false))) + + comparePlans( + parsePlan("SHOW CREATE TABLE a.b.c AS SERDE"), + ShowCreateTable( + UnresolvedTableOrView(Seq("a", "b", "c"), "SHOW CREATE TABLE", allowTempView = false), + asSerde = true)) + } +} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowCreateTableSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowCreateTableSuiteBase.scala new file mode 100644 index 000000000000..53cdec0d2b6c --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowCreateTableSuiteBase.scala @@ -0,0 +1,194 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.command + +import org.apache.spark.sql.QueryTest +import org.apache.spark.sql.sources.SimpleInsertSource +import org.apache.spark.util.Utils + +/** + * This base suite contains unified tests for the `SHOW CREATE TABLE` command that check V1 and V2 + * table catalogs. The tests that cannot run for all supported catalogs are located in more + * specific test suites: + * + * - V2 table catalog tests: `org.apache.spark.sql.execution.command.v2.ShowCreateTableSuite` + * - V1 table catalog tests: + * `org.apache.spark.sql.execution.command.v1.ShowCreateTableSuiteBase` + * - V1 In-Memory catalog: `org.apache.spark.sql.execution.command.v1.ShowCreateTableSuite` + * - V1 Hive External catalog: +* `org.apache.spark.sql.hive.execution.command.ShowCreateTableSuite` + */ +trait ShowCreateTableSuiteBase extends QueryTest with DDLCommandTestUtils { + override val command = "SHOW CREATE TABLE" + protected def ns: String = "ns1" + protected def table: String = "tbl" + protected def fullName: String + + test("SPARK-36012: add null flag when show create table") { + withNamespaceAndTable(ns, table) { t => + sql( + s""" + |CREATE TABLE $t ( + | a bigint NOT NULL, + | b bigint + |) + |USING ${classOf[SimpleInsertSource].getName} + """.stripMargin) + val showDDL = getShowCreateDDL(t) + assert(showDDL(0) == s"CREATE TABLE $fullName (") + assert(showDDL(1) == "`a` BIGINT NOT NULL,") + assert(showDDL(2) == "`b` BIGINT)") + assert(showDDL(3) == s"USING ${classOf[SimpleInsertSource].getName}") + } + } + + test("data source table with user specified schema") { + withNamespaceAndTable(ns, table) { t => + val jsonFilePath = Utils.getSparkClassLoader.getResource("sample.json").getFile + sql( + s"""CREATE TABLE $t ( + | a STRING, + | b STRING, + | `extra col` ARRAY, + | `` STRUCT> + |) + |USING json + |OPTIONS ( + | PATH '$jsonFilePath' + |) + """.stripMargin + ) + val showDDL = getShowCreateDDL(t) + assert(showDDL(0) == s"CREATE TABLE $fullName (") + assert(showDDL(1) == "`a` STRING,") + assert(showDDL(2) == "`b` STRING,") + assert(showDDL(3) == "`extra col` ARRAY,") + assert(showDDL(4) == "`` STRUCT<`x`: INT, `y`: ARRAY>)") + assert(showDDL(5) == "USING json") + assert(showDDL(6).startsWith("LOCATION 'file:") && showDDL(6).endsWith("sample.json'")) + } + } + + test("SPARK-24911: keep quotes for nested fields") { + withNamespaceAndTable(ns, table) { t => + sql( + s""" + |CREATE TABLE $t ( + | `a` STRUCT<`b`: STRING> + |) + |USING json + """.stripMargin) + val showDDL = getShowCreateDDL(t) + assert(showDDL(0) == s"CREATE TABLE $fullName (") + assert(showDDL(1) == "`a` STRUCT<`b`: STRING>)") + assert(showDDL(2) == "USING json") + } + } + + test("SPARK-37494: Unify v1 and v2 option output") { + withNamespaceAndTable(ns, table) { t => + sql( + s"""CREATE TABLE $t ( + | a STRING + |) + |USING json + |TBLPROPERTIES ( + | 'b' = '1', + | 'a' = '2') + |OPTIONS ( + | k4 'v4', + | `k3` 'v3', + | 'k5' 'v5', + | 'k1' = 'v1', + | k2 = 'v2' + |) + """.stripMargin + ) + val expected = s"CREATE TABLE $fullName ( `a` STRING) USING json" + + " OPTIONS ( 'k1' = 'v1', 'k2' = 'v2', 'k3' = 'v3', 'k4' = 'v4', 'k5' = 'v5')" + + " TBLPROPERTIES ( 'a' = '2', 'b' = '1')" + assert(getShowCreateDDL(t).mkString(" ") == expected) + } + } + + test("data source table CTAS") { + withNamespaceAndTable(ns, table) { t => + sql( + s"""CREATE TABLE $t + |USING json + |AS SELECT 1 AS a, "foo" AS b + """.stripMargin + ) + val expected = s"CREATE TABLE $fullName ( `a` INT, `b` STRING) USING json" + assert(getShowCreateDDL(t).mkString(" ") == expected) + } + } + + test("partitioned data source table") { + withNamespaceAndTable(ns, table) { t => + sql( + s"""CREATE TABLE $t + |USING json + |PARTITIONED BY (b) + |AS SELECT 1 AS a, "foo" AS b + """.stripMargin + ) + val expected = s"CREATE TABLE $fullName ( `a` INT, `b` STRING) USING json PARTITIONED BY (b)" + assert(getShowCreateDDL(t).mkString(" ") == expected) + } + } + + test("data source table with a comment") { + withNamespaceAndTable(ns, table) { t => + sql( + s"""CREATE TABLE $t + |USING json + |COMMENT 'This is a comment' + |AS SELECT 1 AS a, "foo" AS b, 2.5 AS c + """.stripMargin + ) + val expected = s"CREATE TABLE $fullName ( `a` INT, `b` STRING, `c` DECIMAL(2,1)) USING json" + + s" COMMENT 'This is a comment'" + assert(getShowCreateDDL(t).mkString(" ") == expected) + } + } + + test("data source table with table properties") { + withNamespaceAndTable(ns, table) { t => + sql( + s"""CREATE TABLE $t + |USING json + |TBLPROPERTIES ('a' = '1') + |AS SELECT 1 AS a, "foo" AS b, 2.5 AS c + """.stripMargin + ) + val expected = s"CREATE TABLE $fullName ( `a` INT, `b` STRING, `c` DECIMAL(2,1)) USING json" + + s" TBLPROPERTIES ( 'a' = '1')" + assert(getShowCreateDDL(t).mkString(" ") == expected) + } + } + + def getShowCreateDDL(table: String, serde: Boolean = false): Array[String] = { + val result = if (serde) { + sql(s"SHOW CREATE TABLE $table AS SERDE") + } else { + sql(s"SHOW CREATE TABLE $table") + } + result.head().getString(0).split("\n").map(_.trim) + } +} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala new file mode 100644 index 000000000000..208ed4c08afc --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowCreateTableSuite.scala @@ -0,0 +1,141 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.command.v1 + +import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.execution.command + +/** + * This base suite contains unified tests for the `SHOW CREATE TABLE` command that checks V1 + * table catalogs. The tests that cannot run for all V1 catalogs are located in more + * specific test suites: + * + * - V1 In-Memory catalog: `org.apache.spark.sql.execution.command.v1.ShowCreateTableSuite` + * - V1 Hive External catalog: + * `org.apache.spark.sql.hive.execution.command.ShowCreateTableSuite` + */ +trait ShowCreateTableSuiteBase extends command.ShowCreateTableSuiteBase + with command.TestsV1AndV2Commands { + override def fullName: String = s"`$ns`.`$table`" + + test("show create table[simple]") { + // todo After SPARK-37517 unify the testcase both v1 and v2 + withNamespaceAndTable(ns, table) { t => + sql( + s""" + |CREATE TABLE $t ( + | a bigint NOT NULL, + | b bigint, + | c bigint, + | `extraCol` ARRAY, + | `` STRUCT> + |) + |using parquet + |OPTIONS ( + | from = 0, + | to = 1, + | via = 2) + |COMMENT 'This is a comment' + |TBLPROPERTIES ('prop1' = '1', 'prop2' = '2', 'prop3' = 3, 'prop4' = 4) + |PARTITIONED BY (a) + |LOCATION '/tmp' + """.stripMargin) + val showDDL = getShowCreateDDL(t) + assert(showDDL === Array( + s"CREATE TABLE $fullName (", + "`b` BIGINT,", + "`c` BIGINT,", + "`extraCol` ARRAY,", + "`` STRUCT<`x`: INT, `y`: ARRAY>,", + "`a` BIGINT NOT NULL)", + "USING parquet", + "OPTIONS (", + "'from' = '0',", + "'to' = '1',", + "'via' = '2')", + "PARTITIONED BY (a)", + "COMMENT 'This is a comment'", + "LOCATION 'file:/tmp'", + "TBLPROPERTIES (", + "'prop1' = '1',", + "'prop2' = '2',", + "'prop3' = '3',", + "'prop4' = '4')" + )) + } + } + + test("bucketed data source table") { + withNamespaceAndTable(ns, table) { t => + sql( + s"""CREATE TABLE $t + |USING json + |CLUSTERED BY (a) SORTED BY (b) INTO 2 BUCKETS + |AS SELECT 1 AS a, "foo" AS b + """.stripMargin + ) + val expected = s"CREATE TABLE $fullName ( `a` INT, `b` STRING) USING json" + + s" CLUSTERED BY (a) SORTED BY (b) INTO 2 BUCKETS" + assert(getShowCreateDDL(t).mkString(" ") == expected) + } + } + + test("partitioned bucketed data source table") { + withNamespaceAndTable(ns, table) { t => + sql( + s"""CREATE TABLE $t + |USING json + |PARTITIONED BY (c) + |CLUSTERED BY (a) SORTED BY (b) INTO 2 BUCKETS + |AS SELECT 1 AS a, "foo" AS b, 2.5 AS c + """.stripMargin + ) + val expected = s"CREATE TABLE $fullName ( `a` INT, `b` STRING, `c` DECIMAL(2,1)) USING json" + + s" PARTITIONED BY (c) CLUSTERED BY (a) SORTED BY (b) INTO 2 BUCKETS" + assert(getShowCreateDDL(t).mkString(" ") == expected) + } + } + + test("show create table as serde can't work on data source table") { + withNamespaceAndTable(ns, table) { t => + sql( + s""" + |CREATE TABLE $t ( + | c1 STRING COMMENT 'bla', + | c2 STRING + |) + |USING orc + """.stripMargin + ) + + val cause = intercept[AnalysisException] { + getShowCreateDDL(t, true) + } + + assert(cause.getMessage.contains("Use `SHOW CREATE TABLE` without `AS SERDE` instead")) + } + } +} + +/** + * The class contains tests for the `SHOW CREATE TABLE` command to check V1 In-Memory + * table catalog. + */ +class ShowCreateTableSuite extends ShowCreateTableSuiteBase with CommandSuiteBase { + override def commandVersion: String = super[ShowCreateTableSuiteBase].commandVersion +} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala new file mode 100644 index 000000000000..35b196fe0d8b --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowCreateTableSuite.scala @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.command.v2 + +import org.apache.spark.sql.AnalysisException +import org.apache.spark.sql.execution.command + +/** + * The class contains tests for the `SHOW CREATE TABLE` command to check V2 table catalogs. + */ +class ShowCreateTableSuite extends command.ShowCreateTableSuiteBase with CommandSuiteBase { + override def fullName: String = s"$catalog.$ns.$table" + + test("SPARK-33898: show create table as serde") { + withNamespaceAndTable(ns, table) { t => + spark.sql(s"CREATE TABLE $t (id bigint, data string) $defaultUsing") + val e = intercept[AnalysisException] { + sql(s"SHOW CREATE TABLE $t AS SERDE") + } + assert(e.message.contains(s"SHOW CREATE TABLE AS SERDE is not supported for v2 tables.")) + } + } + + test("SPARK-33898: show create table[CTAS]") { + // does not work with hive, also different order between v2 with v1/hive + withNamespaceAndTable(ns, table) { t => + sql( + s"""CREATE TABLE $t + |$defaultUsing + |PARTITIONED BY (a) + |COMMENT 'This is a comment' + |TBLPROPERTIES ('a' = '1') + |AS SELECT 1 AS a, "foo" AS b + """.stripMargin + ) + val showDDL = getShowCreateDDL(t, false) + assert(showDDL === Array( + s"CREATE TABLE $t (", + "`a` INT,", + "`b` STRING)", + defaultUsing, + "PARTITIONED BY (a)", + "COMMENT 'This is a comment'", + "TBLPROPERTIES (", + "'a' = '1')" + )) + } + } + + test("SPARK-33898: show create table[simple]") { + // TODO: After SPARK-37517, we can move the test case to base to test for v2/v1/hive + val db = "ns1" + val table = "tbl" + withNamespaceAndTable(db, table) { t => + sql( + s""" + |CREATE TABLE $t ( + | a bigint NOT NULL, + | b bigint, + | c bigint, + | `extraCol` ARRAY, + | `` STRUCT> + |) + |$defaultUsing + |OPTIONS ( + | from = 0, + | to = 1, + | via = 2) + |COMMENT 'This is a comment' + |TBLPROPERTIES ('prop1' = '1', 'prop2' = '2', 'prop3' = 3, 'prop4' = 4) + |PARTITIONED BY (a) + |LOCATION '/tmp' + """.stripMargin) + val showDDL = getShowCreateDDL(t, false) + assert(showDDL === Array( + s"CREATE TABLE $t (", + "`a` BIGINT NOT NULL,", + "`b` BIGINT,", + "`c` BIGINT,", + "`extraCol` ARRAY,", + "`` STRUCT<`x`: INT, `y`: ARRAY>)", + defaultUsing, + "OPTIONS (", + "'from' = '0',", + "'to' = '1',", + "'via' = '2')", + "PARTITIONED BY (a)", + "COMMENT 'This is a comment'", + "LOCATION 'file:/tmp'", + "TBLPROPERTIES (", + "'prop1' = '1',", + "'prop2' = '2',", + "'prop3' = '3',", + "'prop4' = '4')" + )) + } + } + + test("SPARK-33898: show create table[multi-partition]") { + withNamespaceAndTable(ns, table) { t => + sql( + s""" + |CREATE TABLE $t (a INT, b STRING, ts TIMESTAMP) $defaultUsing + |PARTITIONED BY ( + | a, + | bucket(16, b), + | years(ts), + | months(ts), + | days(ts), + | hours(ts)) + """.stripMargin) + // V1 transforms cannot be converted to partition columns: bucket,year,...) + val showDDL = getShowCreateDDL(t, false) + assert(showDDL === Array( + s"CREATE TABLE $t (", + "`a` INT,", + "`b` STRING,", + "`ts` TIMESTAMP)", + defaultUsing, + "PARTITIONED BY (a, bucket(16, b), years(ts), months(ts), days(ts), hours(ts))" + )) + } + } +} diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSQLViewSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSQLViewSuite.scala index 189a5c5768f6..cbf5e640db46 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSQLViewSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveSQLViewSuite.scala @@ -21,7 +21,7 @@ import org.apache.spark.sql.{AnalysisException, Row} import org.apache.spark.sql.catalyst.TableIdentifier import org.apache.spark.sql.catalyst.catalog.{CatalogStorageFormat, CatalogTable, CatalogTableType, HiveTableRelation} import org.apache.spark.sql.execution.SQLViewSuite -import org.apache.spark.sql.hive.HiveUtils +import org.apache.spark.sql.hive.{HiveExternalCatalog, HiveUtils} import org.apache.spark.sql.hive.test.TestHiveSingleton import org.apache.spark.sql.types.{NullType, StructType} import org.apache.spark.tags.SlowHiveTest @@ -181,4 +181,43 @@ class HiveSQLViewSuite extends SQLViewSuite with TestHiveSingleton { } } } + + test("hive partitioned view is not supported") { + withTable("test") { + withView("v1") { + sql( + s""" + |CREATE TABLE test (c1 INT, c2 STRING) + |PARTITIONED BY ( + | p1 BIGINT COMMENT 'bla', + | p2 STRING ) + """.stripMargin) + + createRawHiveTable( + s""" + |CREATE VIEW v1 + |PARTITIONED ON (p1, p2) + |AS SELECT * from test + """.stripMargin + ) + + val cause = intercept[AnalysisException] { + sql("SHOW CREATE TABLE v1") + } + + assert(cause.getMessage.contains(" - partitioned view")) + + val causeForSpark = intercept[AnalysisException] { + sql("SHOW CREATE TABLE v1 AS SERDE") + } + + assert(causeForSpark.getMessage.contains(" - partitioned view")) + } + } + } + + private def createRawHiveTable(ddl: String): Unit = { + hiveContext.sharedState.externalCatalog.unwrapped.asInstanceOf[HiveExternalCatalog] + .client.runSqlHive(ddl) + } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowCreateTableSuite.scala similarity index 63% rename from sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala rename to sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowCreateTableSuite.scala index e3a1034ad4f1..58145b03fd3c 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveShowCreateTableSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowCreateTableSuite.scala @@ -15,77 +15,30 @@ * limitations under the License. */ -package org.apache.spark.sql.hive +package org.apache.spark.sql.hive.execution.command -import org.apache.spark.sql.{AnalysisException, ShowCreateTableSuite} +import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.TableIdentifier -import org.apache.spark.sql.catalyst.catalog.CatalogTable -import org.apache.spark.sql.hive.test.TestHiveSingleton -import org.apache.spark.sql.internal.{HiveSerDe, SQLConf} - -class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSingleton { - - private var origCreateHiveTableConfig = false - - protected override def beforeAll(): Unit = { - super.beforeAll() - origCreateHiveTableConfig = - spark.conf.get(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT) - spark.conf.set(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT.key, true) - } - - protected override def afterAll(): Unit = { - spark.conf.set( - SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT.key, - origCreateHiveTableConfig) - super.afterAll() - } - - test("view") { - Seq(true, false).foreach { serde => - withView("v1") { - sql("CREATE VIEW v1 AS SELECT 1 AS a") - checkCreateView("v1", serde) - } - } - } - - test("view with output columns") { - Seq(true, false).foreach { serde => - withView("v1") { - sql("CREATE VIEW v1 (a, b COMMENT 'b column') AS SELECT 1 AS a, 2 AS b") - checkCreateView("v1", serde) - } - } - } - - test("view with table comment and properties") { - Seq(true, false).foreach { serde => - withView("v1") { - sql( - s""" - |CREATE VIEW v1 ( - | c1 COMMENT 'bla', - | c2 - |) - |COMMENT 'table comment' - |TBLPROPERTIES ( - | 'prop1' = 'value1', - | 'prop2' = 'value2' - |) - |AS SELECT 1 AS c1, '2' AS c2 - """.stripMargin - ) +import org.apache.spark.sql.catalyst.catalog.{CatalogTable, CatalogUtils} +import org.apache.spark.sql.catalyst.util.escapeSingleQuotedString +import org.apache.spark.sql.execution.command.v1 +import org.apache.spark.sql.internal.HiveSerDe + +/** + * The class contains tests for the `SHOW CREATE TABLE` command to check V1 Hive external + * table catalog. + */ +class ShowCreateTableSuite extends v1.ShowCreateTableSuiteBase with CommandSuiteBase { + override def commandVersion: String = super[ShowCreateTableSuiteBase].commandVersion - checkCreateView("v1", serde) - } - } + override def getShowCreateDDL(table: String, serde: Boolean = false): Array[String] = { + super.getShowCreateDDL(table, serde).filter(!_.startsWith("'transient_lastDdlTime'")) } test("simple hive table") { - withTable("t1") { + withNamespaceAndTable(ns, table) { t => sql( - s"""CREATE TABLE t1 ( + s"""CREATE TABLE $t ( | c1 INT COMMENT 'bla', | c2 STRING |) @@ -95,16 +48,21 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet |) """.stripMargin ) - - checkCreateTable("t1", serde = true) + val expected = s"CREATE TABLE $fullName ( `c1` INT COMMENT 'bla', `c2` STRING)" + + " ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'" + + " WITH SERDEPROPERTIES ( 'serialization.format' = '1')" + + " STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'" + + " OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'" + + " TBLPROPERTIES ( 'prop1' = 'value1', 'prop2' = 'value2'," + assert(getShowCreateDDL(t, true).mkString(" ") == expected) } } test("simple external hive table") { withTempDir { dir => - withTable("t1") { + withNamespaceAndTable(ns, table) { t => sql( - s"""CREATE TABLE t1 ( + s"""CREATE TABLE $t ( | c1 INT COMMENT 'bla', | c2 STRING |) @@ -115,16 +73,23 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet |) """.stripMargin ) - - checkCreateTable("t1", serde = true) + val expected = s"CREATE EXTERNAL TABLE $fullName ( `c1` INT COMMENT 'bla', `c2` STRING)" + + s" ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'" + + s" WITH SERDEPROPERTIES ( 'serialization.format' = '1')" + + s" STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'" + + s" OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'" + + s" LOCATION" + + s" '${escapeSingleQuotedString(CatalogUtils.URIToString(dir.toURI)).dropRight(1)}'" + + s" TBLPROPERTIES ( 'prop1' = 'value1', 'prop2' = 'value2'," + assert(getShowCreateDDL(t, true).mkString(" ") == expected) } } } test("partitioned hive table") { - withTable("t1") { + withNamespaceAndTable(ns, table) { t => sql( - s"""CREATE TABLE t1 ( + s"""CREATE TABLE $t ( | c1 INT COMMENT 'bla', | c2 STRING |) @@ -135,15 +100,21 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet |) """.stripMargin ) - - checkCreateTable("t1", serde = true) + val expected = s"CREATE TABLE $fullName ( `c1` INT COMMENT 'bla', `c2` STRING)" + + " COMMENT 'bla' PARTITIONED BY (`p1` BIGINT COMMENT 'bla', `p2` STRING)" + + " ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'" + + " WITH SERDEPROPERTIES ( 'serialization.format' = '1')" + + " STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'" + + " OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'" + + " TBLPROPERTIES (" + assert(getShowCreateDDL(t, true).mkString(" ") == expected) } } test("hive table with explicit storage info") { - withTable("t1") { + withNamespaceAndTable(ns, table) { t => sql( - s"""CREATE TABLE t1 ( + s"""CREATE TABLE $t ( | c1 INT COMMENT 'bla', | c2 STRING |) @@ -153,30 +124,44 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet |NULL DEFINED AS 'NaN' """.stripMargin ) - - checkCreateTable("t1", serde = true) + val expected = s"CREATE TABLE $fullName ( `c1` INT COMMENT 'bla', `c2` STRING)" + + " ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'" + + " WITH SERDEPROPERTIES (" + + " 'colelction.delim' = '@'," + + " 'mapkey.delim' = '#'," + + " 'serialization.format' = ','," + + " 'field.delim' = ',')" + + " STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'" + + " OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'" + + " TBLPROPERTIES (" + assert(getShowCreateDDL(t, true).mkString(" ") == expected) } } test("hive table with STORED AS clause") { - withTable("t1") { + withNamespaceAndTable(ns, table) { t => sql( - s"""CREATE TABLE t1 ( + s"""CREATE TABLE $t ( | c1 INT COMMENT 'bla', | c2 STRING |) |STORED AS PARQUET """.stripMargin ) - - checkCreateTable("t1", serde = true) + val expected = s"CREATE TABLE $fullName ( `c1` INT COMMENT 'bla', `c2` STRING)" + + " ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'" + + " WITH SERDEPROPERTIES ( 'serialization.format' = '1')" + + " STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'" + + " OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'" + + " TBLPROPERTIES (" + assert(getShowCreateDDL(t, true).mkString(" ") == expected) } } test("hive table with serde info") { - withTable("t1") { + withNamespaceAndTable(ns, table) { t => sql( - s"""CREATE TABLE t1 ( + s"""CREATE TABLE $t ( | c1 INT COMMENT 'bla', | c2 STRING |) @@ -190,75 +175,39 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet | OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' """.stripMargin ) - - checkCreateTable("t1", serde = true) + val expected = s"CREATE TABLE $fullName ( `c1` INT COMMENT 'bla', `c2` STRING)" + + " ROW FORMAT SERDE 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'" + + " WITH SERDEPROPERTIES (" + + " 'mapkey.delim' = ','," + + " 'serialization.format' = '1'," + + " 'field.delim' = ',')" + + " STORED AS INPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'" + + " OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'" + + " TBLPROPERTIES (" + assert(getShowCreateDDL(t, true).mkString(" ") == expected) } } test("hive bucketing is supported") { - withTable("t1") { + withNamespaceAndTable(ns, table) { t => sql( - s"""CREATE TABLE t1 (a INT, b STRING) + s"""CREATE TABLE $t (a INT, b STRING) |CLUSTERED BY (a) |SORTED BY (b) |INTO 2 BUCKETS """.stripMargin ) - checkCreateTable("t1", serde = true) - } - } - - test("hive partitioned view is not supported") { - withTable("t1") { - withView("v1") { - sql( - s""" - |CREATE TABLE t1 (c1 INT, c2 STRING) - |PARTITIONED BY ( - | p1 BIGINT COMMENT 'bla', - | p2 STRING ) - """.stripMargin) - - createRawHiveTable( - s""" - |CREATE VIEW v1 - |PARTITIONED ON (p1, p2) - |AS SELECT * from t1 - """.stripMargin - ) - - val cause = intercept[AnalysisException] { - sql("SHOW CREATE TABLE v1") - } - - assert(cause.getMessage.contains(" - partitioned view")) - - val causeForSpark = intercept[AnalysisException] { - sql("SHOW CREATE TABLE v1 AS SERDE") - } - - assert(causeForSpark.getMessage.contains(" - partitioned view")) - } + val expected = s"CREATE TABLE $fullName ( `a` INT, `b` STRING)" + + " CLUSTERED BY (a) SORTED BY (b ASC) INTO 2 BUCKETS" + + " ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'" + + " WITH SERDEPROPERTIES ( 'serialization.format' = '1')" + + " STORED AS INPUTFORMAT 'org.apache.hadoop.mapred.TextInputFormat'" + + " OUTPUTFORMAT 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'" + + " TBLPROPERTIES (" + assert(getShowCreateDDL(t, true).mkString(" ") == expected) } } - test("SPARK-24911: keep quotes for nested fields in hive") { - withTable("t1") { - val createTable = "CREATE TABLE `t1` (`a` STRUCT<`b`: STRING>) USING hive" - sql(createTable) - val shownDDL = getShowDDL("SHOW CREATE TABLE t1") - assert(shownDDL.substring(0, shownDDL.indexOf(" USING")) == - "CREATE TABLE `default`.`t1` ( `a` STRUCT<`b`: STRING>)") - - checkCreateTable("t1", serde = true) - } - } - - private def createRawHiveTable(ddl: String): Unit = { - hiveContext.sharedState.externalCatalog.unwrapped.asInstanceOf[HiveExternalCatalog] - .client.runSqlHive(ddl) - } - private def checkCreateSparkTableAsHive(tableName: String): Unit = { val table = TableIdentifier(tableName, Some("default")) val db = table.database.get @@ -339,26 +288,6 @@ class HiveShowCreateTableSuite extends ShowCreateTableSuite with TestHiveSinglet } } - test("show create table as serde can't work on data source table") { - withTable("t1") { - sql( - s""" - |CREATE TABLE t1 ( - | c1 STRING COMMENT 'bla', - | c2 STRING - |) - |USING orc - """.stripMargin - ) - - val cause = intercept[AnalysisException] { - checkCreateTable("t1", serde = true) - } - - assert(cause.getMessage.contains("Use `SHOW CREATE TABLE` without `AS SERDE` instead")) - } - } - test("simple external hive table in Spark DDL") { withTempDir { dir => withTable("t1") {