From d44f56e1e1064f7ecf422fbcb59be780da8a3a8c Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Sat, 16 Oct 2021 21:50:30 -0700 Subject: [PATCH 1/8] initial commit --- .../sql/catalyst/parser/DDLParserSuite.scala | 9 --- .../sql/connector/DataSourceV2SQLSuite.scala | 39 ----------- .../sql/execution/command/DDLSuite.scala | 58 +++++++--------- .../DescribeNamespaceParserSuite.scala | 33 +++++++++ .../command/DescribeNamespaceSuiteBase.scala | 51 ++++++++++++++ .../command/v1/DescribeNamespaceSuite.scala | 60 ++++++++++++++++ .../command/v2/DescribeNamespaceSuite.scala | 68 +++++++++++++++++++ .../command/DescribeNamespaceSuite.scala | 26 +++++++ 8 files changed, 262 insertions(+), 82 deletions(-) create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceParserSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceSuiteBase.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeNamespaceSuite.scala create mode 100644 sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeNamespaceSuite.scala create mode 100644 sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/DescribeNamespaceSuite.scala diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala index 7bcc2b7f0c3d..44b2e20ee821 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala @@ -1193,15 +1193,6 @@ class DDLParserSuite extends AnalysisTest { "DESC TABLE COLUMN for a specific partition is not supported")) } - test("describe database") { - val sql1 = "DESCRIBE DATABASE EXTENDED a.b" - val sql2 = "DESCRIBE DATABASE a.b" - comparePlans(parsePlan(sql1), - DescribeNamespace(UnresolvedNamespace(Seq("a", "b")), extended = true)) - comparePlans(parsePlan(sql2), - DescribeNamespace(UnresolvedNamespace(Seq("a", "b")), extended = false)) - } - test("SPARK-17328 Fix NPE with EXPLAIN DESCRIBE TABLE") { comparePlans(parsePlan("describe t"), DescribeRelation( diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala index 360c8beff436..7c4a5ea22b16 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala @@ -1233,26 +1233,6 @@ class DataSourceV2SQLSuite assert(exception.getMessage.contains("Namespace 'ns1' not found")) } - test("DescribeNamespace using v2 catalog") { - withNamespace("testcat.ns1.ns2") { - sql("CREATE NAMESPACE IF NOT EXISTS testcat.ns1.ns2 COMMENT " + - "'test namespace' LOCATION '/tmp/ns_test'") - val descriptionDf = sql("DESCRIBE NAMESPACE testcat.ns1.ns2") - assert(descriptionDf.schema.map(field => (field.name, field.dataType)) === - Seq( - ("info_name", StringType), - ("info_value", StringType) - )) - val description = descriptionDf.collect() - assert(description === Seq( - Row("Namespace Name", "ns2"), - Row(SupportsNamespaces.PROP_COMMENT.capitalize, "test namespace"), - Row(SupportsNamespaces.PROP_LOCATION.capitalize, "/tmp/ns_test"), - Row(SupportsNamespaces.PROP_OWNER.capitalize, defaultUser)) - ) - } - } - test("ALTER NAMESPACE .. SET PROPERTIES using v2 catalog") { withNamespace("testcat.ns1.ns2") { sql("CREATE NAMESPACE IF NOT EXISTS testcat.ns1.ns2 COMMENT " + @@ -2810,25 +2790,6 @@ class DataSourceV2SQLSuite } } - test("SPARK-34577: drop/add columns to a dataset of `DESCRIBE NAMESPACE`") { - withNamespace("ns") { - sql("CREATE NAMESPACE ns") - val description = sql(s"DESCRIBE NAMESPACE ns") - val noCommentDataset = description.drop("info_name") - val expectedSchema = new StructType() - .add( - name = "info_value", - dataType = StringType, - nullable = true, - metadata = new MetadataBuilder() - .putString("comment", "value of the namespace info").build()) - assert(noCommentDataset.schema === expectedSchema) - val isNullDataset = noCommentDataset - .withColumn("is_null", noCommentDataset("info_value").isNull) - assert(isNullDataset.schema === expectedSchema.add("is_null", BooleanType, false)) - } - } - test("SPARK-34923: do not propagate metadata columns through Project") { val t1 = s"${catalogAndNamespace}table" withTable(t1) { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index ba87883b64e0..798b0f2c8ada 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -225,7 +225,29 @@ class InMemoryCatalogedDDLSuite extends DDLSuite with SharedSparkSession { } } -abstract class DDLSuite extends QueryTest with SQLTestUtils { +trait CommandTestUtils extends SQLTestUtils { + private val escapedIdentifier = "`(.+)`".r + + /** + * Strip backticks, if any, from the string. + */ + protected def cleanIdentifier(ident: String): String = { + ident match { + case escapedIdentifier(i) => i + case plainIdent => plainIdent + } + } + + /** + * Returns the path to the database of the given name. + */ + protected def getDBPath(dbName: String): URI = { + val warehousePath = makeQualifiedPath(spark.sessionState.conf.warehousePath) + new Path(CatalogUtils.URIToString(warehousePath), s"$dbName.db").toUri + } +} + +abstract class DDLSuite extends QueryTest with CommandTestUtils { protected val reversedProperties = Seq(PROP_OWNER) @@ -239,8 +261,6 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils { isDataSource: Boolean = true, partitionCols: Seq[String] = Seq("a", "b")): CatalogTable - private val escapedIdentifier = "`(.+)`".r - private def dataSource: String = { if (isUsingHiveMetastore) { "HIVE" @@ -258,16 +278,6 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils { assert(normalizeCatalogTable(actual) == normalizeCatalogTable(expected)) } - /** - * Strip backticks, if any, from the string. - */ - private def cleanIdentifier(ident: String): String = { - ident match { - case escapedIdentifier(i) => i - case plainIdent => plainIdent - } - } - private def assertUnsupported(query: String): Unit = { val e = intercept[AnalysisException] { sql(query) @@ -304,11 +314,6 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils { catalog.createPartitions(tableName, Seq(part), ignoreIfExists = false) } - private def getDBPath(dbName: String): URI = { - val warehousePath = makeQualifiedPath(spark.sessionState.conf.warehousePath) - new Path(CatalogUtils.URIToString(warehousePath), s"$dbName.db").toUri - } - test("alter table: set location (datasource table)") { testSetLocation(isDatasourceTable = true) } @@ -757,14 +762,6 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils { sql(s"CREATE DATABASE $dbName") - checkAnswer( - sql(s"DESCRIBE DATABASE EXTENDED $dbName").toDF("key", "value") - .where("key not like 'Owner%'"), // filter for consistency with in-memory catalog - Row("Database Name", dbNameWithoutBackTicks) :: - Row("Comment", "") :: - Row("Location", CatalogUtils.URIToString(location)) :: - Row("Properties", "") :: Nil) - sql(s"ALTER DATABASE $dbName SET DBPROPERTIES ('a'='a', 'b'='b', 'c'='c')") checkAnswer( @@ -813,7 +810,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils { } } - test("Drop/Alter/Describe Database - database does not exists") { + test("Drop/Alter Database - database does not exists") { val databaseNames = Seq("db1", "`database`") databaseNames.foreach { dbName => @@ -829,13 +826,6 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils { sql(s"ALTER DATABASE $dbName SET DBPROPERTIES ('d'='d')") }.getMessage assert(message.contains(s"Database '$dbNameWithoutBackTicks' not found")) - - message = intercept[AnalysisException] { - sql(s"DESCRIBE DATABASE EXTENDED $dbName") - }.getMessage - assert(message.contains(s"Database '$dbNameWithoutBackTicks' not found")) - - sql(s"DROP DATABASE IF EXISTS $dbName") } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceParserSuite.scala new file mode 100644 index 000000000000..5b0ed9f90e9f --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceParserSuite.scala @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.command + +import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedNamespace} +import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan +import org.apache.spark.sql.catalyst.plans.logical.DescribeNamespace + +class DescribeNamespaceParserSuite extends AnalysisTest { + test("describe namespace") { + val sql1 = "DESCRIBE NAMESPACE EXTENDED a.b" + val sql2 = "DESCRIBE NAMESPACE a.b" + comparePlans(parsePlan(sql1), + DescribeNamespace(UnresolvedNamespace(Seq("a", "b")), extended = true)) + comparePlans(parsePlan(sql2), + DescribeNamespace(UnresolvedNamespace(Seq("a", "b")), extended = false)) + } +} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceSuiteBase.scala new file mode 100644 index 000000000000..4ff6d2fc8737 --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceSuiteBase.scala @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.command + +import org.apache.spark.sql.{AnalysisException, QueryTest} + +/** + * This base suite contains unified tests for the `DESCRIBE NAMESPACE` command that check V1 and V2 + * table catalogs. The tests that cannot run for all supported catalogs are located in more + * specific test suites: + * + * - V2 table catalog tests: `org.apache.spark.sql.execution.command.v2.DescribeNamespaceSuite` + * - V1 table catalog tests: + * `org.apache.spark.sql.execution.command.v1.DescribeNamespaceSuiteBase` + * - V1 In-Memory catalog: `org.apache.spark.sql.execution.command.v1.DescribeNamespaceSuite` + * - V1 Hive External catalog: +* `org.apache.spark.sql.hive.execution.command.DescribeNamespaceSuite` + */ +trait DescribeNamespaceSuiteBase extends QueryTest with DDLCommandTestUtils with CommandTestUtils { + override val command = "DESCRIBE NAMESPACE" + + test("namespace does not exists") { + val databaseNames = Seq("db1", "`database`") + databaseNames.foreach { dbName => + val dbNameWithoutBackTicks = cleanIdentifier(dbName) + val message = intercept[AnalysisException] { + sql(s"DESCRIBE NAMESPACE EXTENDED $catalog.$dbName") + }.getMessage + + val prefix = if (catalogVersion == "V2") "Namespace" else "Database" + assert(message.contains(s"$prefix '$dbNameWithoutBackTicks' not found")) + + sql(s"DROP NAMESPACE IF EXISTS $catalog.$dbName") + } + } +} diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeNamespaceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeNamespaceSuite.scala new file mode 100644 index 000000000000..e29999a552f7 --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeNamespaceSuite.scala @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.command.v1 + +import org.apache.spark.sql.Row +import org.apache.spark.sql.catalyst.catalog.CatalogUtils +import org.apache.spark.sql.execution.command + +/** + * This base suite contains unified tests for the `DESCRIBE NAMESPACE` command that checks V1 + * table catalogs. The tests that cannot run for all V1 catalogs are located in more + * specific test suites: + * + * - V1 In-Memory catalog: `org.apache.spark.sql.execution.command.v1.DescribeNamespaceSuite` + * - V1 Hive External catalog: + * `org.apache.spark.sql.hive.execution.command.DescribeNamespaceSuite` + */ +trait DescribeNamespaceSuiteBase extends command.DescribeNamespaceSuiteBase { + test("basic") { + val namespaceNames = Seq("db1", "`database`") + withNamespace(namespaceNames: _*) { + namespaceNames.foreach { ns => + val dbNameWithoutBackTicks = cleanIdentifier(ns) + val location = getDBPath(dbNameWithoutBackTicks) + + sql(s"CREATE NAMESPACE $ns") + + checkAnswer( + sql(s"DESCRIBE NAMESPACE EXTENDED $ns") + .toDF("key", "value") + .where("key not like 'Owner%'"), // filter for consistency with in-memory catalog + Row("Database Name", dbNameWithoutBackTicks) :: + Row("Comment", "") :: + Row("Location", CatalogUtils.URIToString(location)) :: + Row("Properties", "") :: Nil) + } + } + } +} + +/** + * The class contains tests for the `DESCRIBE NAMESPACE` command to check V1 In-Memory + * table catalog. + */ +class DescribeNamespaceSuite extends DescribeNamespaceSuiteBase with CommandSuiteBase diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeNamespaceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeNamespaceSuite.scala new file mode 100644 index 000000000000..5a096401d50a --- /dev/null +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeNamespaceSuite.scala @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.execution.command.v2 + +import org.apache.spark.sql.Row +import org.apache.spark.sql.connector.catalog.SupportsNamespaces +import org.apache.spark.sql.execution.command +import org.apache.spark.sql.types.{BooleanType, MetadataBuilder, StringType, StructType} +import org.apache.spark.util.Utils + +/** + * The class contains tests for the `DESCRIBE NAMESPACE` command to check V2 table catalogs. + */ +class DescribeNamespaceSuite extends command.DescribeNamespaceSuiteBase with CommandSuiteBase { + test("DescribeNamespace using v2 catalog") { + withNamespace("test_catalog.ns1.ns2") { + sql("CREATE NAMESPACE IF NOT EXISTS test_catalog.ns1.ns2 COMMENT " + + "'test namespace' LOCATION '/tmp/ns_test'") + val descriptionDf = sql("DESCRIBE NAMESPACE test_catalog.ns1.ns2") + assert(descriptionDf.schema.map(field => (field.name, field.dataType)) === + Seq( + ("info_name", StringType), + ("info_value", StringType) + )) + val description = descriptionDf.collect() + assert(description === Seq( + Row("Namespace Name", "ns2"), + Row(SupportsNamespaces.PROP_COMMENT.capitalize, "test namespace"), + Row(SupportsNamespaces.PROP_LOCATION.capitalize, "/tmp/ns_test"), + Row(SupportsNamespaces.PROP_OWNER.capitalize, Utils.getCurrentUserName())) + ) + } + } + + test("SPARK-34577: drop/add columns to a dataset of `DESCRIBE NAMESPACE`") { + withNamespace("ns") { + sql("CREATE NAMESPACE ns") + val description = sql(s"DESCRIBE NAMESPACE ns") + val noCommentDataset = description.drop("info_name") + val expectedSchema = new StructType() + .add( + name = "info_value", + dataType = StringType, + nullable = true, + metadata = new MetadataBuilder() + .putString("comment", "value of the namespace info").build()) + assert(noCommentDataset.schema === expectedSchema) + val isNullDataset = noCommentDataset + .withColumn("is_null", noCommentDataset("info_value").isNull) + assert(isNullDataset.schema === expectedSchema.add("is_null", BooleanType, false)) + } + } +} diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/DescribeNamespaceSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/DescribeNamespaceSuite.scala new file mode 100644 index 000000000000..866cc178012d --- /dev/null +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/DescribeNamespaceSuite.scala @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive.execution.command + +import org.apache.spark.sql.execution.command.v1 + +/** + * The class contains tests for the `DESCRIBE NAMESPACE` command to check V1 Hive external + * table catalog. + */ +class DescribeNamespaceSuite extends v1.DescribeNamespaceSuiteBase with CommandSuiteBase From 69ab022108a1586b44227d0d3ea30101529579c1 Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Sun, 24 Oct 2021 11:44:23 -0700 Subject: [PATCH 2/8] address PR comments --- .../sql/execution/command/DDLSuite.scala | 44 +++++++++---------- .../DescribeNamespaceParserSuite.scala | 3 ++ .../command/DescribeNamespaceSuiteBase.scala | 20 ++++----- .../command/v1/DescribeNamespaceSuite.scala | 31 +++++++------ .../command/v2/DescribeNamespaceSuite.scala | 8 ++-- 5 files changed, 53 insertions(+), 53 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 798b0f2c8ada..1b09a5dd6a45 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -225,29 +225,7 @@ class InMemoryCatalogedDDLSuite extends DDLSuite with SharedSparkSession { } } -trait CommandTestUtils extends SQLTestUtils { - private val escapedIdentifier = "`(.+)`".r - - /** - * Strip backticks, if any, from the string. - */ - protected def cleanIdentifier(ident: String): String = { - ident match { - case escapedIdentifier(i) => i - case plainIdent => plainIdent - } - } - - /** - * Returns the path to the database of the given name. - */ - protected def getDBPath(dbName: String): URI = { - val warehousePath = makeQualifiedPath(spark.sessionState.conf.warehousePath) - new Path(CatalogUtils.URIToString(warehousePath), s"$dbName.db").toUri - } -} - -abstract class DDLSuite extends QueryTest with CommandTestUtils { +abstract class DDLSuite extends QueryTest with SQLTestUtils { protected val reversedProperties = Seq(PROP_OWNER) @@ -261,6 +239,8 @@ abstract class DDLSuite extends QueryTest with CommandTestUtils { isDataSource: Boolean = true, partitionCols: Seq[String] = Seq("a", "b")): CatalogTable + private val escapedIdentifier = "`(.+)`".r + private def dataSource: String = { if (isUsingHiveMetastore) { "HIVE" @@ -278,6 +258,16 @@ abstract class DDLSuite extends QueryTest with CommandTestUtils { assert(normalizeCatalogTable(actual) == normalizeCatalogTable(expected)) } + /** + * Strip backticks, if any, from the string. + */ + protected def cleanIdentifier(ident: String): String = { + ident match { + case escapedIdentifier(i) => i + case plainIdent => plainIdent + } + } + private def assertUnsupported(query: String): Unit = { val e = intercept[AnalysisException] { sql(query) @@ -314,6 +304,14 @@ abstract class DDLSuite extends QueryTest with CommandTestUtils { catalog.createPartitions(tableName, Seq(part), ignoreIfExists = false) } + /** + * Returns the path to the database of the given name. + */ + protected def getDBPath(dbName: String): URI = { + val warehousePath = makeQualifiedPath(spark.sessionState.conf.warehousePath) + new Path(CatalogUtils.URIToString(warehousePath), s"$dbName.db").toUri + } + test("alter table: set location (datasource table)") { testSetLocation(isDatasourceTable = true) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceParserSuite.scala index 5b0ed9f90e9f..f6254da23a21 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceParserSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceParserSuite.scala @@ -25,9 +25,12 @@ class DescribeNamespaceParserSuite extends AnalysisTest { test("describe namespace") { val sql1 = "DESCRIBE NAMESPACE EXTENDED a.b" val sql2 = "DESCRIBE NAMESPACE a.b" + val sql3 = "DESCRIBE NAMESPACE cat.`database`" comparePlans(parsePlan(sql1), DescribeNamespace(UnresolvedNamespace(Seq("a", "b")), extended = true)) comparePlans(parsePlan(sql2), DescribeNamespace(UnresolvedNamespace(Seq("a", "b")), extended = false)) + comparePlans(parsePlan(sql3), + DescribeNamespace(UnresolvedNamespace(Seq("cat", "database")), extended = false)) } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceSuiteBase.scala index 4ff6d2fc8737..3e58be07f5d0 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceSuiteBase.scala @@ -31,21 +31,19 @@ import org.apache.spark.sql.{AnalysisException, QueryTest} * - V1 Hive External catalog: * `org.apache.spark.sql.hive.execution.command.DescribeNamespaceSuite` */ -trait DescribeNamespaceSuiteBase extends QueryTest with DDLCommandTestUtils with CommandTestUtils { +trait DescribeNamespaceSuiteBase extends QueryTest with DDLCommandTestUtils { override val command = "DESCRIBE NAMESPACE" + protected def notFoundMsgPrefix: String + test("namespace does not exists") { - val databaseNames = Seq("db1", "`database`") - databaseNames.foreach { dbName => - val dbNameWithoutBackTicks = cleanIdentifier(dbName) - val message = intercept[AnalysisException] { - sql(s"DESCRIBE NAMESPACE EXTENDED $catalog.$dbName") - }.getMessage + val dbName = "db1" + val message = intercept[AnalysisException] { + sql(s"DESCRIBE NAMESPACE EXTENDED $catalog.$dbName") + }.getMessage - val prefix = if (catalogVersion == "V2") "Namespace" else "Database" - assert(message.contains(s"$prefix '$dbNameWithoutBackTicks' not found")) + assert(message.contains(s"$notFoundMsgPrefix '$dbName' not found")) - sql(s"DROP NAMESPACE IF EXISTS $catalog.$dbName") - } + sql(s"DROP NAMESPACE IF EXISTS $catalog.$dbName") } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeNamespaceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeNamespaceSuite.scala index e29999a552f7..a86e4a51b052 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeNamespaceSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/DescribeNamespaceSuite.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.execution.command.v1 import org.apache.spark.sql.Row -import org.apache.spark.sql.catalyst.catalog.CatalogUtils import org.apache.spark.sql.execution.command /** @@ -31,24 +30,24 @@ import org.apache.spark.sql.execution.command * `org.apache.spark.sql.hive.execution.command.DescribeNamespaceSuite` */ trait DescribeNamespaceSuiteBase extends command.DescribeNamespaceSuiteBase { + override def notFoundMsgPrefix: String = "Database" + test("basic") { - val namespaceNames = Seq("db1", "`database`") - withNamespace(namespaceNames: _*) { - namespaceNames.foreach { ns => - val dbNameWithoutBackTicks = cleanIdentifier(ns) - val location = getDBPath(dbNameWithoutBackTicks) + val ns = "db1" + withNamespace(ns) { + sql(s"CREATE NAMESPACE $ns") - sql(s"CREATE NAMESPACE $ns") + val result = sql(s"DESCRIBE NAMESPACE EXTENDED $ns") + .toDF("key", "value") + .where("key not like 'Owner%'") // filter for consistency with in-memory catalog + .collect() - checkAnswer( - sql(s"DESCRIBE NAMESPACE EXTENDED $ns") - .toDF("key", "value") - .where("key not like 'Owner%'"), // filter for consistency with in-memory catalog - Row("Database Name", dbNameWithoutBackTicks) :: - Row("Comment", "") :: - Row("Location", CatalogUtils.URIToString(location)) :: - Row("Properties", "") :: Nil) - } + assert(result.length == 4) + assert(result(0) === Row("Database Name", ns)) + assert(result(1) === Row("Comment", "")) + // Check only the key for "Location" since its value depends on warehouse path, etc. + assert(result(2).getString(0) === "Location") + assert(result(3) === Row("Properties", "")) } } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeNamespaceSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeNamespaceSuite.scala index 5a096401d50a..a98c6a486ae5 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeNamespaceSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/DescribeNamespaceSuite.scala @@ -27,11 +27,13 @@ import org.apache.spark.util.Utils * The class contains tests for the `DESCRIBE NAMESPACE` command to check V2 table catalogs. */ class DescribeNamespaceSuite extends command.DescribeNamespaceSuiteBase with CommandSuiteBase { + override def notFoundMsgPrefix: String = "Namespace" + test("DescribeNamespace using v2 catalog") { - withNamespace("test_catalog.ns1.ns2") { - sql("CREATE NAMESPACE IF NOT EXISTS test_catalog.ns1.ns2 COMMENT " + + withNamespace(s"$catalog.ns1.ns2") { + sql(s"CREATE NAMESPACE IF NOT EXISTS $catalog.ns1.ns2 COMMENT " + "'test namespace' LOCATION '/tmp/ns_test'") - val descriptionDf = sql("DESCRIBE NAMESPACE test_catalog.ns1.ns2") + val descriptionDf = sql(s"DESCRIBE NAMESPACE $catalog.ns1.ns2") assert(descriptionDf.schema.map(field => (field.name, field.dataType)) === Seq( ("info_name", StringType), From 002b4e718ac5d7407ce3e1405057eb56a1006a27 Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Sun, 24 Oct 2021 11:45:49 -0700 Subject: [PATCH 3/8] minor fix --- .../org/apache/spark/sql/execution/command/DDLSuite.scala | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 1b09a5dd6a45..a3868666f979 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -261,7 +261,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils { /** * Strip backticks, if any, from the string. */ - protected def cleanIdentifier(ident: String): String = { + private def cleanIdentifier(ident: String): String = { ident match { case escapedIdentifier(i) => i case plainIdent => plainIdent @@ -304,10 +304,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils { catalog.createPartitions(tableName, Seq(part), ignoreIfExists = false) } - /** - * Returns the path to the database of the given name. - */ - protected def getDBPath(dbName: String): URI = { + private def getDBPath(dbName: String): URI = { val warehousePath = makeQualifiedPath(spark.sessionState.conf.warehousePath) new Path(CatalogUtils.URIToString(warehousePath), s"$dbName.db").toUri } From 2640444646e41788788fd28ba604fe706eedb974 Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Sun, 24 Oct 2021 11:47:16 -0700 Subject: [PATCH 4/8] update variable name --- .../execution/command/DescribeNamespaceSuiteBase.scala | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceSuiteBase.scala index 3e58be07f5d0..fe484fc76be2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceSuiteBase.scala @@ -37,13 +37,13 @@ trait DescribeNamespaceSuiteBase extends QueryTest with DDLCommandTestUtils { protected def notFoundMsgPrefix: String test("namespace does not exists") { - val dbName = "db1" + val ns = "db1" val message = intercept[AnalysisException] { - sql(s"DESCRIBE NAMESPACE EXTENDED $catalog.$dbName") + sql(s"DESCRIBE NAMESPACE EXTENDED $catalog.$ns") }.getMessage - assert(message.contains(s"$notFoundMsgPrefix '$dbName' not found")) + assert(message.contains(s"$notFoundMsgPrefix '$ns' not found")) - sql(s"DROP NAMESPACE IF EXISTS $catalog.$dbName") + sql(s"DROP NAMESPACE IF EXISTS $catalog.$ns") } } From 174aeddcc2ced1e014b9ab2ce8d7d4ded7e42bf4 Mon Sep 17 00:00:00 2001 From: Wenchen Fan Date: Mon, 25 Oct 2021 17:00:16 +0800 Subject: [PATCH 5/8] Update DescribeNamespaceParserSuite.scala --- .../command/DescribeNamespaceParserSuite.scala | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceParserSuite.scala index f6254da23a21..7d054e76f46d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceParserSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceParserSuite.scala @@ -23,14 +23,17 @@ import org.apache.spark.sql.catalyst.plans.logical.DescribeNamespace class DescribeNamespaceParserSuite extends AnalysisTest { test("describe namespace") { - val sql1 = "DESCRIBE NAMESPACE EXTENDED a.b" - val sql2 = "DESCRIBE NAMESPACE a.b" - val sql3 = "DESCRIBE NAMESPACE cat.`database`" - comparePlans(parsePlan(sql1), + def check(sql: String, expected: LogicalPlan): Unit = { + comparePlans(parsePlan(sql), expected) + comparePlans(parsePlan(sql.replace("NAMESPACE", "DATABASE")), expected) + comparePlans(parsePlan(sql.replace("NAMESPACE", "SCHEMA")), expected) + } + + check("DESCRIBE NAMESPACE EXTENDED a.b", DescribeNamespace(UnresolvedNamespace(Seq("a", "b")), extended = true)) - comparePlans(parsePlan(sql2), + check("DESCRIBE NAMESPACE a.b", DescribeNamespace(UnresolvedNamespace(Seq("a", "b")), extended = false)) - comparePlans(parsePlan(sql3), - DescribeNamespace(UnresolvedNamespace(Seq("cat", "database")), extended = false)) + check("DESCRIBE NAMESPACE `a.b`", + DescribeNamespace(UnresolvedNamespace(Seq("a.b")), extended = false)) } } From cecd0a65b6472de6baea362f30887d4f19028845 Mon Sep 17 00:00:00 2001 From: Wenchen Fan Date: Mon, 25 Oct 2021 17:00:27 +0800 Subject: [PATCH 6/8] Update sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceSuiteBase.scala --- .../sql/execution/command/DescribeNamespaceSuiteBase.scala | 2 -- 1 file changed, 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceSuiteBase.scala index fe484fc76be2..d8462a2c9618 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceSuiteBase.scala @@ -43,7 +43,5 @@ trait DescribeNamespaceSuiteBase extends QueryTest with DDLCommandTestUtils { }.getMessage assert(message.contains(s"$notFoundMsgPrefix '$ns' not found")) - - sql(s"DROP NAMESPACE IF EXISTS $catalog.$ns") } } From a5c1492bfcaf55613039b4343f9e3f8c4303cc47 Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Mon, 25 Oct 2021 09:22:30 -0700 Subject: [PATCH 7/8] fix compilation error --- .../sql/execution/command/DescribeNamespaceParserSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceParserSuite.scala index 7d054e76f46d..93da7aa0248f 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceParserSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceParserSuite.scala @@ -19,7 +19,7 @@ package org.apache.spark.sql.execution.command import org.apache.spark.sql.catalyst.analysis.{AnalysisTest, UnresolvedNamespace} import org.apache.spark.sql.catalyst.parser.CatalystSqlParser.parsePlan -import org.apache.spark.sql.catalyst.plans.logical.DescribeNamespace +import org.apache.spark.sql.catalyst.plans.logical.{DescribeNamespace, LogicalPlan} class DescribeNamespaceParserSuite extends AnalysisTest { test("describe namespace") { @@ -28,7 +28,7 @@ class DescribeNamespaceParserSuite extends AnalysisTest { comparePlans(parsePlan(sql.replace("NAMESPACE", "DATABASE")), expected) comparePlans(parsePlan(sql.replace("NAMESPACE", "SCHEMA")), expected) } - + check("DESCRIBE NAMESPACE EXTENDED a.b", DescribeNamespace(UnresolvedNamespace(Seq("a", "b")), extended = true)) check("DESCRIBE NAMESPACE a.b", From a8820abd45be0c10a7296acc00a66aa54020df60 Mon Sep 17 00:00:00 2001 From: Terry Kim Date: Mon, 25 Oct 2021 09:35:36 -0700 Subject: [PATCH 8/8] add back with TODO --- .../sql/execution/command/DescribeNamespaceSuiteBase.scala | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceSuiteBase.scala index d8462a2c9618..727bb90bb0cf 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DescribeNamespaceSuiteBase.scala @@ -43,5 +43,8 @@ trait DescribeNamespaceSuiteBase extends QueryTest with DDLCommandTestUtils { }.getMessage assert(message.contains(s"$notFoundMsgPrefix '$ns' not found")) + + // TODO: Move this to DropNamespaceSuite when the test suite is introduced. + sql(s"DROP NAMESPACE IF EXISTS $catalog.$ns") } }