Skip to content

refine tests #28

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MaxGekk merged 1 commit into from
Nov 10, 2020
Merged

refine tests #28

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
84 changes: 57 additions & 27 deletions sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,14 @@
package org.apache.spark.sql.execution.command

import org.apache.spark.sql.{QueryTest, Row}
import org.apache.spark.sql.connector.catalog.CatalogV2Implicits._
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.sql.types.StructType

trait ShowTablesSuite extends QueryTest with SharedSparkSession {
protected def catalog: String
protected def defaultNamespace: Seq[String]
protected def defaultUsing: String
case class ShowRow(namespace: String, table: String, isTemporary: Boolean)
protected def getRows(showRows: Seq[ShowRow]): Seq[Row]
Expand All @@ -36,47 +39,74 @@ trait ShowTablesSuite extends QueryTest with SharedSparkSession {
}

test("show an existing table") {
val namespace = "test"
val table = "people"
withDatabase(s"$catalog.$namespace") {
sql(s"CREATE DATABASE $catalog.$namespace")
withTable(s"$catalog.$namespace.$table") {
sql(s"CREATE TABLE $catalog.$namespace.$table (name STRING, id INT) $defaultUsing")
runShowTablesSql(s"SHOW TABLES IN $catalog.test", Seq(ShowRow(namespace, table, false)))
withNamespace(s"$catalog.ns") {
sql(s"CREATE NAMESPACE $catalog.ns")
withTable(s"$catalog.ns.table") {
sql(s"CREATE TABLE $catalog.ns.table (name STRING, id INT) $defaultUsing")
runShowTablesSql(s"SHOW TABLES IN $catalog.ns", Seq(ShowRow("ns", "table", false)))
}
}
}

test("show tables with a pattern") {
withDatabase(s"$catalog.db", s"$catalog.db2") {
sql(s"CREATE DATABASE $catalog.db")
sql(s"CREATE DATABASE $catalog.db2")
withNamespace(s"$catalog.ns1", s"$catalog.ns2") {
sql(s"CREATE NAMESPACE $catalog.ns1")
sql(s"CREATE NAMESPACE $catalog.ns2")
withTable(
s"$catalog.db.table",
s"$catalog.db.table_name_1",
s"$catalog.db.table_name_2",
s"$catalog.db2.table_name_2") {
sql(s"CREATE TABLE $catalog.db.table (id bigint, data string) $defaultUsing")
sql(s"CREATE TABLE $catalog.db.table_name_1 (id bigint, data string) $defaultUsing")
sql(s"CREATE TABLE $catalog.db.table_name_2 (id bigint, data string) $defaultUsing")
sql(s"CREATE TABLE $catalog.db2.table_name_2 (id bigint, data string) $defaultUsing")
s"$catalog.ns1.table",
s"$catalog.ns1.table_name_1",
s"$catalog.ns1.table_name_2",
s"$catalog.ns2.table_name_2") {
sql(s"CREATE TABLE $catalog.ns1.table (id bigint, data string) $defaultUsing")
sql(s"CREATE TABLE $catalog.ns1.table_name_1 (id bigint, data string) $defaultUsing")
sql(s"CREATE TABLE $catalog.ns1.table_name_2 (id bigint, data string) $defaultUsing")
sql(s"CREATE TABLE $catalog.ns2.table_name_2 (id bigint, data string) $defaultUsing")

runShowTablesSql(
s"SHOW TABLES FROM $catalog.db",
s"SHOW TABLES FROM $catalog.ns1",
Seq(
ShowRow("db", "table", false),
ShowRow("db", "table_name_1", false),
ShowRow("db", "table_name_2", false)))
ShowRow("ns1", "table", false),
ShowRow("ns1", "table_name_1", false),
ShowRow("ns1", "table_name_2", false)))

runShowTablesSql(
s"SHOW TABLES FROM $catalog.db LIKE '*name*'",
s"SHOW TABLES FROM $catalog.ns1 LIKE '*name*'",
Seq(
ShowRow("db", "table_name_1", false),
ShowRow("db", "table_name_2", false)))
ShowRow("ns1", "table_name_1", false),
ShowRow("ns1", "table_name_2", false)))

runShowTablesSql(
s"SHOW TABLES FROM $catalog.db LIKE '*2'",
Seq(ShowRow("db", "table_name_2", false)))
s"SHOW TABLES FROM $catalog.ns1 LIKE '*2'",
Seq(ShowRow("ns1", "table_name_2", false)))
}
}
}

test("show tables with current catalog and namespace") {
withSQLConf(SQLConf.DEFAULT_CATALOG.key -> catalog) {
val tblName = (catalog +: defaultNamespace :+ "table").quoted
withTable(tblName) {
sql(s"CREATE TABLE $tblName (name STRING, id INT) $defaultUsing")
val ns = defaultNamespace.mkString(".")
runShowTablesSql("SHOW TABLES", Seq(ShowRow(ns, "table", false)))
}
}
}

test("change current catalog and namespace with USE statements") {
withNamespace(s"$catalog.ns") {
sql(s"CREATE NAMESPACE $catalog.ns")
withTable(s"$catalog.ns.table") {
sql(s"CREATE TABLE $catalog.ns.table (name STRING, id INT) $defaultUsing")

sql(s"USE $catalog")
// No table is matched since the current namespace is not ["ns"]
assert(defaultNamespace != Seq("ns"))
runShowTablesSql("SHOW TABLES", Seq())

// Update the current namespace to match "ns.tbl".
sql(s"USE $catalog.ns")
runShowTablesSql("SHOW TABLES", Seq(ShowRow("ns", "table", false)))
}
}
}
Expand Down
24 changes: 3 additions & 21 deletions sql/core/src/test/scala/org/apache/spark/sql/execution/command/v1/ShowTablesSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,11 @@ import org.apache.spark.sql.{AnalysisException, Row}
import org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException
import org.apache.spark.sql.connector.catalog.CatalogManager
import org.apache.spark.sql.execution.command.{ShowTablesSuite => CommonShowTablesSuite}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.{BooleanType, StringType, StructType}

class ShowTablesSuite extends CommonShowTablesSuite {
override def catalog: String = CatalogManager.SESSION_CATALOG_NAME
override def defaultNamespace: Seq[String] = Seq("default")
override protected def defaultUsing: String = "USING parquet"
override protected def showSchema: StructType = {
new StructType()
Expand Down Expand Up @@ -57,15 +57,6 @@ class ShowTablesSuite extends CommonShowTablesSuite {
assert(msg.contains("Database 'unknown' not found"))
}

test("namespace is not specified and the default catalog is set") {
withSQLConf(SQLConf.DEFAULT_CATALOG.key -> catalog) {
withTable("table") {
spark.sql(s"CREATE TABLE table (id bigint, data string) $defaultUsing")
runShowTablesSql("SHOW TABLES", Seq(ShowRow("default", "table", false)))
}
}
}

// `SHOW TABLES` from v2 catalog returns empty result.
test("v1 SHOW TABLES list the temp views") {
withSourceViews {
Expand All @@ -75,23 +66,14 @@ class ShowTablesSuite extends CommonShowTablesSuite {
}
}

test("using v1 catalog, db name with multipartIdentifier ('a.b') is not allowed.") {
test("v1 SHOW TABLES only support single-level namespace") {
val exception = intercept[AnalysisException] {
runShowTablesSql("SHOW TABLES FROM a.b", Seq())
}
assert(exception.getMessage.contains("The database name is not valid: a.b"))
}

test("namespace not specified and default v2 catalog not set - fallback to v1") {
withSourceViews {
runShowTablesSql(
"SHOW TABLES",
Seq(ShowRow("", "source", true), ShowRow("", "source2", true)))
runShowTablesSql("SHOW TABLES LIKE '*2'", Seq(ShowRow("", "source2", true)))
}
}

test("SHOW TABLE EXTENDED for default") {
test("SHOW TABLE EXTENDED from default") {
withSourceViews {
val expected = Seq(Row("", "source", true), Row("", "source2", true))
val schema = new StructType()
Expand Down
29 changes: 1 addition & 28 deletions sql/core/src/test/scala/org/apache/spark/sql/execution/command/v2/ShowTablesSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,12 @@ import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
import org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException
import org.apache.spark.sql.connector.InMemoryTableCatalog
import org.apache.spark.sql.execution.command.{ShowTablesSuite => CommonShowTablesSuite}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.sql.types.{StringType, StructType}

class ShowTablesSuite extends QueryTest with SharedSparkSession with CommonShowTablesSuite {
override def catalog: String = "test_catalog_v2"
override def defaultNamespace: Seq[String] = Nil
override protected def defaultUsing: String = "USING _"
override protected def showSchema: StructType = {
new StructType()
Expand Down Expand Up @@ -71,16 +71,6 @@ class ShowTablesSuite extends QueryTest with SharedSparkSession with CommonShowT
}
}

test("namespace is not specified and the default catalog is set") {
withSQLConf(SQLConf.DEFAULT_CATALOG.key -> catalog) {
withTable("table") {
spark.sql(s"CREATE TABLE table (id bigint, data string) $defaultUsing")
// TODO(SPARK-33403): DSv2 SHOW TABLES doesn't show `default`
runShowTablesSql("SHOW TABLES", Seq(ShowRow("", "table", false)))
}
}
}

// The test fails for V1 catalog with the error:
// org.apache.spark.sql.AnalysisException:
// The namespace in session catalog must have exactly one name part: spark_catalog.ns1.ns2.tbl
Expand Down Expand Up @@ -111,23 +101,6 @@ class ShowTablesSuite extends QueryTest with SharedSparkSession with CommonShowT
}
}

// The test fails with the error in V1 session catalog:
// org.apache.spark.sql.AnalysisException:
// The namespace in session catalog must have exactly one name part: spark_catalog.ns1.ns2.table
test("change current catalog and namespace with USE statements") {
withTable(s"$catalog.ns1.ns2.table") {
sql(s"CREATE TABLE $catalog.ns1.ns2.table (id bigint) $defaultUsing")

// Update the current catalog, and no table is matched since the current namespace is Array().
sql(s"USE $catalog")
runShowTablesSql("SHOW TABLES", Seq())

// Update the current namespace to match ns1.ns2.table.
sql(s"USE $catalog.ns1.ns2")
runShowTablesSql("SHOW TABLES", Seq(ShowRow("ns1.ns2", "table", false)))
}
}

// TODO(SPARK-33393): Support SHOW TABLE EXTENDED in DSv2
test("SHOW TABLE EXTENDED: an existing table") {
val table = "people"
Expand Down