Skip to content

[SPARK-33426][SQL][TESTS] Unify Hive SHOW TABLES tests #30340

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
MaxGekk wants to merge 7 commits into from
Closed

[SPARK-33426][SQL][TESTS] Unify Hive SHOW TABLES tests #30340

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
65fd9c3
Add Hive ShowTablesSuite
MaxGekk Nov 11, 2020
4fb6ddc
Set the default namespace
MaxGekk Nov 11, 2020
8dfd9c7
Move Hive tests to the common trait
MaxGekk Nov 12, 2020
c856b89
Revert back ns2 in a test
MaxGekk Nov 12, 2020
fb74566
Use checkAnswer() in runShowTablesSql()
MaxGekk Nov 12, 2020
bd0f785
Avoid the hask with overriding the beforeAll() method
MaxGekk Nov 12, 2020
7eab687
Refactoring: rename classes/traits
MaxGekk Nov 12, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 18 additions & 15 deletions sql/core/src/test/scala/org/apache/spark/sql/execution/command/ShowTablesSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ trait ShowTablesSuite extends SharedSparkSession {
protected def runShowTablesSql(sqlText: String, expected: Seq[ShowRow]): Unit = {
val df = spark.sql(sqlText)
assert(df.schema === showSchema)
assert(df.collect() === getRows(expected))
assert(df.collect().toSet === getRows(expected).toSet)

@MaxGekk MaxGekk Nov 12, 2020

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

After modifications in show tables with a pattern, the order of rows was changed.

@cloud-fan cloud-fan Nov 12, 2020

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

how about checkAnswer(df, getRows(expected))

@MaxGekk MaxGekk Nov 12, 2020

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Replaced it by checkAnswer()

}

override def test(testName: String, testTags: Tag*)(testFun: => Any)
Expand All @@ -58,35 +58,38 @@ trait ShowTablesSuite extends SharedSparkSession {
}

test("show tables with a pattern") {
withNamespace(s"$catalog.ns1", s"$catalog.ns2") {

@MaxGekk MaxGekk Nov 12, 2020

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ns2 is not used in the test actually

@MaxGekk MaxGekk Nov 12, 2020

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will revert ns2 back to check that tables from the namespace are not showed.

withNamespace(s"$catalog.ns1") {
sql(s"CREATE NAMESPACE $catalog.ns1")
sql(s"CREATE NAMESPACE $catalog.ns2")
withTable(
s"$catalog.ns1.table",
s"$catalog.ns1.table_name_1",
s"$catalog.ns1.table_name_2",
s"$catalog.ns2.table_name_2") {
s"$catalog.ns1.table_name_1a",
s"$catalog.ns1.table_name_2b") {
sql(s"CREATE TABLE $catalog.ns1.table (id bigint, data string) $defaultUsing")
sql(s"CREATE TABLE $catalog.ns1.table_name_1 (id bigint, data string) $defaultUsing")
sql(s"CREATE TABLE $catalog.ns1.table_name_2 (id bigint, data string) $defaultUsing")
sql(s"CREATE TABLE $catalog.ns2.table_name_2 (id bigint, data string) $defaultUsing")
sql(s"CREATE TABLE $catalog.ns1.table_name_1a (id bigint, data string) $defaultUsing")
sql(s"CREATE TABLE $catalog.ns1.table_name_2b (id bigint, data string) $defaultUsing")

runShowTablesSql(
s"SHOW TABLES FROM $catalog.ns1",
Seq(
ShowRow("ns1", "table", false),
ShowRow("ns1", "table_name_1", false),
ShowRow("ns1", "table_name_2", false)))
ShowRow("ns1", "table_name_1a", false),
ShowRow("ns1", "table_name_2b", false)))

runShowTablesSql(
s"SHOW TABLES FROM $catalog.ns1 LIKE '*name*'",
Seq(
ShowRow("ns1", "table_name_1", false),
ShowRow("ns1", "table_name_2", false)))
ShowRow("ns1", "table_name_1a", false),
ShowRow("ns1", "table_name_2b", false)))

runShowTablesSql(
s"SHOW TABLES FROM $catalog.ns1 LIKE '*2'",
Seq(ShowRow("ns1", "table_name_2", false)))
s"SHOW TABLES FROM $catalog.ns1 LIKE 'table_name_1*|table_name_2*'",
Seq(
ShowRow("ns1", "table_name_1a", false),
ShowRow("ns1", "table_name_2b", false)))

runShowTablesSql(
s"SHOW TABLES FROM $catalog.ns1 LIKE '*2b'",
Seq(ShowRow("ns1", "table_name_2b", false)))
}
}
}
Expand Down
22 changes: 0 additions & 22 deletions sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveCommandSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,28 +95,6 @@ class HiveCommandSuite extends QueryTest with SQLTestUtils with TestHiveSingleto
}
}

test("show tables") {
withTable("show1a", "show2b") {
sql("CREATE TABLE show1a(c1 int)")
sql("CREATE TABLE show2b(c2 int)")
checkAnswer(
sql("SHOW TABLES IN default 'show1*'"),
Row("default", "show1a", false) :: Nil)
checkAnswer(
sql("SHOW TABLES IN default 'show1*|show2*'"),
Row("default", "show1a", false) ::
Row("default", "show2b", false) :: Nil)
checkAnswer(
sql("SHOW TABLES 'show1*|show2*'"),
Row("default", "show1a", false) ::
Row("default", "show2b", false) :: Nil)
assert(
sql("SHOW TABLES").count() >= 2)
assert(
sql("SHOW TABLES IN default").count() >= 2)
}
}

test("show views") {
withView("show1a", "show2b", "global_temp.temp1", "temp2") {
sql("CREATE VIEW show1a AS SELECT 1 AS id")
Expand Down
29 changes: 29 additions & 0 deletions sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowTablesSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.apache.spark.sql.hive.execution.command

import org.apache.spark.sql.SparkSession
import org.apache.spark.sql.execution.command.v1
import org.apache.spark.sql.hive.test.TestHive

class ShowTablesSuite extends v1.ShowTablesSuite {
override def version: String = "Hive V1"
override def defaultUsing: String = "USING HIVE"
override protected val spark: SparkSession = TestHive.sparkSession

@MaxGekk MaxGekk Nov 12, 2020

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We could take this from TestHiveSingleton but when I mix the trait:

[error] /Users/maximgekk/proj/show-tables-hive-tests/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/command/ShowTablesSuite.scala:24:7: class ShowTablesSuite inherits conflicting members:
[error]   method spark in trait SharedSparkSessionBase of type => org.apache.spark.sql.SparkSession  and
[error]   value spark in trait TestHiveSingleton of type org.apache.spark.sql.SparkSession
[error] (Note: this can be resolved by declaring an override in class ShowTablesSuite.)
[error] class ShowTablesSuite extends v1.ShowTablesSuite with TestHiveSingleton {
[error]       ^
[error] one error found

protected override def beforeAll(): Unit = {}

@MaxGekk MaxGekk Nov 12, 2020

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I had to override this otherwise tests failed with:

 SHOW TABLES Hive V1: change current catalog and namespace with USE statements *** FAILED *** (17 milliseconds)
[info]   java.lang.IllegalStateException: LiveListenerBus is stopped.
[info]   at org.apache.spark.scheduler.LiveListenerBus.addToQueue(LiveListenerBus.scala:97)
[info]   at org.apache.spark.scheduler.LiveListenerBus.addToStatusQueue(LiveListenerBus.scala:80)
...
[info]   Cause: java.lang.IllegalStateException: LiveListenerBus is stopped.
[info]   at org.apache.spark.scheduler.LiveListenerBus.addToQueue(LiveListenerBus.scala:97)
[info]   at org.apache.spark.scheduler.LiveListenerBus.addToStatusQueue(LiveListenerBus.scala:80)
[info]   at org.apache.spark.sql.internal.SharedState.<init>(SharedState.scala:103)
[info]   at org.apache.spark.sql.hive.test.TestHiveSharedState.<init>(TestHive.scala:99)
[info]   at org.apache.spark.sql.hive.test.TestHiveSparkSession.$anonfun$sharedState$1(TestHive.scala:222)
[info]   at scala.Option.getOrElse(Option.scala:189)
[info]   at org.apache.spark.sql.hive.test.TestHiveSparkSession.sharedState$lzycompute(TestHive.scala:222)
[info]   at org.apache.spark.sql.hive.test.TestHiveSparkSession.sharedState(TestHive.scala:221)
[info]   at org.apache.spark.sql.hive.test.TestHiveSparkSession.sharedState(TestHive.scala:174)
[info]   at org.apache.spark.sql.internal.BaseSessionStateBuilder.build(BaseSessionStateBuilder.scala:333)
[info]   at org.apache.spark.sql.hive.test.TestHiveSparkSession.sessionState$lzycompute(TestHive.scala:227)
[info]   at org.apache.spark.sql.hive.test.TestHiveSparkSession.sessionState(TestHive.scala:226)
[info]   at org.apache.spark.sql.hive.test.TestHiveSparkSession.$anonfun$sql$1(TestHive.scala:240)
[info]   at org.apache.spark.sql.SparkSession.withActive(SparkSession.scala:768)
[info]   at org.apache.spark.sql.hive.test.TestHiveSparkSession.sql(TestHive.scala:239)

}