[Spark-25993][SQL][TEST]Add test cases for CREATE EXTERNAL TABLE with subdirectories #23108
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -190,4 +190,155 @@ class HiveOrcSourceSuite extends OrcSuite with TestHiveSingleton { | |
| } | ||
| } | ||
| } | ||
|
|
||
| test("SPARK-25993 CREATE EXTERNAL TABLE with subdirectories") { | ||
| Seq(true, false).foreach { convertMetastore => | ||
| withSQLConf(HiveUtils.CONVERT_METASTORE_ORC.key -> s"$convertMetastore") { | ||
| withTempDir { dir => | ||
| try { | ||
| hiveClient.runSqlHive("USE default") | ||
| hiveClient.runSqlHive( | ||
| """ | ||
| |CREATE EXTERNAL TABLE hive_orc( | ||
| | C1 INT, | ||
| | C2 INT, | ||
| | C3 STRING) | ||
| |STORED AS orc""".stripMargin) | ||
| // Hive throws an exception if I assign the location in the create table statement. | ||
| hiveClient.runSqlHive( | ||
| s"ALTER TABLE hive_orc SET LOCATION " + | ||
| s"'${new File(s"${dir.getCanonicalPath}/l1/").toURI}'") | ||
| hiveClient.runSqlHive( | ||
| """ | ||
| |INSERT INTO TABLE hive_orc | ||
| |VALUES (1, 1, 'orc1'), (2, 2, 'orc2')""".stripMargin) | ||
|
|
||
| hiveClient.runSqlHive( | ||
| s"ALTER TABLE hive_orc SET LOCATION " + | ||
| s"'${new File(s"${dir.getCanonicalPath}/l1/l2/").toURI}'") | ||
| hiveClient.runSqlHive( | ||
| """ | ||
| |INSERT INTO TABLE hive_orc | ||
| |VALUES (3, 3, 'orc3'), (4, 4, 'orc4')""".stripMargin) | ||
|
|
||
| hiveClient.runSqlHive( | ||
| s"ALTER TABLE hive_orc SET LOCATION " + | ||
| s"'${new File(s"${dir.getCanonicalPath}/l1/l2/l3/").toURI}'") | ||
| hiveClient.runSqlHive( | ||
| """ | ||
| |INSERT INTO TABLE hive_orc | ||
| |VALUES (5, 5, 'orc5'), (6, 6, 'orc6')""".stripMargin) | ||
|
|
||
| withTable("tbl1", "tbl2", "tbl3", "tbl4", "tbl5", "tbl6") { | ||
| val topDirStatement = | ||
| s""" | ||
| |CREATE EXTERNAL TABLE tbl1( | ||
| | c1 int, | ||
| | c2 int, | ||
| | c3 string) | ||
| |STORED AS orc | ||
| |LOCATION '${s"${dir.getCanonicalPath}"}'""".stripMargin | ||
| sql(topDirStatement) | ||
| val topDirSqlStatement = s"select * from tbl1" | ||
| if (convertMetastore) { | ||
| checkAnswer(sql(topDirSqlStatement), Nil) | ||
| } else { | ||
| checkAnswer(sql(topDirSqlStatement), | ||
| (1 to 6).map(i => Row(i, i, s"orc$i"))) | ||
| } | ||
|
|
||
| val l1DirStatement = | ||
| s""" | ||
| |CREATE EXTERNAL TABLE tbl2( | ||
| | c1 int, | ||
| | c2 int, | ||
| | c3 string) | ||
| |STORED AS orc | ||
| |LOCATION '${s"${dir.getCanonicalPath}/l1/"}'""".stripMargin | ||
| sql(l1DirStatement) | ||
| val l1DirSqlStatement = s"select * from tbl2" | ||
| if (convertMetastore) { | ||
| checkAnswer(sql(l1DirSqlStatement), | ||
| (1 to 2).map(i => Row(i, i, s"orc$i"))) | ||
| } else { | ||
| checkAnswer(sql(l1DirSqlStatement), | ||
| (1 to 6).map(i => Row(i, i, s"orc$i"))) | ||
| } | ||
|
|
||
| val l2DirStatement = | ||
| s""" | ||
| |CREATE EXTERNAL TABLE tbl3( | ||
| | c1 int, | ||
| | c2 int, | ||
| | c3 string) | ||
| |STORED AS orc | ||
| |LOCATION '${s"${dir.getCanonicalPath}/l1/l2/"}'""".stripMargin | ||
| sql(l2DirStatement) | ||
| val l2DirSqlStatement = s"select * from tbl3" | ||
| if (convertMetastore) { | ||
| checkAnswer(sql(l2DirSqlStatement), | ||
| (3 to 4).map(i => Row(i, i, s"orc$i"))) | ||
| } else { | ||
| checkAnswer(sql(l2DirSqlStatement), | ||
| (3 to 6).map(i => Row(i, i, s"orc$i"))) | ||
| } | ||
|
|
||
| val wildcardTopDirStatement = | ||
| s""" | ||
| |CREATE EXTERNAL TABLE tbl4( | ||
| | c1 int, | ||
| | c2 int, | ||
| | c3 string) | ||
| |STORED AS orc | ||
| |LOCATION '${new File(s"${dir}/*").toURI}'""".stripMargin | ||
| sql(wildcardTopDirStatement) | ||
| val wildcardTopDirSqlStatement = s"select * from tbl4" | ||
| if (convertMetastore) { | ||
| checkAnswer(sql(wildcardTopDirSqlStatement), | ||
| (1 to 2).map(i => Row(i, i, s"orc$i"))) | ||
| } else { | ||
| checkAnswer(sql(wildcardTopDirSqlStatement), Nil) | ||
| } | ||
|
|
||
| val wildcardL1DirStatement = | ||
| s""" | ||
| |CREATE EXTERNAL TABLE tbl5( | ||
| | c1 int, | ||
| | c2 int, | ||
| | c3 string) | ||
| |STORED AS orc | ||
| |LOCATION '${new File(s"${dir}/l1/*").toURI}'""".stripMargin | ||
| sql(wildcardL1DirStatement) | ||
| val wildcardL1DirSqlStatement = s"select * from tbl5" | ||
| if (convertMetastore) { | ||
| checkAnswer(sql(wildcardL1DirSqlStatement), | ||
| (1 to 4).map(i => Row(i, i, s"orc$i"))) | ||
| } else { | ||
| checkAnswer(sql(wildcardL1DirSqlStatement), Nil) | ||
| } | ||
|
|
||
| val wildcardL2Statement = | ||
| s""" | ||
| |CREATE EXTERNAL TABLE tbl6( | ||
| | c1 int, | ||
| | c2 int, | ||
| | c3 string) | ||
| |STORED AS orc | ||
| |LOCATION '${new File(s"${dir}/l1/l2/*").toURI}'""".stripMargin | ||
| sql(wildcardL2Statement) | ||
| val wildcardL2SqlStatement = s"select * from tbl6" | ||
| if (convertMetastore) { | ||
| checkAnswer(sql(wildcardL2SqlStatement), | ||
| (3 to 6).map(i => Row(i, i, s"orc$i"))) | ||
| } else { | ||
| checkAnswer(sql(wildcardL2SqlStatement), Nil) | ||
| } | ||
| } | ||
| } finally { | ||
| hiveClient.runSqlHive("DROP TABLE IF EXISTS hive_orc") | ||
|
Member
There was a problem hiding this comment. It seems that we need to clean up
Contributor
Author
There was a problem hiding this comment. @dongjoon-hyun at line 221, I put the |
||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Please fix this first for the first and second review comments.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@dongjoon-hyun Thanks for the comments, I have tried to make the changes for the first and second review comments, I changed both suites to make it looks similar, also add more test cases. For the 3rd comments, I haven't found a common place to both suites, when you say the help function missing in the previous commit, can you help to point what kind of help function I missed? Thanks.