Skip to content

Commit 71f1d12

Browse files
author
ouyangxiaochen
committed
1.remove EXTERNAL key word in sqlbase.g4 file
2.simplify the logic: if location is specified, we create an external table internally. Else, create managed table 3.update test cases
1 parent b80f8e6 commit 71f1d12

File tree

4 files changed

+63
-172
lines changed

4 files changed

+63
-172
lines changed

‎sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ statement
8181
rowFormat? createFileFormat? locationSpec?
8282
(TBLPROPERTIES tablePropertyList)?
8383
(AS? query)? #createHiveTable
84-
| CREATE EXTERNAL? TABLE (IF NOT EXISTS)? target=tableIdentifier
84+
| CREATE TABLE (IF NOT EXISTS)? target=tableIdentifier
8585
LIKE source=tableIdentifier locationSpec? #createTableLike
8686
| ANALYZE TABLE tableIdentifier partitionSpec? COMPUTE STATISTICS
8787
(identifier | FOR COLUMNS identifierSeq)? #analyze

‎sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala‎

Lines changed: 1 addition & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1140,18 +1140,14 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder {
11401140
*
11411141
* For example:
11421142
* {{{
1143-
* CREATE [EXTERNAL] TABLE [IF NOT EXISTS] [db_name.]table_name
1143+
* CREATE TABLE [IF NOT EXISTS] [db_name.]table_name
11441144
* LIKE [other_db_name.]existing_table_name [locationSpec]
11451145
* }}}
11461146
*/
11471147
override def visitCreateTableLike(ctx: CreateTableLikeContext): LogicalPlan = withOrigin(ctx) {
11481148
val targetTable = visitTableIdentifier(ctx.target)
11491149
val sourceTable = visitTableIdentifier(ctx.source)
11501150
val location = Option(ctx.locationSpec).map(visitLocationSpec)
1151-
if (ctx.EXTERNAL != null && location.isEmpty) {
1152-
// If we are creating an EXTERNAL table, then the LOCATION field is required
1153-
operationNotAllowed("CREATE EXTERNAL TABLE LIKE must be accompanied by LOCATION", ctx)
1154-
}
11551151
CreateTableLikeCommand(targetTable, sourceTable, location, ctx.EXISTS != null)
11561152
}
11571153

‎sql/core/src/main/scala/org/apache/spark/sql/execution/command/tables.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,8 @@ case class CreateTableLikeCommand(
7171
sourceTableDesc.provider
7272
}
7373

74+
// If location is specified, we create an external table internally.
75+
// Else create managed table.
7476
val tblType = if (location.isEmpty) {
7577
CatalogTableType.MANAGED
7678
} else {
@@ -81,8 +83,6 @@ case class CreateTableLikeCommand(
8183
CatalogTable(
8284
identifier = targetTable,
8385
tableType = tblType,
84-
// If location is not empty the table we are creating is a new external table
85-
// otherwise managed table.
8686
storage = sourceTableDesc.storage.copy(locationUri = location),
8787
schema = sourceTableDesc.schema,
8888
provider = newProvider,

‎sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala‎

Lines changed: 59 additions & 164 deletions
Original file line numberDiff line numberDiff line change
@@ -812,26 +812,8 @@ class HiveDDLSuite
812812
}
813813
}
814814

815-
test("CREATE TABLE LIKE a temporary view") {
816-
val sourceViewName = "tab1"
817-
val targetTabName = "tab2"
818-
withTempView(sourceViewName) {
819-
withTable(targetTabName) {
820-
spark.range(10).select('id as 'a, 'id as 'b, 'id as 'c, 'id as 'd)
821-
.createTempView(sourceViewName)
822-
sql(s"CREATE TABLE $targetTabName LIKE $sourceViewName")
823-
824-
val sourceTable = spark.sessionState.catalog.getTempViewOrPermanentTableMetadata(
825-
TableIdentifier(sourceViewName))
826-
val targetTable = spark.sessionState.catalog.getTableMetadata(
827-
TableIdentifier(targetTabName, Some("default")))
828-
829-
checkCreateTableLike(sourceTable, targetTable, "MANAGED")
830-
}
831-
}
832-
}
833-
834-
test("CREATE [EXTERNAL] TABLE LIKE a temporary view LOCATION...") {
815+
test("CREATE TABLE LIKE a temporary view [LOCATION]...") {
816+
var createdTableType = "MANAGED"
835817
for ( i <- 0 to 1 ) {
836818
withTempDir {tmpDir =>
837819
val sourceViewName = "tab1"
@@ -841,42 +823,28 @@ class HiveDDLSuite
841823
withTable(targetTabName) {
842824
spark.range(10).select('id as 'a, 'id as 'b, 'id as 'c, 'id as 'd)
843825
.createTempView(sourceViewName)
844-
val tblType = if (i == 0) "" else "EXTERNAL"
845-
sql(s"CREATE $tblType TABLE $targetTabName LIKE $sourceViewName LOCATION '$basePath'")
826+
if (i == 0) {
827+
sql(s"CREATE TABLE $targetTabName LIKE $sourceViewName ")
828+
} else {
829+
createdTableType = "EXTERNAL"
830+
sql(s"CREATE TABLE $targetTabName " +
831+
s"LIKE $sourceViewName LOCATION '$basePath'")
832+
}
846833

847834
val sourceTable = spark.sessionState.catalog.getTempViewOrPermanentTableMetadata(
848835
TableIdentifier(sourceViewName))
849836
val targetTable = spark.sessionState.catalog.getTableMetadata(
850837
TableIdentifier(targetTabName, Some("default")))
851838

852-
checkCreateTableLike(sourceTable, targetTable, "EXTERNAL")
839+
checkCreateTableLike(sourceTable, targetTable, createdTableType)
853840
}
854841
}
855842
}
856843
}
857844
}
858845

859-
test("CREATE TABLE LIKE a data source table") {
860-
val sourceTabName = "tab1"
861-
val targetTabName = "tab2"
862-
withTable(sourceTabName, targetTabName) {
863-
spark.range(10).select('id as 'a, 'id as 'b, 'id as 'c, 'id as 'd)
864-
.write.format("json").saveAsTable(sourceTabName)
865-
sql(s"CREATE TABLE $targetTabName LIKE $sourceTabName")
866-
867-
val sourceTable =
868-
spark.sessionState.catalog.getTableMetadata(TableIdentifier(sourceTabName, Some("default")))
869-
val targetTable =
870-
spark.sessionState.catalog.getTableMetadata(TableIdentifier(targetTabName, Some("default")))
871-
// The table type of the source table should be a Hive-managed data source table
872-
assert(DDLUtils.isDatasourceTable(sourceTable))
873-
assert(sourceTable.tableType == CatalogTableType.MANAGED)
874-
875-
checkCreateTableLike(sourceTable, targetTable, "MANAGED")
876-
}
877-
}
878-
879-
test("CREATE [EXTERNAL] TABLE LIKE a data source table LOCATION...") {
846+
test("CREATE TABLE LIKE a data source table [LOCATION]...") {
847+
var createdTableType = "MANAGED"
880848
for ( i <- 0 to 1 ) {
881849
withTempDir { tmpDir =>
882850
val sourceTabName = "tab1"
@@ -885,8 +853,12 @@ class HiveDDLSuite
885853
withTable(sourceTabName, targetTabName) {
886854
spark.range(10).select('id as 'a, 'id as 'b, 'id as 'c, 'id as 'd)
887855
.write.format("json").saveAsTable(sourceTabName)
888-
val tblType = if (i == 0) "" else "EXTERNAL"
889-
sql(s"CREATE $tblType TABLE $targetTabName LIKE $sourceTabName LOCATION '$basePath'")
856+
if ( i == 0 ) {
857+
sql(s"CREATE TABLE $targetTabName LIKE $sourceTabName")
858+
} else {
859+
createdTableType = "EXTERNAL"
860+
sql(s"CREATE TABLE $targetTabName LIKE $sourceTabName LOCATION '$basePath'")
861+
}
890862

891863
val sourceTable =
892864
spark.sessionState.catalog.getTableMetadata(
@@ -898,38 +870,14 @@ class HiveDDLSuite
898870
assert(DDLUtils.isDatasourceTable(sourceTable))
899871
assert(sourceTable.tableType == CatalogTableType.MANAGED)
900872

901-
checkCreateTableLike(sourceTable, targetTable, "EXTERNAL")
873+
checkCreateTableLike(sourceTable, targetTable, createdTableType)
902874
}
903875
}
904876
}
905877
}
906878

907-
test("CREATE TABLE LIKE an external data source table") {
908-
val sourceTabName = "tab1"
909-
val targetTabName = "tab2"
910-
withTable(sourceTabName, targetTabName) {
911-
withTempPath { dir =>
912-
val path = dir.getCanonicalPath
913-
spark.range(10).select('id as 'a, 'id as 'b, 'id as 'c, 'id as 'd)
914-
.write.format("parquet").save(path)
915-
sql(s"CREATE TABLE $sourceTabName USING parquet OPTIONS (PATH '${dir.toURI}')")
916-
sql(s"CREATE TABLE $targetTabName LIKE $sourceTabName")
917-
918-
// The source table should be an external data source table
919-
val sourceTable = spark.sessionState.catalog.getTableMetadata(
920-
TableIdentifier(sourceTabName, Some("default")))
921-
val targetTable = spark.sessionState.catalog.getTableMetadata(
922-
TableIdentifier(targetTabName, Some("default")))
923-
// The table type of the source table should be an external data source table
924-
assert(DDLUtils.isDatasourceTable(sourceTable))
925-
assert(sourceTable.tableType == CatalogTableType.EXTERNAL)
926-
927-
checkCreateTableLike(sourceTable, targetTable, "MANAGED")
928-
}
929-
}
930-
}
931-
932-
test("CREATE [EXTERNAL] TABLE LIKE an external data source table LOCATION...") {
879+
test("CREATE TABLE LIKE an external data source table [LOCATION]...") {
880+
var createdTableType = "MANAGED"
933881
for ( i <- 0 to 1 ) {
934882
withTempDir { tmpDir =>
935883
val sourceTabName = "tab1"
@@ -941,8 +889,12 @@ class HiveDDLSuite
941889
spark.range(10).select('id as 'a, 'id as 'b, 'id as 'c, 'id as 'd)
942890
.write.format("parquet").save(path)
943891
sql(s"CREATE TABLE $sourceTabName USING parquet OPTIONS (PATH '${dir.toURI}')")
944-
val tblType = if (i == 0) "" else "EXTERNAL"
945-
sql(s"CREATE $tblType TABLE $targetTabName LIKE $sourceTabName LOCATION '$basePath'")
892+
if ( i == 0 ) {
893+
sql(s"CREATE TABLE $targetTabName LIKE $sourceTabName")
894+
} else {
895+
createdTableType = "EXTERNAL"
896+
sql(s"CREATE TABLE $targetTabName LIKE $sourceTabName LOCATION '$basePath'")
897+
}
946898

947899
// The source table should be an external data source table
948900
val sourceTable = spark.sessionState.catalog.getTableMetadata(
@@ -953,31 +905,15 @@ class HiveDDLSuite
953905
assert(DDLUtils.isDatasourceTable(sourceTable))
954906
assert(sourceTable.tableType == CatalogTableType.EXTERNAL)
955907

956-
checkCreateTableLike(sourceTable, targetTable, "EXTERNAL")
908+
checkCreateTableLike(sourceTable, targetTable, createdTableType)
957909
}
958910
}
959911
}
960912
}
961913
}
962914

963-
test("CREATE TABLE LIKE a managed Hive serde table") {
964-
val catalog = spark.sessionState.catalog
965-
val sourceTabName = "tab1"
966-
val targetTabName = "tab2"
967-
withTable(sourceTabName, targetTabName) {
968-
sql(s"CREATE TABLE $sourceTabName TBLPROPERTIES('prop1'='value1') AS SELECT 1 key, 'a'")
969-
sql(s"CREATE TABLE $targetTabName LIKE $sourceTabName")
970-
971-
val sourceTable = catalog.getTableMetadata(TableIdentifier(sourceTabName, Some("default")))
972-
assert(sourceTable.tableType == CatalogTableType.MANAGED)
973-
assert(sourceTable.properties.get("prop1").nonEmpty)
974-
val targetTable = catalog.getTableMetadata(TableIdentifier(targetTabName, Some("default")))
975-
976-
checkCreateTableLike(sourceTable, targetTable, "MANAGED")
977-
}
978-
}
979-
980-
test("CREATE [EXTERNAL] TABLE LIKE a managed Hive serde table LOCATION...") {
915+
test("CREATE TABLE LIKE a managed Hive serde table [LOCATION]...") {
916+
var createdTableType = "MANAGED"
981917
for ( i <- 0 to 1 ) {
982918
val catalog = spark.sessionState.catalog
983919
withTempDir { tmpDir =>
@@ -986,8 +922,13 @@ class HiveDDLSuite
986922
val basePath = tmpDir.toURI
987923
withTable(sourceTabName, targetTabName) {
988924
sql(s"CREATE TABLE $sourceTabName TBLPROPERTIES('prop1'='value1') AS SELECT 1 key, 'a'")
989-
val tblType = if (i == 0) "" else "EXTERNAL"
990-
sql(s"CREATE $tblType TABLE $targetTabName LIKE $sourceTabName LOCATION '$basePath'")
925+
926+
if ( i == 0 ) {
927+
sql(s"CREATE TABLE $targetTabName LIKE $sourceTabName")
928+
} else {
929+
createdTableType = "EXTERNAL"
930+
sql(s"CREATE TABLE $targetTabName LIKE $sourceTabName LOCATION '$basePath'")
931+
}
991932

992933
val sourceTable = catalog.getTableMetadata(
993934
TableIdentifier(sourceTabName, Some("default")))
@@ -996,48 +937,14 @@ class HiveDDLSuite
996937
val targetTable = catalog.getTableMetadata(
997938
TableIdentifier(targetTabName, Some("default")))
998939

999-
checkCreateTableLike(sourceTable, targetTable, "EXTERNAL")
1000-
}
1001-
}
1002-
}
1003-
}
1004-
1005-
test("CREATE TABLE LIKE an external Hive serde table") {
1006-
val catalog = spark.sessionState.catalog
1007-
withTempDir { tmpDir =>
1008-
val basePath = tmpDir.toURI
1009-
val sourceTabName = "tab1"
1010-
val targetTabName = "tab2"
1011-
withTable(sourceTabName, targetTabName) {
1012-
assert(tmpDir.listFiles.isEmpty)
1013-
sql(
1014-
s"""
1015-
|CREATE EXTERNAL TABLE $sourceTabName (key INT comment 'test', value STRING)
1016-
|COMMENT 'Apache Spark'
1017-
|PARTITIONED BY (ds STRING, hr STRING)
1018-
|LOCATION '$basePath'
1019-
""".stripMargin)
1020-
for (ds <- Seq("2008-04-08", "2008-04-09"); hr <- Seq("11", "12")) {
1021-
sql(
1022-
s"""
1023-
|INSERT OVERWRITE TABLE $sourceTabName
1024-
|partition (ds='$ds',hr='$hr')
1025-
|SELECT 1, 'a'
1026-
""".stripMargin)
940+
checkCreateTableLike(sourceTable, targetTable, createdTableType)
1027941
}
1028-
sql(s"CREATE TABLE $targetTabName LIKE $sourceTabName")
1029-
1030-
val sourceTable = catalog.getTableMetadata(TableIdentifier(sourceTabName, Some("default")))
1031-
assert(sourceTable.tableType == CatalogTableType.EXTERNAL)
1032-
assert(sourceTable.comment == Option("Apache Spark"))
1033-
val targetTable = catalog.getTableMetadata(TableIdentifier(targetTabName, Some("default")))
1034-
1035-
checkCreateTableLike(sourceTable, targetTable, "MANAGED")
1036942
}
1037943
}
1038944
}
1039945

1040-
test("CREATE [EXTERNAL] TABLE LIKE an external Hive serde table LOCATION...") {
946+
test("CREATE TABLE LIKE an external Hive serde table [LOCATION]...") {
947+
var createdTableType = "MANAGED"
1041948
for ( i <- 0 to 1 ) {
1042949
val catalog = spark.sessionState.catalog
1043950
withTempDir { tmpDir =>
@@ -1063,8 +970,14 @@ class HiveDDLSuite
1063970
|SELECT 1, 'a'
1064971
""".stripMargin)
1065972
}
1066-
val tblType = if (i == 0) "" else "EXTERNAL"
1067-
sql(s"CREATE $tblType TABLE $targetTabName LIKE $sourceTabName LOCATION '$basePath1'")
973+
974+
if ( i == 0 ) {
975+
sql(s"CREATE TABLE $targetTabName LIKE $sourceTabName")
976+
} else {
977+
createdTableType = "EXTERNAL"
978+
sql(s"CREATE TABLE $targetTabName " +
979+
s"LIKE $sourceTabName LOCATION '$basePath1'")
980+
}
1068981

1069982
val sourceTable = catalog.getTableMetadata(
1070983
TableIdentifier(sourceTabName, Some("default")))
@@ -1073,39 +986,16 @@ class HiveDDLSuite
1073986
val targetTable = catalog.getTableMetadata(
1074987
TableIdentifier(targetTabName, Some("default")))
1075988

1076-
checkCreateTableLike(sourceTable, targetTable, "EXTERNAL")
989+
checkCreateTableLike(sourceTable, targetTable, createdTableType)
1077990
}
1078991
}
1079992
}
1080993
}
1081994

1082995
}
1083996

1084-
test("CREATE TABLE LIKE a view") {
1085-
val sourceTabName = "tab1"
1086-
val sourceViewName = "view"
1087-
val targetTabName = "tab2"
1088-
withTable(sourceTabName, targetTabName) {
1089-
withView(sourceViewName) {
1090-
spark.range(10).select('id as 'a, 'id as 'b, 'id as 'c, 'id as 'd)
1091-
.write.format("json").saveAsTable(sourceTabName)
1092-
sql(s"CREATE VIEW $sourceViewName AS SELECT * FROM $sourceTabName")
1093-
sql(s"CREATE TABLE $targetTabName LIKE $sourceViewName")
1094-
1095-
val sourceView = spark.sessionState.catalog.getTableMetadata(
1096-
TableIdentifier(sourceViewName, Some("default")))
1097-
// The original source should be a VIEW with an empty path
1098-
assert(sourceView.tableType == CatalogTableType.VIEW)
1099-
assert(sourceView.viewText.nonEmpty && sourceView.viewOriginalText.nonEmpty)
1100-
val targetTable = spark.sessionState.catalog.getTableMetadata(
1101-
TableIdentifier(targetTabName, Some("default")))
1102-
1103-
checkCreateTableLike(sourceView, targetTable, "MANAGED")
1104-
}
1105-
}
1106-
}
1107-
1108-
test("CREATE [EXTERNAL] TABLE LIKE a view LOCATION...") {
997+
test("CREATE TABLE LIKE a view [LOCATION]...") {
998+
var createdTableType = "MANAGED"
1109999
for ( i <- 0 to 1 ) {
11101000
withTempDir { tmpDir =>
11111001
val sourceTabName = "tab1"
@@ -1117,8 +1007,13 @@ class HiveDDLSuite
11171007
spark.range(10).select('id as 'a, 'id as 'b, 'id as 'c, 'id as 'd)
11181008
.write.format("json").saveAsTable(sourceTabName)
11191009
sql(s"CREATE VIEW $sourceViewName AS SELECT * FROM $sourceTabName")
1120-
val tblType = if (i == 0) "" else "EXTERNAL"
1121-
sql(s"CREATE $tblType TABLE $targetTabName LIKE $sourceViewName LOCATION '$basePath'")
1010+
1011+
if ( i == 0 ) {
1012+
sql(s"CREATE TABLE $targetTabName LIKE $sourceViewName")
1013+
} else {
1014+
createdTableType = "EXTERNAL"
1015+
sql(s"CREATE TABLE $targetTabName LIKE $sourceViewName LOCATION '$basePath'")
1016+
}
11221017

11231018
val sourceView = spark.sessionState.catalog.getTableMetadata(
11241019
TableIdentifier(sourceViewName, Some("default")))
@@ -1128,7 +1023,7 @@ class HiveDDLSuite
11281023
val targetTable = spark.sessionState.catalog.getTableMetadata(
11291024
TableIdentifier(targetTabName, Some("default")))
11301025

1131-
checkCreateTableLike(sourceView, targetTable, "EXTERNAL")
1026+
checkCreateTableLike(sourceView, targetTable, createdTableType)
11321027
}
11331028
}
11341029
}

0 commit comments

Comments
 (0)