Skip to content

Commit 8bf400d

Browse files
rdbluerdblue
committed
Fix remaining broken v1 tests.
1 parent d99b5c3 commit 8bf400d

File tree

4 files changed

+114
-123
lines changed

4 files changed

+114
-123
lines changed

‎sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1613,7 +1613,7 @@ class DataSourceV2SQLSuite
16131613
"""
16141614
|CREATE TABLE testcat.t (id int, `a.b` string) USING foo
16151615
|CLUSTERED BY (`a.b`) INTO 4 BUCKETS
1616-
|OPTIONS ('allow-unsupported-transforms'=true)
1616+
|TBLPROPERTIES ('allow-unsupported-transforms'=true)
16171617
""".stripMargin)
16181618

16191619
val testCatalog = catalog("testcat").asTableCatalog.asInstanceOf[InMemoryTableCatalog]

‎sql/core/src/test/scala/org/apache/spark/sql/execution/SparkSqlParserSuite.scala‎

Lines changed: 1 addition & 120 deletions
Original file line numberDiff line numberDiff line change
@@ -39,23 +39,8 @@ class SparkSqlParserSuite extends AnalysisTest {
3939
val newConf = new SQLConf
4040
private lazy val parser = new SparkSqlParser(newConf)
4141

42-
/**
43-
* Normalizes plans:
44-
* - CreateTable the createTime in tableDesc will replaced by -1L.
45-
*/
46-
override def normalizePlan(plan: LogicalPlan): LogicalPlan = {
47-
plan match {
48-
case CreateTable(tableDesc, mode, query) =>
49-
val newTableDesc = tableDesc.copy(createTime = -1L)
50-
CreateTable(newTableDesc, mode, query)
51-
case _ => plan // Don't transform
52-
}
53-
}
54-
5542
private def assertEqual(sqlCommand: String, plan: LogicalPlan): Unit = {
56-
val normalized1 = normalizePlan(parser.parsePlan(sqlCommand))
57-
val normalized2 = normalizePlan(plan)
58-
comparePlans(normalized1, normalized2)
43+
comparePlans(parser.parsePlan(sqlCommand), plan)
5944
}
6045

6146
private def intercept(sqlCommand: String, messages: String*): Unit =
@@ -80,110 +65,6 @@ class SparkSqlParserSuite extends AnalysisTest {
8065
intercept("REFRESH", "Resource paths cannot be empty in REFRESH statements")
8166
}
8267

83-
private def createTableUsing(
84-
table: String,
85-
database: Option[String] = None,
86-
tableType: CatalogTableType = CatalogTableType.MANAGED,
87-
storage: CatalogStorageFormat = CatalogStorageFormat.empty,
88-
schema: StructType = new StructType,
89-
provider: Option[String] = Some("parquet"),
90-
partitionColumnNames: Seq[String] = Seq.empty,
91-
bucketSpec: Option[BucketSpec] = None,
92-
mode: SaveMode = SaveMode.ErrorIfExists,
93-
query: Option[LogicalPlan] = None): CreateTable = {
94-
CreateTable(
95-
CatalogTable(
96-
identifier = TableIdentifier(table, database),
97-
tableType = tableType,
98-
storage = storage,
99-
schema = schema,
100-
provider = provider,
101-
partitionColumnNames = partitionColumnNames,
102-
bucketSpec = bucketSpec
103-
), mode, query
104-
)
105-
}
106-
107-
private def createTable(
108-
table: String,
109-
database: Option[String] = None,
110-
tableType: CatalogTableType = CatalogTableType.MANAGED,
111-
storage: CatalogStorageFormat = CatalogStorageFormat.empty.copy(
112-
inputFormat = HiveSerDe.sourceToSerDe("textfile").get.inputFormat,
113-
outputFormat = HiveSerDe.sourceToSerDe("textfile").get.outputFormat,
114-
serde = Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe")),
115-
schema: StructType = new StructType,
116-
provider: Option[String] = Some("hive"),
117-
partitionColumnNames: Seq[String] = Seq.empty,
118-
comment: Option[String] = None,
119-
mode: SaveMode = SaveMode.ErrorIfExists,
120-
query: Option[LogicalPlan] = None): CreateTable = {
121-
CreateTable(
122-
CatalogTable(
123-
identifier = TableIdentifier(table, database),
124-
tableType = tableType,
125-
storage = storage,
126-
schema = schema,
127-
provider = provider,
128-
partitionColumnNames = partitionColumnNames,
129-
comment = comment
130-
), mode, query
131-
)
132-
}
133-
134-
test("create table - schema") {
135-
assertEqual("CREATE TABLE my_tab(a INT COMMENT 'test', b STRING) STORED AS textfile",
136-
createTable(
137-
table = "my_tab",
138-
schema = (new StructType)
139-
.add("a", IntegerType, nullable = true, "test")
140-
.add("b", StringType)
141-
)
142-
)
143-
assertEqual("CREATE TABLE my_tab(a INT COMMENT 'test', b STRING) " +
144-
"PARTITIONED BY (c INT, d STRING COMMENT 'test2')",
145-
createTable(
146-
table = "my_tab",
147-
schema = (new StructType)
148-
.add("a", IntegerType, nullable = true, "test")
149-
.add("b", StringType)
150-
.add("c", IntegerType)
151-
.add("d", StringType, nullable = true, "test2"),
152-
partitionColumnNames = Seq("c", "d")
153-
)
154-
)
155-
assertEqual("CREATE TABLE my_tab(id BIGINT, nested STRUCT<col1: STRING,col2: INT>) " +
156-
"STORED AS textfile",
157-
createTable(
158-
table = "my_tab",
159-
schema = (new StructType)
160-
.add("id", LongType)
161-
.add("nested", (new StructType)
162-
.add("col1", StringType)
163-
.add("col2", IntegerType)
164-
)
165-
)
166-
)
167-
// Partitioned by a StructType should be accepted by `SparkSqlParser` but will fail an analyze
168-
// rule in `AnalyzeCreateTable`.
169-
assertEqual("CREATE TABLE my_tab(a INT COMMENT 'test', b STRING) " +
170-
"PARTITIONED BY (nested STRUCT<col1: STRING,col2: INT>)",
171-
createTable(
172-
table = "my_tab",
173-
schema = (new StructType)
174-
.add("a", IntegerType, nullable = true, "test")
175-
.add("b", StringType)
176-
.add("nested", (new StructType)
177-
.add("col1", StringType)
178-
.add("col2", IntegerType)
179-
),
180-
partitionColumnNames = Seq("nested")
181-
)
182-
)
183-
intercept("CREATE TABLE my_tab(a: INT COMMENT 'test', b: STRING)",
184-
"no viable alternative at input")
185-
}
186-
18768
test("describe query") {
18869
val query = "SELECT * FROM t"
18970
assertEqual("DESCRIBE QUERY " + query, DescribeQueryCommand(query, parser.parsePlan(query)))

‎sql/core/src/test/scala/org/apache/spark/sql/execution/command/PlanResolutionSuite.scala‎

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1568,6 +1568,116 @@ class PlanResolutionSuite extends AnalysisTest {
15681568
checkFailure("testcat.tab", "foo")
15691569
}
15701570

1571+
private def compareNormalized(plan1: LogicalPlan, plan2: LogicalPlan): Unit = {
1572+
/**
1573+
* Normalizes plans:
1574+
* - CreateTable the createTime in tableDesc will replaced by -1L.
1575+
*/
1576+
def normalizePlan(plan: LogicalPlan): LogicalPlan = {
1577+
plan match {
1578+
case CreateTable(tableDesc, mode, query) =>
1579+
val newTableDesc = tableDesc.copy(createTime = -1L)
1580+
CreateTable(newTableDesc, mode, query)
1581+
case _ => plan // Don't transform
1582+
}
1583+
}
1584+
comparePlans(normalizePlan(plan1), normalizePlan(plan2))
1585+
}
1586+
1587+
test("create table - schema") {
1588+
def createTable(
1589+
table: String,
1590+
database: Option[String] = None,
1591+
tableType: CatalogTableType = CatalogTableType.MANAGED,
1592+
storage: CatalogStorageFormat = CatalogStorageFormat.empty.copy(
1593+
inputFormat = HiveSerDe.sourceToSerDe("textfile").get.inputFormat,
1594+
outputFormat = HiveSerDe.sourceToSerDe("textfile").get.outputFormat,
1595+
serde = Some("org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe")),
1596+
schema: StructType = new StructType,
1597+
provider: Option[String] = Some("hive"),
1598+
partitionColumnNames: Seq[String] = Seq.empty,
1599+
comment: Option[String] = None,
1600+
mode: SaveMode = SaveMode.ErrorIfExists,
1601+
query: Option[LogicalPlan] = None): CreateTable = {
1602+
CreateTable(
1603+
CatalogTable(
1604+
identifier = TableIdentifier(table, database),
1605+
tableType = tableType,
1606+
storage = storage,
1607+
schema = schema,
1608+
provider = provider,
1609+
partitionColumnNames = partitionColumnNames,
1610+
comment = comment
1611+
), mode, query
1612+
)
1613+
}
1614+
1615+
def compare(sql: String, plan: LogicalPlan): Unit = {
1616+
compareNormalized(parseAndResolve(sql), plan)
1617+
}
1618+
1619+
compare("CREATE TABLE my_tab(a INT COMMENT 'test', b STRING) STORED AS textfile",
1620+
createTable(
1621+
table = "my_tab",
1622+
database = Some("default"),
1623+
schema = (new StructType)
1624+
.add("a", IntegerType, nullable = true, "test")
1625+
.add("b", StringType)
1626+
)
1627+
)
1628+
withSQLConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key -> "true") {
1629+
compare("CREATE TABLE my_tab(a INT COMMENT 'test', b STRING) " +
1630+
"PARTITIONED BY (c INT, d STRING COMMENT 'test2')",
1631+
createTable(
1632+
table = "my_tab",
1633+
database = Some("default"),
1634+
schema = (new StructType)
1635+
.add("a", IntegerType, nullable = true, "test")
1636+
.add("b", StringType)
1637+
.add("c", IntegerType)
1638+
.add("d", StringType, nullable = true, "test2"),
1639+
partitionColumnNames = Seq("c", "d")
1640+
)
1641+
)
1642+
}
1643+
compare("CREATE TABLE my_tab(id BIGINT, nested STRUCT<col1: STRING,col2: INT>) " +
1644+
"STORED AS textfile",
1645+
createTable(
1646+
table = "my_tab",
1647+
database = Some("default"),
1648+
schema = (new StructType)
1649+
.add("id", LongType)
1650+
.add("nested", (new StructType)
1651+
.add("col1", StringType)
1652+
.add("col2", IntegerType)
1653+
)
1654+
)
1655+
)
1656+
// Partitioned by a StructType should be accepted by `SparkSqlParser` but will fail an analyze
1657+
// rule in `AnalyzeCreateTable`.
1658+
withSQLConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT_ENABLED.key -> "true") {
1659+
compare("CREATE TABLE my_tab(a INT COMMENT 'test', b STRING) " +
1660+
"PARTITIONED BY (nested STRUCT<col1: STRING,col2: INT>)",
1661+
createTable(
1662+
table = "my_tab",
1663+
database = Some("default"),
1664+
schema = (new StructType)
1665+
.add("a", IntegerType, nullable = true, "test")
1666+
.add("b", StringType)
1667+
.add("nested", (new StructType)
1668+
.add("col1", StringType)
1669+
.add("col2", IntegerType)
1670+
),
1671+
partitionColumnNames = Seq("nested")
1672+
)
1673+
)
1674+
}
1675+
1676+
interceptParseException(parsePlan)(
1677+
"CREATE TABLE my_tab(a: INT COMMENT 'test', b: STRING)",
1678+
"extraneous input ':'")
1679+
}
1680+
15711681
test("create hive table - table file format") {
15721682
val allSources = Seq("parquet", "parquetfile", "orc", "orcfile", "avro", "avrofile",
15731683
"sequencefile", "rcfile", "textfile")

‎sql/core/src/test/scala/org/apache/spark/sql/sources/CreateTableAsSelectSuite.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -166,13 +166,13 @@ class CreateTableAsSelectSuite extends DataSourceTest with SharedSparkSession {
166166
)
167167
}.getMessage
168168
assert(error.contains("Operation not allowed") &&
169-
error.contains("CREATE TEMPORARY TABLE ... USING ... AS query"))
169+
error.contains("CREATE TEMPORARY TABLE"))
170170
}
171171
}
172172

173173
test("disallows CREATE EXTERNAL TABLE ... USING ... AS query") {
174174
withTable("t") {
175-
val error = intercept[ParseException] {
175+
val error = intercept[AnalysisException] {
176176
sql(
177177
s"""
178178
|CREATE EXTERNAL TABLE t USING PARQUET

0 commit comments

Comments
 (0)