diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index 15573c9ce50a..a8d4a3f808e6 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -149,11 +149,8 @@ statement | ALTER (TABLE | VIEW) multipartIdentifier UNSET TBLPROPERTIES (IF EXISTS)? tablePropertyList #unsetTableProperties | ALTER TABLE multipartIdentifier - (ALTER | CHANGE) COLUMN? qualifiedName + (ALTER | CHANGE) COLUMN? multipartIdentifier (TYPE dataType)? (COMMENT comment=STRING)? colPosition? #alterTableColumn - | ALTER TABLE tableIdentifier partitionSpec? - CHANGE COLUMN? - colName=errorCapturingIdentifier colType colPosition? #changeColumn | ALTER TABLE multipartIdentifier (partitionSpec)? SET SERDE STRING (WITH SERDEPROPERTIES tablePropertyList)? #setTableSerDe | ALTER TABLE multipartIdentifier (partitionSpec)? diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 39cf171829b3..e30dd8cf9e48 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -2677,9 +2677,12 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging operationNotAllowed(s"ALTER TABLE table $verb COLUMN requires a TYPE or a COMMENT", ctx) } + val tableIdentifier = ctx.multipartIdentifier(0) + val qualifiedColumn = ctx.multipartIdentifier(1) + AlterTableAlterColumnStatement( - visitMultipartIdentifier(ctx.multipartIdentifier), - typedVisit[Seq[String]](ctx.qualifiedName), + visitMultipartIdentifier(tableIdentifier), + typedVisit[Seq[String]](qualifiedColumn), Option(ctx.dataType).map(typedVisit[DataType]), Option(ctx.comment).map(string)) } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ErrorParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ErrorParserSuite.scala index 478953fb1b57..837edc5fc9dd 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ErrorParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ErrorParserSuite.scala @@ -95,7 +95,7 @@ class ErrorParserSuite extends AnalysisTest { """ |ALTER TABLE t |CHANGE COLUMN - |test-col BIGINT + |test-col TYPE BIGINT """.stripMargin, 4, 4, 5, msg + " test-col") intercept("CREATE TABLE test (attri-bute INT)", 1, 24, 25, msg + " attri-bute") intercept( diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index 6902890c5d17..de32a7f30342 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -408,33 +408,6 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { ctx.VIEW != null) } - /** - * Create a [[AlterTableChangeColumnCommand]] command. - * - * For example: - * {{{ - * ALTER TABLE table [PARTITION partition_spec] - * CHANGE [COLUMN] column_old_name column_new_name column_dataType [COMMENT column_comment] - * [FIRST | AFTER column_name]; - * }}} - */ - override def visitChangeColumn(ctx: ChangeColumnContext): LogicalPlan = withOrigin(ctx) { - if (ctx.partitionSpec != null) { - operationNotAllowed("ALTER TABLE table PARTITION partition_spec CHANGE COLUMN", ctx) - } - - if (ctx.colPosition != null) { - operationNotAllowed( - "ALTER TABLE table [PARTITION partition_spec] CHANGE COLUMN ... FIRST | AFTER otherCol", - ctx) - } - - AlterTableChangeColumnCommand( - tableName = visitTableIdentifier(ctx.tableIdentifier), - columnName = ctx.colName.getText, - newColumn = visitColType(ctx.colType)) - } - /** * Convert a nested constants list into a sequence of string sequences. */ diff --git a/sql/core/src/test/resources/sql-tests/inputs/change-column.sql b/sql/core/src/test/resources/sql-tests/inputs/change-column.sql index 6f5ac221ce79..dd2fc660b53e 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/change-column.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/change-column.sql @@ -2,56 +2,43 @@ CREATE TABLE test_change(a INT, b STRING, c INT) using parquet; DESC test_change; --- Change column name (not supported yet) -ALTER TABLE test_change CHANGE a a1 INT; +-- ALTER TABLE CHANGE COLUMN must change either type or comment +ALTER TABLE test_change CHANGE a; +DESC test_change; + +-- Change column name (not supported on v1 table) +ALTER TABLE test_change RENAME COLUMN a TO a1; DESC test_change; -- Change column dataType (not supported yet) -ALTER TABLE test_change CHANGE a a STRING; +ALTER TABLE test_change CHANGE a TYPE STRING; DESC test_change; -- Change column position (not supported yet) -ALTER TABLE test_change CHANGE a a INT AFTER b; -ALTER TABLE test_change CHANGE b b STRING FIRST; +ALTER TABLE test_change CHANGE a TYPE INT AFTER b; +ALTER TABLE test_change CHANGE b TYPE STRING FIRST; DESC test_change; -- Change column comment -ALTER TABLE test_change CHANGE a a INT COMMENT 'this is column a'; -ALTER TABLE test_change CHANGE b b STRING COMMENT '#*02?`'; -ALTER TABLE test_change CHANGE c c INT COMMENT ''; +ALTER TABLE test_change CHANGE a TYPE INT COMMENT 'this is column a'; +ALTER TABLE test_change CHANGE b TYPE STRING COMMENT '#*02?`'; +ALTER TABLE test_change CHANGE c TYPE INT COMMENT ''; DESC test_change; -- Don't change anything. -ALTER TABLE test_change CHANGE a a INT COMMENT 'this is column a'; +ALTER TABLE test_change CHANGE a TYPE INT COMMENT 'this is column a'; DESC test_change; -- Change a invalid column -ALTER TABLE test_change CHANGE invalid_col invalid_col INT; -DESC test_change; - --- Change column name/dataType/position/comment together (not supported yet) -ALTER TABLE test_change CHANGE a a1 STRING COMMENT 'this is column a1' AFTER b; -DESC test_change; - --- Check the behavior with different values of CASE_SENSITIVE -SET spark.sql.caseSensitive=false; -ALTER TABLE test_change CHANGE a A INT COMMENT 'this is column A'; -SET spark.sql.caseSensitive=true; -ALTER TABLE test_change CHANGE a A INT COMMENT 'this is column A1'; +ALTER TABLE test_change CHANGE invalid_col TYPE INT; DESC test_change; -- Change column can't apply to a temporary/global_temporary view CREATE TEMPORARY VIEW temp_view(a, b) AS SELECT 1, "one"; -ALTER TABLE temp_view CHANGE a a INT COMMENT 'this is column a'; +ALTER TABLE temp_view CHANGE a TYPE INT COMMENT 'this is column a'; CREATE GLOBAL TEMPORARY VIEW global_temp_view(a, b) AS SELECT 1, "one"; -ALTER TABLE global_temp.global_temp_view CHANGE a a INT COMMENT 'this is column a'; - --- Change column in partition spec (not supported yet) -CREATE TABLE partition_table(a INT, b STRING, c INT, d STRING) USING parquet PARTITIONED BY (c, d); -ALTER TABLE partition_table PARTITION (c = 1) CHANGE COLUMN a new_a INT; -ALTER TABLE partition_table CHANGE COLUMN c c INT COMMENT 'this is column C'; +ALTER TABLE global_temp.global_temp_view CHANGE a TYPE INT COMMENT 'this is column a'; -- DROP TEST TABLE DROP TABLE test_change; -DROP TABLE partition_table; DROP VIEW global_temp.global_temp_view; diff --git a/sql/core/src/test/resources/sql-tests/results/change-column.sql.out b/sql/core/src/test/resources/sql-tests/results/change-column.sql.out index 114617873af4..21a344c071bc 100644 --- a/sql/core/src/test/resources/sql-tests/results/change-column.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/change-column.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 34 +-- Number of queries: 25 -- !query 0 @@ -21,12 +21,17 @@ c int -- !query 2 -ALTER TABLE test_change CHANGE a a1 INT +ALTER TABLE test_change CHANGE a -- !query 2 schema struct<> -- !query 2 output -org.apache.spark.sql.AnalysisException -ALTER TABLE CHANGE COLUMN is not supported for changing column 'a' with type 'IntegerType' to 'a1' with type 'IntegerType'; +org.apache.spark.sql.catalyst.parser.ParseException + +Operation not allowed: ALTER TABLE table CHANGE COLUMN requires a TYPE or a COMMENT(line 1, pos 0) + +== SQL == +ALTER TABLE test_change CHANGE a +^^^ -- !query 3 @@ -40,12 +45,12 @@ c int -- !query 4 -ALTER TABLE test_change CHANGE a a STRING +ALTER TABLE test_change RENAME COLUMN a TO a1 -- !query 4 schema struct<> -- !query 4 output org.apache.spark.sql.AnalysisException -ALTER TABLE CHANGE COLUMN is not supported for changing column 'a' with type 'IntegerType' to 'a' with type 'StringType'; +RENAME COLUMN is only supported with v2 tables.; -- !query 5 @@ -59,61 +64,64 @@ c int -- !query 6 -ALTER TABLE test_change CHANGE a a INT AFTER b +ALTER TABLE test_change CHANGE a TYPE STRING -- !query 6 schema struct<> -- !query 6 output -org.apache.spark.sql.catalyst.parser.ParseException - -Operation not allowed: ALTER TABLE table [PARTITION partition_spec] CHANGE COLUMN ... FIRST | AFTER otherCol(line 1, pos 0) - -== SQL == -ALTER TABLE test_change CHANGE a a INT AFTER b -^^^ +org.apache.spark.sql.AnalysisException +ALTER TABLE CHANGE COLUMN is not supported for changing column 'a' with type 'IntegerType' to 'a' with type 'StringType'; -- !query 7 -ALTER TABLE test_change CHANGE b b STRING FIRST +DESC test_change -- !query 7 schema -struct<> +struct -- !query 7 output -org.apache.spark.sql.catalyst.parser.ParseException - -Operation not allowed: ALTER TABLE table [PARTITION partition_spec] CHANGE COLUMN ... FIRST | AFTER otherCol(line 1, pos 0) - -== SQL == -ALTER TABLE test_change CHANGE b b STRING FIRST -^^^ +a int +b string +c int -- !query 8 -DESC test_change +ALTER TABLE test_change CHANGE a TYPE INT AFTER b -- !query 8 schema -struct +struct<> -- !query 8 output -a int -b string -c int +org.apache.spark.sql.catalyst.parser.ParseException + +Operation not allowed: ALTER TABLE table CHANGE COLUMN ... FIRST | AFTER otherCol(line 1, pos 0) + +== SQL == +ALTER TABLE test_change CHANGE a TYPE INT AFTER b +^^^ -- !query 9 -ALTER TABLE test_change CHANGE a a INT COMMENT 'this is column a' +ALTER TABLE test_change CHANGE b TYPE STRING FIRST -- !query 9 schema struct<> -- !query 9 output +org.apache.spark.sql.catalyst.parser.ParseException +Operation not allowed: ALTER TABLE table CHANGE COLUMN ... FIRST | AFTER otherCol(line 1, pos 0) + +== SQL == +ALTER TABLE test_change CHANGE b TYPE STRING FIRST +^^^ -- !query 10 -ALTER TABLE test_change CHANGE b b STRING COMMENT '#*02?`' +DESC test_change -- !query 10 schema -struct<> +struct -- !query 10 output - +a int +b string +c int -- !query 11 -ALTER TABLE test_change CHANGE c c INT COMMENT '' +ALTER TABLE test_change CHANGE a TYPE INT COMMENT 'this is column a' -- !query 11 schema struct<> -- !query 11 output @@ -121,17 +129,15 @@ struct<> -- !query 12 -DESC test_change +ALTER TABLE test_change CHANGE b TYPE STRING COMMENT '#*02?`' -- !query 12 schema -struct +struct<> -- !query 12 output -a int this is column a -b string #*02?` -c int + -- !query 13 -ALTER TABLE test_change CHANGE a a INT COMMENT 'this is column a' +ALTER TABLE test_change CHANGE c TYPE INT COMMENT '' -- !query 13 schema struct<> -- !query 13 output @@ -149,12 +155,11 @@ c int -- !query 15 -ALTER TABLE test_change CHANGE invalid_col invalid_col INT +ALTER TABLE test_change CHANGE a TYPE INT COMMENT 'this is column a' -- !query 15 schema struct<> -- !query 15 output -org.apache.spark.sql.AnalysisException -Can't find column `invalid_col` given table data columns [`a`, `b`, `c`]; + -- !query 16 @@ -168,17 +173,12 @@ c int -- !query 17 -ALTER TABLE test_change CHANGE a a1 STRING COMMENT 'this is column a1' AFTER b +ALTER TABLE test_change CHANGE invalid_col TYPE INT -- !query 17 schema struct<> -- !query 17 output -org.apache.spark.sql.catalyst.parser.ParseException - -Operation not allowed: ALTER TABLE table [PARTITION partition_spec] CHANGE COLUMN ... FIRST | AFTER otherCol(line 1, pos 0) - -== SQL == -ALTER TABLE test_change CHANGE a a1 STRING COMMENT 'this is column a1' AFTER b -^^^ +org.apache.spark.sql.AnalysisException +Can't find column `invalid_col` given table data columns [`a`, `b`, `c`]; -- !query 18 @@ -192,132 +192,50 @@ c int -- !query 19 -SET spark.sql.caseSensitive=false +CREATE TEMPORARY VIEW temp_view(a, b) AS SELECT 1, "one" -- !query 19 schema -struct +struct<> -- !query 19 output -spark.sql.caseSensitive false + -- !query 20 -ALTER TABLE test_change CHANGE a A INT COMMENT 'this is column A' +ALTER TABLE temp_view CHANGE a TYPE INT COMMENT 'this is column a' -- !query 20 schema struct<> -- !query 20 output - +org.apache.spark.sql.AnalysisException +Invalid command: 'temp_view' is a view not a table.; line 1 pos 0 -- !query 21 -SET spark.sql.caseSensitive=true +CREATE GLOBAL TEMPORARY VIEW global_temp_view(a, b) AS SELECT 1, "one" -- !query 21 schema -struct +struct<> -- !query 21 output -spark.sql.caseSensitive true + -- !query 22 -ALTER TABLE test_change CHANGE a A INT COMMENT 'this is column A1' +ALTER TABLE global_temp.global_temp_view CHANGE a TYPE INT COMMENT 'this is column a' -- !query 22 schema struct<> -- !query 22 output org.apache.spark.sql.AnalysisException -ALTER TABLE CHANGE COLUMN is not supported for changing column 'a' with type 'IntegerType' to 'A' with type 'IntegerType'; +Invalid command: 'global_temp.global_temp_view' is a view not a table.; line 1 pos 0 -- !query 23 -DESC test_change +DROP TABLE test_change -- !query 23 schema -struct +struct<> -- !query 23 output -a int this is column A -b string #*02?` -c int + -- !query 24 -CREATE TEMPORARY VIEW temp_view(a, b) AS SELECT 1, "one" +DROP VIEW global_temp.global_temp_view -- !query 24 schema struct<> -- !query 24 output - - --- !query 25 -ALTER TABLE temp_view CHANGE a a INT COMMENT 'this is column a' --- !query 25 schema -struct<> --- !query 25 output -org.apache.spark.sql.catalyst.analysis.NoSuchTableException -Table or view 'temp_view' not found in database 'default'; - - --- !query 26 -CREATE GLOBAL TEMPORARY VIEW global_temp_view(a, b) AS SELECT 1, "one" --- !query 26 schema -struct<> --- !query 26 output - - - --- !query 27 -ALTER TABLE global_temp.global_temp_view CHANGE a a INT COMMENT 'this is column a' --- !query 27 schema -struct<> --- !query 27 output -org.apache.spark.sql.catalyst.analysis.NoSuchDatabaseException -Database 'global_temp' not found; - - --- !query 28 -CREATE TABLE partition_table(a INT, b STRING, c INT, d STRING) USING parquet PARTITIONED BY (c, d) --- !query 28 schema -struct<> --- !query 28 output - - - --- !query 29 -ALTER TABLE partition_table PARTITION (c = 1) CHANGE COLUMN a new_a INT --- !query 29 schema -struct<> --- !query 29 output -org.apache.spark.sql.catalyst.parser.ParseException - -Operation not allowed: ALTER TABLE table PARTITION partition_spec CHANGE COLUMN(line 1, pos 0) - -== SQL == -ALTER TABLE partition_table PARTITION (c = 1) CHANGE COLUMN a new_a INT -^^^ - - --- !query 30 -ALTER TABLE partition_table CHANGE COLUMN c c INT COMMENT 'this is column C' --- !query 30 schema -struct<> --- !query 30 output -org.apache.spark.sql.AnalysisException -Can't find column `c` given table data columns [`a`, `b`]; - - --- !query 31 -DROP TABLE test_change --- !query 31 schema -struct<> --- !query 31 output - - - --- !query 32 -DROP TABLE partition_table --- !query 32 schema -struct<> --- !query 32 output - - - --- !query 33 -DROP VIEW global_temp.global_temp_view --- !query 33 schema -struct<> --- !query 33 output - diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala index df81f46390e2..940df21131e0 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala @@ -489,34 +489,6 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession { "SET FILEFORMAT PARQUET") } - test("alter table: change column name/type/comment") { - val sql1 = "ALTER TABLE table_name CHANGE COLUMN col_old_name col_new_name INT" - val sql2 = "ALTER TABLE table_name CHANGE COLUMN col_name col_name INT COMMENT 'new_comment'" - val parsed1 = parser.parsePlan(sql1) - val parsed2 = parser.parsePlan(sql2) - val tableIdent = TableIdentifier("table_name", None) - val expected1 = AlterTableChangeColumnCommand( - tableIdent, - "col_old_name", - StructField("col_new_name", IntegerType)) - val expected2 = AlterTableChangeColumnCommand( - tableIdent, - "col_name", - StructField("col_name", IntegerType).withComment("new_comment")) - comparePlans(parsed1, expected1) - comparePlans(parsed2, expected2) - } - - test("alter table: change column position (not supported)") { - assertUnsupported("ALTER TABLE table_name CHANGE COLUMN col_old_name col_new_name INT FIRST") - assertUnsupported( - "ALTER TABLE table_name CHANGE COLUMN col_old_name col_new_name INT AFTER other_col") - } - - test("alter table: change column in partition spec") { - assertUnsupported("ALTER TABLE table_name PARTITION (a='1', a='2') CHANGE COLUMN a new_a INT") - } - test("alter table: touch (not supported)") { assertUnsupported("ALTER TABLE table_name TOUCH") assertUnsupported("ALTER TABLE table_name TOUCH PARTITION (dt='2008-08-08', country='us')") diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index 514062919dfc..2df4537a743f 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -1725,7 +1725,7 @@ abstract class DDLSuite extends QueryTest with SQLTestUtils { column.map(_.metadata).getOrElse(Metadata.empty) } // Ensure that change column will preserve other metadata fields. - sql("ALTER TABLE dbx.tab1 CHANGE COLUMN col1 col1 INT COMMENT 'this is col1'") + sql("ALTER TABLE dbx.tab1 CHANGE COLUMN col1 TYPE INT COMMENT 'this is col1'") assert(getMetadata("col1").getString("key") == "value") assert(getMetadata("col1").getString("comment") == "this is col1") }