diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index a9e3ca654986..bf85fc4da5bd 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -158,10 +158,8 @@ statement SET SERDE STRING (WITH SERDEPROPERTIES tablePropertyList)? #setTableSerDe | ALTER TABLE tableIdentifier (partitionSpec)? SET SERDEPROPERTIES tablePropertyList #setTableSerDe - | ALTER TABLE tableIdentifier ADD (IF NOT EXISTS)? + | ALTER (TABLE | VIEW) multipartIdentifier ADD (IF NOT EXISTS)? partitionSpecLocation+ #addTablePartition - | ALTER VIEW tableIdentifier ADD (IF NOT EXISTS)? - partitionSpec+ #addTablePartition | ALTER TABLE multipartIdentifier from=partitionSpec RENAME TO to=partitionSpec #renameTablePartition | ALTER (TABLE | VIEW) multipartIdentifier diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 3de13abf5456..55931c9ec61d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -2970,6 +2970,35 @@ class AstBuilder(conf: SQLConf) extends SqlBaseBaseVisitor[AnyRef] with Logging AlterTableRecoverPartitionsStatement(visitMultipartIdentifier(ctx.multipartIdentifier)) } + /** + * Create an [[AlterTableAddPartitionStatement]]. + * + * For example: + * {{{ + * ALTER TABLE multi_part_name ADD [IF NOT EXISTS] PARTITION spec [LOCATION 'loc1'] + * ALTER VIEW multi_part_name ADD [IF NOT EXISTS] PARTITION spec + * }}} + * + * ALTER VIEW ... ADD PARTITION ... is not supported because the concept of partitioning + * is associated with physical tables + */ + override def visitAddTablePartition( + ctx: AddTablePartitionContext): LogicalPlan = withOrigin(ctx) { + if (ctx.VIEW != null) { + operationNotAllowed("ALTER VIEW ... ADD PARTITION", ctx) + } + // Create partition spec to location mapping. + val specsAndLocs = ctx.partitionSpecLocation.asScala.map { splCtx => + val spec = visitNonOptionalPartitionSpec(splCtx.partitionSpec) + val location = Option(splCtx.locationSpec).map(visitLocationSpec) + spec -> location + } + AlterTableAddPartitionStatement( + visitMultipartIdentifier(ctx.multipartIdentifier), + specsAndLocs, + ctx.EXISTS != null) + } + /** * Create an [[AlterTableRenamePartitionStatement]] * diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala index dd3f8f8ce6c4..e0303e883dee 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/statements.scala @@ -188,6 +188,14 @@ case class AlterTableSetLocationStatement( case class AlterTableRecoverPartitionsStatement( tableName: Seq[String]) extends ParsedStatement +/** + * ALTER TABLE ... ADD PARTITION command, as parsed from SQL + */ +case class AlterTableAddPartitionStatement( + tableName: Seq[String], + partitionSpecsAndLocs: Seq[(TablePartitionSpec, Option[String])], + ifNotExists: Boolean) extends ParsedStatement + /** * ALTER TABLE ... RENAME PARTITION command, as parsed from SQL. */ diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala index 10125627ba17..63196e0df7cd 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DDLParserSuite.scala @@ -1202,6 +1202,42 @@ class DDLParserSuite extends AnalysisTest { AlterTableRecoverPartitionsStatement(Seq("a", "b", "c"))) } + test("alter table: add partition") { + val sql1 = + """ + |ALTER TABLE a.b.c ADD IF NOT EXISTS PARTITION + |(dt='2008-08-08', country='us') LOCATION 'location1' PARTITION + |(dt='2009-09-09', country='uk') + """.stripMargin + val sql2 = "ALTER TABLE a.b.c ADD PARTITION (dt='2008-08-08') LOCATION 'loc'" + + val parsed1 = parsePlan(sql1) + val parsed2 = parsePlan(sql2) + + val expected1 = AlterTableAddPartitionStatement( + Seq("a", "b", "c"), + Seq( + (Map("dt" -> "2008-08-08", "country" -> "us"), Some("location1")), + (Map("dt" -> "2009-09-09", "country" -> "uk"), None)), + ifNotExists = true) + val expected2 = AlterTableAddPartitionStatement( + Seq("a", "b", "c"), + Seq((Map("dt" -> "2008-08-08"), Some("loc"))), + ifNotExists = false) + + comparePlans(parsed1, expected1) + comparePlans(parsed2, expected2) + } + + test("alter view: add partition (not supported)") { + assertUnsupported( + """ + |ALTER VIEW a.b.c ADD IF NOT EXISTS PARTITION + |(dt='2008-08-08', country='us') PARTITION + |(dt='2009-09-09', country='uk') + """.stripMargin) + } + test("alter table: rename partition") { val sql1 = """ diff --git a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala index 8076bacefb32..cf80031f46c5 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/catalyst/analysis/ResolveSessionCatalog.scala @@ -370,6 +370,13 @@ class ResolveSessionCatalog( v1TableName.asTableIdentifier, "ALTER TABLE RECOVER PARTITIONS") + case AlterTableAddPartitionStatement(tableName, partitionSpecsAndLocs, ifNotExists) => + val v1TableName = parseV1Table(tableName, "ALTER TABLE ADD PARTITION") + AlterTableAddPartitionCommand( + v1TableName.asTableIdentifier, + partitionSpecsAndLocs, + ifNotExists) + case AlterTableRenamePartitionStatement(tableName, from, to) => val v1TableName = parseV1Table(tableName, "ALTER TABLE RENAME PARTITION") AlterTableRenamePartitionCommand( diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index 84641f2e9391..8bbf549309ad 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -426,41 +426,6 @@ class SparkSqlAstBuilder(conf: SQLConf) extends AstBuilder(conf) { Option(ctx.partitionSpec).map(visitNonOptionalPartitionSpec)) } - /** - * Create an [[AlterTableAddPartitionCommand]] command. - * - * For example: - * {{{ - * ALTER TABLE table ADD [IF NOT EXISTS] PARTITION spec [LOCATION 'loc1'] - * ALTER VIEW view ADD [IF NOT EXISTS] PARTITION spec - * }}} - * - * ALTER VIEW ... ADD PARTITION ... is not supported because the concept of partitioning - * is associated with physical tables - */ - override def visitAddTablePartition( - ctx: AddTablePartitionContext): LogicalPlan = withOrigin(ctx) { - if (ctx.VIEW != null) { - operationNotAllowed("ALTER VIEW ... ADD PARTITION", ctx) - } - // Create partition spec to location mapping. - val specsAndLocs = if (ctx.partitionSpec.isEmpty) { - ctx.partitionSpecLocation.asScala.map { - splCtx => - val spec = visitNonOptionalPartitionSpec(splCtx.partitionSpec) - val location = Option(splCtx.locationSpec).map(visitLocationSpec) - spec -> location - } - } else { - // Alter View: the location clauses are not allowed. - ctx.partitionSpec.asScala.map(visitNonOptionalPartitionSpec(_) -> None) - } - AlterTableAddPartitionCommand( - visitTableIdentifier(ctx.tableIdentifier), - specsAndLocs, - ctx.EXISTS != null) - } - /** * Create a [[AlterTableChangeColumnCommand]] command. * diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala index 096ae9690cbd..ce6bcbda9775 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala @@ -1385,6 +1385,17 @@ class DataSourceV2SQLSuite } } + test("ALTER TABLE ADD PARTITION") { + val t = "testcat.ns1.ns2.tbl" + withTable(t) { + spark.sql(s"CREATE TABLE $t (id bigint, data string) USING foo PARTITIONED BY (id)") + val e = intercept[AnalysisException] { + sql(s"ALTER TABLE $t ADD PARTITION (id=1) LOCATION 'loc'") + } + assert(e.message.contains("ALTER TABLE ADD PARTITION is only supported with v1 tables")) + } + } + test("ALTER TABLE RENAME PARTITION") { val t = "testcat.ns1.ns2.tbl" withTable(t) { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala index d82752f53215..9ea4a8cc38a6 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLParserSuite.scala @@ -520,44 +520,6 @@ class DDLParserSuite extends AnalysisTest with SharedSparkSession { containsThesePhrases = Seq("key_without_value")) } - // ALTER TABLE table_name ADD [IF NOT EXISTS] PARTITION partition_spec - // [LOCATION 'location1'] partition_spec [LOCATION 'location2'] ...; - test("alter table: add partition") { - val sql1 = - """ - |ALTER TABLE table_name ADD IF NOT EXISTS PARTITION - |(dt='2008-08-08', country='us') LOCATION 'location1' PARTITION - |(dt='2009-09-09', country='uk') - """.stripMargin - val sql2 = "ALTER TABLE table_name ADD PARTITION (dt='2008-08-08') LOCATION 'loc'" - - val parsed1 = parser.parsePlan(sql1) - val parsed2 = parser.parsePlan(sql2) - - val expected1 = AlterTableAddPartitionCommand( - TableIdentifier("table_name", None), - Seq( - (Map("dt" -> "2008-08-08", "country" -> "us"), Some("location1")), - (Map("dt" -> "2009-09-09", "country" -> "uk"), None)), - ifNotExists = true) - val expected2 = AlterTableAddPartitionCommand( - TableIdentifier("table_name", None), - Seq((Map("dt" -> "2008-08-08"), Some("loc"))), - ifNotExists = false) - - comparePlans(parsed1, expected1) - comparePlans(parsed2, expected2) - } - - test("alter view: add partition (not supported)") { - assertUnsupported( - """ - |ALTER VIEW view_name ADD IF NOT EXISTS PARTITION - |(dt='2008-08-08', country='us') PARTITION - |(dt='2009-09-09', country='uk') - """.stripMargin) - } - test("alter table: exchange partition (not supported)") { assertUnsupported( """