diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 960d7b4599b2..447664201b77 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -507,6 +507,9 @@ class AstBuilder extends SqlBaseParserBaseVisitor[AnyRef] with SQLConfHelper wit ctx: PartitionSpecContext): Map[String, Option[String]] = withOrigin(ctx) { val legacyNullAsString = conf.getConf(SQLConf.LEGACY_PARSE_NULL_PARTITION_SPEC_AS_STRING_LITERAL) + val keepPartitionSpecAsString = + conf.getConf(SQLConf.LEGACY_KEEP_PARTITION_SPEC_AS_STRING_LITERAL) + val parts = ctx.partitionVal.asScala.map { pVal => // Check if the query attempted to refer to a DEFAULT column value within the PARTITION clause // and return a specific error to help guide the user, since this is not allowed. @@ -514,7 +517,9 @@ class AstBuilder extends SqlBaseParserBaseVisitor[AnyRef] with SQLConfHelper wit throw QueryParsingErrors.defaultColumnReferencesNotAllowedInPartitionSpec(ctx) } val name = pVal.identifier.getText - val value = Option(pVal.constant).map(v => visitStringConstant(v, legacyNullAsString)) + val value = Option(pVal.constant).map(v => { + visitStringConstant(v, legacyNullAsString, keepPartitionSpecAsString) + }) name -> value } // Before calling `toMap`, we check duplicated keys to avoid silently ignore partition values @@ -546,14 +551,19 @@ class AstBuilder extends SqlBaseParserBaseVisitor[AnyRef] with SQLConfHelper wit */ protected def visitStringConstant( ctx: ConstantContext, - legacyNullAsString: Boolean): String = withOrigin(ctx) { + legacyNullAsString: Boolean = false, + keepPartitionSpecAsString: Boolean = false): String = withOrigin(ctx) { expression(ctx) match { case Literal(null, _) if !legacyNullAsString => null case l @ Literal(null, _) => l.toString case l: Literal => - // TODO For v2 commands, we will cast the string back to its actual value, - // which is a waste and can be improved in the future. - Cast(l, StringType, Some(conf.sessionLocalTimeZone)).eval().toString + if (keepPartitionSpecAsString && !ctx.isInstanceOf[StringLiteralContext]) { + ctx.getText + } else { + // TODO For v2 commands, we will cast the string back to its actual value, + // which is a waste and can be improved in the future. + Cast(l, StringType, Some(conf.sessionLocalTimeZone)).eval().toString + } case other => throw new IllegalArgumentException(s"Only literals are allowed in the " + s"partition spec, but got ${other.sql}") diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index abde198617d3..7f1cb04d5c6f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -3288,6 +3288,16 @@ object SQLConf { .booleanConf .createWithDefault(false) + val LEGACY_KEEP_PARTITION_SPEC_AS_STRING_LITERAL = + buildConf("spark.sql.legacy.keepPartitionSpecAsStringLiteral") + .internal() + .doc("If it is set to true, `PARTITION(col=05)` is parsed as a string literal of its " + + "text representation, e.g., string '05', when the partition column is string type. " + + "Otherwise, it is always parsed as a numeric literal in the partition spec.") + .version("3.4.0") + .booleanConf + .createWithDefault(false) + val LEGACY_REPLACE_DATABRICKS_SPARK_AVRO_ENABLED = buildConf("spark.sql.legacy.replaceDatabricksSparkAvro.enabled") .internal() diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala index e67ffa606efd..3f9acf103ba2 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/SparkSqlParser.scala @@ -363,7 +363,7 @@ class SparkSqlAstBuilder extends AstBuilder { * Convert a constants list into a String sequence. */ override def visitConstantList(ctx: ConstantListContext): Seq[String] = withOrigin(ctx) { - ctx.constant.asScala.map(v => visitStringConstant(v, legacyNullAsString = false)).toSeq + ctx.constant.asScala.map(v => visitStringConstant(v)).toSeq } /** diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLInsertTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLInsertTestSuite.scala index 051ac0f31418..24ebfd75bd56 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLInsertTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLInsertTestSuite.scala @@ -351,6 +351,35 @@ trait SQLInsertTestSuite extends QueryTest with SQLTestUtils { } } } + + test("SPARK-41982: treat the partition field as string literal " + + "when keepPartitionSpecAsStringLiteral is enabled") { + withSQLConf(SQLConf.LEGACY_KEEP_PARTITION_SPEC_AS_STRING_LITERAL.key -> "true") { + withTable("t") { + sql("create table t(i string, j int) using orc partitioned by (dt string)") + sql("insert into t partition(dt=08) values('a', 10)") + Seq( + "select * from t where dt='08'", + "select * from t where dt=08" + ).foreach { query => + checkAnswer(sql(query), Seq(Row("a", 10, "08"))) + } + val e = intercept[AnalysisException](sql("alter table t drop partition(dt='8')")) + assert(e.getMessage.contains("PARTITIONS_NOT_FOUND")) + } + } + + withSQLConf(SQLConf.LEGACY_KEEP_PARTITION_SPEC_AS_STRING_LITERAL.key -> "false") { + withTable("t") { + sql("create table t(i string, j int) using orc partitioned by (dt string)") + sql("insert into t partition(dt=08) values('a', 10)") + checkAnswer(sql("select * from t where dt='08'"), sql("select * from t where dt='07'")) + checkAnswer(sql("select * from t where dt=08"), Seq(Row("a", 10, "8"))) + val e = intercept[AnalysisException](sql("alter table t drop partition(dt='08')")) + assert(e.getMessage.contains("PARTITIONS_NOT_FOUND")) + } + } + } } class FileSourceSQLInsertTestSuite extends SQLInsertTestSuite with SharedSparkSession { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableAddPartitionSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableAddPartitionSuiteBase.scala index f414de1b87c4..3feb4f13c73f 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableAddPartitionSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableAddPartitionSuiteBase.scala @@ -257,4 +257,26 @@ trait AlterTableAddPartitionSuiteBase extends QueryTest with DDLCommandTestUtils } } } + + test("SPARK-41982: add partition when keepPartitionSpecAsString set `true`") { + withSQLConf(SQLConf.LEGACY_KEEP_PARTITION_SPEC_AS_STRING_LITERAL.key -> "true") { + withNamespaceAndTable("ns", "tbl") { t => + sql(s"CREATE TABLE $t(name STRING, age INT) USING PARQUET PARTITIONED BY (dt STRING)") + sql(s"ALTER TABLE $t ADD PARTITION(dt = 08)") + checkPartitions(t, Map("dt" -> "08")) + sql(s"ALTER TABLE $t ADD PARTITION(dt = '09')") + checkPartitions(t, Map("dt" -> "09"), Map("dt" -> "08")) + } + } + + withSQLConf(SQLConf.LEGACY_KEEP_PARTITION_SPEC_AS_STRING_LITERAL.key -> "false") { + withNamespaceAndTable("ns", "tb2") { t => + sql(s"CREATE TABLE $t(name STRING, age INT) USING PARQUET PARTITIONED BY (dt STRING)") + sql(s"ALTER TABLE $t ADD PARTITION(dt = 08)") + checkPartitions(t, Map("dt" -> "8")) + sql(s"ALTER TABLE $t ADD PARTITION(dt = '09')") + checkPartitions(t, Map("dt" -> "09"), Map("dt" -> "8")) + } + } + } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableDropPartitionSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableDropPartitionSuiteBase.scala index b38f5b2dfeb5..3f15533ca5fe 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableDropPartitionSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableDropPartitionSuiteBase.scala @@ -254,4 +254,34 @@ trait AlterTableDropPartitionSuiteBase extends QueryTest with DDLCommandTestUtil checkPartitions(t) } } + + test("SPARK-41982: drop partition when keepPartitionSpecAsString set `true`") { + withSQLConf(SQLConf.LEGACY_KEEP_PARTITION_SPEC_AS_STRING_LITERAL.key -> "true") { + withNamespaceAndTable("ns", "tbl") { t => + sql(s"CREATE TABLE $t(name STRING, age INT) using orc PARTITIONED BY (dt STRING)") + sql(s"ALTER TABLE $t ADD PARTITION(dt = 08)") + checkPartitions(t, Map("dt" -> "08")) + sql(s"ALTER TABLE $t DROP PARTITION (dt = 08)") + checkPartitions(t) + sql(s"ALTER TABLE $t ADD PARTITION(dt = '08')") + checkPartitions(t, Map("dt" -> "08")) + sql(s"ALTER TABLE $t DROP PARTITION (dt = '08')") + checkPartitions(t) + } + } + + withSQLConf(SQLConf.LEGACY_KEEP_PARTITION_SPEC_AS_STRING_LITERAL.key -> "false") { + withNamespaceAndTable("ns", "tb2") { t => + sql(s"CREATE TABLE $t(name STRING, age INT) using orc PARTITIONED BY (dt STRING)") + sql(s"ALTER TABLE $t ADD PARTITION(dt = 08)") + checkPartitions(t, Map("dt" -> "8")) + sql(s"ALTER TABLE $t DROP PARTITION (dt = 08)") + checkPartitions(t) + sql(s"ALTER TABLE $t ADD PARTITION(dt = 08)") + checkPartitions(t, Map("dt" -> "8")) + sql(s"ALTER TABLE $t DROP PARTITION (dt = 8)") + checkPartitions(t) + } + } + } } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableRenamePartitionSuiteBase.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableRenamePartitionSuiteBase.scala index f24cebbf1387..0aaeb8d2160c 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableRenamePartitionSuiteBase.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/AlterTableRenamePartitionSuiteBase.scala @@ -242,4 +242,40 @@ trait AlterTableRenamePartitionSuiteBase extends QueryTest with DDLCommandTestUt checkPartitions(t, Map("part" -> "2020-01-02")) } } + + test("SPARK-41982: rename partition when keepPartitionSpecAsString set `true`") { + withSQLConf(SQLConf.LEGACY_KEEP_PARTITION_SPEC_AS_STRING_LITERAL.key -> "true") { + withNamespaceAndTable("ns", "tbl") { t => + sql(s"CREATE TABLE $t(name STRING, age INT) USING PARQUET PARTITIONED BY (dt STRING)") + sql(s"ALTER TABLE $t ADD PARTITION(dt = 08)") + checkPartitions(t, Map("dt" -> "08")) + sql(s"ALTER TABLE $t PARTITION (dt = 08)" + + s" RENAME TO PARTITION (dt = 09)") + checkPartitions(t, Map("dt" -> "09")) + sql(s"ALTER TABLE $t PARTITION (dt = 09)" + + s" RENAME TO PARTITION (dt = '08')") + checkPartitions(t, Map("dt" -> "08")) + sql(s"ALTER TABLE $t PARTITION (dt = '08')" + + s" RENAME TO PARTITION (dt = '09')") + checkPartitions(t, Map("dt" -> "09")) + } + } + + withSQLConf(SQLConf.LEGACY_KEEP_PARTITION_SPEC_AS_STRING_LITERAL.key -> "false") { + withNamespaceAndTable("ns", "tb2") { t => + sql(s"CREATE TABLE $t(name STRING, age INT) USING PARQUET PARTITIONED BY (dt STRING)") + sql(s"ALTER TABLE $t ADD PARTITION(dt = 08)") + checkPartitions(t, Map("dt" -> "8")) + sql(s"ALTER TABLE $t PARTITION (dt = 08)" + + s" RENAME TO PARTITION (dt = 09)") + checkPartitions(t, Map("dt" -> "9")) + sql(s"ALTER TABLE $t PARTITION (dt = 09)" + + s" RENAME TO PARTITION (dt = '08')") + checkPartitions(t, Map("dt" -> "08")) + sql(s"ALTER TABLE $t PARTITION (dt = '08')" + + s" RENAME TO PARTITION (dt = '09')") + checkPartitions(t, Map("dt" -> "09")) + } + } + } }