From a5f7d216e5fb0377dd64eeab0dd9dde87b499f50 Mon Sep 17 00:00:00 2001 From: PengLei Date: Wed, 19 Jan 2022 15:37:53 +0800 Subject: [PATCH] add draft --- docs/sql-migration-guide.md | 2 ++ .../org/apache/spark/sql/catalyst/parser/AstBuilder.scala | 4 ++++ .../apache/spark/sql/connector/catalog/CatalogV2Util.scala | 3 ++- .../sql/execution/datasources/v2/ShowCreateTableExec.scala | 4 +--- .../sql/execution/datasources/v2/V2SessionCatalogSuite.scala | 3 +-- .../org/apache/spark/sql/hive/execution/HiveDDLSuite.scala | 2 +- 6 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md index 5edf83935dcd..01c828a0f69b 100644 --- a/docs/sql-migration-guide.md +++ b/docs/sql-migration-guide.md @@ -56,6 +56,8 @@ license: | - Since Spark 3.3, DESCRIBE FUNCTION fails if the function does not exist. In Spark 3.2 or earlier, DESCRIBE FUNCTION can still run and print "Function: func_name not found". + - Since Spark 3.3, the table property `external` becomes reserved. Certain commands will fail if you specify the `external` property, such as `CREATE TABLE ... TBLPROPERTIES` and `ALTER TABLE ... SET TBLPROPERTIES`. In Spark 3.2 and earlier, the table property `external` is silently ignored. You can set `spark.sql.legacy.notReserveProperties` to `true` to restore the old behavior. + ## Upgrading from Spark SQL 3.1 to 3.2 - Since Spark 3.2, ADD FILE/JAR/ARCHIVE commands require each path to be enclosed by `"` or `'` if the path contains whitespaces. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index 6a509db73718..35fe084e1a4e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -3203,6 +3203,10 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logg throw QueryParsingErrors.cannotCleanReservedTablePropertyError( PROP_OWNER, ctx, "it will be set to the current user") case (PROP_OWNER, _) => false + case (PROP_EXTERNAL, _) if !legacyOn => + throw QueryParsingErrors.cannotCleanReservedTablePropertyError( + PROP_EXTERNAL, ctx, "please use CREATE EXTERNAL TABLE") + case (PROP_EXTERNAL, _) => false case _ => true } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala index 597b3c3884c6..4092674046ec 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogV2Util.scala @@ -47,7 +47,8 @@ private[sql] object CatalogV2Util { Seq(TableCatalog.PROP_COMMENT, TableCatalog.PROP_LOCATION, TableCatalog.PROP_PROVIDER, - TableCatalog.PROP_OWNER) + TableCatalog.PROP_OWNER, + TableCatalog.PROP_EXTERNAL) /** * The list of reserved namespace properties, which can not be removed or changed directly by diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateTableExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateTableExec.scala index 8b3ad9521648..abe8d67adb85 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateTableExec.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/ShowCreateTableExec.scala @@ -127,9 +127,7 @@ case class ShowCreateTableExec( val showProps = table.properties.asScala .filterKeys(key => !CatalogV2Util.TABLE_RESERVED_PROPERTIES.contains(key) && !key.startsWith(TableCatalog.OPTION_PREFIX) - && !tableOptions.contains(key) - && !key.equals(TableCatalog.PROP_EXTERNAL) - ) + && !tableOptions.contains(key)) if (showProps.nonEmpty) { val props = showProps.toSeq.sortBy(_._1).map { case (key, value) => diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala index 86f4dc467638..68b2b9f5c40b 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/v2/V2SessionCatalogSuite.scala @@ -785,8 +785,7 @@ class V2SessionCatalogTableSuite extends V2SessionCatalogBaseSuite { private def filterV2TableProperties( properties: util.Map[String, String]): Map[String, String] = { properties.asScala.filter(kv => !CatalogV2Util.TABLE_RESERVED_PROPERTIES.contains(kv._1)) - .filter(!_._1.startsWith(TableCatalog.OPTION_PREFIX)) - .filter(_._1 != TableCatalog.PROP_EXTERNAL).toMap + .filter(!_._1.startsWith(TableCatalog.OPTION_PREFIX)).toMap } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala index 014feb33df5e..a41a43599661 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveDDLSuite.scala @@ -844,7 +844,7 @@ class HiveDDLSuite assert( catalog.getTableMetadata(TableIdentifier(tabName)).tableType == CatalogTableType.MANAGED) // The table property is case sensitive. Thus, external is allowed - sql(s"ALTER TABLE $tabName SET TBLPROPERTIES ('external' = 'TRUE')") + sql(s"ALTER TABLE $tabName SET TBLPROPERTIES ('External' = 'TRUE')") // The table type is not changed to external assert( catalog.getTableMetadata(TableIdentifier(tabName)).tableType == CatalogTableType.MANAGED)