From d299dde892fd51d29479c47ba489a697044e7385 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Mon, 29 Jul 2024 21:25:48 -0700 Subject: [PATCH 1/3] fix --- .../util/ResolveDefaultColumnsUtil.scala | 11 ++++++++-- .../sql/ResolveDefaultColumnsSuite.scala | 21 +++++++++++++++++++ 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala index 6b4f29bea757..ed94a113ee90 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala @@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.analysis._ import org.apache.spark.sql.catalyst.catalog.{CatalogDatabase, InMemoryCatalog, SessionCatalog} import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.expressions.{Literal => ExprLiteral} -import org.apache.spark.sql.catalyst.optimizer.{ConstantFolding, ReplaceExpressions} +import org.apache.spark.sql.catalyst.optimizer.{ConstantFolding, Optimizer} import org.apache.spark.sql.catalyst.parser.{CatalystSqlParser, ParseException} import org.apache.spark.sql.catalyst.plans.logical._ import org.apache.spark.sql.catalyst.trees.TreePattern.PLAN_EXPRESSION @@ -290,7 +290,9 @@ object ResolveDefaultColumns extends QueryErrorsBase val analyzer: Analyzer = DefaultColumnAnalyzer val analyzed = analyzer.execute(Project(Seq(Alias(parsed, colName)()), OneRowRelation())) analyzer.checkAnalysis(analyzed) - ConstantFolding(ReplaceExpressions(analyzed)) + // Eagerly execute finish-analysis and constant-folding rules before checking whether the + // expression is foldable and resolved. + ConstantFolding(DefaultColumnOptimizer.FinishAnalysis(analyzed)) } catch { case ex: AnalysisException => throw QueryCompilationErrors.defaultValuesUnresolvedExprError( @@ -517,6 +519,11 @@ object ResolveDefaultColumns extends QueryErrorsBase new CatalogManager(BuiltInFunctionCatalog, BuiltInFunctionCatalog.v1Catalog)) { } + /** + * This is an Analyzer for processing default column values using built-in functions only. + */ + object DefaultColumnOptimizer extends Optimizer(DefaultColumnAnalyzer.catalogManager) + /** * This is a FunctionCatalog for performing analysis using built-in functions only. It is a helper * for the DefaultColumnAnalyzer above. diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ResolveDefaultColumnsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ResolveDefaultColumnsSuite.scala index bca147279993..203b752838cc 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/ResolveDefaultColumnsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/ResolveDefaultColumnsSuite.scala @@ -287,4 +287,25 @@ class ResolveDefaultColumnsSuite extends QueryTest with SharedSparkSession { checkAnswer(sql("select v from t"), sql("select parse_json('1')").collect()) } } + + test("Create table with current_user() default") { + val tableName = "test_current_user" + val user = spark.sparkContext.sparkUser + withTable(tableName) { + sql(s"CREATE TABLE $tableName(i int, s string default current_user()) USING parquet") + sql(s"INSERT INTO $tableName (i) VALUES ((0))") + checkAnswer(sql(s"SELECT * FROM $tableName"), Seq(Row(0, user))) + } + } + + test("Alter table with current_user() default") { + val tableName = "test_current_user" + val user = spark.sparkContext.sparkUser + withTable(tableName) { + sql(s"CREATE TABLE $tableName(i int, s string) USING parquet") + sql(s"ALTER TABLE $tableName ALTER COLUMN s SET DEFAULT current_user()") + sql(s"INSERT INTO $tableName (i) VALUES ((0))") + checkAnswer(sql(s"SELECT * FROM $tableName"), Seq(Row(0, user))) + } + } } From d15f3a29c53bfa5d314ccc69dd9ee5c6c15b7087 Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Mon, 29 Jul 2024 21:45:20 -0700 Subject: [PATCH 2/3] add jira in test cases --- .../org/apache/spark/sql/ResolveDefaultColumnsSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ResolveDefaultColumnsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ResolveDefaultColumnsSuite.scala index 203b752838cc..e3ebbadbb829 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/ResolveDefaultColumnsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/ResolveDefaultColumnsSuite.scala @@ -288,7 +288,7 @@ class ResolveDefaultColumnsSuite extends QueryTest with SharedSparkSession { } } - test("Create table with current_user() default") { + test("SPARK-49054: Create table with current_user() default") { val tableName = "test_current_user" val user = spark.sparkContext.sparkUser withTable(tableName) { @@ -298,7 +298,7 @@ class ResolveDefaultColumnsSuite extends QueryTest with SharedSparkSession { } } - test("Alter table with current_user() default") { + test("SPARK-49054: Alter table with current_user() default") { val tableName = "test_current_user" val user = spark.sparkContext.sparkUser withTable(tableName) { From d4803a33a722e774e054e383b852fbec08bcc43a Mon Sep 17 00:00:00 2001 From: Gengliang Wang Date: Mon, 29 Jul 2024 21:49:38 -0700 Subject: [PATCH 3/3] update comment --- .../spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala index ed94a113ee90..8b7392e71249 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/ResolveDefaultColumnsUtil.scala @@ -520,7 +520,7 @@ object ResolveDefaultColumns extends QueryErrorsBase } /** - * This is an Analyzer for processing default column values using built-in functions only. + * This is an Optimizer for convert default column expressions to foldable literals. */ object DefaultColumnOptimizer extends Optimizer(DefaultColumnAnalyzer.catalogManager)