From 8042cd653f339f226adbccdc7318f36fa8a75189 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 7 Jan 2021 20:45:45 +0800 Subject: [PATCH 1/2] set analyzed --- .../org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 1 + .../scala/org/apache/spark/sql/CharVarcharTestSuite.scala | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 883ff46148ca6..081401708a03d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -2384,6 +2384,7 @@ class Analyzer(override val catalogManager: CatalogManager) val aliasedOrdering = unresolvedSortOrders.map(o => Alias(o.child, "aggOrder")()) val aggregatedOrdering = aggregate.copy(aggregateExpressions = aliasedOrdering) + aggregatedOrdering.setAnalyzed() val resolvedAggregate: Aggregate = executeSameContext(aggregatedOrdering).asInstanceOf[Aggregate] val resolvedAliasedOrdering: Seq[Alias] = diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala index d20cee0815d4d..fb35d6cf8dacb 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/CharVarcharTestSuite.scala @@ -466,6 +466,14 @@ trait CharVarcharTestSuite extends QueryTest with SQLTestUtils { Row("c")) } } + + test("SPARK-34003: fix char/varchar fails w/ both group by and order by ") { + withTable("t") { + sql(s"CREATE TABLE t(v VARCHAR(3), i INT) USING $format") + sql("INSERT INTO t VALUES ('c', 1)") + checkAnswer(sql("SELECT v, sum(i) FROM t GROUP BY v ORDER BY v"), Row("c", 1)) + } + } } // Some basic char/varchar tests which doesn't rely on table implementation. From daa85499eafafbe37f40401f780663437f46cb31 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Thu, 7 Jan 2021 22:53:43 +0800 Subject: [PATCH 2/2] hack --- .../spark/sql/catalyst/analysis/Analyzer.scala | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 081401708a03d..bf5dbb8200e87 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -2384,7 +2384,6 @@ class Analyzer(override val catalogManager: CatalogManager) val aliasedOrdering = unresolvedSortOrders.map(o => Alias(o.child, "aggOrder")()) val aggregatedOrdering = aggregate.copy(aggregateExpressions = aliasedOrdering) - aggregatedOrdering.setAnalyzed() val resolvedAggregate: Aggregate = executeSameContext(aggregatedOrdering).asInstanceOf[Aggregate] val resolvedAliasedOrdering: Seq[Alias] = @@ -2401,16 +2400,22 @@ class Analyzer(override val catalogManager: CatalogManager) // to push down this ordering expression and can reference the original aggregate // expression instead. val needsPushDown = ArrayBuffer.empty[NamedExpression] - val evaluatedOrderings = resolvedAliasedOrdering.zip(unresolvedSortOrders).map { - case (evaluated, order) => + val orderToAlias = unresolvedSortOrders.zip(aliasedOrdering) + val evaluatedOrderings = resolvedAliasedOrdering.zip(orderToAlias).map { + case (evaluated, (order, aliasOrder)) => val index = originalAggExprs.indexWhere { case Alias(child, _) => child semanticEquals evaluated.child case other => other semanticEquals evaluated.child } if (index == -1) { - needsPushDown += evaluated - order.copy(child = evaluated.toAttribute) + if (CharVarcharUtils.getRawType(evaluated.metadata).nonEmpty) { + needsPushDown += aliasOrder + order.copy(child = aliasOrder) + } else { + needsPushDown += evaluated + order.copy(child = evaluated.toAttribute) + } } else { order.copy(child = originalAggExprs(index).toAttribute) }