diff --git a/backends-velox/src/test/scala/io/glutenproject/execution/TestOperator.scala b/backends-velox/src/test/scala/io/glutenproject/execution/TestOperator.scala index 486aac830c94..42c38b2ac16b 100644 --- a/backends-velox/src/test/scala/io/glutenproject/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/io/glutenproject/execution/TestOperator.scala @@ -516,6 +516,12 @@ class TestOperator extends WholeStageTransformerSuite { }} } + test("test df.describe() method") { + val d = 3 + val df = Seq(d, d, d, d, d, d, d, d, d, d).toDF("IntegerCol") + df.describe().show() + } + test("bit_and and bit_or") { runQueryAndCompare( """ diff --git a/gluten-core/src/main/scala/io/glutenproject/extension/ColumnarOverrides.scala b/gluten-core/src/main/scala/io/glutenproject/extension/ColumnarOverrides.scala index 471f554f559f..630db307ec4d 100644 --- a/gluten-core/src/main/scala/io/glutenproject/extension/ColumnarOverrides.scala +++ b/gluten-core/src/main/scala/io/glutenproject/extension/ColumnarOverrides.scala @@ -613,7 +613,7 @@ case class ColumnarOverrideRules(session: SparkSession) List( (_: SparkSession) => FallbackEmptySchemaRelation(), (_: SparkSession) => StoreExpandGroupExpression(), - (_: SparkSession) => AddTransformHintRule(), + (_: SparkSession) => AddTransformHintRule(isTopParentExchange), (_: SparkSession) => TransformPreOverrides( this.isTopParentExchange || this.isAdaptiveContext), (_: SparkSession) => RemoveTransformHintRule()) ::: diff --git a/gluten-core/src/main/scala/io/glutenproject/extension/columnar/TransformHintRule.scala b/gluten-core/src/main/scala/io/glutenproject/extension/columnar/TransformHintRule.scala index 9ab36d83d6d3..19edb3f9a43e 100644 --- a/gluten-core/src/main/scala/io/glutenproject/extension/columnar/TransformHintRule.scala +++ b/gluten-core/src/main/scala/io/glutenproject/extension/columnar/TransformHintRule.scala @@ -238,7 +238,7 @@ case class FallbackEmptySchemaRelation() extends Rule[SparkPlan] { // The doValidate function will be called to check if the conversion is supported. // If false is returned or any unsupported exception is thrown, a row guard will // be added on the top of that plan to prevent actual conversion. -case class AddTransformHintRule() extends Rule[SparkPlan] { +case class AddTransformHintRule(isTopExchange: Boolean = true) extends Rule[SparkPlan] { val columnarConf: GlutenConfig = GlutenConfig.getConf val preferColumnar: Boolean = columnarConf.enablePreferColumnar val optimizeLevel: Integer = columnarConf.physicalJoinOptimizationThrottle @@ -277,6 +277,9 @@ case class AddTransformHintRule() extends Rule[SparkPlan] { * Inserts a transformable tag on top of those that are not supported. */ private def addTransformableTags(plan: SparkPlan): SparkPlan = { + if (!isTopExchange) { + TransformHints.tagNotTransformable(plan) + } addTransformableTag(plan) plan.withNewChildren(plan.children.map(addTransformableTags)) } diff --git a/gluten-ut/spark32/src/test/scala/io/glutenproject/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark32/src/test/scala/io/glutenproject/utils/velox/VeloxTestSettings.scala index 6f5cd25c2068..d9b505040424 100644 --- a/gluten-ut/spark32/src/test/scala/io/glutenproject/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark32/src/test/scala/io/glutenproject/utils/velox/VeloxTestSettings.scala @@ -88,9 +88,7 @@ class VeloxTestSettings extends BackendTestSettings { // Mismatch when max NaN and infinite value "NaN is greater than all other non-NaN numeric values", // Rewrite this test because the describe functions creates unmatched plan. - "describe", - // decimal failed ut. - "SPARK-22271: mean overflows and returns null for some decimal variables" + "describe" ) enableSuite[GlutenDataFrameNaFunctionsSuite]