Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -516,6 +516,12 @@ class TestOperator extends WholeStageTransformerSuite {
}}
}

test("test df.describe() method") {
val d = 3
val df = Seq(d, d, d, d, d, d, d, d, d, d).toDF("IntegerCol")
df.describe().show()
}

test("bit_and and bit_or") {
runQueryAndCompare(
"""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -613,7 +613,7 @@ case class ColumnarOverrideRules(session: SparkSession)
List(
(_: SparkSession) => FallbackEmptySchemaRelation(),
(_: SparkSession) => StoreExpandGroupExpression(),
(_: SparkSession) => AddTransformHintRule(),
(_: SparkSession) => AddTransformHintRule(isTopParentExchange),
(_: SparkSession) => TransformPreOverrides(
this.isTopParentExchange || this.isAdaptiveContext),
(_: SparkSession) => RemoveTransformHintRule()) :::
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -238,7 +238,7 @@ case class FallbackEmptySchemaRelation() extends Rule[SparkPlan] {
// The doValidate function will be called to check if the conversion is supported.
// If false is returned or any unsupported exception is thrown, a row guard will
// be added on the top of that plan to prevent actual conversion.
case class AddTransformHintRule() extends Rule[SparkPlan] {
case class AddTransformHintRule(isTopExchange: Boolean = true) extends Rule[SparkPlan] {
val columnarConf: GlutenConfig = GlutenConfig.getConf
val preferColumnar: Boolean = columnarConf.enablePreferColumnar
val optimizeLevel: Integer = columnarConf.physicalJoinOptimizationThrottle
Expand Down Expand Up @@ -277,6 +277,9 @@ case class AddTransformHintRule() extends Rule[SparkPlan] {
* Inserts a transformable tag on top of those that are not supported.
*/
private def addTransformableTags(plan: SparkPlan): SparkPlan = {
if (!isTopExchange) {
TransformHints.tagNotTransformable(plan)
}
addTransformableTag(plan)
plan.withNewChildren(plan.children.map(addTransformableTags))
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -88,9 +88,7 @@ class VeloxTestSettings extends BackendTestSettings {
// Mismatch when max NaN and infinite value
"NaN is greater than all other non-NaN numeric values",
// Rewrite this test because the describe functions creates unmatched plan.
"describe",
// decimal failed ut.
"SPARK-22271: mean overflows and returns null for some decimal variables"
"describe"
)

enableSuite[GlutenDataFrameNaFunctionsSuite]
Expand Down