diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala index ac3ea61ff8..7ed333aec2 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/backendsapi/clickhouse/CHSparkPlanExecApi.scala @@ -612,13 +612,6 @@ class CHSparkPlanExecApi extends SparkPlanExecApi { CHStringTranslateTransformer(substraitExprName, srcExpr, matchingExpr, replaceExpr, original) } - override def genSizeExpressionTransformer( - substraitExprName: String, - child: ExpressionTransformer, - original: Size): ExpressionTransformer = { - CHSizeExpressionTransformer(substraitExprName, child, original) - } - override def genLikeTransformer( substraitExprName: String, left: ExpressionTransformer, diff --git a/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala b/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala index 5ca4e02339..7b389ead00 100644 --- a/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala +++ b/backends-clickhouse/src/main/scala/org/apache/gluten/expression/CHExpressionTransformer.scala @@ -30,16 +30,6 @@ import com.google.common.collect.Lists import java.util.Locale -case class CHSizeExpressionTransformer( - substraitExprName: String, - expr: ExpressionTransformer, - original: Size) - extends BinaryExpressionTransformer { - override def left: ExpressionTransformer = expr - // Pass legacyLiteral as second argument in substrait function - override def right: ExpressionTransformer = LiteralTransformer(original.legacySizeOfNull) -} - case class CHTruncTimestampTransformer( substraitExprName: String, format: ExpressionTransformer, diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala index bd32a799c3..3db0f5e79b 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/ScalarFunctionsValidateSuite.scala @@ -101,6 +101,12 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest { } } + testWithSpecifiedSparkVersion("null input for array_size", Some("3.3")) { + runQueryAndCompare("SELECT array_size(null)") { + checkGlutenOperatorMatch[ProjectExecTransformer] + } + } + test("chr function") { val df = runQueryAndCompare( "SELECT chr(l_orderkey + 64) " + diff --git a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala index d84f5e7cc3..9b47a519cd 100644 --- a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala +++ b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala @@ -1017,7 +1017,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla } } - ignore("test explode/posexplode function") { + test("test explode/posexplode function") { Seq("explode", "posexplode").foreach { func => // Literal: func(literal) @@ -1190,7 +1190,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla |""".stripMargin)(_) } - ignore("test multi-generate") { + test("test multi-generate") { withTable("t") { sql("CREATE TABLE t (col1 array>, col2 array) using parquet") sql("INSERT INTO t VALUES (array(struct(1, 'a'), struct(2, 'b')), array(1, 2))") @@ -1588,7 +1588,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla } } - ignore("test array literal") { + test("test array literal") { withTable("array_table") { sql("create table array_table(a array) using parquet") sql("insert into table array_table select array(1)") @@ -1601,7 +1601,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla } } - ignore("test map literal") { + test("test map literal") { withTable("map_table") { sql("create table map_table(a map) using parquet") sql("insert into table map_table select map(1, 'hello')") diff --git a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala index 3ca5e03139..ff7449e2d3 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/backendsapi/SparkPlanExecApi.scala @@ -464,13 +464,6 @@ trait SparkPlanExecApi { original) } - def genSizeExpressionTransformer( - substraitExprName: String, - child: ExpressionTransformer, - original: Size): ExpressionTransformer = { - GenericExpressionTransformer(substraitExprName, Seq(child), original) - } - def genLikeTransformer( substraitExprName: String, left: ExpressionTransformer, diff --git a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala index d5222cfc63..b5bcb6876e 100644 --- a/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala +++ b/gluten-core/src/main/scala/org/apache/gluten/expression/ExpressionConverter.scala @@ -396,14 +396,12 @@ object ExpressionConverter extends SQLConfHelper with Logging { r ) case size: Size => - if (size.legacySizeOfNull != SQLConf.get.legacySizeOfNull) { - throw new GlutenNotSupportException( - "The value of legacySizeOfNull field of size is " + - "not equals to legacySizeOfNull of SQLConf, this case is not supported yet") - } - BackendsApiManager.getSparkPlanExecApiInstance.genSizeExpressionTransformer( + // Covers Spark ArraySize which is replaced by Size(child, false). + val child = + replaceWithExpressionTransformerInternal(size.child, attributeSeq, expressionsMap) + GenericExpressionTransformer( substraitExprName, - replaceWithExpressionTransformerInternal(size.child, attributeSeq, expressionsMap), + Seq(child, LiteralTransformer(size.legacySizeOfNull)), size) case namedStruct: CreateNamedStruct => BackendsApiManager.getSparkPlanExecApiInstance.genNamedStructTransformer(