Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -612,13 +612,6 @@ class CHSparkPlanExecApi extends SparkPlanExecApi {
CHStringTranslateTransformer(substraitExprName, srcExpr, matchingExpr, replaceExpr, original)
}

override def genSizeExpressionTransformer(
substraitExprName: String,
child: ExpressionTransformer,
original: Size): ExpressionTransformer = {
CHSizeExpressionTransformer(substraitExprName, child, original)
}

override def genLikeTransformer(
substraitExprName: String,
left: ExpressionTransformer,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,16 +30,6 @@ import com.google.common.collect.Lists

import java.util.Locale

case class CHSizeExpressionTransformer(
substraitExprName: String,
expr: ExpressionTransformer,
original: Size)
extends BinaryExpressionTransformer {
override def left: ExpressionTransformer = expr
// Pass legacyLiteral as second argument in substrait function
override def right: ExpressionTransformer = LiteralTransformer(original.legacySizeOfNull)
}

case class CHTruncTimestampTransformer(
substraitExprName: String,
format: ExpressionTransformer,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -101,6 +101,12 @@ class ScalarFunctionsValidateSuite extends FunctionsValidateTest {
}
}

testWithSpecifiedSparkVersion("null input for array_size", Some("3.3")) {
runQueryAndCompare("SELECT array_size(null)") {
checkGlutenOperatorMatch[ProjectExecTransformer]
}
}

test("chr function") {
val df = runQueryAndCompare(
"SELECT chr(l_orderkey + 64) " +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1017,7 +1017,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla
}
}

ignore("test explode/posexplode function") {
test("test explode/posexplode function") {
Seq("explode", "posexplode").foreach {
func =>
// Literal: func(literal)
Expand Down Expand Up @@ -1190,7 +1190,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla
|""".stripMargin)(_)
}

ignore("test multi-generate") {
test("test multi-generate") {
withTable("t") {
sql("CREATE TABLE t (col1 array<struct<a int, b string>>, col2 array<int>) using parquet")
sql("INSERT INTO t VALUES (array(struct(1, 'a'), struct(2, 'b')), array(1, 2))")
Expand Down Expand Up @@ -1588,7 +1588,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla
}
}

ignore("test array literal") {
test("test array literal") {
withTable("array_table") {
sql("create table array_table(a array<bigint>) using parquet")
sql("insert into table array_table select array(1)")
Expand All @@ -1601,7 +1601,7 @@ class TestOperator extends VeloxWholeStageTransformerSuite with AdaptiveSparkPla
}
}

ignore("test map literal") {
test("test map literal") {
withTable("map_table") {
sql("create table map_table(a map<bigint, string>) using parquet")
sql("insert into table map_table select map(1, 'hello')")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -464,13 +464,6 @@ trait SparkPlanExecApi {
original)
}

def genSizeExpressionTransformer(
substraitExprName: String,
child: ExpressionTransformer,
original: Size): ExpressionTransformer = {
GenericExpressionTransformer(substraitExprName, Seq(child), original)
}

def genLikeTransformer(
substraitExprName: String,
left: ExpressionTransformer,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -396,14 +396,12 @@ object ExpressionConverter extends SQLConfHelper with Logging {
r
)
case size: Size =>
if (size.legacySizeOfNull != SQLConf.get.legacySizeOfNull) {
throw new GlutenNotSupportException(
"The value of legacySizeOfNull field of size is " +
"not equals to legacySizeOfNull of SQLConf, this case is not supported yet")
}
BackendsApiManager.getSparkPlanExecApiInstance.genSizeExpressionTransformer(
// Covers Spark ArraySize which is replaced by Size(child, false).
val child =
replaceWithExpressionTransformerInternal(size.child, attributeSeq, expressionsMap)
GenericExpressionTransformer(
substraitExprName,
replaceWithExpressionTransformerInternal(size.child, attributeSeq, expressionsMap),
Seq(child, LiteralTransformer(size.legacySizeOfNull)),
size)
case namedStruct: CreateNamedStruct =>
BackendsApiManager.getSparkPlanExecApiInstance.genNamedStructTransformer(
Expand Down