From 804394b44e65142c0af3eb3dc19bab29ed13875d Mon Sep 17 00:00:00 2001 From: Marco Gaido Date: Mon, 5 Mar 2018 14:15:06 +0100 Subject: [PATCH 1/3] [SPARK-23586][SQL] Add interpreted execution to UnwrapOption --- .../sql/catalyst/expressions/objects/objects.scala | 10 ++++++++-- .../expressions/ObjectExpressionsSuite.scala | 14 +++++++++++++- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala index 80618af1e859f..03cc8eaceb4e6 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala @@ -382,8 +382,14 @@ case class UnwrapOption( override def inputTypes: Seq[AbstractDataType] = ObjectType :: Nil - override def eval(input: InternalRow): Any = - throw new UnsupportedOperationException("Only code-generated evaluation is supported") + override def eval(input: InternalRow): Any = { + val inputObject = child.eval(input) + if (inputObject == null) { + null + } else { + inputObject.asInstanceOf[Option[_]].orNull + } + } override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = { val javaType = CodeGenerator.javaType(dataType) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala index 3edcc02f15264..d054b8ec0f245 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala @@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.expressions import org.apache.spark.SparkFunSuite import org.apache.spark.sql.catalyst.InternalRow import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder -import org.apache.spark.sql.catalyst.expressions.objects.Invoke +import org.apache.spark.sql.catalyst.expressions.objects.{Invoke, UnwrapOption} import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, GenericArrayData} import org.apache.spark.sql.types.{IntegerType, ObjectType} @@ -66,4 +66,16 @@ class ObjectExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvalutionWithUnsafeProjection( mapEncoder.serializer.head, mapExpected, mapInputRow) } + + test("SPARK-23586: UnwrapOption should support interpreted execution") { + val inputRowWithSome = InternalRow.fromSeq(Seq(Some(1))) + val inputRowWithNone = InternalRow.fromSeq(Seq(None)) + val inputRowWithNull = InternalRow.fromSeq(Seq(null)) + val cls = classOf[Option[Int]] + val inputObject = BoundReference(0, ObjectType(cls), nullable = true) + val unwrapObject = UnwrapOption(IntegerType, inputObject) + assert(unwrapObject.eval(inputRowWithSome) == 1) + assert(unwrapObject.eval(inputRowWithNone) == null) + assert(unwrapObject.eval(inputRowWithNull) == null) + } } From d899f44b213da4e3e819a308a2442a9202901ad3 Mon Sep 17 00:00:00 2001 From: Marco Gaido Date: Mon, 5 Mar 2018 15:04:33 +0100 Subject: [PATCH 2/3] use checkEvaluation --- .../catalyst/expressions/ObjectExpressionsSuite.scala | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala index d054b8ec0f245..b050e1992fa61 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala @@ -68,14 +68,11 @@ class ObjectExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { } test("SPARK-23586: UnwrapOption should support interpreted execution") { - val inputRowWithSome = InternalRow.fromSeq(Seq(Some(1))) - val inputRowWithNone = InternalRow.fromSeq(Seq(None)) - val inputRowWithNull = InternalRow.fromSeq(Seq(null)) val cls = classOf[Option[Int]] val inputObject = BoundReference(0, ObjectType(cls), nullable = true) val unwrapObject = UnwrapOption(IntegerType, inputObject) - assert(unwrapObject.eval(inputRowWithSome) == 1) - assert(unwrapObject.eval(inputRowWithNone) == null) - assert(unwrapObject.eval(inputRowWithNull) == null) + Seq((Some(1), 1), (None, null), (null, null)).foreach { case (input, expected) => + checkEvaluation(unwrapObject, expected, InternalRow.fromSeq(Seq(input))) + } } } From fc202c6fc5846b65e39d2fce6f4cdd13e6cd927a Mon Sep 17 00:00:00 2001 From: Marco Gaido Date: Mon, 5 Mar 2018 17:04:21 +0100 Subject: [PATCH 3/3] fix jira nunber --- .../spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala index b050e1992fa61..d95db5867b19c 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala @@ -67,7 +67,7 @@ class ObjectExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { mapEncoder.serializer.head, mapExpected, mapInputRow) } - test("SPARK-23586: UnwrapOption should support interpreted execution") { + test("SPARK-23585: UnwrapOption should support interpreted execution") { val cls = classOf[Option[Int]] val inputObject = BoundReference(0, ObjectType(cls), nullable = true) val unwrapObject = UnwrapOption(IntegerType, inputObject)