Skip to content

[SPARK-39496][SQL] Handle null struct in Inline.eval #36903

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
bersprockets wants to merge 2 commits into from
Closed

[SPARK-39496][SQL] Handle null struct in Inline.eval #36903

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
77e7c6b
Add failing test
bersprockets Jun 16, 2022
d5c1389
Proposed fix
bersprockets Jun 17, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/generators.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -452,13 +452,17 @@ case class Inline(child: Expression) extends UnaryExpression with CollectionGene

private lazy val numFields = elementSchema.fields.length

private lazy val generatorNullRow = new GenericInternalRow(elementSchema.length)

override def eval(input: InternalRow): TraversableOnce[InternalRow] = {
val inputArray = child.eval(input).asInstanceOf[ArrayData]
if (inputArray == null) {
Nil
} else {
for (i <- 0 until inputArray.numElements())
yield inputArray.getStruct(i, numFields)
for (i <- 0 until inputArray.numElements()) yield {
val s = inputArray.getStruct(i, numFields)
if (s == null) generatorNullRow else s
}
}
}

Expand Down
13 changes: 12 additions & 1 deletion sql/core/src/test/scala/org/apache/spark/sql/GeneratorFunctionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.{CodegenContext, ExprCo
import org.apache.spark.sql.catalyst.expressions.codegen.Block._
import org.apache.spark.sql.catalyst.trees.LeafLike
import org.apache.spark.sql.functions._
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.sql.types.{IntegerType, StructType}

Expand Down Expand Up @@ -390,7 +391,7 @@ class GeneratorFunctionSuite extends QueryTest with SharedSparkSession {
}
}

test("SPARK-39061: inline should handle null struct") {
def testNullStruct(): Unit = {
val df = sql(
"""select * from values
|(
Expand All @@ -414,6 +415,16 @@ class GeneratorFunctionSuite extends QueryTest with SharedSparkSession {
sql("select a, inline(b) from t1"),
Row(1, 0, 1) :: Row(1, null, null) :: Row(1, 2, 3) :: Row(1, null, null) :: Nil)
}

test("SPARK-39061: inline should handle null struct") {
testNullStruct
}

test("SPARK-39496: inline eval path should handle null struct") {
withSQLConf(SQLConf.WHOLESTAGE_CODEGEN_ENABLED.key -> "false") {
testNullStruct
}
}
}

case class EmptyGenerator() extends Generator with LeafLike[Expression] {
Expand Down