Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -555,7 +555,6 @@ class CodegenContext {
addNewFunction(compareFunc, funcCode)
s"this.$compareFunc($c1, $c2)"
case schema: StructType =>
INPUT_ROW = "i"
val comparisons = GenerateOrdering.genComparisons(this, schema)
val compareFunc = freshName("compareStruct")
val funcCode: String =
Expand All @@ -566,7 +565,6 @@ class CodegenContext {
if (a instanceof UnsafeRow && b instanceof UnsafeRow && a.equals(b)) {
return 0;
}
InternalRow i = null;
$comparisons
return 0;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,12 @@ object GenerateOrdering extends CodeGenerator[Seq[SortOrder], Ordering[InternalR
*/
def genComparisons(ctx: CodegenContext, ordering: Seq[SortOrder]): String = {
val comparisons = ordering.map { order =>
val oldCurrentVars = ctx.currentVars
ctx.INPUT_ROW = "i"
// to use INPUT_ROW we must make sure currentVars is null
ctx.currentVars = null
val eval = order.child.genCode(ctx)
ctx.currentVars = oldCurrentVars
val asc = order.isAscending
val isNullA = ctx.freshName("isNullA")
val primitiveA = ctx.freshName("primitiveA")
Expand Down Expand Up @@ -119,7 +124,7 @@ object GenerateOrdering extends CodeGenerator[Seq[SortOrder], Ordering[InternalR
"""
}

ctx.splitExpressions(
val code = ctx.splitExpressions(
expressions = comparisons,
funcName = "compare",
arguments = Seq(("InternalRow", "a"), ("InternalRow", "b")),
Expand All @@ -142,6 +147,12 @@ object GenerateOrdering extends CodeGenerator[Seq[SortOrder], Ordering[InternalR
"""
}.mkString
})
// make sure INPUT_ROW is declared even if splitExpressions
// returns an inlined block
s"""
|InternalRow ${ctx.INPUT_ROW} = null;
|$code
""".stripMargin
}

protected def create(ordering: Seq[SortOrder]): BaseOrdering = {
Expand All @@ -165,7 +176,6 @@ object GenerateOrdering extends CodeGenerator[Seq[SortOrder], Ordering[InternalR
${ctx.declareAddedFunctions()}

public int compare(InternalRow a, InternalRow b) {
InternalRow ${ctx.INPUT_ROW} = null; // Holds current row being evaluated.
$comparisons
return 0;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -143,4 +143,16 @@ class WholeStageCodegenSuite extends SparkPlanTest with SharedSQLContext {
assert(createStackGenerator(50).find(isCodeGenerated).isDefined)
assert(createStackGenerator(100).find(isCodeGenerated).isEmpty)
}

test("SPARK-19512 codegen for comparing structs is incorrect") {
// this would raise CompileException before the fix
spark.range(10)
.selectExpr("named_struct('a', id) as col1", "named_struct('a', id+2) as col2")
.filter("col1 = col2").count()
// this would raise java.lang.IndexOutOfBoundsException before the fix
spark.range(10)
.selectExpr("named_struct('a', id, 'b', id) as col1",
"named_struct('a',id+2, 'b',id+2) as col2")
.filter("col1 = col2").count()
}
}