Skip to content

Commit c8392cd

Browse files
cloud-fandavies
authored andcommitted
[SPARK-10934] [SQL] handle hashCode of unsafe array correctly
`Murmur3_x86_32.hashUnsafeWords` only accepts word-aligned bytes, but unsafe array is not. Author: Wenchen Fan <[email protected]> Closes #8987 from cloud-fan/hash.
1 parent d323e5e commit c8392cd

File tree

2 files changed

+12
-1
lines changed

2 files changed

+12
-1
lines changed

sql/catalyst/src/main/java/org/apache/spark/sql/catalyst/expressions/UnsafeArrayData.java

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -286,7 +286,11 @@ public MapData getMap(int ordinal) {
286286

287287
@Override
288288
public int hashCode() {
289-
return Murmur3_x86_32.hashUnsafeWords(baseObject, baseOffset, sizeInBytes, 42);
289+
int result = 37;
290+
for (int i = 0; i < sizeInBytes; i++) {
291+
result = 37 * result + Platform.getByte(baseObject, baseOffset + i);
292+
}
293+
return result;
290294
}
291295

292296
@Override

sql/core/src/test/scala/org/apache/spark/sql/UnsafeRowSuite.scala

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,4 +130,11 @@ class UnsafeRowSuite extends SparkFunSuite {
130130
assert(emptyRow.getInt(0) === unsafeRow.getInt(0))
131131
assert(emptyRow.getUTF8String(1) === unsafeRow.getUTF8String(1))
132132
}
133+
134+
test("calling hashCode on unsafe array returned by getArray(ordinal)") {
135+
val row = InternalRow.apply(new GenericArrayData(Array(1L)))
136+
val unsafeRow = UnsafeProjection.create(Array[DataType](ArrayType(LongType))).apply(row)
137+
// Makes sure hashCode on unsafe array won't crash
138+
unsafeRow.getArray(0).hashCode()
139+
}
133140
}

0 commit comments

Comments
 (0)