Get UnsafeInMemorySorterSuite to pass (WIP)

Author: JoshRosen

core/pom.xml

@@ -343,28 +343,28 @@
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>org.mockito</groupId>
-      <artifactId>mockito-core</artifactId>
+      <groupId>org.hamcrest</groupId>
+      <artifactId>hamcrest-core</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>org.scalacheck</groupId>
-      <artifactId>scalacheck_${scala.binary.version}</artifactId>
+      <groupId>org.hamcrest</groupId>
+      <artifactId>hamcrest-library</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>junit</groupId>
-      <artifactId>junit</artifactId>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-core</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>org.hamcrest</groupId>
-      <artifactId>hamcrest-core</artifactId>
+      <groupId>org.scalacheck</groupId>
+      <artifactId>scalacheck_${scala.binary.version}</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>
-      <groupId>org.hamcrest</groupId>
-      <artifactId>hamcrest-library</artifactId>
+      <groupId>junit</groupId>
+      <artifactId>junit</artifactId>
       <scope>test</scope>
     </dependency>
     <dependency>

core/src/test/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorterSuite.java

@@ -35,11 +35,11 @@
 public class UnsafeInMemorySorterSuite {
 
   private static String getStringFromDataPage(Object baseObject, long baseOffset) {
-    final int strLength = (int) PlatformDependent.UNSAFE.getLong(baseObject, baseOffset);
+    final int strLength = PlatformDependent.UNSAFE.getInt(baseObject, baseOffset);
     final byte[] strBytes = new byte[strLength];
     PlatformDependent.copyMemory(
       baseObject,
-      baseOffset + 8,
+      baseOffset + 4,
       strBytes,
       PlatformDependent.BYTE_ARRAY_OFFSET, strLength);
     return new String(strBytes);
@@ -77,8 +77,8 @@ public void testSortingOnlyByIntegerPrefix() throws Exception {
     long position = dataPage.getBaseOffset();
     for (String str : dataToSort) {
       final byte[] strBytes = str.getBytes("utf-8");
-      PlatformDependent.UNSAFE.putLong(baseObject, position, strBytes.length);
-      position += 8;
+      PlatformDependent.UNSAFE.putInt(baseObject, position, strBytes.length);
+      position += 4;
       PlatformDependent.copyMemory(
         strBytes,
         PlatformDependent.BYTE_ARRAY_OFFSET,
@@ -114,22 +114,24 @@ public int compare(long prefix1, long prefix2) {
     position = dataPage.getBaseOffset();
     for (int i = 0; i < dataToSort.length; i++) {
       // position now points to the start of a record (which holds its length).
-      final long recordLength = PlatformDependent.UNSAFE.getLong(baseObject, position);
+      final int recordLength = PlatformDependent.UNSAFE.getInt(baseObject, position);
       final long address = memoryManager.encodePageNumberAndOffset(dataPage, position);
       final String str = getStringFromDataPage(baseObject, position);
       final int partitionId = hashPartitioner.getPartition(str);
       sorter.insertRecord(address, partitionId);
-      position += 8 + recordLength;
+      position += 4 + recordLength;
     }
     final UnsafeSorterIterator iter = sorter.getSortedIterator();
     int iterLength = 0;
     long prevPrefix = -1;
     Arrays.sort(dataToSort);
     while (iter.hasNext()) {
       iter.loadNext();
-      final String str = getStringFromDataPage(iter.getBaseObject(), iter.getBaseOffset());
+      // TODO: the logic for how we manipulate record length offsets here is confusing; clean
+      // this up and clarify it in comments.
+      final String str = getStringFromDataPage(iter.getBaseObject(), iter.getBaseOffset() - 4);
       final long keyPrefix = iter.getKeyPrefix();
-      assertTrue(Arrays.asList(dataToSort).contains(str));
+      assertThat(str, isIn(Arrays.asList(dataToSort)));
       assertThat(keyPrefix, greaterThanOrEqualTo(prevPrefix));
       prevPrefix = keyPrefix;
       iterLength++;