ORC-546. Fix reading timestamps with duplicated millis within a second.

This caused SPARK-27594.

Fixes #420

Signed-off-by: Owen O'Malley <[email protected]>

Author: omalley

java/core/src/java/org/apache/orc/impl/TreeReaderFactory.java

@@ -1013,9 +1013,9 @@ public void nextVector(ColumnVector previousVector,
 
       for (int i = 0; i < batchSize; i++) {
         if (result.noNulls || !result.isNull[i]) {
-          final int newNanos = parseNanos(nanos.next());
+          int newNanos = parseNanos(nanos.next());
           long millis = (data.next() + base_timestamp)
-              * TimestampTreeWriter.MILLIS_PER_SECOND + newNanos / 1_000_000;
+              * TimestampTreeWriter.MILLIS_PER_SECOND;
           if (millis < 0 && newNanos > 999_999) {
             millis -= TimestampTreeWriter.MILLIS_PER_SECOND;
           }

java/core/src/test/org/apache/orc/TestVectorOrcFile.java

@@ -74,6 +74,7 @@
 import java.nio.charset.StandardCharsets;
 import java.sql.Date;
 import java.sql.Timestamp;
+import java.time.format.DateTimeFormatter;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
@@ -1442,16 +1443,17 @@ public void createOrcDateFile(Path file, int minYear, int maxYear
     batch = reader.getSchema().createRowBatch(1000);
     TimestampColumnVector times = (TimestampColumnVector) batch.cols[0];
     LongColumnVector dates = (LongColumnVector) batch.cols[1];
+    DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSSS");
     for (int year = minYear; year < maxYear; ++year) {
       rows.nextBatch(batch);
       assertEquals(1000, batch.size);
       for(int row = 0; row < 1000; ++row) {
-        Timestamp expected = Timestamp.valueOf(
-            String.format("%04d-05-05 12:34:56.%04d", year, 2*row));
-        assertEquals("ms row " + row + " " + expected, expected.getTime(),
-            times.time[row]);
-        assertEquals("nanos row " + row + " " + expected, expected.getNanos(),
-            times.nanos[row]);
+        String expectedStr = String.format("%04d-05-05 12:34:56.%04d", year, 2*row);
+        assertEquals("row " + row, expectedStr,
+            formatter.format(times.asScratchTimestamp(row).toLocalDateTime()));
+        assertEquals(0, times.time[row] % 1000);
+        assertTrue("nano " + row + " = " + times.nanos[row],
+            times.nanos[row] >= 0 && times.nanos[row] < 1_000_000_000);
         assertEquals("year " + year + " row " + row,
             Integer.toString(year) + "-12-25",
             new DateWritable((int) dates.vector[row]).toString());