Hive compatibility

Author: tarekbecker

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala

@@ -277,12 +277,18 @@ object DateTimeUtils {
     }
     segments(i) = currentSegmentValue
 
+    // Hive compatibility 2011-05-06 07:08:09.1000 == 2011-05-06 07:08:09.1
+    if (digitsMilli == 4) {
+      segments(6) = segments(6) / 10
+    }
+
     // 18:3:1.1 is equals to 18:3:1:100
     if (digitsMilli == 1) {
       segments(6) = segments(6) * 100
     } else if (digitsMilli == 2) {
       segments(6) = segments(6) * 10
     }
+
     if (segments(0) < 0 || segments(0) > 9999 || segments(1) < 1 || segments(1) > 12 ||
         segments(2) < 1 || segments(2) > 31 || segments(3) < 0 || segments(3) > 23 ||
         segments(4) < 0 || segments(4) > 59 || segments(5) < 0 || segments(5) > 59 ||

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala

@@ -128,6 +128,10 @@ class DateTimeUtilsSuite extends SparkFunSuite {
   test("string to timestamp") {
 
     var c = Calendar.getInstance()
+    c.set(2011, 4, 6, 7, 8, 9)
+    c.set(Calendar.MILLISECOND, 100)
+    assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("2011-05-06 07:08:09.1000")) ==
+      new Timestamp(c.getTimeInMillis))
     c.set(1969, 11, 31, 16, 0, 0)
     c.set(Calendar.MILLISECOND, 0)
     assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("1969-12-31 16:00:00")) ==