Skip to content

[SPARK-4077][SQL] Spark SQL return wrong values for valid string timestamp values #3019

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
gvramana wants to merge 2 commits into from
Closed

[SPARK-4077][SQL] Spark SQL return wrong values for valid string timestamp values #3019

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fa01e71
cloned timestamp object as org.apache.hadoop.hive.serde2.io.Timestamp…
gvramana Oct 30, 2014
32d818f
fixed check style
gvramana Oct 30, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions sql/hive/src/main/scala/org/apache/spark/sql/hive/HiveInspectors.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,9 @@ private[hive] trait HiveInspectors {
if (data == null) null else hvoi.getPrimitiveJavaObject(data).getValue
case hdoi: HiveDecimalObjectInspector =>
if (data == null) null else BigDecimal(hdoi.getPrimitiveJavaObject(data).bigDecimalValue())
// org.apache.hadoop.hive.serde2.io.TimestampWritable.set will reset current time object
// if next timestamp is null, so Timestamp object is cloned
case ti: TimestampObjectInspector => ti.getPrimitiveJavaObject(data).clone()
case pi: PrimitiveObjectInspector => pi.getPrimitiveJavaObject(data)
case li: ListObjectInspector =>
Option(li.getList(data))
Expand Down
2 changes: 2 additions & 0 deletions sql/hive/src/test/resources/data/files/issue-4077-data.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
2014-12-11 00:00:00,1
2014-12-11astring00:00:00,2
22 changes: 22 additions & 0 deletions sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveTableScanSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,9 @@
package org.apache.spark.sql.hive.execution

import org.apache.spark.sql.hive.test.TestHive
import org.apache.spark.sql.{Row, SchemaRDD}

import org.apache.spark.util.Utils

class HiveTableScanSuite extends HiveComparisonTest {

Expand Down Expand Up @@ -47,4 +50,23 @@ class HiveTableScanSuite extends HiveComparisonTest {
TestHive.sql("select KEY from tb where VALUE='just_for_test' limit 5").collect()
TestHive.sql("drop table tb")
}

test("Spark-4077: timestamp query for null value") {
TestHive.sql("DROP TABLE IF EXISTS timestamp_query_null")
TestHive.sql(
"""
CREATE EXTERNAL TABLE timestamp_query_null (time TIMESTAMP,id INT)
ROW FORMAT DELIMITED
FIELDS TERMINATED BY ','
LINES TERMINATED BY '\n'
""".stripMargin)
val location =
Utils.getSparkClassLoader.getResource("data/files/issue-4077-data.txt").getFile()

TestHive.sql(s"LOAD DATA LOCAL INPATH '$location' INTO TABLE timestamp_query_null")
assert(TestHive.sql("SELECT time from timestamp_query_null limit 2").collect()
=== Array(Row(java.sql.Timestamp.valueOf("2014-12-11 00:00:00")),Row(null)))
TestHive.sql("DROP TABLE timestamp_query_null")
}

}