Skip to content

Commit 14f333b

Browse files
tarekbeckertarekbecker
committed
[SPARK-8995] added tests for daylight saving time
1 parent ca1ae69 commit 14f333b

File tree

3 files changed

+71
-40
lines changed

3 files changed

+71
-40
lines changed

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -329,8 +329,7 @@ object DateTimeUtils {
329329
c.set(segments(0), segments(1) - 1, segments(2), segments(3), segments(4), segments(5))
330330
}
331331

332-
c.set(Calendar.MILLISECOND, segments(6) / 1000)
333-
Some(c.getTimeInMillis * 1000 + segments(6) % 1000)
332+
Some(c.getTimeInMillis / 1000 * 1000000 + segments(6))
334333
}
335334

336335
/**
@@ -377,7 +376,6 @@ object DateTimeUtils {
377376
}
378377
val c = Calendar.getInstance()
379378
c.set(segments(0), segments(1) - 1, segments(2), 0, 0, 0)
380-
c.set(Calendar.MILLISECOND, 0)
381379
Some((c.getTimeInMillis / 1000 / 3600 / 24).toInt)
382380
}
383381
}

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ import org.apache.spark.SparkFunSuite
2424
import org.apache.spark.sql.catalyst.InternalRow
2525
import org.apache.spark.sql.catalyst.util.DateTimeUtils
2626
import org.apache.spark.sql.types._
27-
import org.apache.spark.unsafe.types.UTF8String
2827

2928
/**
3029
* Test suite for data type casting expression [[Cast]].
@@ -282,6 +281,18 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper {
282281
val nts = sts + ".1"
283282
val ts = Timestamp.valueOf(nts)
284283

284+
val defaultTimeZone = TimeZone.getDefault
285+
TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
286+
var c = Calendar.getInstance()
287+
c.set(2015, 2, 8, 2, 30, 0)
288+
checkEvaluation(cast(cast(new Timestamp(c.getTimeInMillis), StringType), TimestampType),
289+
c.getTimeInMillis * 1000)
290+
c = Calendar.getInstance()
291+
c.set(2015, 10, 1, 2, 30, 0)
292+
checkEvaluation(cast(cast(new Timestamp(c.getTimeInMillis), StringType), TimestampType),
293+
c.getTimeInMillis * 1000)
294+
TimeZone.setDefault(defaultTimeZone)
295+
285296
checkEvaluation(cast("abdef", StringType), "abdef")
286297
checkEvaluation(cast("abdef", DecimalType.Unlimited), null)
287298
checkEvaluation(cast("abdef", TimestampType), null)

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala‎

Lines changed: 58 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -94,29 +94,29 @@ class DateTimeUtilsSuite extends SparkFunSuite {
9494
var c = Calendar.getInstance()
9595
c.set(2015, 0, 28, 0, 0, 0)
9696
c.set(Calendar.MILLISECOND, 0)
97-
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-01-28")).get ==
97+
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-01-28")).get ===
9898
c.getTimeInMillis / millisPerDay)
9999
c.set(2015, 0, 1, 0, 0, 0)
100100
c.set(Calendar.MILLISECOND, 0)
101-
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015")).get ==
101+
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015")).get ===
102102
c.getTimeInMillis / millisPerDay)
103103
c = Calendar.getInstance()
104104
c.set(2015, 2, 1, 0, 0, 0)
105105
c.set(Calendar.MILLISECOND, 0)
106-
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-03")).get ==
106+
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-03")).get ===
107107
c.getTimeInMillis / millisPerDay)
108108
c = Calendar.getInstance()
109109
c.set(2015, 2, 18, 0, 0, 0)
110110
c.set(Calendar.MILLISECOND, 0)
111-
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-03-18")).get ==
111+
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-03-18")).get ===
112112
c.getTimeInMillis / millisPerDay)
113-
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-03-18 ")).get ==
113+
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-03-18 ")).get ===
114114
c.getTimeInMillis / millisPerDay)
115-
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-03-18 123142")).get ==
115+
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-03-18 123142")).get ===
116116
c.getTimeInMillis / millisPerDay)
117-
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-03-18T123123")).get ==
117+
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-03-18T123123")).get ===
118118
c.getTimeInMillis / millisPerDay)
119-
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-03-18T")).get ==
119+
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-03-18T")).get ===
120120
c.getTimeInMillis / millisPerDay)
121121

122122
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-03-18X")).isEmpty)
@@ -130,117 +130,117 @@ class DateTimeUtilsSuite extends SparkFunSuite {
130130
var c = Calendar.getInstance()
131131
c.set(1969, 11, 31, 16, 0, 0)
132132
c.set(Calendar.MILLISECOND, 0)
133-
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("1969-12-31 16:00:00")).get ==
133+
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("1969-12-31 16:00:00")).get ===
134134
c.getTimeInMillis * 1000)
135135
c.set(2015, 0, 1, 0, 0, 0)
136136
c.set(Calendar.MILLISECOND, 0)
137-
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("2015")).get ==
137+
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("2015")).get ===
138138
c.getTimeInMillis * 1000)
139139
c = Calendar.getInstance()
140140
c.set(2015, 2, 1, 0, 0, 0)
141141
c.set(Calendar.MILLISECOND, 0)
142-
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("2015-03")).get ==
142+
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("2015-03")).get ===
143143
c.getTimeInMillis * 1000)
144144
c = Calendar.getInstance()
145145
c.set(2015, 2, 18, 0, 0, 0)
146146
c.set(Calendar.MILLISECOND, 0)
147-
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("2015-03-18")).get ==
147+
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("2015-03-18")).get ===
148148
c.getTimeInMillis * 1000)
149-
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("2015-03-18 ")).get ==
149+
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("2015-03-18 ")).get ===
150150
c.getTimeInMillis * 1000)
151-
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("2015-03-18T")).get ==
151+
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("2015-03-18T")).get ===
152152
c.getTimeInMillis * 1000)
153153

154154
c = Calendar.getInstance()
155155
c.set(2015, 2, 18, 12, 3, 17)
156156
c.set(Calendar.MILLISECOND, 0)
157-
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("2015-03-18 12:03:17")).get ==
157+
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("2015-03-18 12:03:17")).get ===
158158
c.getTimeInMillis * 1000)
159-
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("2015-03-18T12:03:17")).get ==
159+
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("2015-03-18T12:03:17")).get ===
160160
c.getTimeInMillis * 1000)
161161

162162
c = Calendar.getInstance(TimeZone.getTimeZone("GMT-13:53"))
163163
c.set(2015, 2, 18, 12, 3, 17)
164164
c.set(Calendar.MILLISECOND, 0)
165165
assert(DateTimeUtils.stringToTimestamp(
166-
UTF8String.fromString("2015-03-18T12:03:17-13:53")).get == c.getTimeInMillis * 1000)
166+
UTF8String.fromString("2015-03-18T12:03:17-13:53")).get === c.getTimeInMillis * 1000)
167167

168168
c = Calendar.getInstance(TimeZone.getTimeZone("UTC"))
169169
c.set(2015, 2, 18, 12, 3, 17)
170170
c.set(Calendar.MILLISECOND, 0)
171-
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("2015-03-18T12:03:17Z")).get ==
171+
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("2015-03-18T12:03:17Z")).get ===
172172
c.getTimeInMillis * 1000)
173-
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("2015-03-18 12:03:17Z")).get ==
173+
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("2015-03-18 12:03:17Z")).get ===
174174
c.getTimeInMillis * 1000)
175175

176176
c = Calendar.getInstance(TimeZone.getTimeZone("GMT-01:00"))
177177
c.set(2015, 2, 18, 12, 3, 17)
178178
c.set(Calendar.MILLISECOND, 0)
179-
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("2015-03-18T12:03:17-1:0")).get ==
179+
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("2015-03-18T12:03:17-1:0")).get ===
180180
c.getTimeInMillis * 1000)
181181
assert(DateTimeUtils.stringToTimestamp(
182-
UTF8String.fromString("2015-03-18T12:03:17-01:00")).get == c.getTimeInMillis * 1000)
182+
UTF8String.fromString("2015-03-18T12:03:17-01:00")).get === c.getTimeInMillis * 1000)
183183

184184
c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:30"))
185185
c.set(2015, 2, 18, 12, 3, 17)
186186
c.set(Calendar.MILLISECOND, 0)
187187
assert(DateTimeUtils.stringToTimestamp(
188-
UTF8String.fromString("2015-03-18T12:03:17+07:30")).get == c.getTimeInMillis * 1000)
188+
UTF8String.fromString("2015-03-18T12:03:17+07:30")).get === c.getTimeInMillis * 1000)
189189

190190
c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:03"))
191191
c.set(2015, 2, 18, 12, 3, 17)
192192
c.set(Calendar.MILLISECOND, 0)
193193
assert(DateTimeUtils.stringToTimestamp(
194-
UTF8String.fromString("2015-03-18T12:03:17+07:03")).get == c.getTimeInMillis * 1000)
194+
UTF8String.fromString("2015-03-18T12:03:17+07:03")).get === c.getTimeInMillis * 1000)
195195

196196
c = Calendar.getInstance()
197197
c.set(2015, 2, 18, 12, 3, 17)
198198
c.set(Calendar.MILLISECOND, 123)
199199
assert(DateTimeUtils.stringToTimestamp(
200-
UTF8String.fromString("2015-03-18 12:03:17.123")).get == c.getTimeInMillis * 1000)
200+
UTF8String.fromString("2015-03-18 12:03:17.123")).get === c.getTimeInMillis * 1000)
201201
assert(DateTimeUtils.stringToTimestamp(
202-
UTF8String.fromString("2015-03-18T12:03:17.123")).get == c.getTimeInMillis * 1000)
202+
UTF8String.fromString("2015-03-18T12:03:17.123")).get === c.getTimeInMillis * 1000)
203203

204204
c = Calendar.getInstance(TimeZone.getTimeZone("UTC"))
205205
c.set(2015, 2, 18, 12, 3, 17)
206206
c.set(Calendar.MILLISECOND, 456)
207207
assert(DateTimeUtils.stringToTimestamp(
208-
UTF8String.fromString("2015-03-18T12:03:17.456Z")).get == c.getTimeInMillis * 1000)
208+
UTF8String.fromString("2015-03-18T12:03:17.456Z")).get === c.getTimeInMillis * 1000)
209209
assert(DateTimeUtils.stringToTimestamp(
210-
UTF8String.fromString("2015-03-18 12:03:17.456Z")).get == c.getTimeInMillis * 1000)
210+
UTF8String.fromString("2015-03-18 12:03:17.456Z")).get === c.getTimeInMillis * 1000)
211211

212212
c = Calendar.getInstance(TimeZone.getTimeZone("GMT-01:00"))
213213
c.set(2015, 2, 18, 12, 3, 17)
214214
c.set(Calendar.MILLISECOND, 123)
215215
assert(DateTimeUtils.stringToTimestamp(
216-
UTF8String.fromString("2015-03-18T12:03:17.123-1:0")).get == c.getTimeInMillis * 1000)
216+
UTF8String.fromString("2015-03-18T12:03:17.123-1:0")).get === c.getTimeInMillis * 1000)
217217
assert(DateTimeUtils.stringToTimestamp(
218-
UTF8String.fromString("2015-03-18T12:03:17.123-01:00")).get == c.getTimeInMillis * 1000)
218+
UTF8String.fromString("2015-03-18T12:03:17.123-01:00")).get === c.getTimeInMillis * 1000)
219219

220220
c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:30"))
221221
c.set(2015, 2, 18, 12, 3, 17)
222222
c.set(Calendar.MILLISECOND, 123)
223223
assert(DateTimeUtils.stringToTimestamp(
224-
UTF8String.fromString("2015-03-18T12:03:17.123+07:30")).get == c.getTimeInMillis * 1000)
224+
UTF8String.fromString("2015-03-18T12:03:17.123+07:30")).get === c.getTimeInMillis * 1000)
225225

226226
c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:30"))
227227
c.set(2015, 2, 18, 12, 3, 17)
228228
c.set(Calendar.MILLISECOND, 123)
229229
assert(DateTimeUtils.stringToTimestamp(
230-
UTF8String.fromString("2015-03-18T12:03:17.123+07:30")).get == c.getTimeInMillis * 1000)
230+
UTF8String.fromString("2015-03-18T12:03:17.123+07:30")).get === c.getTimeInMillis * 1000)
231231

232232
c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:30"))
233233
c.set(2015, 2, 18, 12, 3, 17)
234234
c.set(Calendar.MILLISECOND, 123)
235235
assert(DateTimeUtils.stringToTimestamp(
236-
UTF8String.fromString("2015-03-18T12:03:17.123121+7:30")).get ==
236+
UTF8String.fromString("2015-03-18T12:03:17.123121+7:30")).get ===
237237
c.getTimeInMillis * 1000 + 121)
238238

239239
c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:30"))
240240
c.set(2015, 2, 18, 12, 3, 17)
241241
c.set(Calendar.MILLISECOND, 123)
242242
assert(DateTimeUtils.stringToTimestamp(
243-
UTF8String.fromString("2015-03-18T12:03:17.12312+7:30")).get ==
243+
UTF8String.fromString("2015-03-18T12:03:17.12312+7:30")).get ===
244244
c.getTimeInMillis * 1000 + 120)
245245

246246
c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:30"))
@@ -249,7 +249,7 @@ class DateTimeUtilsSuite extends SparkFunSuite {
249249
c.set(Calendar.SECOND, 15)
250250
c.set(Calendar.MILLISECOND, 123)
251251
assert(DateTimeUtils.stringToTimestamp(
252-
UTF8String.fromString("T18:12:15.12312+7:30")).get ==
252+
UTF8String.fromString("T18:12:15.12312+7:30")).get ===
253253
c.getTimeInMillis * 1000 + 120)
254254

255255
c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:30"))
@@ -258,14 +258,36 @@ class DateTimeUtilsSuite extends SparkFunSuite {
258258
c.set(Calendar.SECOND, 15)
259259
c.set(Calendar.MILLISECOND, 123)
260260
assert(DateTimeUtils.stringToTimestamp(
261-
UTF8String.fromString("18:12:15.12312+7:30")).get ==
261+
UTF8String.fromString("18:12:15.12312+7:30")).get ===
262262
c.getTimeInMillis * 1000 + 120)
263263

264264
c = Calendar.getInstance()
265265
c.set(2011, 4, 6, 7, 8, 9)
266266
c.set(Calendar.MILLISECOND, 100)
267267
assert(DateTimeUtils.stringToTimestamp(
268-
UTF8String.fromString("2011-05-06 07:08:09.1000")).get == c.getTimeInMillis * 1000)
268+
UTF8String.fromString("2011-05-06 07:08:09.1000")).get === c.getTimeInMillis * 1000)
269+
270+
val defaultTimeZone = TimeZone.getDefault
271+
TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
272+
273+
c = Calendar.getInstance()
274+
c.set(2015, 2, 8, 2, 0, 0)
275+
c.set(Calendar.MILLISECOND, 0)
276+
assert(DateTimeUtils.stringToTimestamp(
277+
UTF8String.fromString("2015-3-8 2:0:0")).get === c.getTimeInMillis * 1000)
278+
c.add(Calendar.MINUTE, 30)
279+
assert(DateTimeUtils.stringToTimestamp(
280+
UTF8String.fromString("2015-3-8 3:30:0")).get === c.getTimeInMillis * 1000)
281+
assert(DateTimeUtils.stringToTimestamp(
282+
UTF8String.fromString("2015-3-8 2:30:0")).get === c.getTimeInMillis * 1000)
283+
284+
c = Calendar.getInstance()
285+
c.set(2015, 10, 1, 1, 59, 0)
286+
c.set(Calendar.MILLISECOND, 0)
287+
c.add(Calendar.MINUTE, 31)
288+
assert(DateTimeUtils.stringToTimestamp(
289+
UTF8String.fromString("2015-11-1 2:30:0")).get === c.getTimeInMillis * 1000)
290+
TimeZone.setDefault(defaultTimeZone)
269291

270292
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("238")).isEmpty)
271293
assert(DateTimeUtils.stringToTimestamp(UTF8String.fromString("2015-03-18 123142")).isEmpty)

0 commit comments

Comments
 (0)