Skip to content

Commit 077cf2a

Browse files
AngersZhuuuuMaxGekk
authored andcommitted
[SPARK-35733][SQL][TESTS] Check all day-time interval types in HiveInspectors tests
### What changes were proposed in this pull request? Check all day-time interval types in HiveInspectors tests. ### Why are the changes needed? New tests should improve test coverage for day-time interval types. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Added UT. Closes #33036 from AngersZhuuuu/SPARK-35733. Authored-by: Angerszhuuuu <[email protected]> Signed-off-by: Max Gekk <[email protected]>
1 parent 66d5a00 commit 077cf2a

File tree

1 file changed

+37
-16
lines changed

1 file changed

+37
-16
lines changed

sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveScriptTransformationSuite.scala

Lines changed: 37 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import org.apache.spark.sql.execution._
3232
import org.apache.spark.sql.functions._
3333
import org.apache.spark.sql.hive.test.TestHiveSingleton
3434
import org.apache.spark.sql.types._
35+
import org.apache.spark.sql.types.DayTimeIntervalType._
3536
import org.apache.spark.sql.types.YearMonthIntervalType._
3637
import org.apache.spark.unsafe.types.CalendarInterval
3738

@@ -522,36 +523,57 @@ class HiveScriptTransformationSuite extends BaseScriptTransformationSuite with T
522523

523524
}
524525

525-
test("SPARK-34879: HiveInspectors supports DayTimeIntervalType") {
526+
test("SPARK-34879, SPARK-35733: HiveInspectors supports all type of DayTimeIntervalType") {
526527
assume(TestUtils.testCommandAvailable("/bin/bash"))
527528
withTempView("v") {
528-
val df = Seq(
529-
(Duration.ofDays(1),
529+
val schema = StructType(Seq(
530+
StructField("a", DayTimeIntervalType(DAY)),
531+
StructField("b", DayTimeIntervalType(DAY, HOUR)),
532+
StructField("c", DayTimeIntervalType(DAY, MINUTE)),
533+
StructField("d", DayTimeIntervalType(DAY, SECOND)),
534+
StructField("e", DayTimeIntervalType(HOUR)),
535+
StructField("f", DayTimeIntervalType(HOUR, MINUTE)),
536+
StructField("g", DayTimeIntervalType(HOUR, SECOND)),
537+
StructField("h", DayTimeIntervalType(MINUTE)),
538+
StructField("i", DayTimeIntervalType(MINUTE, SECOND)),
539+
StructField("j", DayTimeIntervalType(SECOND))
540+
))
541+
val df = spark.createDataFrame(sparkContext.parallelize(Seq(
542+
Row(Duration.ofDays(1),
543+
Duration.ofHours(11),
544+
Duration.ofMinutes(1),
530545
Duration.ofSeconds(100).plusNanos(123456),
531-
Duration.of(Long.MaxValue, ChronoUnit.MICROS)),
532-
(Duration.ofDays(1),
533-
Duration.ofSeconds(100).plusNanos(1123456789),
534-
Duration.ofSeconds(Long.MaxValue / DateTimeConstants.MICROS_PER_SECOND))
535-
).toDF("a", "b", "c")
536-
df.createTempView("v")
546+
Duration.of(Long.MaxValue, ChronoUnit.MICROS),
547+
Duration.ofDays(1),
548+
Duration.ofHours(11),
549+
Duration.ofMinutes(1),
550+
Duration.ofSeconds(100).plusNanos(123456),
551+
Duration.ofSeconds(Long.MaxValue / DateTimeConstants.MICROS_PER_SECOND)
552+
))), schema)
537553

538554
// Hive serde supports DayTimeIntervalType as input and output data type
539555
checkAnswer(
540556
df,
541557
(child: SparkPlan) => createScriptTransformationExec(
542558
script = "cat",
543559
output = Seq(
544-
// TODO(SPARK-35733): Check all day-time interval types in HiveInspectors tests
545-
AttributeReference("a", DayTimeIntervalType())(),
546-
AttributeReference("b", DayTimeIntervalType())(),
547-
AttributeReference("c", DayTimeIntervalType())()),
560+
AttributeReference("a", DayTimeIntervalType(DAY))(),
561+
AttributeReference("b", DayTimeIntervalType(DAY, HOUR))(),
562+
AttributeReference("c", DayTimeIntervalType(DAY, MINUTE))(),
563+
AttributeReference("d", DayTimeIntervalType(DAY, SECOND))(),
564+
AttributeReference("e", DayTimeIntervalType(HOUR))(),
565+
AttributeReference("f", DayTimeIntervalType(HOUR, MINUTE))(),
566+
AttributeReference("g", DayTimeIntervalType(HOUR, SECOND))(),
567+
AttributeReference("h", DayTimeIntervalType(MINUTE))(),
568+
AttributeReference("i", DayTimeIntervalType(MINUTE, SECOND))(),
569+
AttributeReference("j", DayTimeIntervalType(SECOND))()),
548570
child = child,
549571
ioschema = hiveIOSchema),
550-
df.select($"a", $"b", $"c").collect())
572+
df.select($"a", $"b", $"c", $"d", $"e", $"f", $"g", $"h", $"i", $"j").collect())
551573
}
552574
}
553575

554-
test("SPARK-35722: HiveInspectors supports all type of YearMonthIntervalType") {
576+
test("SPARK-34879, SPARK-35722: HiveInspectors supports all type of YearMonthIntervalType") {
555577
assume(TestUtils.testCommandAvailable("/bin/bash"))
556578
withTempView("v") {
557579
val schema = StructType(Seq(
@@ -589,7 +611,6 @@ class HiveScriptTransformationSuite extends BaseScriptTransformationSuite with T
589611
df,
590612
(child: SparkPlan) => createScriptTransformationExec(
591613
script = "cat",
592-
// TODO(SPARK-35733): Check all day-time interval types in HiveInspectors tests
593614
output = Seq(AttributeReference("a", DayTimeIntervalType())()),
594615
child = child,
595616
ioschema = hiveIOSchema),

0 commit comments

Comments
 (0)