@@ -32,6 +32,7 @@ import org.apache.spark.sql.execution._
3232import org .apache .spark .sql .functions ._
3333import org .apache .spark .sql .hive .test .TestHiveSingleton
3434import org .apache .spark .sql .types ._
35+ import org .apache .spark .sql .types .DayTimeIntervalType ._
3536import org .apache .spark .sql .types .YearMonthIntervalType ._
3637import org .apache .spark .unsafe .types .CalendarInterval
3738
@@ -522,36 +523,57 @@ class HiveScriptTransformationSuite extends BaseScriptTransformationSuite with T
522523
523524 }
524525
525- test(" SPARK-34879: HiveInspectors supports DayTimeIntervalType" ) {
526+ test(" SPARK-34879, SPARK-35733 : HiveInspectors supports all type of DayTimeIntervalType" ) {
526527 assume(TestUtils .testCommandAvailable(" /bin/bash" ))
527528 withTempView(" v" ) {
528- val df = Seq (
529- (Duration .ofDays(1 ),
529+ val schema = StructType (Seq (
530+ StructField (" a" , DayTimeIntervalType (DAY )),
531+ StructField (" b" , DayTimeIntervalType (DAY , HOUR )),
532+ StructField (" c" , DayTimeIntervalType (DAY , MINUTE )),
533+ StructField (" d" , DayTimeIntervalType (DAY , SECOND )),
534+ StructField (" e" , DayTimeIntervalType (HOUR )),
535+ StructField (" f" , DayTimeIntervalType (HOUR , MINUTE )),
536+ StructField (" g" , DayTimeIntervalType (HOUR , SECOND )),
537+ StructField (" h" , DayTimeIntervalType (MINUTE )),
538+ StructField (" i" , DayTimeIntervalType (MINUTE , SECOND )),
539+ StructField (" j" , DayTimeIntervalType (SECOND ))
540+ ))
541+ val df = spark.createDataFrame(sparkContext.parallelize(Seq (
542+ Row (Duration .ofDays(1 ),
543+ Duration .ofHours(11 ),
544+ Duration .ofMinutes(1 ),
530545 Duration .ofSeconds(100 ).plusNanos(123456 ),
531- Duration .of(Long .MaxValue , ChronoUnit .MICROS )),
532- (Duration .ofDays(1 ),
533- Duration .ofSeconds(100 ).plusNanos(1123456789 ),
534- Duration .ofSeconds(Long .MaxValue / DateTimeConstants .MICROS_PER_SECOND ))
535- ).toDF(" a" , " b" , " c" )
536- df.createTempView(" v" )
546+ Duration .of(Long .MaxValue , ChronoUnit .MICROS ),
547+ Duration .ofDays(1 ),
548+ Duration .ofHours(11 ),
549+ Duration .ofMinutes(1 ),
550+ Duration .ofSeconds(100 ).plusNanos(123456 ),
551+ Duration .ofSeconds(Long .MaxValue / DateTimeConstants .MICROS_PER_SECOND )
552+ ))), schema)
537553
538554 // Hive serde supports DayTimeIntervalType as input and output data type
539555 checkAnswer(
540556 df,
541557 (child : SparkPlan ) => createScriptTransformationExec(
542558 script = " cat" ,
543559 output = Seq (
544- // TODO(SPARK-35733): Check all day-time interval types in HiveInspectors tests
545- AttributeReference (" a" , DayTimeIntervalType ())(),
546- AttributeReference (" b" , DayTimeIntervalType ())(),
547- AttributeReference (" c" , DayTimeIntervalType ())()),
560+ AttributeReference (" a" , DayTimeIntervalType (DAY ))(),
561+ AttributeReference (" b" , DayTimeIntervalType (DAY , HOUR ))(),
562+ AttributeReference (" c" , DayTimeIntervalType (DAY , MINUTE ))(),
563+ AttributeReference (" d" , DayTimeIntervalType (DAY , SECOND ))(),
564+ AttributeReference (" e" , DayTimeIntervalType (HOUR ))(),
565+ AttributeReference (" f" , DayTimeIntervalType (HOUR , MINUTE ))(),
566+ AttributeReference (" g" , DayTimeIntervalType (HOUR , SECOND ))(),
567+ AttributeReference (" h" , DayTimeIntervalType (MINUTE ))(),
568+ AttributeReference (" i" , DayTimeIntervalType (MINUTE , SECOND ))(),
569+ AttributeReference (" j" , DayTimeIntervalType (SECOND ))()),
548570 child = child,
549571 ioschema = hiveIOSchema),
550- df.select($" a" , $" b" , $" c" ).collect())
572+ df.select($" a" , $" b" , $" c" , $ " d " , $ " e " , $ " f " , $ " g " , $ " h " , $ " i " , $ " j " ).collect())
551573 }
552574 }
553575
554- test(" SPARK-35722: HiveInspectors supports all type of YearMonthIntervalType" ) {
576+ test(" SPARK-34879, SPARK- 35722: HiveInspectors supports all type of YearMonthIntervalType" ) {
555577 assume(TestUtils .testCommandAvailable(" /bin/bash" ))
556578 withTempView(" v" ) {
557579 val schema = StructType (Seq (
@@ -589,7 +611,6 @@ class HiveScriptTransformationSuite extends BaseScriptTransformationSuite with T
589611 df,
590612 (child : SparkPlan ) => createScriptTransformationExec(
591613 script = " cat" ,
592- // TODO(SPARK-35733): Check all day-time interval types in HiveInspectors tests
593614 output = Seq (AttributeReference (" a" , DayTimeIntervalType ())()),
594615 child = child,
595616 ioschema = hiveIOSchema),
0 commit comments