[SPARK-29783][SQL] Support SQL Standard/ISO_8601 output style for interval type

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala

@@ -31,6 +31,7 @@ import org.apache.spark.sql.catalyst.util._
 import org.apache.spark.sql.catalyst.util.DateTimeConstants._
 import org.apache.spark.sql.catalyst.util.DateTimeUtils._
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.IntervalStyle
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.UTF8StringBuilder
 import org.apache.spark.unsafe.types.{CalendarInterval, UTF8String}
@@ -281,6 +282,9 @@ abstract class CastBase extends UnaryExpression with TimeZoneAwareExpression wit
 
   // UDFToString
   private[this] def castToString(from: DataType): Any => Any = from match {
+    case CalendarIntervalType if SQLConf.get.intervalOutputStyle == IntervalStyle.SQL_STANDARD =>
+      buildCast[CalendarInterval](_, i =>
+        UTF8String.fromString(IntervalUtils.toSqlStandardString(i)))
     case BinaryType => buildCast[Array[Byte]](_, UTF8String.fromBytes)
     case DateType => buildCast[Int](_, d => UTF8String.fromString(dateFormatter.format(d)))
     case TimestampType => buildCast[Long](_,

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/IntervalUtils.scala

@@ -17,6 +17,7 @@
 
 package org.apache.spark.sql.catalyst.util
 
+import java.math.BigDecimal
 import java.util.concurrent.TimeUnit
 
 import scala.util.control.NonFatal
@@ -378,6 +379,43 @@ object IntervalUtils {
     fromDoubles(interval.months / num, interval.days / num, interval.microseconds / num)
   }
 
+  def toSqlStandardString(interval: CalendarInterval): String = {
+    val yearMonthPart = if (interval.months != 0) {
+      interval.months / 12 + "-" + math.abs(interval.months) % 12
+    } else {
+      ""
+    }
+
+    val dayPart = if (interval.days != 0) interval.days.toString else ""
+
+    val timePart = if (interval.microseconds != 0) {
+      val sb = new StringBuilder()
+      var rest = interval.microseconds
+      sb.append(rest / MICROS_PER_HOUR)
+      sb.append(':')
+      rest = math.abs(rest % MICROS_PER_HOUR)
+      val minutes = rest / MICROS_PER_MINUTE;
+      if (minutes < 10) {
+        sb.append(0)
+      }
+      sb.append(minutes)
+      sb.append(':')
+      rest %= MICROS_PER_MINUTE
+      val db = BigDecimal.valueOf(rest, 6)
+      if (db.compareTo(new BigDecimal(10)) < 0) {
+        sb.append(0)
+      }
+      val s = db.stripTrailingZeros().toPlainString
+      sb.append(s)
+      sb.toString()
+    } else {
+      ""
+    }
+
+    val intervalList = Seq(yearMonthPart, dayPart, timePart).filter(_.nonEmpty)
+    if (intervalList.nonEmpty) intervalList.mkString(" ") else "0"
+  }
+
   private object ParseState extends Enumeration {
     val PREFIX,
         BEGIN_VALUE,

sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala

@@ -37,7 +37,6 @@ import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode
 import org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator
 import org.apache.spark.sql.catalyst.plans.logical.HintErrorHandler
 import org.apache.spark.sql.connector.catalog.CatalogManager.SESSION_CATALOG_NAME
-import org.apache.spark.sql.internal.SQLConf.StoreAssignmentPolicy
 import org.apache.spark.unsafe.array.ByteArrayMethods
 import org.apache.spark.util.Utils
 
@@ -1774,6 +1773,19 @@ object SQLConf {
     .booleanConf
     .createWithDefault(false)
 
+  object IntervalStyle extends Enumeration {
+    val SQL_STANDARD, MULTI_UNITS = Value
+  }
+
+  val INTERVAL_STYLE = buildConf("spark.sql.IntervalOutputStyle")
+    .doc("Display format for interval values. The value SQL_STANDARD will produce output" +
+      " matching SQL standard interval literals. The value MULTI_UNITS (which is the default)" +
+      " will produce output in form of value unit pairs, i.e. '3 year 2 months 10 days'")
+    .stringConf
+    .transform(_.toUpperCase(Locale.ROOT))
+    .checkValues(IntervalStyle.values.map(_.toString))
+    .createWithDefault(IntervalStyle.MULTI_UNITS.toString)
+
   val SORT_BEFORE_REPARTITION =
     buildConf("spark.sql.execution.sortBeforeRepartition")
       .internal()
@@ -2502,6 +2514,8 @@ class SQLConf extends Serializable with Logging {
   def storeAssignmentPolicy: StoreAssignmentPolicy.Value =
     StoreAssignmentPolicy.withName(getConf(STORE_ASSIGNMENT_POLICY))
 
+  def intervalOutputStyle: IntervalStyle.Value = IntervalStyle.withName(getConf(INTERVAL_STYLE))
+
   def ansiEnabled: Boolean = getConf(ANSI_ENABLED)
 
   def usePostgreSQLDialect: Boolean = getConf(DIALECT) == Dialect.POSTGRESQL.toString()

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/IntervalUtilsSuite.scala

@@ -241,4 +241,25 @@ class IntervalUtilsSuite extends SparkFunSuite {
         assert(e.getMessage.contains("divide by zero"))
     }
   }
+
+  test("to ansi sql standard string") {
+    val i1 = new CalendarInterval(0, 0, 0)
+    assert(IntervalUtils.toSqlStandardString(i1) === "0")
+    val i2 = new CalendarInterval(34, 0, 0)
+    assert(IntervalUtils.toSqlStandardString(i2) === "2-10")
+    val i3 = new CalendarInterval(-34, 0, 0)
+    assert(IntervalUtils.toSqlStandardString(i3) === "-2-10")
+    val i4 = new CalendarInterval(0, 31, 0)
+    assert(IntervalUtils.toSqlStandardString(i4) === "31")
+    val i5 = new CalendarInterval(0, -31, 0)
+    assert(IntervalUtils.toSqlStandardString(i5) === "-31")
+    val i6 = new CalendarInterval(0, 0, 3 * MICROS_PER_HOUR + 13 * MICROS_PER_MINUTE + 123)
+    assert(IntervalUtils.toSqlStandardString(i6) === "3:13:00.000123")
+    val i7 = new CalendarInterval(0, 0, -3 * MICROS_PER_HOUR - 13 * MICROS_PER_MINUTE - 123)
+    assert(IntervalUtils.toSqlStandardString(i7) === "-3:13:00.000123")
+    val i8 = new CalendarInterval(-34, 31, 3 * MICROS_PER_HOUR + 13 * MICROS_PER_MINUTE + 123)
+    assert(IntervalUtils.toSqlStandardString(i8) === "-2-10 31 3:13:00.000123")
+    val i9 = new CalendarInterval(0, 0, -3000 * MICROS_PER_HOUR)
+    assert(IntervalUtils.toSqlStandardString(i9) === "-3000:00:00")
+  }
 }

sql/core/src/main/scala/org/apache/spark/sql/execution/HiveResult.scala

@@ -21,10 +21,12 @@ import java.nio.charset.StandardCharsets
 import java.sql.{Date, Timestamp}
 
 import org.apache.spark.sql.Row
-import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils, TimestampFormatter}
+import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils, IntervalUtils, TimestampFormatter}
 import org.apache.spark.sql.execution.command.{DescribeCommandBase, ExecutedCommandExec, ShowTablesCommand}
 import org.apache.spark.sql.internal.SQLConf
+import org.apache.spark.sql.internal.SQLConf.IntervalStyle
 import org.apache.spark.sql.types._
+import org.apache.spark.unsafe.types.CalendarInterval
 
 /**
  * Runs a query returning the result in Hive compatible form.
@@ -97,6 +99,9 @@ object HiveResult {
     case (null, _) => "null"
     case (s: String, StringType) => "\"" + s + "\""
     case (decimal, DecimalType()) => decimal.toString
+    case (interval: CalendarInterval, CalendarIntervalType)
+      if SQLConf.get.intervalOutputStyle == IntervalStyle.SQL_STANDARD =>
+      IntervalUtils.toSqlStandardString(interval)
     case (interval, CalendarIntervalType) => interval.toString
     case (other, tpe) if primitiveTypes contains tpe => other.toString
   }
@@ -120,6 +125,9 @@ object HiveResult {
       DateTimeUtils.timestampToString(timestampFormatter, DateTimeUtils.fromJavaTimestamp(t))
     case (bin: Array[Byte], BinaryType) => new String(bin, StandardCharsets.UTF_8)
     case (decimal: java.math.BigDecimal, DecimalType()) => formatDecimal(decimal)
+    case (interval: CalendarInterval, CalendarIntervalType)
+      if SQLConf.get.intervalOutputStyle == IntervalStyle.SQL_STANDARD =>
+      IntervalUtils.toSqlStandardString(interval)
     case (interval, CalendarIntervalType) => interval.toString
     case (other, _ : UserDefinedType[_]) => other.toString
     case (other, tpe) if primitiveTypes.contains(tpe) => other.toString

sql/core/src/test/resources/sql-tests/inputs/datetime.sql

@@ -36,14 +36,3 @@ select date '2001-10-01' - 7;
 select date '2001-10-01' - date '2001-09-28';
 select date'2020-01-01' - timestamp'2019-10-06 10:11:12.345678';
 select timestamp'2019-10-06 10:11:12.345678' - date'2020-01-01';
-
--- interval operations
-select 3 * (timestamp'2019-10-15 10:11:12.001002' - date'2019-10-15');
-select interval 4 month 2 weeks 3 microseconds * 1.5;
-select (timestamp'2019-10-15' - timestamp'2019-10-14') / 1.5;
-
--- interval operation with null and zero case
-select interval '2 seconds' / 0;
-select interval '2 seconds' / null;
-select interval '2 seconds' * null;
-select null * interval '2 seconds';

sql/core/src/test/resources/sql-tests/inputs/interval.sql

@@ -0,0 +1,40 @@
+--
+--   Licensed to the Apache Software Foundation (ASF) under one or more
+--   contributor license agreements.  See the NOTICE file distributed with
+--   this work for additional information regarding copyright ownership.
+--   The ASF licenses this file to You under the Apache License, Version 2.0
+--   (the "License"); you may not use this file except in compliance with
+--   the License.  You may obtain a copy of the License at
+--
+--      http://www.apache.org/licenses/LICENSE-2.0
+--
+--   Unless required by applicable law or agreed to in writing, software
+--   distributed under the License is distributed on an "AS IS" BASIS,
+--   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+--   See the License for the specific language governing permissions and
+--   limitations under the License.
+--
+
+-- test for interacting with intervals
+
+-- using SQL_STANDARD output style
+set spark.sql.IntervalOutputStyle=SQL_STANDARD;
+select interval 4 month 2 weeks 3 microseconds;
+select interval '1 year 20 month';
+select interval '-1 year -20 month';
+select interval '20 month 30 days -21 hours 10 minutes 999 milliseconds';
+select date'2019-10-15' - timestamp'2019-10-15 10:11:12.001002';
+
+-- using MULTI_UNITS (which is default) output style
+set spark.sql.IntervalOutputStyle=MULTI_UNITS;
+
+-- interval operations
+select 3 * (timestamp'2019-10-15 10:11:12.001002' - date'2019-10-15');
+select interval 4 month 2 weeks 3 microseconds * 1.5;
+select (timestamp'2019-10-15' - timestamp'2019-10-14') / 1.5;
+
+-- interval operation with null and zero case
+select interval '2 seconds' / 0;
+select interval '2 seconds' / null;
+select interval '2 seconds' * null;
+select null * interval '2 seconds';

sql/core/src/test/resources/sql-tests/results/datetime.sql.out

@@ -1,5 +1,5 @@
 -- Automatically generated by SQLQueryTestSuite
--- Number of queries: 24
+-- Number of queries: 17
 
 
 -- !query 0
@@ -145,59 +145,3 @@ select timestamp'2019-10-06 10:11:12.345678' - date'2020-01-01'
 struct<subtracttimestamps(TIMESTAMP('2019-10-06 10:11:12.345678'), CAST(DATE '2020-01-01' AS TIMESTAMP)):interval>
 -- !query 16 output
 -2078 hours -48 minutes -47.654322 seconds
-
-
--- !query 17
-select 3 * (timestamp'2019-10-15 10:11:12.001002' - date'2019-10-15')
--- !query 17 schema
-struct<multiply_interval(subtracttimestamps(TIMESTAMP('2019-10-15 10:11:12.001002'), CAST(DATE '2019-10-15' AS TIMESTAMP)), CAST(3 AS DOUBLE)):interval>
--- !query 17 output
-30 hours 33 minutes 36.003006 seconds
-
-
--- !query 18
-select interval 4 month 2 weeks 3 microseconds * 1.5
--- !query 18 schema
-struct<multiply_interval(4 months 14 days 0.000003 seconds, CAST(1.5 AS DOUBLE)):interval>
--- !query 18 output
-6 months 21 days 0.000005 seconds
-
-
--- !query 19
-select (timestamp'2019-10-15' - timestamp'2019-10-14') / 1.5
--- !query 19 schema
-struct<divide_interval(subtracttimestamps(TIMESTAMP('2019-10-15 00:00:00'), TIMESTAMP('2019-10-14 00:00:00')), CAST(1.5 AS DOUBLE)):interval>
--- !query 19 output
-16 hours
-
-
--- !query 20
-select interval '2 seconds' / 0
--- !query 20 schema
-struct<divide_interval(2 seconds, CAST(0 AS DOUBLE)):interval>
--- !query 20 output
-NULL
-
-
--- !query 21
-select interval '2 seconds' / null
--- !query 21 schema
-struct<divide_interval(2 seconds, CAST(NULL AS DOUBLE)):interval>
--- !query 21 output
-NULL
-
-
--- !query 22
-select interval '2 seconds' * null
--- !query 22 schema
-struct<multiply_interval(2 seconds, CAST(NULL AS DOUBLE)):interval>
--- !query 22 output
-NULL
-
-
--- !query 23
-select null * interval '2 seconds'
--- !query 23 schema
-struct<multiply_interval(2 seconds, CAST(NULL AS DOUBLE)):interval>
--- !query 23 output
-NULL