diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md index e9cf49a7ca4bf..1138afa45d085 100644 --- a/docs/sql-migration-guide.md +++ b/docs/sql-migration-guide.md @@ -81,6 +81,8 @@ license: | - In Spark 3.2, `TRANSFORM` operator can support `ArrayType/MapType/StructType` without Hive SerDe, in this mode, we use `StructsToJosn` to convert `ArrayType/MapType/StructType` column to `STRING` and use `JsonToStructs` to parse `STRING` to `ArrayType/MapType/StructType`. In Spark 3.1, Spark just support case `ArrayType/MapType/StructType` column as `STRING` but can't support parse `STRING` to `ArrayType/MapType/StructType` output columns. + - In Spark 3.2, the unit-to-unit interval literals like `INTERVAL '1-1' YEAR TO MONTH` are converted to ANSI interval types: `YearMonthIntervalType` or `DayTimeIntervalType`. In Spark 3.1 and earlier, such interval literals are converted to `CalendarIntervalType`. To restore the behavior before Spark 3.2, you can set `spark.sql.legacy.interval.enabled` to `true`. + ## Upgrading from Spark SQL 3.0 to 3.1 - In Spark 3.1, statistical aggregation function includes `std`, `stddev`, `stddev_samp`, `variance`, `var_samp`, `skewness`, `kurtosis`, `covar_samp`, `corr` will return `NULL` instead of `Double.NaN` when `DivideByZero` occurs during expression evaluation, for example, when `stddev_samp` applied on a single element set. In Spark version 3.0 and earlier, it will return `Double.NaN` in such case. To restore the behavior before Spark 3.1, you can set `spark.sql.legacy.statisticalAggregate` to `true`. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala index f9317e865f275..63290d3547c01 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.catalyst.parser import java.util.Locale +import java.util.concurrent.TimeUnit import javax.xml.bind.DatatypeConverter import scala.collection.JavaConverters._ @@ -2306,12 +2307,30 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logg } /** - * Create a [[CalendarInterval]] literal expression. Two syntaxes are supported: + * Create a [[CalendarInterval]] or ANSI interval literal expression. + * Two syntaxes are supported: * - multiple unit value pairs, for instance: interval 2 months 2 days. * - from-to unit, for instance: interval '1-2' year to month. */ override def visitInterval(ctx: IntervalContext): Literal = withOrigin(ctx) { - Literal(parseIntervalLiteral(ctx), CalendarIntervalType) + val calendarInterval = parseIntervalLiteral(ctx) + if (ctx.errorCapturingUnitToUnitInterval != null && !conf.legacyIntervalEnabled) { + // Check the `to` unit to distinguish year-month and day-time intervals because + // `CalendarInterval` doesn't have enough info. For instance, new CalendarInterval(0, 0, 0) + // can be derived from INTERVAL '0-0' YEAR TO MONTH as well as from + // INTERVAL '0 00:00:00' DAY TO SECOND. + val toUnit = ctx.errorCapturingUnitToUnitInterval.body.to.getText.toLowerCase(Locale.ROOT) + if (toUnit == "month") { + assert(calendarInterval.days == 0 && calendarInterval.microseconds == 0) + Literal(calendarInterval.months, YearMonthIntervalType) + } else { + assert(calendarInterval.months == 0) + val micros = IntervalUtils.getDuration(calendarInterval, TimeUnit.MICROSECONDS) + Literal(micros, DayTimeIntervalType) + } + } else { + Literal(calendarInterval, CalendarIntervalType) + } } /** diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala index 5c64e572bbd77..9a5653e0bf62a 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala @@ -714,37 +714,39 @@ class ExpressionParserSuite extends AnalysisTest { // Non Existing unit intercept("interval 10 nanoseconds", "invalid unit 'nanoseconds'") - // Year-Month intervals. - val yearMonthValues = Seq("123-10", "496-0", "-2-3", "-123-0", "\t -1-2\t") - yearMonthValues.foreach { value => - val result = Literal(IntervalUtils.fromYearMonthString(value)) - checkIntervals(s"'$value' year to month", result) - } + withSQLConf(SQLConf.LEGACY_INTERVAL_ENABLED.key -> "true") { + // Year-Month intervals. + val yearMonthValues = Seq("123-10", "496-0", "-2-3", "-123-0", "\t -1-2\t") + yearMonthValues.foreach { value => + val result = Literal(IntervalUtils.fromYearMonthString(value)) + checkIntervals(s"'$value' year to month", result) + } - // Day-Time intervals. - val datTimeValues = Seq( - "99 11:22:33.123456789", - "-99 11:22:33.123456789", - "10 9:8:7.123456789", - "1 0:0:0", - "-1 0:0:0", - "1 0:0:1", - "\t 1 0:0:1 ") - datTimeValues.foreach { value => - val result = Literal(IntervalUtils.fromDayTimeString(value)) - checkIntervals(s"'$value' day to second", result) - } + // Day-Time intervals. + val datTimeValues = Seq( + "99 11:22:33.123456789", + "-99 11:22:33.123456789", + "10 9:8:7.123456789", + "1 0:0:0", + "-1 0:0:0", + "1 0:0:1", + "\t 1 0:0:1 ") + datTimeValues.foreach { value => + val result = Literal(IntervalUtils.fromDayTimeString(value)) + checkIntervals(s"'$value' day to second", result) + } - // Hour-Time intervals. - val hourTimeValues = Seq( - "11:22:33.123456789", - "9:8:7.123456789", - "-19:18:17.123456789", - "0:0:0", - "0:0:1") - hourTimeValues.foreach { value => - val result = Literal(IntervalUtils.fromDayTimeString(value, HOUR, SECOND)) - checkIntervals(s"'$value' hour to second", result) + // Hour-Time intervals. + val hourTimeValues = Seq( + "11:22:33.123456789", + "9:8:7.123456789", + "-19:18:17.123456789", + "0:0:0", + "0:0:1") + hourTimeValues.foreach { value => + val result = Literal(IntervalUtils.fromDayTimeString(value, HOUR, SECOND)) + checkIntervals(s"'$value' hour to second", result) + } } // Unknown FROM TO intervals diff --git a/sql/core/src/test/resources/sql-tests/inputs/interval.sql b/sql/core/src/test/resources/sql-tests/inputs/interval.sql index dc0fd3e63f828..3fcbdacda60da 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/interval.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/interval.sql @@ -51,6 +51,8 @@ select cast('- +1 second' as interval); select interval 13.123456789 seconds, interval -13.123456789 second; select interval 1 year 2 month 3 week 4 day 5 hour 6 minute 7 seconds 8 millisecond 9 microsecond; select interval '30' year '25' month '-100' day '40' hour '80' minute '299.889987299' second; +select interval '0-0' year to month; +select interval '0 0:0:0' day to second; select interval '0 0:0:0.1' day to second; select interval '10-9' year to month; select interval '20 15' day to hour; diff --git a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out index 324d1510a4862..d5ace9f8adec0 100644 --- a/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/ansi/interval.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 116 +-- Number of queries: 118 -- !query @@ -130,49 +130,49 @@ struct<(+ INTERVAL '-1 months 1 days -1 seconds'):interval> -- !query select interval -'1-1' year to month -- !query schema -struct +struct -- !query output --1 years -1 months +-1-1 -- !query select interval -'-1-1' year to month -- !query schema -struct +struct -- !query output -1 years 1 months +1-1 -- !query select interval +'-1-1' year to month -- !query schema -struct +struct -- !query output --1 years -1 months +-1-1 -- !query select interval - '1 2:3:4.001' day to second -- !query schema -struct +struct -- !query output --1 days -2 hours -3 minutes -4.001 seconds +-1 02:03:04.001000000 -- !query select interval +'1 2:3:4.001' day to second -- !query schema -struct +struct -- !query output -1 days 2 hours 3 minutes 4.001 seconds +1 02:03:04.001000000 -- !query select interval -'-1 2:3:4.001' day to second -- !query schema -struct +struct -- !query output -1 days 2 hours 3 minutes 4.001 seconds +1 02:03:04.001000000 -- !query @@ -328,76 +328,92 @@ struct +-- !query output +0-0 + + +-- !query +select interval '0 0:0:0' day to second +-- !query schema +struct +-- !query output +0 00:00:00.000000000 + + -- !query select interval '0 0:0:0.1' day to second -- !query schema -struct +struct -- !query output -0.1 seconds +0 00:00:00.100000000 -- !query select interval '10-9' year to month -- !query schema -struct +struct -- !query output -10 years 9 months +10-9 -- !query select interval '20 15' day to hour -- !query schema -struct +struct -- !query output -20 days 15 hours +20 15:00:00.000000000 -- !query select interval '20 15:40' day to minute -- !query schema -struct +struct -- !query output -20 days 15 hours 40 minutes +20 15:40:00.000000000 -- !query select interval '20 15:40:32.99899999' day to second -- !query schema -struct +struct -- !query output -20 days 15 hours 40 minutes 32.998999 seconds +20 15:40:32.998999000 -- !query select interval '15:40' hour to minute -- !query schema -struct +struct -- !query output -15 hours 40 minutes +0 15:40:00.000000000 -- !query select interval '15:40:32.99899999' hour to second -- !query schema -struct +struct -- !query output -15 hours 40 minutes 32.998999 seconds +0 15:40:32.998999000 -- !query select interval '40:32.99899999' minute to second -- !query schema -struct +struct -- !query output -40 minutes 32.998999 seconds +0 00:40:32.998999000 -- !query select interval '40:32' minute to second -- !query schema -struct +struct -- !query output -40 minutes 32 seconds +0 00:40:32.000000000 -- !query @@ -786,7 +802,7 @@ select interval '2-2' year to month + dateval from interval_arithmetic -- !query schema -struct +struct -- !query output 2012-01-01 2009-11-01 2014-03-01 2014-03-01 2009-11-01 2009-11-01 2014-03-01 @@ -802,7 +818,7 @@ select interval '2-2' year to month + tsval from interval_arithmetic -- !query schema -struct +struct -- !query output 2012-01-01 00:00:00 2009-11-01 00:00:00 2014-03-01 00:00:00 2014-03-01 00:00:00 2009-11-01 00:00:00 2009-11-01 00:00:00 2014-03-01 00:00:00 @@ -813,9 +829,9 @@ select interval '2-2' year to month - interval '3-3' year to month from interval_arithmetic -- !query schema -struct<(INTERVAL '2 years 2 months' + INTERVAL '3 years 3 months'):interval,(INTERVAL '2 years 2 months' - INTERVAL '3 years 3 months'):interval> +struct<(INTERVAL '2-2' YEAR TO MONTH + INTERVAL '3-3' YEAR TO MONTH):year-month interval,(INTERVAL '2-2' YEAR TO MONTH - INTERVAL '3-3' YEAR TO MONTH):year-month interval> -- !query output -5 years 5 months -1 years -1 months +5-5 -1-1 -- !query @@ -829,10 +845,9 @@ select interval '99 11:22:33.123456789' day to second + dateval from interval_arithmetic -- !query schema -struct<> +struct -- !query output -java.lang.IllegalArgumentException -requirement failed: Cannot add hours, minutes or seconds, milliseconds, microseconds to a date +2012-01-01 2011-09-23 12:37:26.876544 2012-04-09 11:22:33.123456 2012-04-09 11:22:33.123456 2011-09-23 12:37:26.876544 2011-09-23 12:37:26.876544 2012-04-09 11:22:33.123456 -- !query @@ -846,7 +861,7 @@ select interval '99 11:22:33.123456789' day to second + tsval from interval_arithmetic -- !query schema -struct +struct -- !query output 2012-01-01 00:00:00 2011-09-23 12:37:26.876544 2012-04-09 11:22:33.123456 2012-04-09 11:22:33.123456 2011-09-23 12:37:26.876544 2011-09-23 12:37:26.876544 2012-04-09 11:22:33.123456 @@ -862,7 +877,7 @@ select interval '99 11:22:33.123456789' day to second + strval from interval_arithmetic -- !query schema -struct +struct -- !query output 2012-01-01 2011-09-23 12:37:26.876544 2012-04-09 11:22:33.123456 2012-04-09 11:22:33.123456 2011-09-23 12:37:26.876544 2011-09-23 12:37:26.876544 2012-04-09 11:22:33.123456 @@ -873,9 +888,9 @@ select interval '99 11:22:33.123456789' day to second - interval '10 9:8:7.123456789' day to second from interval_arithmetic -- !query schema -struct<(INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' + INTERVAL '10 days 9 hours 8 minutes 7.123456 seconds'):interval,(INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' - INTERVAL '10 days 9 hours 8 minutes 7.123456 seconds'):interval> +struct<(INTERVAL '99 11:22:33.123456' DAY TO SECOND + INTERVAL '10 09:08:07.123456' DAY TO SECOND):day-time interval,(INTERVAL '99 11:22:33.123456' DAY TO SECOND - INTERVAL '10 09:08:07.123456' DAY TO SECOND):day-time interval> -- !query output -109 days 20 hours 30 minutes 40.246912 seconds 89 days 2 hours 14 minutes 26 seconds +109 20:30:40.246912000 89 02:14:26.000000000 -- !query @@ -921,9 +936,9 @@ struct -- !query select interval '2-2\t' year to month -- !query schema -struct +struct -- !query output -2 years 2 months +2-2 -- !query @@ -943,9 +958,9 @@ select interval '-\t2-2\t' year to month -- !query select interval '\n0 12:34:46.789\t' day to second -- !query schema -struct +struct -- !query output -12 hours 34 minutes 46.789 seconds +0 12:34:46.789000000 -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/interval.sql.out b/sql/core/src/test/resources/sql-tests/results/interval.sql.out index 33a3c63e5e07b..4855252811725 100644 --- a/sql/core/src/test/resources/sql-tests/results/interval.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/interval.sql.out @@ -1,5 +1,5 @@ -- Automatically generated by SQLQueryTestSuite --- Number of queries: 116 +-- Number of queries: 118 -- !query @@ -125,49 +125,49 @@ struct<(+ INTERVAL '-1 months 1 days -1 seconds'):interval> -- !query select interval -'1-1' year to month -- !query schema -struct +struct -- !query output --1 years -1 months +-1-1 -- !query select interval -'-1-1' year to month -- !query schema -struct +struct -- !query output -1 years 1 months +1-1 -- !query select interval +'-1-1' year to month -- !query schema -struct +struct -- !query output --1 years -1 months +-1-1 -- !query select interval - '1 2:3:4.001' day to second -- !query schema -struct +struct -- !query output --1 days -2 hours -3 minutes -4.001 seconds +-1 02:03:04.001000000 -- !query select interval +'1 2:3:4.001' day to second -- !query schema -struct +struct -- !query output -1 days 2 hours 3 minutes 4.001 seconds +1 02:03:04.001000000 -- !query select interval -'-1 2:3:4.001' day to second -- !query schema -struct +struct -- !query output -1 days 2 hours 3 minutes 4.001 seconds +1 02:03:04.001000000 -- !query @@ -322,76 +322,92 @@ struct +-- !query output +0-0 + + +-- !query +select interval '0 0:0:0' day to second +-- !query schema +struct +-- !query output +0 00:00:00.000000000 + + -- !query select interval '0 0:0:0.1' day to second -- !query schema -struct +struct -- !query output -0.1 seconds +0 00:00:00.100000000 -- !query select interval '10-9' year to month -- !query schema -struct +struct -- !query output -10 years 9 months +10-9 -- !query select interval '20 15' day to hour -- !query schema -struct +struct -- !query output -20 days 15 hours +20 15:00:00.000000000 -- !query select interval '20 15:40' day to minute -- !query schema -struct +struct -- !query output -20 days 15 hours 40 minutes +20 15:40:00.000000000 -- !query select interval '20 15:40:32.99899999' day to second -- !query schema -struct +struct -- !query output -20 days 15 hours 40 minutes 32.998999 seconds +20 15:40:32.998999000 -- !query select interval '15:40' hour to minute -- !query schema -struct +struct -- !query output -15 hours 40 minutes +0 15:40:00.000000000 -- !query select interval '15:40:32.99899999' hour to second -- !query schema -struct +struct -- !query output -15 hours 40 minutes 32.998999 seconds +0 15:40:32.998999000 -- !query select interval '40:32.99899999' minute to second -- !query schema -struct +struct -- !query output -40 minutes 32.998999 seconds +0 00:40:32.998999000 -- !query select interval '40:32' minute to second -- !query schema -struct +struct -- !query output -40 minutes 32 seconds +0 00:40:32.000000000 -- !query @@ -780,7 +796,7 @@ select interval '2-2' year to month + dateval from interval_arithmetic -- !query schema -struct +struct -- !query output 2012-01-01 2009-11-01 2014-03-01 2014-03-01 2009-11-01 2009-11-01 2014-03-01 @@ -796,7 +812,7 @@ select interval '2-2' year to month + tsval from interval_arithmetic -- !query schema -struct +struct -- !query output 2012-01-01 00:00:00 2009-11-01 00:00:00 2014-03-01 00:00:00 2014-03-01 00:00:00 2009-11-01 00:00:00 2009-11-01 00:00:00 2014-03-01 00:00:00 @@ -807,9 +823,9 @@ select interval '2-2' year to month - interval '3-3' year to month from interval_arithmetic -- !query schema -struct<(INTERVAL '2 years 2 months' + INTERVAL '3 years 3 months'):interval,(INTERVAL '2 years 2 months' - INTERVAL '3 years 3 months'):interval> +struct<(INTERVAL '2-2' YEAR TO MONTH + INTERVAL '3-3' YEAR TO MONTH):year-month interval,(INTERVAL '2-2' YEAR TO MONTH - INTERVAL '3-3' YEAR TO MONTH):year-month interval> -- !query output -5 years 5 months -1 years -1 months +5-5 -1-1 -- !query @@ -823,9 +839,9 @@ select interval '99 11:22:33.123456789' day to second + dateval from interval_arithmetic -- !query schema -struct +struct -- !query output -2012-01-01 2011-09-23 2012-04-09 2012-04-09 2011-09-23 2011-09-23 2012-04-09 +2012-01-01 2011-09-23 12:37:26.876544 2012-04-09 11:22:33.123456 2012-04-09 11:22:33.123456 2011-09-23 12:37:26.876544 2011-09-23 12:37:26.876544 2012-04-09 11:22:33.123456 -- !query @@ -839,7 +855,7 @@ select interval '99 11:22:33.123456789' day to second + tsval from interval_arithmetic -- !query schema -struct +struct -- !query output 2012-01-01 00:00:00 2011-09-23 12:37:26.876544 2012-04-09 11:22:33.123456 2012-04-09 11:22:33.123456 2011-09-23 12:37:26.876544 2011-09-23 12:37:26.876544 2012-04-09 11:22:33.123456 @@ -855,7 +871,7 @@ select interval '99 11:22:33.123456789' day to second + strval from interval_arithmetic -- !query schema -struct +struct -- !query output 2012-01-01 2011-09-23 12:37:26.876544 2012-04-09 11:22:33.123456 2012-04-09 11:22:33.123456 2011-09-23 12:37:26.876544 2011-09-23 12:37:26.876544 2012-04-09 11:22:33.123456 @@ -866,9 +882,9 @@ select interval '99 11:22:33.123456789' day to second - interval '10 9:8:7.123456789' day to second from interval_arithmetic -- !query schema -struct<(INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' + INTERVAL '10 days 9 hours 8 minutes 7.123456 seconds'):interval,(INTERVAL '99 days 11 hours 22 minutes 33.123456 seconds' - INTERVAL '10 days 9 hours 8 minutes 7.123456 seconds'):interval> +struct<(INTERVAL '99 11:22:33.123456' DAY TO SECOND + INTERVAL '10 09:08:07.123456' DAY TO SECOND):day-time interval,(INTERVAL '99 11:22:33.123456' DAY TO SECOND - INTERVAL '10 09:08:07.123456' DAY TO SECOND):day-time interval> -- !query output -109 days 20 hours 30 minutes 40.246912 seconds 89 days 2 hours 14 minutes 26 seconds +109 20:30:40.246912000 89 02:14:26.000000000 -- !query @@ -914,9 +930,9 @@ struct -- !query select interval '2-2\t' year to month -- !query schema -struct +struct -- !query output -2 years 2 months +2-2 -- !query @@ -936,9 +952,9 @@ select interval '-\t2-2\t' year to month -- !query select interval '\n0 12:34:46.789\t' day to second -- !query schema -struct +struct -- !query output -12 hours 34 minutes 46.789 seconds +0 12:34:46.789000000 -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out index 3b383c0f42719..056c961e911f9 100644 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/create_view.sql.out @@ -257,7 +257,7 @@ View Text SELECT * FROM base_table View Original Text SELECT * FROM base_table View Catalog and Namespace spark_catalog.temp_view_test View Query Output Columns [a, id] -Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test, view.query.out.col.0=a, view.query.out.col.1=id, view.query.out.numCols=2, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true] +Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test, view.query.out.col.0=a, view.query.out.col.1=id, view.query.out.numCols=2, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true, view.sqlConfig.spark.sql.legacy.interval.enabled=true] -- !query @@ -313,7 +313,7 @@ View Text SELECT * FROM base_table View Original Text SELECT * FROM base_table View Catalog and Namespace spark_catalog.temp_view_test View Query Output Columns [a, id] -Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test, view.query.out.col.0=a, view.query.out.col.1=id, view.query.out.numCols=2, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true] +Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test, view.query.out.col.0=a, view.query.out.col.1=id, view.query.out.numCols=2, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true, view.sqlConfig.spark.sql.legacy.interval.enabled=true] -- !query @@ -359,7 +359,7 @@ View Original Text SELECT t1.a AS t1_a, t2.a AS t2_a WHERE t1.id = t2.id View Catalog and Namespace spark_catalog.temp_view_test View Query Output Columns [t1_a, t2_a] -Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test, view.query.out.col.0=t1_a, view.query.out.col.1=t2_a, view.query.out.numCols=2, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true] +Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test, view.query.out.col.0=t1_a, view.query.out.col.1=t2_a, view.query.out.numCols=2, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true, view.sqlConfig.spark.sql.legacy.interval.enabled=true] -- !query @@ -413,7 +413,7 @@ View Text SELECT * FROM base_table WHERE id IN (SELECT id FROM base_t View Original Text SELECT * FROM base_table WHERE id IN (SELECT id FROM base_table2) View Catalog and Namespace spark_catalog.temp_view_test View Query Output Columns [a, id] -Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test, view.query.out.col.0=a, view.query.out.col.1=id, view.query.out.numCols=2, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true] +Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test, view.query.out.col.0=a, view.query.out.col.1=id, view.query.out.numCols=2, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true, view.sqlConfig.spark.sql.legacy.interval.enabled=true] -- !query @@ -443,7 +443,7 @@ View Text SELECT t1.id, t2.a FROM base_table t1, (SELECT * FROM base_ View Original Text SELECT t1.id, t2.a FROM base_table t1, (SELECT * FROM base_table2) t2 View Catalog and Namespace spark_catalog.temp_view_test View Query Output Columns [id, a] -Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test, view.query.out.col.0=id, view.query.out.col.1=a, view.query.out.numCols=2, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true] +Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test, view.query.out.col.0=id, view.query.out.col.1=a, view.query.out.numCols=2, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true, view.sqlConfig.spark.sql.legacy.interval.enabled=true] -- !query @@ -473,7 +473,7 @@ View Text SELECT * FROM base_table WHERE EXISTS (SELECT 1 FROM base_t View Original Text SELECT * FROM base_table WHERE EXISTS (SELECT 1 FROM base_table2) View Catalog and Namespace spark_catalog.temp_view_test View Query Output Columns [a, id] -Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test, view.query.out.col.0=a, view.query.out.col.1=id, view.query.out.numCols=2, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true] +Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test, view.query.out.col.0=a, view.query.out.col.1=id, view.query.out.numCols=2, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true, view.sqlConfig.spark.sql.legacy.interval.enabled=true] -- !query @@ -503,7 +503,7 @@ View Text SELECT * FROM base_table WHERE NOT EXISTS (SELECT 1 FROM ba View Original Text SELECT * FROM base_table WHERE NOT EXISTS (SELECT 1 FROM base_table2) View Catalog and Namespace spark_catalog.temp_view_test View Query Output Columns [a, id] -Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test, view.query.out.col.0=a, view.query.out.col.1=id, view.query.out.numCols=2, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true] +Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test, view.query.out.col.0=a, view.query.out.col.1=id, view.query.out.numCols=2, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true, view.sqlConfig.spark.sql.legacy.interval.enabled=true] -- !query @@ -533,7 +533,7 @@ View Text SELECT * FROM base_table WHERE EXISTS (SELECT 1) View Original Text SELECT * FROM base_table WHERE EXISTS (SELECT 1) View Catalog and Namespace spark_catalog.temp_view_test View Query Output Columns [a, id] -Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test, view.query.out.col.0=a, view.query.out.col.1=id, view.query.out.numCols=2, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true] +Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=temp_view_test, view.query.out.col.0=a, view.query.out.col.1=id, view.query.out.numCols=2, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true, view.sqlConfig.spark.sql.legacy.interval.enabled=true] -- !query @@ -669,7 +669,7 @@ View Text SELECT * FROM t1 CROSS JOIN t2 View Original Text SELECT * FROM t1 CROSS JOIN t2 View Catalog and Namespace spark_catalog.testviewschm2 View Query Output Columns [num, name, num2, value] -Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=testviewschm2, view.query.out.col.0=num, view.query.out.col.1=name, view.query.out.col.2=num2, view.query.out.col.3=value, view.query.out.numCols=4, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true] +Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=testviewschm2, view.query.out.col.0=num, view.query.out.col.1=name, view.query.out.col.2=num2, view.query.out.col.3=value, view.query.out.numCols=4, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true, view.sqlConfig.spark.sql.legacy.interval.enabled=true] -- !query @@ -710,7 +710,7 @@ View Text SELECT * FROM t1 INNER JOIN t2 ON t1.num = t2.num2 View Original Text SELECT * FROM t1 INNER JOIN t2 ON t1.num = t2.num2 View Catalog and Namespace spark_catalog.testviewschm2 View Query Output Columns [num, name, num2, value] -Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=testviewschm2, view.query.out.col.0=num, view.query.out.col.1=name, view.query.out.col.2=num2, view.query.out.col.3=value, view.query.out.numCols=4, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true] +Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=testviewschm2, view.query.out.col.0=num, view.query.out.col.1=name, view.query.out.col.2=num2, view.query.out.col.3=value, view.query.out.numCols=4, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true, view.sqlConfig.spark.sql.legacy.interval.enabled=true] -- !query @@ -751,7 +751,7 @@ View Text SELECT * FROM t1 LEFT JOIN t2 ON t1.num = t2.num2 View Original Text SELECT * FROM t1 LEFT JOIN t2 ON t1.num = t2.num2 View Catalog and Namespace spark_catalog.testviewschm2 View Query Output Columns [num, name, num2, value] -Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=testviewschm2, view.query.out.col.0=num, view.query.out.col.1=name, view.query.out.col.2=num2, view.query.out.col.3=value, view.query.out.numCols=4, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true] +Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=testviewschm2, view.query.out.col.0=num, view.query.out.col.1=name, view.query.out.col.2=num2, view.query.out.col.3=value, view.query.out.numCols=4, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true, view.sqlConfig.spark.sql.legacy.interval.enabled=true] -- !query @@ -792,7 +792,7 @@ View Text SELECT * FROM t1 LEFT JOIN t2 ON t1.num = t2.num2 AND t2.va View Original Text SELECT * FROM t1 LEFT JOIN t2 ON t1.num = t2.num2 AND t2.value = 'xxx' View Catalog and Namespace spark_catalog.testviewschm2 View Query Output Columns [num, name, num2, value] -Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=testviewschm2, view.query.out.col.0=num, view.query.out.col.1=name, view.query.out.col.2=num2, view.query.out.col.3=value, view.query.out.numCols=4, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true] +Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=testviewschm2, view.query.out.col.0=num, view.query.out.col.1=name, view.query.out.col.2=num2, view.query.out.col.3=value, view.query.out.numCols=4, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true, view.sqlConfig.spark.sql.legacy.interval.enabled=true] -- !query @@ -894,7 +894,7 @@ BETWEEN (SELECT d FROM tbl2 WHERE c = 1) AND (SELECT e FROM tbl3 WHERE f = 2) AND EXISTS (SELECT g FROM tbl4 LEFT JOIN tbl3 ON tbl4.h = tbl3.f) View Catalog and Namespace spark_catalog.testviewschm2 View Query Output Columns [a, b] -Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=testviewschm2, view.query.out.col.0=a, view.query.out.col.1=b, view.query.out.numCols=2, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true] +Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=testviewschm2, view.query.out.col.0=a, view.query.out.col.1=b, view.query.out.numCols=2, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true, view.sqlConfig.spark.sql.legacy.interval.enabled=true] -- !query @@ -933,7 +933,7 @@ AND EXISTS (SELECT g FROM tbl4 LEFT JOIN tbl3 ON tbl4.h = tbl3.f) AND NOT EXISTS (SELECT g FROM tbl4 LEFT JOIN tmptbl ON tbl4.h = tmptbl.j) View Catalog and Namespace spark_catalog.testviewschm2 View Query Output Columns [a, b] -Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=testviewschm2, view.query.out.col.0=a, view.query.out.col.1=b, view.query.out.numCols=2, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true] +Table Properties [view.catalogAndNamespace.numParts=2, view.catalogAndNamespace.part.0=spark_catalog, view.catalogAndNamespace.part.1=testviewschm2, view.query.out.col.0=a, view.query.out.col.1=b, view.query.out.numCols=2, view.referredTempFunctionsNames=[], view.referredTempViewNames=[], view.sqlConfig.spark.sql.ansi.enabled=true, view.sqlConfig.spark.sql.legacy.interval.enabled=true] -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out index 7c8da456be2b1..a959284750483 100755 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/date.sql.out @@ -482,93 +482,93 @@ SELECT date '5874898-01-01' -- !query SELECT f1 - date '2000-01-01' AS `Days From 2K` FROM DATE_TBL -- !query schema -struct --- !query output --1035 00:00:00.000000000 --1036 00:00:00.000000000 --1037 00:00:00.000000000 --1400 00:00:00.000000000 --1401 00:00:00.000000000 --1402 00:00:00.000000000 --1403 00:00:00.000000000 --15542 00:00:00.000000000 --15607 00:00:00.000000000 -13977 00:00:00.000000000 -14343 00:00:00.000000000 -14710 00:00:00.000000000 -91 00:00:00.000000000 -92 00:00:00.000000000 -93 00:00:00.000000000 +struct +-- !query output +-2 years -10 months +-2 years -10 months -1 days +-2 years -9 months -30 days +-3 years -10 months +-3 years -10 months -1 days +-3 years -10 months -2 days +-3 years -9 months -30 days +-42 years -6 months -18 days +-42 years -8 months -22 days +3 months +3 months 1 days +3 months 2 days +38 years 3 months 7 days +39 years 3 months 8 days +40 years 3 months 9 days -- !query SELECT f1 - date 'epoch' AS `Days From Epoch` FROM DATE_TBL -- !query schema -struct +struct -- !query output --4585 00:00:00.000000000 --4650 00:00:00.000000000 -11048 00:00:00.000000000 -11049 00:00:00.000000000 -11050 00:00:00.000000000 -24934 00:00:00.000000000 -25300 00:00:00.000000000 -25667 00:00:00.000000000 -9554 00:00:00.000000000 -9555 00:00:00.000000000 -9556 00:00:00.000000000 -9557 00:00:00.000000000 -9920 00:00:00.000000000 -9921 00:00:00.000000000 -9922 00:00:00.000000000 +-12 years -6 months -18 days +-12 years -8 months -22 days +26 years 1 months 27 days +26 years 1 months 28 days +26 years 2 months +26 years 2 months 1 days +27 years 1 months 27 days +27 years 2 months +27 years 2 months 1 days +30 years 3 months +30 years 3 months 1 days +30 years 3 months 2 days +68 years 3 months 7 days +69 years 3 months 8 days +70 years 3 months 9 days -- !query SELECT date 'yesterday' - date 'today' AS `One day` -- !query schema -struct +struct -- !query output --1 00:00:00.000000000 +-1 days -- !query SELECT date 'today' - date 'tomorrow' AS `One day` -- !query schema -struct +struct -- !query output --1 00:00:00.000000000 +-1 days -- !query SELECT date 'yesterday' - date 'tomorrow' AS `Two days` -- !query schema -struct +struct -- !query output --2 00:00:00.000000000 +-2 days -- !query SELECT date 'tomorrow' - date 'today' AS `One day` -- !query schema -struct +struct -- !query output -1 00:00:00.000000000 +1 days -- !query SELECT date 'today' - date 'yesterday' AS `One day` -- !query schema -struct +struct -- !query output -1 00:00:00.000000000 +1 days -- !query SELECT date 'tomorrow' - date 'yesterday' AS `Two days` -- !query schema -struct +struct -- !query output -2 00:00:00.000000000 +2 days -- !query diff --git a/sql/core/src/test/resources/sql-tests/results/postgreSQL/timestamp.sql.out b/sql/core/src/test/resources/sql-tests/results/postgreSQL/timestamp.sql.out index d4293a24c6f16..68d2b5c9ce1bf 100644 --- a/sql/core/src/test/resources/sql-tests/results/postgreSQL/timestamp.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/postgreSQL/timestamp.sql.out @@ -217,13 +217,13 @@ struct<49:string,d1:timestamp> SELECT '' AS `54`, d1 - timestamp '1997-01-02' AS diff FROM TIMESTAMP_TBL WHERE d1 BETWEEN '1902-01-01' AND '2038-01-01' -- !query schema -struct<54:string,diff:day-time interval> +struct<54:string,diff:interval> -- !query output - -9863 08:00:00.000000000 - 0 00:00:00.000000000 - 0 03:04:05.000000000 - 1724 18:19:20.000000000 - 39 17:32:01.000000000 + -236720 hours + 0 seconds + 3 hours 4 minutes 5 seconds + 41393 hours 19 minutes 20 seconds + 953 hours 32 minutes 1 seconds -- !query @@ -240,13 +240,13 @@ SELECT '' AS `54`, d1 - timestamp '1997-01-02' AS diff WHERE d1 BETWEEN timestamp '1902-01-01' AND timestamp '2038-01-01' -- !query schema -struct<54:string,diff:day-time interval> +struct<54:string,diff:interval> -- !query output - -9863 08:00:00.000000000 - 0 00:00:00.000000000 - 0 03:04:05.000000000 - 1724 18:19:20.000000000 - 39 17:32:01.000000000 + -236720 hours + 0 seconds + 3 hours 4 minutes 5 seconds + 41393 hours 19 minutes 20 seconds + 953 hours 32 minutes 1 seconds -- !query diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala index 7c414c075f7c4..b4f92af794891 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala @@ -368,6 +368,7 @@ class SQLQueryTestSuite extends QueryTest with SharedSparkSession with SQLHelper // vol used by boolean.sql and case.sql. localSparkSession.udf.register("vol", (s: String) => s) localSparkSession.conf.set(SQLConf.ANSI_ENABLED.key, true) + localSparkSession.conf.set(SQLConf.LEGACY_INTERVAL_ENABLED.key, true) case _: AnsiTest => localSparkSession.conf.set(SQLConf.ANSI_ENABLED.key, true) case _ => diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala index 7ed2b50ed7797..1d03c90cc812b 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/ThriftServerQueryTestSuite.scala @@ -107,7 +107,10 @@ class ThriftServerQueryTestSuite extends SQLQueryTestSuite with SharedThriftServ } testCase match { - case _: PgSQLTest | _: AnsiTest => + case _: PgSQLTest => + statement.execute(s"SET ${SQLConf.ANSI_ENABLED.key} = true") + statement.execute(s"SET ${SQLConf.LEGACY_INTERVAL_ENABLED.key} = true") + case _: AnsiTest => statement.execute(s"SET ${SQLConf.ANSI_ENABLED.key} = true") case _ => statement.execute(s"SET ${SQLConf.ANSI_ENABLED.key} = false")