Skip to content

[SPARK-29508][SQL] Implicitly cast strings in datetime arithmetic operations #26165

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
MaxGekk wants to merge 6 commits into from
Closed

[SPARK-29508][SQL] Implicitly cast strings in datetime arithmetic operations #26165

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
9163066
Add rules and tests
MaxGekk Oct 18, 2019
287824a
Exclude TimestampType and DateType from casting to double
MaxGekk Oct 18, 2019
bef882f
Fix TypeCoercionSuite
MaxGekk Oct 18, 2019
9a2ce13
Add tests to dateTimeOperations
MaxGekk Oct 18, 2019
01a42a4
Regen promoteStrings.sql.out
MaxGekk Oct 18, 2019
b001177
Regen division.sql.out
MaxGekk Oct 18, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -412,10 +412,10 @@ object TypeCoercion {
case e if !e.childrenResolved => e

case a @ BinaryArithmetic(left @ StringType(), right)
if right.dataType != CalendarIntervalType =>
if !Seq(CalendarIntervalType, TimestampType, DateType).contains(right.dataType) =>
a.makeCopy(Array(Cast(left, DoubleType), right))
case a @ BinaryArithmetic(left, right @ StringType())
if left.dataType != CalendarIntervalType =>
if !Seq(CalendarIntervalType, TimestampType, DateType).contains(left.dataType) =>
a.makeCopy(Array(left, Cast(right, DoubleType)))

// For equality between string and timestamp we cast the string to a timestamp
Expand Down Expand Up @@ -846,6 +846,8 @@ object TypeCoercion {
Cast(TimeAdd(l, r), l.dataType)
case Subtract(l, r @ CalendarIntervalType()) if acceptedTypes.contains(l.dataType) =>
Cast(TimeSub(l, r), l.dataType)
case Subtract(l @ CalendarIntervalType(), r @ StringType()) =>
Subtract(l, Cast(r, CalendarIntervalType))

case Add(l @ DateType(), r @ IntegerType()) => DateAdd(l, r)
case Add(l @ IntegerType(), r @ DateType()) => DateAdd(r, l)
Expand All @@ -858,6 +860,17 @@ object TypeCoercion {
SubtractTimestamps(l, Cast(r, TimestampType))
case Subtract(l @ DateType(), r @ TimestampType()) =>
SubtractTimestamps(Cast(l, TimestampType), r)

// Cast StringType to CalendarIntervalType
case Add(l @ StringType(), r @ (TimestampType() | DateType())) =>
TimeAdd(r, Cast(l, CalendarIntervalType))
case Add(l @ (TimestampType() | DateType()), r @ StringType()) =>
TimeAdd(l, Cast(r, CalendarIntervalType))
// Cast StringType to TimestampType
case Subtract(l @ StringType(), r @ (TimestampType() | DateType())) =>
SubtractTimestamps(Cast(l, TimestampType), r)
case Subtract(l @ (TimestampType() | DateType()), r @ StringType()) =>
SubtractTimestamps(l, Cast(r, TimestampType))
}
}

Expand Down
19 changes: 19 additions & 0 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1408,6 +1408,7 @@ class TypeCoercionSuite extends AnalysisTest {
val interval = Literal(new CalendarInterval(0, 0))
val str = Literal("2015-01-01")
val intValue = Literal(0, IntegerType)
val strInterval = Literal("1 day")

ruleTest(dateTimeOperations, Add(date, interval), Cast(TimeAdd(date, interval), DateType))
ruleTest(dateTimeOperations, Add(interval, date), Cast(TimeAdd(date, interval), DateType))
Expand All @@ -1422,6 +1423,8 @@ class TypeCoercionSuite extends AnalysisTest {
ruleTest(dateTimeOperations, Subtract(timestamp, interval),
Cast(TimeSub(timestamp, interval), TimestampType))
ruleTest(dateTimeOperations, Subtract(str, interval), Cast(TimeSub(str, interval), StringType))
ruleTest(dateTimeOperations, Subtract(interval, strInterval),
Subtract(interval, Cast(strInterval, CalendarIntervalType)))

// interval operations should not be effected
ruleTest(dateTimeOperations, Add(interval, interval), Add(interval, interval))
Expand All @@ -1437,6 +1440,22 @@ class TypeCoercionSuite extends AnalysisTest {
SubtractTimestamps(timestamp, Cast(date, TimestampType)))
ruleTest(dateTimeOperations, Subtract(date, timestamp),
SubtractTimestamps(Cast(date, TimestampType), timestamp))

ruleTest(dateTimeOperations, Add(timestamp, str),
TimeAdd(timestamp, Cast(str, CalendarIntervalType)))
ruleTest(dateTimeOperations, Add(date, str), TimeAdd(date, Cast(str, CalendarIntervalType)))
ruleTest(dateTimeOperations, Add(str, timestamp),
TimeAdd(timestamp, Cast(str, CalendarIntervalType)))
ruleTest(dateTimeOperations, Add(str, date), TimeAdd(date, Cast(str, CalendarIntervalType)))

ruleTest(dateTimeOperations, Subtract(timestamp, str),
SubtractTimestamps(timestamp, Cast(str, TimestampType)))
ruleTest(dateTimeOperations, Subtract(date, str),
SubtractTimestamps(date, Cast(str, TimestampType)))
ruleTest(dateTimeOperations, Subtract(str, timestamp),
SubtractTimestamps(Cast(str, TimestampType), timestamp))
ruleTest(dateTimeOperations, Subtract(str, date),
SubtractTimestamps(Cast(str, TimestampType), date))
}

/**
Expand Down
12 changes: 12 additions & 0 deletions sql/core/src/test/resources/sql-tests/inputs/typeCoercion/native/dateTimeOperations.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,3 +58,15 @@ select cast('1' as binary) - interval 2 day;
select cast(1 as boolean) - interval 2 day;
select cast('2017-12-11 09:30:00.0' as timestamp) - interval 2 day;
select cast('2017-12-11 09:30:00' as date) - interval 2 day;

select interval 1 month - '2 seconds';

select '1 month' + date'2019-10-01';
select '1 month 1 second' + timestamp'2019-10-18 10:11:12';
select date'2019-10-01' + '1 month';
select timestamp'2019-10-18 10:11:12' + '1 month 1 second';

select timestamp'2019-10-18 10:11:12' - '2019-10-18';
select date'2019-10-18' - '2019-10-01';
select '2019-10-18 10:11:12' - timestamp'2019-10-18 00:00:00';
select '2019-10-18 10:11:12' - date'2019-10-18';
74 changes: 73 additions & 1 deletion sql/core/src/test/resources/sql-tests/results/typeCoercion/native/dateTimeOperations.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 40
-- Number of queries: 49


-- !query 0
Expand Down Expand Up @@ -347,3 +347,75 @@ select cast('2017-12-11 09:30:00' as date) - interval 2 day
struct<CAST(CAST(CAST(2017-12-11 09:30:00 AS DATE) AS TIMESTAMP) - interval 2 days AS DATE):date>
-- !query 39 output
2017-12-09


-- !query 40
select interval 1 month - '2 seconds'
-- !query 40 schema
struct<(interval 1 months - CAST(2 seconds AS INTERVAL)):interval>
-- !query 40 output
interval 1 months -2 seconds


-- !query 41
select '1 month' + date'2019-10-01'
-- !query 41 schema
struct<CAST(DATE '2019-10-01' AS TIMESTAMP) + CAST(1 month AS INTERVAL):timestamp>
-- !query 41 output
2019-11-01 00:00:00


-- !query 42
select '1 month 1 second' + timestamp'2019-10-18 10:11:12'
-- !query 42 schema
struct<TIMESTAMP('2019-10-18 10:11:12') + CAST(1 month 1 second AS INTERVAL):timestamp>
-- !query 42 output
2019-11-18 10:11:13


-- !query 43
select date'2019-10-01' + '1 month'
-- !query 43 schema
struct<CAST(DATE '2019-10-01' AS TIMESTAMP) + CAST(1 month AS INTERVAL):timestamp>
-- !query 43 output
2019-11-01 00:00:00


-- !query 44
select timestamp'2019-10-18 10:11:12' + '1 month 1 second'
-- !query 44 schema
struct<TIMESTAMP('2019-10-18 10:11:12') + CAST(1 month 1 second AS INTERVAL):timestamp>
-- !query 44 output
2019-11-18 10:11:13


-- !query 45
select timestamp'2019-10-18 10:11:12' - '2019-10-18'
-- !query 45 schema
struct<subtracttimestamps(TIMESTAMP('2019-10-18 10:11:12'), CAST(2019-10-18 AS TIMESTAMP)):interval>
-- !query 45 output
interval 10 hours 11 minutes 12 seconds


-- !query 46
select date'2019-10-18' - '2019-10-01'
-- !query 46 schema
struct<subtracttimestamps(CAST(DATE '2019-10-18' AS TIMESTAMP), CAST(2019-10-01 AS TIMESTAMP)):interval>
-- !query 46 output
interval 2 weeks 3 days


-- !query 47
select '2019-10-18 10:11:12' - timestamp'2019-10-18 00:00:00'
-- !query 47 schema
struct<subtracttimestamps(CAST(2019-10-18 10:11:12 AS TIMESTAMP), TIMESTAMP('2019-10-18 00:00:00')):interval>
-- !query 47 output
interval 10 hours 11 minutes 12 seconds


-- !query 48
select '2019-10-18 10:11:12' - date'2019-10-18'
-- !query 48 schema
struct<subtracttimestamps(CAST(2019-10-18 10:11:12 AS TIMESTAMP), CAST(DATE '2019-10-18' AS TIMESTAMP)):interval>
-- !query 48 output
interval 10 hours 11 minutes 12 seconds
8 changes: 4 additions & 4 deletions sql/core/src/test/resources/sql-tests/results/typeCoercion/native/division.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -798,7 +798,7 @@ SELECT cast(1 as string) / cast('2017-12-11 09:30:00.0' as timestamp) FROM t
struct<>
-- !query 95 output
org.apache.spark.sql.AnalysisException
cannot resolve '(CAST(CAST(1 AS STRING) AS DOUBLE) / CAST('2017-12-11 09:30:00.0' AS TIMESTAMP))' due to data type mismatch: differing types in '(CAST(CAST(1 AS STRING) AS DOUBLE) / CAST('2017-12-11 09:30:00.0' AS TIMESTAMP))' (double and timestamp).; line 1 pos 7
cannot resolve '(CAST(1 AS STRING) / CAST('2017-12-11 09:30:00.0' AS TIMESTAMP))' due to data type mismatch: differing types in '(CAST(1 AS STRING) / CAST('2017-12-11 09:30:00.0' AS TIMESTAMP))' (string and timestamp).; line 1 pos 7


-- !query 96
Expand All @@ -807,7 +807,7 @@ SELECT cast(1 as string) / cast('2017-12-11 09:30:00' as date) FROM t
struct<>
-- !query 96 output
org.apache.spark.sql.AnalysisException
cannot resolve '(CAST(CAST(1 AS STRING) AS DOUBLE) / CAST('2017-12-11 09:30:00' AS DATE))' due to data type mismatch: differing types in '(CAST(CAST(1 AS STRING) AS DOUBLE) / CAST('2017-12-11 09:30:00' AS DATE))' (double and date).; line 1 pos 7
cannot resolve '(CAST(1 AS STRING) / CAST('2017-12-11 09:30:00' AS DATE))' due to data type mismatch: differing types in '(CAST(1 AS STRING) / CAST('2017-12-11 09:30:00' AS DATE))' (string and date).; line 1 pos 7


-- !query 97
Expand Down Expand Up @@ -1095,7 +1095,7 @@ SELECT cast('2017-12-11 09:30:00.0' as timestamp) / cast(1 as string) FROM t
struct<>
-- !query 128 output
org.apache.spark.sql.AnalysisException
cannot resolve '(CAST('2017-12-11 09:30:00.0' AS TIMESTAMP) / CAST(CAST(1 AS STRING) AS DOUBLE))' due to data type mismatch: differing types in '(CAST('2017-12-11 09:30:00.0' AS TIMESTAMP) / CAST(CAST(1 AS STRING) AS DOUBLE))' (timestamp and double).; line 1 pos 7
cannot resolve '(CAST('2017-12-11 09:30:00.0' AS TIMESTAMP) / CAST(1 AS STRING))' due to data type mismatch: differing types in '(CAST('2017-12-11 09:30:00.0' AS TIMESTAMP) / CAST(1 AS STRING))' (timestamp and string).; line 1 pos 7


-- !query 129
Expand Down Expand Up @@ -1203,7 +1203,7 @@ SELECT cast('2017-12-11 09:30:00' as date) / cast(1 as string) FROM t
struct<>
-- !query 140 output
org.apache.spark.sql.AnalysisException
cannot resolve '(CAST('2017-12-11 09:30:00' AS DATE) / CAST(CAST(1 AS STRING) AS DOUBLE))' due to data type mismatch: differing types in '(CAST('2017-12-11 09:30:00' AS DATE) / CAST(CAST(1 AS STRING) AS DOUBLE))' (date and double).; line 1 pos 7
cannot resolve '(CAST('2017-12-11 09:30:00' AS DATE) / CAST(1 AS STRING))' due to data type mismatch: differing types in '(CAST('2017-12-11 09:30:00' AS DATE) / CAST(1 AS STRING))' (date and string).; line 1 pos 7


-- !query 141
Expand Down
Loading