Skip to content

[SPARK-38332][SQL] Add the DATEADD() and DATE_ADD() aliases for TIMESTAMPADD() #35661

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
MaxGekk wants to merge 6 commits into from
Closed

[SPARK-38332][SQL] Add the DATEADD() and DATE_ADD() aliases for TIMESTAMPADD() #35661

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
61a3ed6
Add DATEADD
MaxGekk Feb 25, 2022
2284017
Add DATEADD to docs/sql-ref-ansi-compliance.md
MaxGekk Feb 25, 2022
7f733d7
Add tests to date.sql
MaxGekk Feb 25, 2022
d13caea
Add more checks
MaxGekk Feb 25, 2022
1b15cda
Add the date_add alias
MaxGekk Feb 25, 2022
6ad3adb
Re-gen datetime-legacy.sql.out
MaxGekk Feb 26, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions docs/sql-ref-ansi-compliance.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -390,6 +390,8 @@ Below is a list of all the keywords in Spark SQL.
|DATA|non-reserved|non-reserved|non-reserved|
|DATABASE|non-reserved|non-reserved|non-reserved|
|DATABASES|non-reserved|non-reserved|non-reserved|
|DATEADD|non-reserved|non-reserved|non-reserved|
|DATE_ADD|non-reserved|non-reserved|non-reserved|
|DAY|non-reserved|non-reserved|non-reserved|
|DBPROPERTIES|non-reserved|non-reserved|non-reserved|
|DEFINED|non-reserved|non-reserved|non-reserved|
Expand Down
8 changes: 7 additions & 1 deletion sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -860,7 +860,7 @@ valueExpression

primaryExpression
: name=(CURRENT_DATE | CURRENT_TIMESTAMP | CURRENT_USER) #currentLike
| TIMESTAMPADD '(' unit=identifier ',' unitsAmount=valueExpression ',' timestamp=valueExpression ')' #timestampadd
| name=(TIMESTAMPADD | DATEADD | DATE_ADD) '(' unit=identifier ',' unitsAmount=valueExpression ',' timestamp=valueExpression ')' #timestampadd
| CASE whenClause+ (ELSE elseExpression=expression)? END #searchedCase
| CASE value=expression whenClause+ (ELSE elseExpression=expression)? END #simpleCase
| name=(CAST | TRY_CAST) '(' expression AS dataType ')' #cast
Expand Down Expand Up @@ -1130,6 +1130,8 @@ ansiNonReserved
| DATA
| DATABASE
| DATABASES
| DATEADD
| DATE_ADD
| DAY
| DBPROPERTIES
| DEFINED
Expand Down Expand Up @@ -1377,6 +1379,8 @@ nonReserved
| DATA
| DATABASE
| DATABASES
| DATEADD
| DATE_ADD
| DAY
| DBPROPERTIES
| DEFINED
Expand Down Expand Up @@ -1644,6 +1648,8 @@ DAY: 'DAY';
DATA: 'DATA';
DATABASE: 'DATABASE';
DATABASES: 'DATABASES';
DATEADD: 'DATEADD';
DATE_ADD: 'DATE_ADD';
DBPROPERTIES: 'DBPROPERTIES';
DEFINED: 'DEFINED';
DELETE: 'DELETE';
Expand Down
12 changes: 12 additions & 0 deletions sql/core/src/test/resources/sql-tests/inputs/date.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,3 +140,15 @@ select date '2012-01-01' - interval '2-2' year to month,
select to_date('26/October/2015', 'dd/MMMMM/yyyy');
select from_json('{"d":"26/October/2015"}', 'd Date', map('dateFormat', 'dd/MMMMM/yyyy'));
select from_csv('26/October/2015', 'd Date', map('dateFormat', 'dd/MMMMM/yyyy'));

-- Add a number of units to a timestamp or a date
select dateadd(MICROSECOND, 1001, timestamp'2022-02-25 01:02:03.123');
select date_add(MILLISECOND, -1, timestamp'2022-02-25 01:02:03.456');
select dateadd(SECOND, 58, timestamp'2022-02-25 01:02:03');
select date_add(MINUTE, -100, date'2022-02-25');
select dateadd(HOUR, -1, timestamp'2022-02-25 01:02:03');
select date_add(DAY, 367, date'2022-02-25');
select dateadd(WEEK, -4, timestamp'2022-02-25 01:02:03');
select date_add(MONTH, -1, timestamp'2022-02-25 01:02:03');
select dateadd(QUARTER, 5, date'2022-02-25');
select date_add(YEAR, 1, date'2022-02-25');
82 changes: 81 additions & 1 deletion sql/core/src/test/resources/sql-tests/results/ansi/date.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 77
-- Number of queries: 87


-- !query
Expand Down Expand Up @@ -660,3 +660,83 @@ struct<>
-- !query output
org.apache.spark.SparkUpgradeException
You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html


-- !query
select dateadd(MICROSECOND, 1001, timestamp'2022-02-25 01:02:03.123')
-- !query schema
struct<timestampadd(MICROSECOND, 1001, TIMESTAMP '2022-02-25 01:02:03.123'):timestamp>
-- !query output
2022-02-25 01:02:03.124001


-- !query
select date_add(MILLISECOND, -1, timestamp'2022-02-25 01:02:03.456')
-- !query schema
struct<timestampadd(MILLISECOND, -1, TIMESTAMP '2022-02-25 01:02:03.456'):timestamp>
-- !query output
2022-02-25 01:02:03.455


-- !query
select dateadd(SECOND, 58, timestamp'2022-02-25 01:02:03')
-- !query schema
struct<timestampadd(SECOND, 58, TIMESTAMP '2022-02-25 01:02:03'):timestamp>
-- !query output
2022-02-25 01:03:01


-- !query
select date_add(MINUTE, -100, date'2022-02-25')
-- !query schema
struct<timestampadd(MINUTE, -100, DATE '2022-02-25'):timestamp>
-- !query output
2022-02-24 22:20:00


-- !query
select dateadd(HOUR, -1, timestamp'2022-02-25 01:02:03')
-- !query schema
struct<timestampadd(HOUR, -1, TIMESTAMP '2022-02-25 01:02:03'):timestamp>
-- !query output
2022-02-25 00:02:03


-- !query
select date_add(DAY, 367, date'2022-02-25')
Copy link

@superdupershant superdupershant Feb 28, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think the convention is if the third argument is of type DATE the expression's return type is:

DATE if the unit is DAY or larger
TIMESTAMP if the unit is smaller than DAY i.e. (HOUR, MINUTE, SECOND, etc...)

Copy link
Member Author

@MaxGekk MaxGekk Feb 28, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think we should do such kind of changes in the PR which just introduces aliases. Let's do that separately w/ tests if the feature is supported by other DBMSs as well.

Also I would prefer to not link such changes to DATEADD since they affect TIMESTAMPADD. It is slightly strange to me that a timestamp function will return date type. Let's discuss this separately.

Copy link
Contributor

@cloud-fan cloud-fan Feb 28, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you mean timestampadd can take date inputs but dateadd can't take timestamp inputs? Then it's not a simple alias any more.

Copy link

@superdupershant superdupershant Feb 28, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@cloud-fan no both expressions can take date and timestamp inputs it is just a simple alias.

@MaxGekk yes I agree fixing it in a separate PR makes sense.

-- !query schema
struct<timestampadd(DAY, 367, DATE '2022-02-25'):timestamp>
-- !query output
2023-02-27 00:00:00


-- !query
select dateadd(WEEK, -4, timestamp'2022-02-25 01:02:03')
-- !query schema
struct<timestampadd(WEEK, -4, TIMESTAMP '2022-02-25 01:02:03'):timestamp>
-- !query output
2022-01-28 01:02:03


-- !query
select date_add(MONTH, -1, timestamp'2022-02-25 01:02:03')
-- !query schema
struct<timestampadd(MONTH, -1, TIMESTAMP '2022-02-25 01:02:03'):timestamp>
-- !query output
2022-01-25 01:02:03


-- !query
select dateadd(QUARTER, 5, date'2022-02-25')
-- !query schema
struct<timestampadd(QUARTER, 5, DATE '2022-02-25'):timestamp>
-- !query output
2023-05-25 00:00:00


-- !query
select date_add(YEAR, 1, date'2022-02-25')
-- !query schema
struct<timestampadd(YEAR, 1, DATE '2022-02-25'):timestamp>
-- !query output
2023-02-25 00:00:00
82 changes: 81 additions & 1 deletion sql/core/src/test/resources/sql-tests/results/date.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 77
-- Number of queries: 87


-- !query
Expand Down Expand Up @@ -659,3 +659,83 @@ struct<>
-- !query output
org.apache.spark.SparkUpgradeException
You may get a different result due to the upgrading of Spark 3.0: Fail to recognize 'dd/MMMMM/yyyy' pattern in the DateTimeFormatter. 1) You can set spark.sql.legacy.timeParserPolicy to LEGACY to restore the behavior before Spark 3.0. 2) You can form a valid datetime pattern with the guide from https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html


-- !query
select dateadd(MICROSECOND, 1001, timestamp'2022-02-25 01:02:03.123')
-- !query schema
struct<timestampadd(MICROSECOND, 1001, TIMESTAMP '2022-02-25 01:02:03.123'):timestamp>
-- !query output
2022-02-25 01:02:03.124001


-- !query
select date_add(MILLISECOND, -1, timestamp'2022-02-25 01:02:03.456')
-- !query schema
struct<timestampadd(MILLISECOND, -1, TIMESTAMP '2022-02-25 01:02:03.456'):timestamp>
-- !query output
2022-02-25 01:02:03.455


-- !query
select dateadd(SECOND, 58, timestamp'2022-02-25 01:02:03')
-- !query schema
struct<timestampadd(SECOND, 58, TIMESTAMP '2022-02-25 01:02:03'):timestamp>
-- !query output
2022-02-25 01:03:01


-- !query
select date_add(MINUTE, -100, date'2022-02-25')
-- !query schema
struct<timestampadd(MINUTE, -100, DATE '2022-02-25'):timestamp>
-- !query output
2022-02-24 22:20:00


-- !query
select dateadd(HOUR, -1, timestamp'2022-02-25 01:02:03')
-- !query schema
struct<timestampadd(HOUR, -1, TIMESTAMP '2022-02-25 01:02:03'):timestamp>
-- !query output
2022-02-25 00:02:03


-- !query
select date_add(DAY, 367, date'2022-02-25')
-- !query schema
struct<timestampadd(DAY, 367, DATE '2022-02-25'):timestamp>
-- !query output
2023-02-27 00:00:00


-- !query
select dateadd(WEEK, -4, timestamp'2022-02-25 01:02:03')
-- !query schema
struct<timestampadd(WEEK, -4, TIMESTAMP '2022-02-25 01:02:03'):timestamp>
-- !query output
2022-01-28 01:02:03


-- !query
select date_add(MONTH, -1, timestamp'2022-02-25 01:02:03')
-- !query schema
struct<timestampadd(MONTH, -1, TIMESTAMP '2022-02-25 01:02:03'):timestamp>
-- !query output
2022-01-25 01:02:03


-- !query
select dateadd(QUARTER, 5, date'2022-02-25')
-- !query schema
struct<timestampadd(QUARTER, 5, DATE '2022-02-25'):timestamp>
-- !query output
2023-05-25 00:00:00


-- !query
select date_add(YEAR, 1, date'2022-02-25')
-- !query schema
struct<timestampadd(YEAR, 1, DATE '2022-02-25'):timestamp>
-- !query output
2023-02-25 00:00:00
82 changes: 81 additions & 1 deletion sql/core/src/test/resources/sql-tests/results/datetime-legacy.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
-- Automatically generated by SQLQueryTestSuite
-- Number of queries: 170
-- Number of queries: 180


-- !query
Expand Down Expand Up @@ -658,6 +658,86 @@ struct<from_csv(26/October/2015):struct<d:date>>
{"d":2015-10-26}


-- !query
select dateadd(MICROSECOND, 1001, timestamp'2022-02-25 01:02:03.123')
-- !query schema
struct<timestampadd(MICROSECOND, 1001, TIMESTAMP '2022-02-25 01:02:03.123'):timestamp>
-- !query output
2022-02-25 01:02:03.124001


-- !query
select date_add(MILLISECOND, -1, timestamp'2022-02-25 01:02:03.456')
-- !query schema
struct<timestampadd(MILLISECOND, -1, TIMESTAMP '2022-02-25 01:02:03.456'):timestamp>
-- !query output
2022-02-25 01:02:03.455


-- !query
select dateadd(SECOND, 58, timestamp'2022-02-25 01:02:03')
-- !query schema
struct<timestampadd(SECOND, 58, TIMESTAMP '2022-02-25 01:02:03'):timestamp>
-- !query output
2022-02-25 01:03:01


-- !query
select date_add(MINUTE, -100, date'2022-02-25')
-- !query schema
struct<timestampadd(MINUTE, -100, DATE '2022-02-25'):timestamp>
-- !query output
2022-02-24 22:20:00


-- !query
select dateadd(HOUR, -1, timestamp'2022-02-25 01:02:03')
-- !query schema
struct<timestampadd(HOUR, -1, TIMESTAMP '2022-02-25 01:02:03'):timestamp>
-- !query output
2022-02-25 00:02:03


-- !query
select date_add(DAY, 367, date'2022-02-25')
-- !query schema
struct<timestampadd(DAY, 367, DATE '2022-02-25'):timestamp>
-- !query output
2023-02-27 00:00:00


-- !query
select dateadd(WEEK, -4, timestamp'2022-02-25 01:02:03')
-- !query schema
struct<timestampadd(WEEK, -4, TIMESTAMP '2022-02-25 01:02:03'):timestamp>
-- !query output
2022-01-28 01:02:03


-- !query
select date_add(MONTH, -1, timestamp'2022-02-25 01:02:03')
-- !query schema
struct<timestampadd(MONTH, -1, TIMESTAMP '2022-02-25 01:02:03'):timestamp>
-- !query output
2022-01-25 01:02:03


-- !query
select dateadd(QUARTER, 5, date'2022-02-25')
-- !query schema
struct<timestampadd(QUARTER, 5, DATE '2022-02-25'):timestamp>
-- !query output
2023-05-25 00:00:00


-- !query
select date_add(YEAR, 1, date'2022-02-25')
-- !query schema
struct<timestampadd(YEAR, 1, DATE '2022-02-25'):timestamp>
-- !query output
2023-02-25 00:00:00


-- !query
select timestamp '2019-01-01\t'
-- !query schema
Expand Down