Skip to content

[SPARK-35285][SQL] Parse ANSI interval types in SQL schema #32409

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
MaxGekk wants to merge 8 commits into from
Closed

[SPARK-35285][SQL] Parse ANSI interval types in SQL schema #32409

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
3374f63
Parse interval types
MaxGekk Apr 30, 2021
a335f00
Parse ANSI interval types in SQL schema
MaxGekk Apr 30, 2021
e8cbd14
Re-gen promoteStrings.sql.out
MaxGekk Apr 30, 2021
e8e889a
Re-gen window.sql.out
MaxGekk Apr 30, 2021
7f79210
Re-gen extract.sql.out
MaxGekk Apr 30, 2021
eb3c6fe
Re-gen datetime-legacy.sql.out
MaxGekk Apr 30, 2021
d6fb04a
Fix CollectionExpressionsSuite
MaxGekk Apr 30, 2021
0b4f6bf
Fix SQLKeywordSuite
MaxGekk Apr 30, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/sql-ref-ansi-compliance.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -254,6 +254,7 @@ Below is a list of all the keywords in Spark SQL.
|DATA|non-reserved|non-reserved|non-reserved|
|DATABASE|non-reserved|non-reserved|non-reserved|
|DATABASES|non-reserved|non-reserved|non-reserved|
|DAY|non-reserved|non-reserved|non-reserved|
|DBPROPERTIES|non-reserved|non-reserved|non-reserved|
|DEFINED|non-reserved|non-reserved|non-reserved|
|DELETE|non-reserved|non-reserved|reserved|
Expand Down Expand Up @@ -336,6 +337,7 @@ Below is a list of all the keywords in Spark SQL.
|MATCHED|non-reserved|non-reserved|non-reserved|
|MERGE|non-reserved|non-reserved|non-reserved|
|MINUS|non-reserved|strict-non-reserved|non-reserved|
|MONTH|non-reserved|non-reserved|non-reserved|
|MSCK|non-reserved|non-reserved|non-reserved|
|NAMESPACE|non-reserved|non-reserved|non-reserved|
|NAMESPACES|non-reserved|non-reserved|non-reserved|
Expand Down Expand Up @@ -396,6 +398,7 @@ Below is a list of all the keywords in Spark SQL.
|ROWS|non-reserved|non-reserved|reserved|
|SCHEMA|non-reserved|non-reserved|non-reserved|
|SCHEMAS|non-reserved|non-reserved|not a keyword|
|SECOND|non-reserved|non-reserved|non-reserved|
|SELECT|reserved|non-reserved|reserved|
|SEMI|non-reserved|strict-non-reserved|non-reserved|
|SEPARATED|non-reserved|non-reserved|non-reserved|
Expand Down Expand Up @@ -456,4 +459,5 @@ Below is a list of all the keywords in Spark SQL.
|WHERE|reserved|non-reserved|reserved|
|WINDOW|non-reserved|non-reserved|reserved|
|WITH|reserved|non-reserved|reserved|
|YEAR|non-reserved|non-reserved|non-reserved|
|ZONE|non-reserved|non-reserved|non-reserved|
14 changes: 14 additions & 0 deletions sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -904,6 +904,8 @@ dataType
: complex=ARRAY '<' dataType '>' #complexDataType
| complex=MAP '<' dataType ',' dataType '>' #complexDataType
| complex=STRUCT ('<' complexColTypeList? '>' | NEQ) #complexDataType
| INTERVAL YEAR TO MONTH #yearMonthIntervalDataType
| INTERVAL DAY TO SECOND #dayTimeIntervalDataType
| identifier ('(' INTEGER_VALUE (',' INTEGER_VALUE)* ')')? #primitiveDataType
;

Expand Down Expand Up @@ -1077,6 +1079,7 @@ ansiNonReserved
| DATA
| DATABASE
| DATABASES
| DAY
| DBPROPERTIES
| DEFINED
| DELETE
Expand Down Expand Up @@ -1135,6 +1138,7 @@ ansiNonReserved
| MAP
| MATCHED
| MERGE
| MONTH
| MSCK
| NAMESPACE
| NAMESPACES
Expand Down Expand Up @@ -1181,6 +1185,7 @@ ansiNonReserved
| ROW
| ROWS
| SCHEMA
| SECOND
| SEMI
| SEPARATED
| SERDE
Expand Down Expand Up @@ -1225,6 +1230,7 @@ ansiNonReserved
| VIEW
| VIEWS
| WINDOW
| YEAR
| ZONE
//--ANSI-NON-RESERVED-END
;
Expand Down Expand Up @@ -1308,6 +1314,7 @@ nonReserved
| DATA
| DATABASE
| DATABASES
| DAY
| DBPROPERTIES
| DEFINED
| DELETE
Expand Down Expand Up @@ -1383,6 +1390,7 @@ nonReserved
| MAP
| MATCHED
| MERGE
| MONTH
| MSCK
| NAMESPACE
| NAMESPACES
Expand Down Expand Up @@ -1438,6 +1446,7 @@ nonReserved
| ROW
| ROWS
| SCHEMA
| SECOND
| SELECT
| SEPARATED
| SERDE
Expand Down Expand Up @@ -1494,6 +1503,7 @@ nonReserved
| WHERE
| WINDOW
| WITH
| YEAR
| ZONE
//--DEFAULT-NON-RESERVED-END
;
Expand Down Expand Up @@ -1554,6 +1564,7 @@ CURRENT_DATE: 'CURRENT_DATE';
CURRENT_TIME: 'CURRENT_TIME';
CURRENT_TIMESTAMP: 'CURRENT_TIMESTAMP';
CURRENT_USER: 'CURRENT_USER';
DAY: 'DAY';
DATA: 'DATA';
DATABASE: 'DATABASE';
DATABASES: 'DATABASES' | 'SCHEMAS';
Expand Down Expand Up @@ -1638,6 +1649,7 @@ MACRO: 'MACRO';
MAP: 'MAP';
MATCHED: 'MATCHED';
MERGE: 'MERGE';
MONTH: 'MONTH';
MSCK: 'MSCK';
NAMESPACE: 'NAMESPACE';
NAMESPACES: 'NAMESPACES';
Expand Down Expand Up @@ -1695,6 +1707,7 @@ ROLLBACK: 'ROLLBACK';
ROLLUP: 'ROLLUP';
ROW: 'ROW';
ROWS: 'ROWS';
SECOND: 'SECOND';
SCHEMA: 'SCHEMA';
SELECT: 'SELECT';
SEMI: 'SEMI';
Expand Down Expand Up @@ -1756,6 +1769,7 @@ WHEN: 'WHEN';
WHERE: 'WHERE';
WINDOW: 'WINDOW';
WITH: 'WITH';
YEAR: 'YEAR';
ZONE: 'ZONE';
//--SPARK-KEYWORD-LIST-END
//============================
Expand Down
8 changes: 8 additions & 0 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/parser/AstBuilder.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2484,6 +2484,14 @@ class AstBuilder extends SqlBaseBaseVisitor[AnyRef] with SQLConfHelper with Logg
}
}

override def visitYearMonthIntervalDataType(ctx: YearMonthIntervalDataTypeContext): DataType = {
YearMonthIntervalType
}

override def visitDayTimeIntervalDataType(ctx: DayTimeIntervalDataTypeContext): DataType = {
DayTimeIntervalType
}

/**
* Create a complex DataType. Arrays, Maps and Structures are supported.
*/
Expand Down
2 changes: 1 addition & 1 deletion sql/catalyst/src/main/scala/org/apache/spark/sql/types/DayTimeIntervalType.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ class DayTimeIntervalType private() extends AtomicType {

private[spark] override def asNullable: DayTimeIntervalType = this

override def typeName: String = "day-time interval"
override def typeName: String = "interval day to second"
}

/**
Expand Down
2 changes: 1 addition & 1 deletion sql/catalyst/src/main/scala/org/apache/spark/sql/types/YearMonthIntervalType.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ class YearMonthIntervalType private() extends AtomicType {

private[spark] override def asNullable: YearMonthIntervalType = this

override def typeName: String = "year-month interval"
override def typeName: String = "interval year to month"
}

/**
Expand Down
4 changes: 2 additions & 2 deletions ...t/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,9 +78,9 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite {
assertErrorForDifferingTypes(BitwiseXor(Symbol("intField"), Symbol("booleanField")))

assertError(Add(Symbol("booleanField"), Symbol("booleanField")),
"requires (numeric or interval or day-time interval or year-month interval) type")
"requires (numeric or interval or interval day to second or interval year to month) type")
assertError(Subtract(Symbol("booleanField"), Symbol("booleanField")),
"requires (numeric or interval or day-time interval or year-month interval) type")
"requires (numeric or interval or interval day to second or interval year to month) type")
assertError(Multiply(Symbol("booleanField"), Symbol("booleanField")), "requires numeric type")
assertError(Divide(Symbol("booleanField"), Symbol("booleanField")),
"requires (double or decimal) type")
Expand Down
2 changes: 1 addition & 1 deletion ...src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1098,7 +1098,7 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
Literal(Date.valueOf("2018-01-05")),
Literal(Period.ofDays(2))),
EmptyRow,
"sequence step must be a day year-month interval if start and end values are dates")
"sequence step must be a day interval year to month if start and end values are dates")
Copy link
Member Author

@MaxGekk MaxGekk May 10, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@beliefer The error message confuses me slightly, especially the combination a day interval year to month. Could you open a PR to improve the error, please, something like "... sequence step must be an interval of day granularity ...".

Copy link
Contributor

@beliefer beliefer May 10, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I see.


checkExceptionInExpression[IllegalArgumentException](
new Sequence(
Expand Down
2 changes: 2 additions & 0 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/DataTypeParserSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ class DataTypeParserSuite extends SparkFunSuite {
checkDataType("BINARY", BinaryType)
checkDataType("void", NullType)
checkDataType("interval", CalendarIntervalType)
checkDataType("INTERVAL YEAR TO MONTH", YearMonthIntervalType)
checkDataType("interval day to second", DayTimeIntervalType)

checkDataType("array<doublE>", ArrayType(DoubleType, true))
checkDataType("Array<map<int, tinYint>>", ArrayType(MapType(IntegerType, ByteType, true), true))
Expand Down
3 changes: 3 additions & 0 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,9 @@ class DataTypeSuite extends SparkFunSuite {
checkDataTypeFromJson(VarcharType(10))
checkDataTypeFromDDL(VarcharType(11))

checkDataTypeFromDDL(YearMonthIntervalType)
checkDataTypeFromDDL(DayTimeIntervalType)

val metadata = new MetadataBuilder()
.putString("name", "age")
.build()
Expand Down
8 changes: 8 additions & 0 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/types/StructTypeSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -235,4 +235,12 @@ class StructTypeSuite extends SparkFunSuite with SQLHelper {
.exists(_.sameType(missing4)))
}
}

test("SPARK-35285: ANSI interval types in schema") {
val yearMonthInterval = "`ymi` INTERVAL YEAR TO MONTH"
assert(fromDDL(yearMonthInterval).toDDL === yearMonthInterval)

val dayTimeInterval = "`dti` INTERVAL DAY TO SECOND"
assert(fromDDL(dayTimeInterval).toDDL === dayTimeInterval)
}
}
12 changes: 6 additions & 6 deletions sql/core/src/test/resources/sql-tests/results/ansi/datetime.sql.out
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -359,31 +359,31 @@ cannot resolve '1 + (- INTERVAL '2 seconds')' due to data type mismatch: argumen
-- !query
select date'2020-01-01' - timestamp'2019-10-06 10:11:12.345678'
-- !query schema
struct<(DATE '2020-01-01' - TIMESTAMP '2019-10-06 10:11:12.345678'):day-time interval>
struct<(DATE '2020-01-01' - TIMESTAMP '2019-10-06 10:11:12.345678'):interval day to second>
-- !query output
86 13:48:47.654322000


-- !query
select timestamp'2019-10-06 10:11:12.345678' - date'2020-01-01'
-- !query schema
struct<(TIMESTAMP '2019-10-06 10:11:12.345678' - DATE '2020-01-01'):day-time interval>
struct<(TIMESTAMP '2019-10-06 10:11:12.345678' - DATE '2020-01-01'):interval day to second>
-- !query output
-86 13:48:47.654322000


-- !query
select timestamp'2019-10-06 10:11:12.345678' - null
-- !query schema
struct<(TIMESTAMP '2019-10-06 10:11:12.345678' - NULL):day-time interval>
struct<(TIMESTAMP '2019-10-06 10:11:12.345678' - NULL):interval day to second>
-- !query output
NULL


-- !query
select null - timestamp'2019-10-06 10:11:12.345678'
-- !query schema
struct<(NULL - TIMESTAMP '2019-10-06 10:11:12.345678'):day-time interval>
struct<(NULL - TIMESTAMP '2019-10-06 10:11:12.345678'):interval day to second>
-- !query output
NULL

Expand Down Expand Up @@ -625,15 +625,15 @@ cannot resolve 'date_sub(CAST('2011-11-11' AS DATE), v.str)' due to data type mi
-- !query
select null - date '2019-10-06'
-- !query schema
struct<(NULL - DATE '2019-10-06'):day-time interval>
struct<(NULL - DATE '2019-10-06'):interval day to second>
-- !query output
NULL


-- !query
select date '2001-10-01' - date '2001-09-28'
-- !query schema
struct<(DATE '2001-10-01' - DATE '2001-09-28'):day-time interval>
struct<(DATE '2001-10-01' - DATE '2001-09-28'):interval day to second>
-- !query output
3 00:00:00.000000000

Expand Down
Loading