diff --git a/docs/sql-keywords.md b/docs/sql-keywords.md index 816b4eaa0ca63..13058cba7564f 100644 --- a/docs/sql-keywords.md +++ b/docs/sql-keywords.md @@ -117,6 +117,7 @@ Below is a list of all the keywords in Spark SQL. FIELDSnon-reservednon-reservednon-reserved FILEFORMATnon-reservednon-reservednon-reserved FIRSTnon-reservednon-reservednon-reserved + FIRST_VALUEreservednon-reservedreserved FOLLOWINGnon-reservednon-reservednon-reserved FORreservednon-reservedreserved FOREIGNreservednon-reservedreserved @@ -151,6 +152,7 @@ Below is a list of all the keywords in Spark SQL. JOINreservedstrict-non-reservedreserved KEYSnon-reservednon-reservednon-reserved LASTnon-reservednon-reservednon-reserved + LAST_VALUEreservednon-reservedreserved LATERALnon-reservednon-reservedreserved LAZYnon-reservednon-reservednon-reserved LEADINGreservednon-reservedreserved @@ -219,6 +221,7 @@ Below is a list of all the keywords in Spark SQL. REPAIRnon-reservednon-reservednon-reserved REPLACEnon-reservednon-reservednon-reserved RESETnon-reservednon-reservednon-reserved + RESPECTnon-reservednon-reservednon-reserved RESTRICTnon-reservednon-reservednon-reserved REVOKEnon-reservednon-reservedreserved RIGHTreservedstrict-non-reservedreserved diff --git a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 index a1c11504a9036..d991e7cf7e898 100644 --- a/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 +++ b/sql/catalyst/src/main/antlr4/org/apache/spark/sql/catalyst/parser/SqlBase.g4 @@ -680,8 +680,8 @@ primaryExpression | CASE value=expression whenClause+ (ELSE elseExpression=expression)? END #simpleCase | CAST '(' expression AS dataType ')' #cast | STRUCT '(' (argument+=namedExpression (',' argument+=namedExpression)*)? ')' #struct - | FIRST '(' expression (IGNORE NULLS)? ')' #first - | LAST '(' expression (IGNORE NULLS)? ')' #last + | (FIRST | FIRST_VALUE) '(' expression ((IGNORE | RESPECT) NULLS)? ')' #first + | (LAST | LAST_VALUE) '(' expression ((IGNORE | RESPECT) NULLS)? ')' #last | POSITION '(' substr=valueExpression IN str=valueExpression ')' #position | constant #constantDefault | ASTERISK #star @@ -1023,6 +1023,7 @@ ansiNonReserved | REPAIR | REPLACE | RESET + | RESPECT | RESTRICT | REVOKE | RLIKE @@ -1184,6 +1185,7 @@ nonReserved | FIELDS | FILEFORMAT | FIRST + | FIRST_VALUE | FOLLOWING | FOR | FOREIGN @@ -1214,6 +1216,7 @@ nonReserved | ITEMS | KEYS | LAST + | LAST_VALUE | LATERAL | LAZY | LEADING @@ -1278,6 +1281,7 @@ nonReserved | REPAIR | REPLACE | RESET + | RESPECT | RESTRICT | REVOKE | RLIKE @@ -1435,6 +1439,7 @@ FETCH: 'FETCH'; FIELDS: 'FIELDS'; FILEFORMAT: 'FILEFORMAT'; FIRST: 'FIRST'; +FIRST_VALUE: 'FIRST_VALUE'; FOLLOWING: 'FOLLOWING'; FOR: 'FOR'; FOREIGN: 'FOREIGN'; @@ -1469,6 +1474,7 @@ ITEMS: 'ITEMS'; JOIN: 'JOIN'; KEYS: 'KEYS'; LAST: 'LAST'; +LAST_VALUE: 'LAST_VALUE'; LATERAL: 'LATERAL'; LAZY: 'LAZY'; LEADING: 'LEADING'; @@ -1536,6 +1542,7 @@ RENAME: 'RENAME'; REPAIR: 'REPAIR'; REPLACE: 'REPLACE'; RESET: 'RESET'; +RESPECT: 'RESPECT'; RESTRICT: 'RESTRICT'; REVOKE: 'REVOKE'; RIGHT: 'RIGHT'; diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala index e16262ddb9cd3..6248e5724f063 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/ExpressionParserSuite.scala @@ -737,6 +737,15 @@ class ExpressionParserSuite extends AnalysisTest { assertEqual("last(a)", Last('a, Literal(false)).toAggregateExpression()) } + test("Support respect nulls keywords for first_value and last_value") { + assertEqual("first_value(a ignore nulls)", First('a, Literal(true)).toAggregateExpression()) + assertEqual("first_value(a respect nulls)", First('a, Literal(false)).toAggregateExpression()) + assertEqual("first_value(a)", First('a, Literal(false)).toAggregateExpression()) + assertEqual("last_value(a ignore nulls)", Last('a, Literal(true)).toAggregateExpression()) + assertEqual("last_value(a respect nulls)", Last('a, Literal(false)).toAggregateExpression()) + assertEqual("last_value(a)", Last('a, Literal(false)).toAggregateExpression()) + } + test("timestamp literals") { DateTimeTestUtils.outstandingTimezones.foreach { timeZone => withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timeZone.getID) { diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala index ba01380558530..fc2ce12092190 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/parser/TableIdentifierParserSuite.scala @@ -381,6 +381,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { "fields", "fileformat", "first", + "first_value", "following", "for", "foreign", @@ -415,6 +416,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { "join", "keys", "last", + "last_value", "lateral", "lazy", "leading", @@ -483,6 +485,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { "repair", "replace", "reset", + "respect", "restrict", "revoke", "right", @@ -579,6 +582,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { "except", "false", "fetch", + "first_value", "for", "foreign", "from", @@ -593,6 +597,7 @@ class TableIdentifierParserSuite extends SparkFunSuite with SQLHelper { "into", "join", "is", + "last_value", "leading", "left", "minute",