From 52f4213a62486c3bd543494ecce79e02f397b57c Mon Sep 17 00:00:00 2001 From: Chongguang LIU Date: Mon, 14 Dec 2020 10:59:52 +0100 Subject: [PATCH 1/2] adding an overridden version of the next-day function correct unit test of the next-day function add unit test of the next-day function --- .../org/apache/spark/sql/functions.scala | 21 +++++++++++++++++-- .../apache/spark/sql/DateFunctionsSuite.scala | 18 ++++++++++++---- 2 files changed, 33 insertions(+), 6 deletions(-) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala index ede2b52930a17..8510e6f50a551 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala @@ -3074,8 +3074,25 @@ object functions { * @group datetime_funcs * @since 1.5.0 */ - def next_day(date: Column, dayOfWeek: String): Column = withExpr { - NextDay(date.expr, lit(dayOfWeek).expr) + def next_day(date: Column, dayOfWeek: String): Column = next_day(date, lit(dayOfWeek)) + + /** + * Returns the first date which is later than the value of the `date` column that is on the + * specified day of the week. + * + * For example, `next_day('2015-07-27', "Sunday")` returns 2015-08-02 because that is the first + * Sunday after 2015-07-27. + * + * @param date A date, timestamp or string. If a string, the data must be in a format that + * can be cast to a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS` + * @param dayOfWeek A column of the day of week. Case insensitive, and accepts: "Mon", "Tue", + * "Wed", "Thu", "Fri", "Sat", "Sun" + * @return A date, or null if `date` was a string that could not be cast to a date or if + * `dayOfWeek` was an invalid value + * @group datetime_funcs + */ + def next_day(date: Column, dayOfWeek: Column): Column = withExpr { + NextDay(date.expr, dayOfWeek.expr) } /** diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala index d7bbf597ff983..b545d6097d71d 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DateFunctionsSuite.scala @@ -372,11 +372,21 @@ class DateFunctionsSuite extends QueryTest with SharedSparkSession { val df1 = Seq(("mon", "2015-07-23"), ("tuesday", "2015-07-20")).toDF("dow", "d") val df2 = Seq(("th", "2015-07-23 00:11:22"), ("xx", "2015-07-24 11:22:33")).toDF("dow", "t") checkAnswer( - df1.select(next_day(col("d"), "MONDAY")), - Seq(Row(Date.valueOf("2015-07-27")), Row(Date.valueOf("2015-07-27")))) + df1.select( + next_day(col("d"), "MONDAY"), + next_day(col("d"), col("dow")), + next_day(col("d"), "NonValidDay")), + Seq( + Row(Date.valueOf("2015-07-27"), Date.valueOf("2015-07-27"), null), + Row(Date.valueOf("2015-07-27"), Date.valueOf("2015-07-21"), null))) checkAnswer( - df2.select(next_day(col("t"), "th")), - Seq(Row(Date.valueOf("2015-07-30")), Row(Date.valueOf("2015-07-30")))) + df2.select( + next_day(col("t"), "th"), + next_day(col("t"), col("dow")), + next_day(col("t"), "NonValidDay")), + Seq( + Row(Date.valueOf("2015-07-30"), Date.valueOf("2015-07-30"), null), + Row(Date.valueOf("2015-07-30"), null, null))) } def checkExceptionMessage(df: DataFrame): Unit = { From 7c32905643ec1e072f82936d429a8e445e365356 Mon Sep 17 00:00:00 2001 From: Chongguang LIU Date: Tue, 15 Dec 2020 08:33:12 +0100 Subject: [PATCH 2/2] add version annotation --- sql/core/src/main/scala/org/apache/spark/sql/functions.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala index 8510e6f50a551..4defcb836a978 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala @@ -3090,6 +3090,7 @@ object functions { * @return A date, or null if `date` was a string that could not be cast to a date or if * `dayOfWeek` was an invalid value * @group datetime_funcs + * @since 3.2.0 */ def next_day(date: Column, dayOfWeek: Column): Column = withExpr { NextDay(date.expr, dayOfWeek.expr)