Skip to content

[Spark-20771][SQL] Make weekofyear more intuitive #18080

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
rubenljanssen wants to merge 5 commits into from
Closed

[Spark-20771][SQL] Make weekofyear more intuitive #18080

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
7235f4a
added additional weekofyear function
rubenljanssen May 24, 2017
057ede5
updated desc
rubenljanssen May 24, 2017
d26d1cb
merged functions into 1
rubenljanssen May 24, 2017
0c4618c
added support to specify start of week
rubenljanssen May 27, 2017
c9b5d3c
updated description
rubenljanssen May 27, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 44 additions & 8 deletions ...talyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -402,23 +402,59 @@ case class DayOfMonth(child: Expression) extends UnaryExpression with ImplicitCa
}
}

// scalastyle:off line.size.limit
@ExpressionDescription(
usage = "_FUNC_(date) - Returns the week of the year of the given date.",
usage = "_FUNC_(date[, format[, firstDayOfWeek]) - Returns the week of the year of the given date. Defaults to ISO 8601 standard: weeks start on Monday, and week 1 is defined as the first week in the new year that contains more than half of the days (Thursday in a Monday to Sunday week). Start of week can be overriden, and the week first week can be defined as the week that contains the first day of the new year by setting it to 'gregorian'.",
extended = """
Examples:
> SELECT _FUNC_('2008-02-20');
8
> SELECT _FUNC_('2011-01-01');
52
> SELECT _FUNC_('2011-01-01', 'gregorian');
1
> SELECT _FUNC_('2011-01-01', 'iso');
52
> SELECT _FUNC_('2011-01-01', 'iso', 'monday');
52
> SELECT _FUNC_('2011-01-01', 'iso', 'sunday');
1

""")
case class WeekOfYear(child: Expression) extends UnaryExpression with ImplicitCastInputTypes {
// scalastyle:on line.size.limit
case class WeekOfYear(child: Expression, format: Expression, firstDayOfWeek: Expression) extends
UnaryExpression with ImplicitCastInputTypes {

def this(child: Expression) = {
this(child, Literal("iso"), Literal("monday"))
}

def this(child: Expression, format: Expression) = {
this(child, format, Literal("monday"))
}

override def inputTypes: Seq[AbstractDataType] = Seq(DateType)

override def dataType: DataType = IntegerType

@transient private lazy val minimalDays = {
if ("gregorian".equalsIgnoreCase(format.toString)) 1 else 4
Copy link
Member

@gatorsmile gatorsmile May 26, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

How many formats the other DB/systems allow? Could you do a search?

Copy link
Contributor Author

@rubenljanssen rubenljanssen May 27, 2017 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did a bit of research, and there seem to be no other formats. However, some systems (such as MySQL and Java), allow the first day of the week to be defined as well. Some countries in the middle east have weekends on Friday/Saturday, or even Thursday/Friday.
I will update the PR to allow users to override the first day of the week, as well as specify how the first week is defined (1 iso standard: week with more than half of the days, i.e. Thursday in a Monday-Sunday week. 2 gregorian: week with first day of the new year)

Copy link
Contributor Author

@rubenljanssen rubenljanssen May 27, 2017

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It will still default to ISO stanards with Monday-Sunday week of course, but now users can override it in any way they would like

}

@transient private lazy val startOfWeek = {
firstDayOfWeek.toString match {
case day if "monday".equalsIgnoreCase(day) => Calendar.MONDAY
case day if "tuesday".equalsIgnoreCase(day) => Calendar.TUESDAY
case day if "wednesday".equalsIgnoreCase(day) => Calendar.WEDNESDAY
case day if "thursday".equalsIgnoreCase(day) => Calendar.THURSDAY
case day if "friday".equalsIgnoreCase(day) => Calendar.FRIDAY
case day if "saturday".equalsIgnoreCase(day) => Calendar.SATURDAY
case day if "sunday".equalsIgnoreCase(day) => Calendar.SUNDAY
}
}

@transient private lazy val c = {
val c = Calendar.getInstance(DateTimeUtils.getTimeZone("UTC"))
c.setFirstDayOfWeek(Calendar.MONDAY)
c.setMinimalDaysInFirstWeek(4)
c.setFirstDayOfWeek(startOfWeek)
c.setMinimalDaysInFirstWeek(minimalDays)
c
}

Expand All @@ -435,8 +471,8 @@ case class WeekOfYear(child: Expression) extends UnaryExpression with ImplicitCa
ctx.addMutableState(cal, c,
s"""
$c = $cal.getInstance($dtu.getTimeZone("UTC"));
$c.setFirstDayOfWeek($cal.MONDAY);
$c.setMinimalDaysInFirstWeek(4);
$c.setFirstDayOfWeek($startOfWeek);
$c.setMinimalDaysInFirstWeek($minimalDays);
""")
s"""
$c.setTimeInMillis($time * 1000L * 3600L * 24L);
Expand Down
31 changes: 23 additions & 8 deletions ...alyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/DateExpressionsSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,14 +197,29 @@ class DateExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
}

test("WeekOfYear") {
checkEvaluation(WeekOfYear(Literal.create(null, DateType)), null)
checkEvaluation(WeekOfYear(Literal(d)), 15)
checkEvaluation(WeekOfYear(Cast(Literal(sdfDate.format(d)), DateType, gmtId)), 15)
checkEvaluation(WeekOfYear(Cast(Literal(ts), DateType, gmtId)), 45)
checkEvaluation(WeekOfYear(Cast(Literal("2011-05-06"), DateType, gmtId)), 18)
checkEvaluation(WeekOfYear(Literal(new Date(sdf.parse("1582-10-15 13:10:15").getTime))), 40)
checkEvaluation(WeekOfYear(Literal(new Date(sdf.parse("1582-10-04 13:10:15").getTime))), 40)
checkConsistencyBetweenInterpretedAndCodegen(WeekOfYear, DateType)
checkEvaluation(WeekOfYear(
Literal.create(null, DateType), Literal("iso"), Literal("monday")), null)
checkEvaluation(WeekOfYear(Literal(d), Literal("iso"), Literal("monday")), 15)
checkEvaluation(WeekOfYear(
Cast(Literal(ts), DateType, gmtId), Literal("iso"), Literal("monday")), 45)
checkEvaluation(WeekOfYear(
Cast(Literal(sdfDate.format(d)), DateType, gmtId), Literal("iso"), Literal("monday")), 15)
checkEvaluation(WeekOfYear(
Literal(new Date(sdf.parse("1582-10-15 13:10:15").getTime)),
Literal("iso"), Literal("monday")), 40)
checkEvaluation(WeekOfYear(
Literal(new Date(sdf.parse("1582-10-04 13:10:15").getTime)),
Literal("iso"), Literal("monday")), 40)
checkEvaluation(WeekOfYear(
Cast(Literal("2011-01-01"), DateType, gmtId), Literal("iso"), Literal("monday")), 52)
checkEvaluation(WeekOfYear(
Cast(Literal("2011-01-01"), DateType, gmtId), Literal("gregorian"), Literal("monday")), 1)
checkEvaluation(WeekOfYear(
Cast(Literal("2011-01-01"), DateType, gmtId), Literal("iso"), Literal("sunday")), 52)

checkConsistencyBetweenInterpretedAndCodegen(
(child: Expression) => WeekOfYear(child, Literal("iso"), Literal("monday")), DateType)

}

test("DateFormat") {
Expand Down