Skip to content

[SPARK-32052][SQL] Extract common code from date-time field expressions #28894

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
MaxGekk wants to merge 3 commits into from
Closed

[SPARK-32052][SQL] Extract common code from date-time field expressions #28894

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
dff314e
Extract common code from Hour, Minute, Second and SecondWithFraction
MaxGekk Jun 22, 2020
d9efa6c
Extract common code to GetDateField
MaxGekk Jun 22, 2020
f2a278e
Extract common code from DayWeek to GetDateField
MaxGekk Jun 22, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
290 changes: 73 additions & 217 deletions ...talyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/datetimeExpressions.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -274,39 +274,43 @@ case class DateSub(startDate: Expression, days: Expression)
override def prettyName: String = "date_sub"
}

@ExpressionDescription(
usage = "_FUNC_(timestamp) - Returns the hour component of the string/timestamp.",
examples = """
Examples:
> SELECT _FUNC_('2009-07-30 12:58:59');
12
""",
group = "datetime_funcs",
since = "1.5.0")
case class Hour(child: Expression, timeZoneId: Option[String] = None)
extends UnaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes
with NullIntolerant {
trait GetTimeField extends UnaryExpression
Copy link
Member Author

@MaxGekk MaxGekk Jun 22, 2020 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

When I made it as abstract class with 2 args func and funcName, I got the following exception on tests from DateExpressionsSuite:

org.apache.spark.sql.catalyst.expressions.Second; no valid constructor
java.io.InvalidClassException: org.apache.spark.sql.catalyst.expressions.Second; no valid constructor
	at java.io.ObjectStreamClass$ExceptionInfo.newInvalidClassException(ObjectStreamClass.java:169)
	at java.io.ObjectStreamClass.checkDeserialize(ObjectStreamClass.java:885)
	at java.io.ObjectInputStream.readOrdinaryObject(ObjectInputStream.java:2100)
	at java.io.ObjectInputStream.readObject0(ObjectInputStream.java:1625)
	at java.io.ObjectInputStream.readObject(ObjectInputStream.java:465)
	at java.io.ObjectInputStream.readObject(ObjectInputStream.java:423)
	at org.apache.spark.serializer.JavaDeserializationStream.readObject(JavaSerializer.scala:76)
	at org.apache.spark.serializer.JavaSerializerInstance.deserialize(JavaSerializer.scala:109)
	at org.apache.spark.sql.catalyst.expressions.ExpressionEvalHelper.prepareEvaluation(ExpressionEvalHelper.scala:75)

I decided to leave it as a trait.

with TimeZoneAwareExpression with ImplicitCastInputTypes with NullIntolerant {

def this(child: Expression) = this(child, None)
val func: (Long, ZoneId) => Any
val funcName: String

override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType)

override def dataType: DataType = IntegerType

override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
copy(timeZoneId = Option(timeZoneId))

override protected def nullSafeEval(timestamp: Any): Any = {
DateTimeUtils.getHours(timestamp.asInstanceOf[Long], zoneId)
func(timestamp.asInstanceOf[Long], zoneId)
}

override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
defineCodeGen(ctx, ev, c => s"$dtu.getHours($c, $zid)")
defineCodeGen(ctx, ev, c => s"$dtu.$funcName($c, $zid)")
}
}

@ExpressionDescription(
usage = "_FUNC_(timestamp) - Returns the hour component of the string/timestamp.",
examples = """
Examples:
> SELECT _FUNC_('2009-07-30 12:58:59');
12
""",
group = "datetime_funcs",
since = "1.5.0")
case class Hour(child: Expression, timeZoneId: Option[String] = None) extends GetTimeField {
def this(child: Expression) = this(child, None)
override def withTimeZone(timeZoneId: String): Hour = copy(timeZoneId = Option(timeZoneId))
override val func = DateTimeUtils.getHours
override val funcName = "getHours"
}

@ExpressionDescription(
usage = "_FUNC_(timestamp) - Returns the minute component of the string/timestamp.",
examples = """
Expand All @@ -316,28 +320,11 @@ case class Hour(child: Expression, timeZoneId: Option[String] = None)
""",
group = "datetime_funcs",
since = "1.5.0")
case class Minute(child: Expression, timeZoneId: Option[String] = None)
extends UnaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes
with NullIntolerant {

case class Minute(child: Expression, timeZoneId: Option[String] = None) extends GetTimeField {
def this(child: Expression) = this(child, None)

override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType)

override def dataType: DataType = IntegerType

override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
copy(timeZoneId = Option(timeZoneId))

override protected def nullSafeEval(timestamp: Any): Any = {
DateTimeUtils.getMinutes(timestamp.asInstanceOf[Long], zoneId)
}

override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
defineCodeGen(ctx, ev, c => s"$dtu.getMinutes($c, $zid)")
}
override def withTimeZone(timeZoneId: String): Minute = copy(timeZoneId = Option(timeZoneId))
override val func = DateTimeUtils.getMinutes
override val funcName = "getMinutes"
}

@ExpressionDescription(
Expand All @@ -349,52 +336,39 @@ case class Minute(child: Expression, timeZoneId: Option[String] = None)
""",
group = "datetime_funcs",
since = "1.5.0")
case class Second(child: Expression, timeZoneId: Option[String] = None)
extends UnaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes
with NullIntolerant {

case class Second(child: Expression, timeZoneId: Option[String] = None) extends GetTimeField {
def this(child: Expression) = this(child, None)

override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType)

override def dataType: DataType = IntegerType

override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
copy(timeZoneId = Option(timeZoneId))

override protected def nullSafeEval(timestamp: Any): Any = {
DateTimeUtils.getSeconds(timestamp.asInstanceOf[Long], zoneId)
}

override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
defineCodeGen(ctx, ev, c => s"$dtu.getSeconds($c, $zid)")
}
override def withTimeZone(timeZoneId: String): Second = copy(timeZoneId = Option(timeZoneId))
override val func = DateTimeUtils.getSeconds
override val funcName = "getSeconds"
}

case class SecondWithFraction(child: Expression, timeZoneId: Option[String] = None)
extends UnaryExpression with TimeZoneAwareExpression with ImplicitCastInputTypes
with NullIntolerant {

extends GetTimeField {
def this(child: Expression) = this(child, None)

override def inputTypes: Seq[AbstractDataType] = Seq(TimestampType)

// 2 digits for seconds, and 6 digits for the fractional part with microsecond precision.
override def dataType: DataType = DecimalType(8, 6)

override def withTimeZone(timeZoneId: String): TimeZoneAwareExpression =
override def withTimeZone(timeZoneId: String): SecondWithFraction =
copy(timeZoneId = Option(timeZoneId))
override val func = DateTimeUtils.getSecondsWithFraction
override val funcName = "getSecondsWithFraction"
}

override protected def nullSafeEval(timestamp: Any): Any = {
DateTimeUtils.getSecondsWithFraction(timestamp.asInstanceOf[Long], zoneId)
trait GetDateField extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {
val func: Int => Any
val funcName: String

override def inputTypes: Seq[AbstractDataType] = Seq(DateType)

override def dataType: DataType = IntegerType

override protected def nullSafeEval(date: Any): Any = {
func(date.asInstanceOf[Int])
}

override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val zid = ctx.addReferenceObj("zoneId", zoneId, classOf[ZoneId].getName)
val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
defineCodeGen(ctx, ev, c => s"$dtu.getSecondsWithFraction($c, $zid)")
defineCodeGen(ctx, ev, c => s"$dtu.$funcName($c)")
}
}

Expand All @@ -407,21 +381,9 @@ case class SecondWithFraction(child: Expression, timeZoneId: Option[String] = No
""",
group = "datetime_funcs",
since = "1.5.0")
case class DayOfYear(child: Expression)
extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {

override def inputTypes: Seq[AbstractDataType] = Seq(DateType)

override def dataType: DataType = IntegerType

override protected def nullSafeEval(date: Any): Any = {
DateTimeUtils.getDayInYear(date.asInstanceOf[Int])
}

override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
defineCodeGen(ctx, ev, c => s"$dtu.getDayInYear($c)")
}
case class DayOfYear(child: Expression) extends GetDateField {
override val func = DateTimeUtils.getDayInYear
override val funcName = "getDayInYear"
}

abstract class NumberToTimestampBase extends UnaryExpression
Expand Down Expand Up @@ -510,38 +472,14 @@ case class MicrosToTimestamp(child: Expression)
""",
group = "datetime_funcs",
since = "1.5.0")
case class Year(child: Expression)
extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {

override def inputTypes: Seq[AbstractDataType] = Seq(DateType)

override def dataType: DataType = IntegerType

override protected def nullSafeEval(date: Any): Any = {
DateTimeUtils.getYear(date.asInstanceOf[Int])
}

override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
defineCodeGen(ctx, ev, c => s"$dtu.getYear($c)")
}
case class Year(child: Expression) extends GetDateField {
override val func = DateTimeUtils.getYear
override val funcName = "getYear"
}

case class YearOfWeek(child: Expression)
extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {

override def inputTypes: Seq[AbstractDataType] = Seq(DateType)

override def dataType: DataType = IntegerType

override protected def nullSafeEval(date: Any): Any = {
DateTimeUtils.getWeekBasedYear(date.asInstanceOf[Int])
}

override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
defineCodeGen(ctx, ev, c => s"$dtu.getWeekBasedYear($c)")
}
case class YearOfWeek(child: Expression) extends GetDateField {
override val func = DateTimeUtils.getWeekBasedYear
override val funcName = "getWeekBasedYear"
}

@ExpressionDescription(
Expand All @@ -553,21 +491,9 @@ case class YearOfWeek(child: Expression)
""",
group = "datetime_funcs",
since = "1.5.0")
case class Quarter(child: Expression)
extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {

override def inputTypes: Seq[AbstractDataType] = Seq(DateType)

override def dataType: DataType = IntegerType

override protected def nullSafeEval(date: Any): Any = {
DateTimeUtils.getQuarter(date.asInstanceOf[Int])
}

override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
defineCodeGen(ctx, ev, c => s"$dtu.getQuarter($c)")
}
case class Quarter(child: Expression) extends GetDateField {
override val func = DateTimeUtils.getQuarter
override val funcName = "getQuarter"
}

@ExpressionDescription(
Expand All @@ -579,21 +505,9 @@ case class Quarter(child: Expression)
""",
group = "datetime_funcs",
since = "1.5.0")
case class Month(child: Expression)
extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {

override def inputTypes: Seq[AbstractDataType] = Seq(DateType)

override def dataType: DataType = IntegerType

override protected def nullSafeEval(date: Any): Any = {
DateTimeUtils.getMonth(date.asInstanceOf[Int])
}

override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
defineCodeGen(ctx, ev, c => s"$dtu.getMonth($c)")
}
case class Month(child: Expression) extends GetDateField {
override val func = DateTimeUtils.getMonth
override val funcName = "getMonth"
}

@ExpressionDescription(
Expand All @@ -604,21 +518,9 @@ case class Month(child: Expression)
30
""",
since = "1.5.0")
case class DayOfMonth(child: Expression)
extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {

override def inputTypes: Seq[AbstractDataType] = Seq(DateType)

override def dataType: DataType = IntegerType

override protected def nullSafeEval(date: Any): Any = {
DateTimeUtils.getDayOfMonth(date.asInstanceOf[Int])
}

override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val dtu = DateTimeUtils.getClass.getName.stripSuffix("$")
defineCodeGen(ctx, ev, c => s"$dtu.getDayOfMonth($c)")
}
case class DayOfMonth(child: Expression) extends GetDateField {
override val func = DateTimeUtils.getDayOfMonth
override val funcName = "getDayOfMonth"
}

// scalastyle:off line.size.limit
Expand All @@ -632,20 +534,9 @@ case class DayOfMonth(child: Expression)
group = "datetime_funcs",
since = "2.3.0")
// scalastyle:on line.size.limit
case class DayOfWeek(child: Expression) extends DayWeek {

override protected def nullSafeEval(date: Any): Any = {
val localDate = LocalDate.ofEpochDay(date.asInstanceOf[Int])
localDate.getDayOfWeek.plus(1).getValue
}

override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
nullSafeCodeGen(ctx, ev, days => {
s"""
${ev.value} = java.time.LocalDate.ofEpochDay($days).getDayOfWeek().plus(1).getValue();
"""
})
}
case class DayOfWeek(child: Expression) extends GetDateField {
override val func = DateTimeUtils.getDayOfWeek
override val funcName = "getDayOfWeek"
}

// scalastyle:off line.size.limit
Expand All @@ -659,27 +550,9 @@ case class DayOfWeek(child: Expression) extends DayWeek {
group = "datetime_funcs",
since = "2.4.0")
// scalastyle:on line.size.limit
case class WeekDay(child: Expression) extends DayWeek {

override protected def nullSafeEval(date: Any): Any = {
val localDate = LocalDate.ofEpochDay(date.asInstanceOf[Int])
localDate.getDayOfWeek.ordinal()
}

override protected def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
nullSafeCodeGen(ctx, ev, days => {
s"""
${ev.value} = java.time.LocalDate.ofEpochDay($days).getDayOfWeek().ordinal();
"""
})
}
}

abstract class DayWeek extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {

override def inputTypes: Seq[AbstractDataType] = Seq(DateType)

override def dataType: DataType = IntegerType
case class WeekDay(child: Expression) extends GetDateField {
override val func = DateTimeUtils.getWeekDay
override val funcName = "getWeekDay"
}

// scalastyle:off line.size.limit
Expand All @@ -693,26 +566,9 @@ abstract class DayWeek extends UnaryExpression with ImplicitCastInputTypes with
group = "datetime_funcs",
since = "1.5.0")
// scalastyle:on line.size.limit
case class WeekOfYear(child: Expression)
extends UnaryExpression with ImplicitCastInputTypes with NullIntolerant {

override def inputTypes: Seq[AbstractDataType] = Seq(DateType)

override def dataType: DataType = IntegerType

override protected def nullSafeEval(date: Any): Any = {
val localDate = LocalDate.ofEpochDay(date.asInstanceOf[Int])
localDate.get(IsoFields.WEEK_OF_WEEK_BASED_YEAR)
}

override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
nullSafeCodeGen(ctx, ev, days => {
s"""
|${ev.value} = java.time.LocalDate.ofEpochDay($days).get(
| java.time.temporal.IsoFields.WEEK_OF_WEEK_BASED_YEAR);
""".stripMargin
})
}
case class WeekOfYear(child: Expression) extends GetDateField {
override val func = DateTimeUtils.getWeekOfYear
override val funcName = "getWeekOfYear"
}

// scalastyle:off line.size.limit
Expand Down
Loading