Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -488,7 +488,7 @@ case class DateFormatClass(left: Expression, right: Expression, timeZoneId: Opti
* Deterministic version of [[UnixTimestamp]], must have at least one parameter.
*/
@ExpressionDescription(
usage = "_FUNC_(expr[, pattern]) - Returns the UNIX timestamp of the give time.",
usage = "_FUNC_(expr[, pattern]) - Returns the UNIX timestamp of the given time.",
extended = """
Examples:
> SELECT _FUNC_('2016-04-08', 'yyyy-MM-dd');
Expand Down Expand Up @@ -1225,8 +1225,8 @@ case class ParseToTimestamp(left: Expression, format: Expression, child: Express
extends RuntimeReplaceable {

def this(left: Expression, format: Expression) = {
this(left, format, Cast(UnixTimestamp(left, format), TimestampType))
}
this(left, format, Cast(UnixTimestamp(left, format), TimestampType))
}

override def flatArguments: Iterator[Any] = Iterator(left, format)
override def sql: String = s"$prettyName(${left.sql}, ${format.sql})"
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -423,7 +423,7 @@ object DateTimeUtils {
}

/**
* Parses a given UTF8 date string to the corresponding a corresponding [[Int]] value.
* Parses a given UTF8 date string to a corresponding [[Int]] value.
* The return type is [[Option]] in order to distinguish between 0 and null. The following
* formats are allowed:
*
Expand Down
25 changes: 18 additions & 7 deletions sql/core/src/main/scala/org/apache/spark/sql/functions.scala
Original file line number Diff line number Diff line change
Expand Up @@ -2487,19 +2487,21 @@ object functions {
*/
def current_timestamp(): Column = withExpr { CurrentTimestamp() }

// scalastyle:off line.size.limit
/**
* Converts a date/timestamp/string to a value of string in the format specified by the date
* format given by the second argument.
*
* A pattern could be for instance `dd.MM.yyyy` and could return a string like '18.03.1993'. All
* pattern letters of `java.text.SimpleDateFormat` can be used.
* pattern letters of [[https://docs.oracle.com/javase/8/docs/api/java/text/SimpleDateFormat.html java.text.SimpleDateFormat]] can be used.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We don't generally link to external javadoc, and we've had problems creating a normal javadoc link to the class in many instances. For consistency, I wouldn't bother with this.

*
* @note Use when ever possible specialized functions like [[year]]. These benefit from a
* specialized implementation.
*
* @group datetime_funcs
* @since 1.5.0
*/
// scalastyle:on line.size.limit
def date_format(dateExpr: Column, format: String): Column = withExpr {
DateFormatClass(dateExpr.expr, Literal(format))
}
Expand Down Expand Up @@ -2647,7 +2649,11 @@ object functions {
}

/**
* Gets current Unix timestamp in seconds.
* Returns the current Unix timestamp (in seconds).
*
* NOTE: All calls of `unix_timestamp` within the same query return the same value

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should we use @note and fix the corresponding contents in Python and R if applicable?

* (i.e. the current timestamp is calculated at the start of query evaluation).
*
* @group datetime_funcs
* @since 1.5.0
*/
Expand All @@ -2657,22 +2663,27 @@ object functions {

/**
* Converts time string in format yyyy-MM-dd HH:mm:ss to Unix timestamp (in seconds),
* using the default timezone and the default locale, return null if fail.
* using the default timezone and the default locale.
* Returns `null` if fails.
*
* @group datetime_funcs
* @since 1.5.0
*/
def unix_timestamp(s: Column): Column = withExpr {
UnixTimestamp(s.expr, Literal("yyyy-MM-dd HH:mm:ss"))
}

// scalastyle:off line.size.limit
/**
* Convert time string with given pattern
* (see [http://docs.oracle.com/javase/tutorial/i18n/format/simpleDateFormat.html])
* to Unix time stamp (in seconds), return null if fail.
* Converts time string with given pattern to Unix timestamp (in seconds).
* Returns `null` if fails.
*
* @see [[http://docs.oracle.com/javase/tutorial/i18n/format/simpleDateFormat.html Customizing Formats]]

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It looks the documentation generation for Javadoc 8 is being failed due to @see with link -

[error] /home/jenkins/workspace/SparkPullRequestBuilder/sql/core/target/java/org/apache/spark/sql/functions.java:2996: error: self-closing element not allowed
[error]    * @see <a href="http://docs.oracle.com/javase/tutorial/i18n/format/simpleDateFormat.html Customizing Formats"/>
[error]           ^
[error] /home/jenkins/workspace/SparkPullRequestBuilder/sql/core/target/java/org/apache/spark/sql/functions.java:2996: error: invalid uri: "http://docs.oracle.com/javase/tutorial/i18n/format/simpleDateFormat.html Customizing Formats"
[error]    * @see <a href="http://docs.oracle.com/javase/tutorial/i18n/format/simpleDateFormat.html Customizing Formats"/>
[error]                                                                     ^

Probably, we should wrap it href as I did before - #16013 or find a way to make this link properly (or just remove it).

The other errors seem spurious. Please refer my observation - #17389 (comment)

* @group datetime_funcs
* @since 1.5.0
*/
def unix_timestamp(s: Column, p: String): Column = withExpr {UnixTimestamp(s.expr, Literal(p)) }
// scalastyle:on
def unix_timestamp(s: Column, p: String): Column = withExpr { UnixTimestamp(s.expr, Literal(p)) }

/**
* Convert time string to a Unix timestamp (in seconds).
Expand Down