-
Notifications
You must be signed in to change notification settings - Fork 29k
[SPARK-26653][SQL] Use Proleptic Gregorian calendar in parsing JDBC lower/upper bounds #23597
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 4 commits
f025ae1
6171e9a
0e3691e
0b61076
8bb4f3a
7ba3cbb
a0b23ed
e793970
c6df730
4dc4a2a
af20442
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -17,8 +17,6 @@ | |
|
|
||
| package org.apache.spark.sql.execution.datasources.jdbc | ||
|
|
||
| import java.sql.{Date, Timestamp} | ||
|
|
||
| import scala.collection.mutable.ArrayBuffer | ||
|
|
||
| import org.apache.spark.Partition | ||
|
|
@@ -27,10 +25,12 @@ import org.apache.spark.rdd.RDD | |
| import org.apache.spark.sql.{AnalysisException, DataFrame, Row, SaveMode, SparkSession, SQLContext} | ||
| import org.apache.spark.sql.catalyst.analysis._ | ||
| import org.apache.spark.sql.catalyst.util.{DateFormatter, DateTimeUtils, TimestampFormatter} | ||
| import org.apache.spark.sql.catalyst.util.DateTimeUtils.{getTimeZone, stringToDate, stringToTimestamp} | ||
| import org.apache.spark.sql.internal.SQLConf | ||
| import org.apache.spark.sql.jdbc.JdbcDialects | ||
| import org.apache.spark.sql.sources._ | ||
| import org.apache.spark.sql.types.{DataType, DateType, NumericType, StructType, TimestampType} | ||
| import org.apache.spark.unsafe.types.UTF8String | ||
|
|
||
| /** | ||
| * Instructions on how to partition the table among workers. | ||
|
|
@@ -174,10 +174,19 @@ private[sql] object JDBCRelation extends Logging { | |
| (dialect.quoteIdentifier(column.name), column.dataType) | ||
| } | ||
|
|
||
| private def toInternalBoundValue(value: String, columnType: DataType): Long = columnType match { | ||
| case _: NumericType => value.toLong | ||
| case DateType => DateTimeUtils.fromJavaDate(Date.valueOf(value)).toLong | ||
| case TimestampType => DateTimeUtils.fromJavaTimestamp(Timestamp.valueOf(value)) | ||
| private def toInternalBoundValue(value: String, columnType: DataType): Long = { | ||
| def parse[T](f: UTF8String => Option[T]): T = { | ||
| f(UTF8String.fromString(value)).getOrElse { | ||
HyukjinKwon marked this conversation as resolved.
Show resolved
Hide resolved
|
||
| throw new IllegalArgumentException( | ||
| s"Cannot parse the bound value $value as ${columnType.catalogString}") | ||
| } | ||
| } | ||
| columnType match { | ||
| case _: NumericType => value.toLong | ||
| case DateType => parse(stringToDate).toLong | ||
| case TimestampType => | ||
| parse(stringToTimestamp(_, getTimeZone(SQLConf.get.sessionLocalTimeZone))) | ||
|
||
| } | ||
| } | ||
|
|
||
| private def toBoundValueInWhereClause( | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
we should pass in the timezone id, just like what we did for
toBoundValueInWhereClauseThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
done