-
Notifications
You must be signed in to change notification settings - Fork 29k
[SPARK-49695][SQL] Postgres fix xor push-down #48144
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 2 commits
70cdc0e
7e81576
02cf3d3
6df7f81
f2982ae
de4a5cb
f7aadd7
6d18ec6
7b95659
485a48d
309c130
dc78465
ad10faf
823cf7c
9a9a6a2
b843186
574b7fc
cdf3724
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -23,14 +23,15 @@ import java.util | |
| import java.util.Locale | ||
|
|
||
| import scala.util.Using | ||
| import scala.util.control.NonFatal | ||
|
|
||
| import org.apache.spark.internal.LogKeys.COLUMN_NAME | ||
| import org.apache.spark.internal.MDC | ||
| import org.apache.spark.sql.AnalysisException | ||
| import org.apache.spark.sql.catalyst.SQLConfHelper | ||
| import org.apache.spark.sql.catalyst.analysis.{IndexAlreadyExistsException, NonEmptyNamespaceException, NoSuchIndexException} | ||
| import org.apache.spark.sql.connector.catalog.Identifier | ||
| import org.apache.spark.sql.connector.expressions.NamedReference | ||
| import org.apache.spark.sql.connector.expressions.{Expression, GeneralScalarExpression, NamedReference} | ||
| import org.apache.spark.sql.errors.QueryCompilationErrors | ||
| import org.apache.spark.sql.execution.datasources.jdbc.{JDBCOptions, JdbcUtils} | ||
| import org.apache.spark.sql.execution.datasources.v2.TableSampleInfo | ||
|
|
@@ -379,4 +380,26 @@ private case class PostgresDialect() | |
| case _ => | ||
| } | ||
| } | ||
|
|
||
| override def compileExpression(expr: Expression): Option[String] = { | ||
| val builder = new PostgresSQLBuilder() | ||
| try { | ||
|
||
| Some(builder.build(expr)) | ||
| } catch { | ||
| case NonFatal(e) => | ||
| logWarning("Error occurs while compiling V2 expression", e) | ||
| None | ||
| } | ||
| } | ||
|
|
||
| private class PostgresSQLBuilder extends JDBCSQLBuilder { | ||
| override def build(expr: Expression): String = { | ||
| expr match { | ||
| // Postgres uses '#' for xor, rather then '^'. | ||
| case e: GeneralScalarExpression if e.name() == "^" => | ||
| visitBinaryArithmetic("#", inputToSQL(e.children().head), inputToSQL(e.children()(1))) | ||
| case _ => super.build(expr) | ||
| } | ||
| } | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Maybe you can do explain formatted and check whether string contains
"id" # 3, that will add a little bit of robustness.Another thing we can do is to make unit test, and just invoke compilation of XOR expression and check whether
col # constantis result of compilation.There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
neat, I like it