diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala index 02b0661eb3dd7..cc148d9e247f6 100644 --- a/project/MimaExcludes.scala +++ b/project/MimaExcludes.scala @@ -53,6 +53,8 @@ object MimaExcludes { ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.util.ExecutionListenerManager.this"), // [SPARK-37786][SQL] StreamingQueryListener support use SQLConf.get to get corresponding SessionState's SQLConf ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.sql.streaming.StreamingQueryManager.this"), + // [SPARK-38432][SQL] Reactor framework so as JDBC dialect could compile filter by self way + ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.sources.Filter.toV2"), // [SPARK-37600][BUILD] Upgrade to Hadoop 3.3.2 ProblemFilters.exclude[MissingClassProblem]("org.apache.hadoop.shaded.net.jpountz.lz4.LZ4Compressor"), diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expression.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expression.java index 9f6c0975ae0e1..76dfe73f666cf 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expression.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/Expression.java @@ -17,6 +17,8 @@ package org.apache.spark.sql.connector.expressions; +import java.util.Arrays; + import org.apache.spark.annotation.Evolving; /** @@ -26,8 +28,23 @@ */ @Evolving public interface Expression { + Expression[] EMPTY_EXPRESSION = new Expression[0]; + /** * Format the expression as a human readable SQL-like string. */ default String describe() { return this.toString(); } + + /** + * Returns an array of the children of this node. Children should not change. + */ + Expression[] children(); + + /** + * List of fields or columns that are referenced by this expression. + */ + default NamedReference[] references() { + return Arrays.stream(children()).map(e -> e.references()) + .flatMap(Arrays::stream).distinct().toArray(NamedReference[]::new); + } } diff --git a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/GeneralScalarExpression.java b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/GeneralScalarExpression.java index b3dd2cbfe3d7d..8952761f9ef34 100644 --- a/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/GeneralScalarExpression.java +++ b/sql/catalyst/src/main/java/org/apache/spark/sql/connector/expressions/GeneralScalarExpression.java @@ -19,77 +19,19 @@ import java.io.Serializable; import java.util.Arrays; +import java.util.Objects; import org.apache.spark.annotation.Evolving; +import org.apache.spark.sql.connector.expressions.filter.Predicate; import org.apache.spark.sql.connector.util.V2ExpressionSQLBuilder; -// scalastyle:off line.size.limit /** * The general representation of SQL scalar expressions, which contains the upper-cased - * expression name and all the children expressions. + * expression name and all the children expressions. Please also see {@link Predicate} + * for the supported predicate expressions. *

* The currently supported SQL scalar expressions: *

    - *
  1. Name: IS_NULL - * - *
  2. - *
  3. Name: IS_NOT_NULL - * - *
  4. - *
  5. Name: = - * - *
  6. - *
  7. Name: != - * - *
  8. - *
  9. Name: <> - * - *
  10. - *
  11. Name: <=> - * - *
  12. - *
  13. Name: < - * - *
  14. - *
  15. Name: <= - * - *
  16. - *
  17. Name: > - * - *
  18. - *
  19. Name: >= - * - *
  20. *
  21. Name: + * *
  22. - *
  23. Name: AND - * - *
  24. - *
  25. Name: OR - * - *
  26. - *
  27. Name: NOT - * - *
  28. *
  29. Name: ~ *