Skip to content

Commit d5cc79b

Browse files
committed
Address @rxin 's comments.
1 parent 366b6d9 commit d5cc79b

File tree

3 files changed

+16
-8
lines changed

3 files changed

+16
-8
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -46,12 +46,18 @@ trait PredicateHelper {
4646
}
4747
}
4848

49-
private def combineConjunctivePredicates(predicates: Seq[Expression]) =
50-
predicates.reduceLeft(And)
51-
52-
/** Returns true if `expr` can be evaluated using only the output of `plan`. */
49+
/**
50+
* Returns true if `expr` can be evaluated using only the output of `plan`. This method
51+
* can be used to determine when is is acceptable to move expression evaluation within a query
52+
* plan.
53+
*
54+
* For example consider a join between two relations R(a, b) and S(c, d).
55+
*
56+
* `canEvaluate(Equals(a,b), R)` returns `true` where as `canEvaluate(Equals(a,c), R)` returns
57+
* `false`.
58+
*/
5359
protected def canEvaluate(expr: Expression, plan: LogicalPlan): Boolean =
54-
expr.references subsetOf plan.outputSet
60+
expr.references.subsetOf(plan.outputSet)
5561
}
5662

5763
abstract class BinaryPredicate extends BinaryExpression with Predicate {

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/planning/patterns.scala

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ object PhysicalOperation extends PredicateHelper {
109109
* techniques. For inner joins, any filters on top of the join operator are also matched.
110110
*/
111111
object HashFilteredJoin extends Logging with PredicateHelper {
112-
/** (joinType, rightKeys, leftKeys, condition, left, right) */
112+
/** (joinType, rightKeys, leftKeys, condition, leftChild, rightChild) */
113113
type ReturnType =
114114
(JoinType, Seq[Expression], Seq[Expression], Option[Expression], LogicalPlan, LogicalPlan)
115115

@@ -136,8 +136,8 @@ object HashFilteredJoin extends Logging with PredicateHelper {
136136
}
137137

138138
val joinKeys = joinPredicates.map {
139-
case Equals(l,r) if canEvaluate(l, left) && canEvaluate(r, right) => (l, r)
140-
case Equals(l,r) if canEvaluate(l, right) && canEvaluate(r, left) => (r, l)
139+
case Equals(l, r) if canEvaluate(l, left) && canEvaluate(r, right) => (l, r)
140+
case Equals(l, r) if canEvaluate(l, right) && canEvaluate(r, left) => (r, l)
141141
}
142142

143143
// Do not consider this strategy if there are no join keys.

sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
3030

3131
object HashJoin extends Strategy with PredicateHelper {
3232
def apply(plan: LogicalPlan): Seq[SparkPlan] = plan match {
33+
// Find inner joins where at least some predicates can be evaluated by matching hash keys
34+
// using the HashFilteredJoin pattern.
3335
case HashFilteredJoin(Inner, leftKeys, rightKeys, condition, left, right) =>
3436
val hashJoin =
3537
execution.HashJoin(leftKeys, rightKeys, BuildRight, planLater(left), planLater(right))

0 commit comments

Comments
 (0)