diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala index f7aa6da0a5bd..66c8a35a2512 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala @@ -1212,6 +1212,9 @@ object PushDownPredicate extends Rule[LogicalPlan] with PredicateHelper { case filter @ Filter(_, _: Filter) => filter // should not push predicates through sample, or will generate different results. case filter @ Filter(_, _: Sample) => filter + // should not push predicates through limit, or will generate different results. + case filter @ Filter(_, _: GlobalLimit) => filter + case filter @ Filter(_, _: LocalLimit) => filter case filter @ Filter(condition, u: UnaryNode) if u.expressions.forall(_.deterministic) => pushDownPredicate(filter, u.child) { predicate => diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala index 596b8fcea194..113533347d99 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/FilterPushdownSuite.scala @@ -150,6 +150,17 @@ class FilterPushdownSuite extends PlanTest { comparePlans(optimized, originalQuery) } + test("SPARK-16994: filter should not be pushed down into local limit") { + val originalQuery = testRelation + .limit(1) + .where('a % 2 == 0) + .analyze + + val optimized = Optimize.execute(originalQuery) + + comparePlans(optimized, originalQuery) + } + test("filters: combines filters") { val originalQuery = testRelation .select('a)