Skip to content

Commit f787b18

Browse files
pzzsjeanlyn
authored andcommitted
[SPARK-7225][SQL] CombineLimits optimizer does not work
SQL ``` select key from (select key from src limit 100) t2 limit 10 ``` Optimized Logical Plan before modifying ``` == Optimized Logical Plan == Limit 10 Limit 100 Project key#3 MetastoreRelation default, src, None ``` Optimized Logical Plan after modifying ``` == Optimized Logical Plan == Limit 10 Project [key#1] MetastoreRelation default, src, None ``` Author: Zhongshuai Pei <[email protected]> Author: DoingDone9 <[email protected]> Closes apache#5770 from DoingDone9/limitOptimizer and squashes the following commits: c68eaa7 [Zhongshuai Pei] Update CombiningLimitsSuite.scala 97e18cf [Zhongshuai Pei] Update Optimizer.scala 19ab875 [Zhongshuai Pei] Update CombiningLimitsSuite.scala 7db4566 [Zhongshuai Pei] Update CombiningLimitsSuite.scala e2a491d [Zhongshuai Pei] Update Optimizer.scala f03fe7f [Zhongshuai Pei] Merge pull request #12 from apache/master f12fa50 [Zhongshuai Pei] Merge pull request #10 from apache/master f61210c [Zhongshuai Pei] Merge pull request #9 from apache/master 34b1a9a [Zhongshuai Pei] Merge pull request #8 from apache/master 802261c [DoingDone9] Merge pull request #7 from apache/master d00303b [DoingDone9] Merge pull request #6 from apache/master 98b134f [DoingDone9] Merge pull request #5 from apache/master 161cae3 [DoingDone9] Merge pull request #4 from apache/master c87e8b6 [DoingDone9] Merge pull request #3 from apache/master cb1852d [DoingDone9] Merge pull request #2 from apache/master c3f046f [DoingDone9] Merge pull request #1 from apache/master
1 parent 4584ee1 commit f787b18

File tree

2 files changed

+26
-8
lines changed

2 files changed

+26
-8
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,13 @@ object DefaultOptimizer extends Optimizer {
3636
// SubQueries are only needed for analysis and can be removed before execution.
3737
Batch("Remove SubQueries", FixedPoint(100),
3838
EliminateSubQueries) ::
39-
Batch("Combine Limits", FixedPoint(100),
39+
Batch("Operator Reordering", FixedPoint(100),
40+
UnionPushdown,
41+
CombineFilters,
42+
PushPredicateThroughProject,
43+
PushPredicateThroughJoin,
44+
PushPredicateThroughGenerate,
45+
ColumnPruning,
4046
CombineLimits) ::
4147
Batch("ConstantFolding", FixedPoint(100),
4248
NullPropagation,
@@ -49,13 +55,6 @@ object DefaultOptimizer extends Optimizer {
4955
OptimizeIn) ::
5056
Batch("Decimal Optimizations", FixedPoint(100),
5157
DecimalAggregates) ::
52-
Batch("Filter Pushdown", FixedPoint(100),
53-
UnionPushdown,
54-
CombineFilters,
55-
PushPredicateThroughProject,
56-
PushPredicateThroughJoin,
57-
PushPredicateThroughGenerate,
58-
ColumnPruning) ::
5958
Batch("LocalRelation", FixedPoint(100),
6059
ConvertToLocalRelation) :: Nil
6160
}

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/CombiningLimitsSuite.scala

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ class CombiningLimitsSuite extends PlanTest {
2727

2828
object Optimize extends RuleExecutor[LogicalPlan] {
2929
val batches =
30+
Batch("Filter Pushdown", FixedPoint(100),
31+
ColumnPruning) ::
3032
Batch("Combine Limit", FixedPoint(10),
3133
CombineLimits) ::
3234
Batch("Constant Folding", FixedPoint(10),
@@ -69,4 +71,21 @@ class CombiningLimitsSuite extends PlanTest {
6971

7072
comparePlans(optimized, correctAnswer)
7173
}
74+
75+
test("limits: combines two limits after ColumnPruning") {
76+
val originalQuery =
77+
testRelation
78+
.select('a)
79+
.limit(2)
80+
.select('a)
81+
.limit(5)
82+
83+
val optimized = Optimize.execute(originalQuery.analyze)
84+
val correctAnswer =
85+
testRelation
86+
.select('a)
87+
.limit(2).analyze
88+
89+
comparePlans(optimized, correctAnswer)
90+
}
7291
}

0 commit comments

Comments
 (0)