Skip to content

Commit 8ea872b

Browse files
committed
Fixed the number of tasks when the data of LocalRelation is empty.
Added optimization rule related to bool expression.
1 parent 69c3f44 commit 8ea872b

File tree

2 files changed

+14
-1
lines changed

2 files changed

+14
-1
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -299,6 +299,18 @@ object BooleanSimplification extends Rule[LogicalPlan] {
299299
case (_, _) => or
300300
}
301301

302+
case not @ Not(exp) =>
303+
exp match {
304+
case Literal(true, BooleanType) => Literal(false)
305+
case Literal(false, BooleanType) => Literal(true)
306+
case GreaterThan(l, r) => LessThanOrEqual(l, r)
307+
case GreaterThanOrEqual(l, r) => LessThan(l, r)
308+
case LessThan(l, r) => GreaterThanOrEqual(l, r)
309+
case LessThanOrEqual(l, r) => GreaterThan(l, r)
310+
case Not(e) => e
311+
case _ => not
312+
}
313+
302314
// Turn "if (true) a else b" into "a", and if (false) a else b" into "b".
303315
case e @ If(Literal(v, _), trueValue, falseValue) => if (v == true) trueValue else falseValue
304316
}

sql/core/src/main/scala/org/apache/spark/sql/execution/SparkStrategies.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -274,9 +274,10 @@ private[sql] abstract class SparkStrategies extends QueryPlanner[SparkPlan] {
274274
execution.Sample(fraction, withReplacement, seed, planLater(child)) :: Nil
275275
case SparkLogicalPlan(alreadyPlanned) => alreadyPlanned :: Nil
276276
case logical.LocalRelation(output, data) =>
277+
val nPartitions = if (data.isEmpty) 1 else numPartitions
277278
PhysicalRDD(
278279
output,
279-
RDDConversions.productToRowRdd(sparkContext.parallelize(data, numPartitions))) :: Nil
280+
RDDConversions.productToRowRdd(sparkContext.parallelize(data, nPartitions))) :: Nil
280281
case logical.Limit(IntegerLiteral(limit), child) =>
281282
execution.Limit(limit, planLater(child)) :: Nil
282283
case Unions(unionChildren) =>

0 commit comments

Comments
 (0)