Skip to content

Commit 40df5d4

Browse files
Davies Liurxin
Davies Liu
authored and
rxin
committed
[SPARK-6663] [SQL] use Literal.create instread of constructor
In order to do inbound checking and type conversion, we should use Literal.create() instead of constructor. Author: Davies Liu <[email protected]> Closes #5320 from davies/literal and squashes the following commits: 1667604 [Davies Liu] fix style and add comment 5f8c0fd [Davies Liu] use Literal.create instread of constructor
1 parent 2bc7fe7 commit 40df5d4

File tree

16 files changed

+220
-213
lines changed

16 files changed

+220
-213
lines changed

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -316,13 +316,13 @@ class SqlParser extends AbstractSparkSQLParser with DataTypeParser {
316316
protected lazy val literal: Parser[Literal] =
317317
( numericLiteral
318318
| booleanLiteral
319-
| stringLit ^^ {case s => Literal(s, StringType) }
320-
| NULL ^^^ Literal(null, NullType)
319+
| stringLit ^^ {case s => Literal.create(s, StringType) }
320+
| NULL ^^^ Literal.create(null, NullType)
321321
)
322322

323323
protected lazy val booleanLiteral: Parser[Literal] =
324-
( TRUE ^^^ Literal(true, BooleanType)
325-
| FALSE ^^^ Literal(false, BooleanType)
324+
( TRUE ^^^ Literal.create(true, BooleanType)
325+
| FALSE ^^^ Literal.create(false, BooleanType)
326326
)
327327

328328
protected lazy val numericLiteral: Parser[Literal] =

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -140,10 +140,10 @@ class Analyzer(
140140
case x: Expression if nonSelectedGroupExprSet.contains(x) =>
141141
// if the input attribute in the Invalid Grouping Expression set of for this group
142142
// replace it with constant null
143-
Literal(null, expr.dataType)
143+
Literal.create(null, expr.dataType)
144144
case x if x == g.gid =>
145145
// replace the groupingId with concrete value (the bit mask)
146-
Literal(bitmask, IntegerType)
146+
Literal.create(bitmask, IntegerType)
147147
})
148148

149149
result += GroupExpression(substitution)

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,7 @@ trait HiveTypeCoercion {
115115
* the appropriate numeric equivalent.
116116
*/
117117
object ConvertNaNs extends Rule[LogicalPlan] {
118-
val stringNaN = Literal("NaN", StringType)
118+
val stringNaN = Literal.create("NaN", StringType)
119119

120120
def apply(plan: LogicalPlan): LogicalPlan = plan transform {
121121
case q: LogicalPlan => q transformExpressions {

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregates.scala‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -505,7 +505,8 @@ case class AverageFunction(expr: Expression, base: AggregateExpression)
505505
private var count: Long = _
506506
private val sum = MutableLiteral(zero.eval(null), calcType)
507507

508-
private def addFunction(value: Any) = Add(sum, Cast(Literal(value, expr.dataType), calcType))
508+
private def addFunction(value: Any) = Add(sum,
509+
Cast(Literal.create(value, expr.dataType), calcType))
509510

510511
override def eval(input: Row): Any = {
511512
if (count == 0L) {

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ object Literal {
4141
case _ =>
4242
throw new RuntimeException("Unsupported literal type " + v.getClass + " " + v)
4343
}
44+
45+
def create(v: Any, dataType: DataType): Literal = Literal(v, dataType)
4446
}
4547

4648
/**
@@ -62,7 +64,10 @@ object IntegerLiteral {
6264
}
6365
}
6466

65-
case class Literal(value: Any, dataType: DataType) extends LeafExpression {
67+
/**
68+
* In order to do type checking, use Literal.create() instead of constructor
69+
*/
70+
case class Literal protected (value: Any, dataType: DataType) extends LeafExpression {
6671

6772
override def foldable: Boolean = true
6873
override def nullable: Boolean = value == null

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -218,12 +218,12 @@ object NullPropagation extends Rule[LogicalPlan] {
218218
def apply(plan: LogicalPlan): LogicalPlan = plan transform {
219219
case q: LogicalPlan => q transformExpressionsUp {
220220
case e @ Count(Literal(null, _)) => Cast(Literal(0L), e.dataType)
221-
case e @ IsNull(c) if !c.nullable => Literal(false, BooleanType)
222-
case e @ IsNotNull(c) if !c.nullable => Literal(true, BooleanType)
223-
case e @ GetItem(Literal(null, _), _) => Literal(null, e.dataType)
224-
case e @ GetItem(_, Literal(null, _)) => Literal(null, e.dataType)
225-
case e @ StructGetField(Literal(null, _), _, _) => Literal(null, e.dataType)
226-
case e @ ArrayGetField(Literal(null, _), _, _, _) => Literal(null, e.dataType)
221+
case e @ IsNull(c) if !c.nullable => Literal.create(false, BooleanType)
222+
case e @ IsNotNull(c) if !c.nullable => Literal.create(true, BooleanType)
223+
case e @ GetItem(Literal(null, _), _) => Literal.create(null, e.dataType)
224+
case e @ GetItem(_, Literal(null, _)) => Literal.create(null, e.dataType)
225+
case e @ StructGetField(Literal(null, _), _, _) => Literal.create(null, e.dataType)
226+
case e @ ArrayGetField(Literal(null, _), _, _, _) => Literal.create(null, e.dataType)
227227
case e @ EqualNullSafe(Literal(null, _), r) => IsNull(r)
228228
case e @ EqualNullSafe(l, Literal(null, _)) => IsNull(l)
229229
case e @ Count(expr) if !expr.nullable => Count(Literal(1))
@@ -235,36 +235,36 @@ object NullPropagation extends Rule[LogicalPlan] {
235235
case _ => true
236236
}
237237
if (newChildren.length == 0) {
238-
Literal(null, e.dataType)
238+
Literal.create(null, e.dataType)
239239
} else if (newChildren.length == 1) {
240240
newChildren(0)
241241
} else {
242242
Coalesce(newChildren)
243243
}
244244

245-
case e @ Substring(Literal(null, _), _, _) => Literal(null, e.dataType)
246-
case e @ Substring(_, Literal(null, _), _) => Literal(null, e.dataType)
247-
case e @ Substring(_, _, Literal(null, _)) => Literal(null, e.dataType)
245+
case e @ Substring(Literal(null, _), _, _) => Literal.create(null, e.dataType)
246+
case e @ Substring(_, Literal(null, _), _) => Literal.create(null, e.dataType)
247+
case e @ Substring(_, _, Literal(null, _)) => Literal.create(null, e.dataType)
248248

249249
// Put exceptional cases above if any
250250
case e: BinaryArithmetic => e.children match {
251-
case Literal(null, _) :: right :: Nil => Literal(null, e.dataType)
252-
case left :: Literal(null, _) :: Nil => Literal(null, e.dataType)
251+
case Literal(null, _) :: right :: Nil => Literal.create(null, e.dataType)
252+
case left :: Literal(null, _) :: Nil => Literal.create(null, e.dataType)
253253
case _ => e
254254
}
255255
case e: BinaryComparison => e.children match {
256-
case Literal(null, _) :: right :: Nil => Literal(null, e.dataType)
257-
case left :: Literal(null, _) :: Nil => Literal(null, e.dataType)
256+
case Literal(null, _) :: right :: Nil => Literal.create(null, e.dataType)
257+
case left :: Literal(null, _) :: Nil => Literal.create(null, e.dataType)
258258
case _ => e
259259
}
260260
case e: StringRegexExpression => e.children match {
261-
case Literal(null, _) :: right :: Nil => Literal(null, e.dataType)
262-
case left :: Literal(null, _) :: Nil => Literal(null, e.dataType)
261+
case Literal(null, _) :: right :: Nil => Literal.create(null, e.dataType)
262+
case left :: Literal(null, _) :: Nil => Literal.create(null, e.dataType)
263263
case _ => e
264264
}
265265
case e: StringComparison => e.children match {
266-
case Literal(null, _) :: right :: Nil => Literal(null, e.dataType)
267-
case left :: Literal(null, _) :: Nil => Literal(null, e.dataType)
266+
case Literal(null, _) :: right :: Nil => Literal.create(null, e.dataType)
267+
case left :: Literal(null, _) :: Nil => Literal.create(null, e.dataType)
268268
case _ => e
269269
}
270270
}
@@ -284,13 +284,13 @@ object ConstantFolding extends Rule[LogicalPlan] {
284284
case l: Literal => l
285285

286286
// Fold expressions that are foldable.
287-
case e if e.foldable => Literal(e.eval(null), e.dataType)
287+
case e if e.foldable => Literal.create(e.eval(null), e.dataType)
288288

289289
// Fold "literal in (item1, item2, ..., literal, ...)" into true directly.
290290
case In(Literal(v, _), list) if list.exists {
291291
case Literal(candidate, _) if candidate == v => true
292292
case _ => false
293-
} => Literal(true, BooleanType)
293+
} => Literal.create(true, BooleanType)
294294
}
295295
}
296296
}
@@ -647,7 +647,7 @@ object DecimalAggregates extends Rule[LogicalPlan] {
647647

648648
case Average(e @ DecimalType.Expression(prec, scale)) if prec + 4 <= MAX_DOUBLE_DIGITS =>
649649
Cast(
650-
Divide(Average(UnscaledValue(e)), Literal(math.pow(10.0, scale), DoubleType)),
650+
Divide(Average(UnscaledValue(e)), Literal.create(math.pow(10.0, scale), DoubleType)),
651651
DecimalType(prec + 4, scale + 4))
652652
}
653653
}

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercionSuite.scala‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -127,11 +127,11 @@ class HiveTypeCoercionSuite extends PlanTest {
127127
ruleTest(
128128
Coalesce(Literal(1.0)
129129
:: Literal(1)
130-
:: Literal(1.0, FloatType)
130+
:: Literal.create(1.0, FloatType)
131131
:: Nil),
132132
Coalesce(Cast(Literal(1.0), DoubleType)
133133
:: Cast(Literal(1), DoubleType)
134-
:: Cast(Literal(1.0, FloatType), DoubleType)
134+
:: Cast(Literal.create(1.0, FloatType), DoubleType)
135135
:: Nil))
136136
ruleTest(
137137
Coalesce(Literal(1L)

0 commit comments

Comments
 (0)