From c46e14ca203de5a1de46d788447813cf715ec262 Mon Sep 17 00:00:00 2001 From: Takeshi YAMAMURO Date: Tue, 27 Sep 2016 17:28:39 +0900 Subject: [PATCH 1/7] Support ArrayType in Literal.apply --- .../sql/catalyst/expressions/literals.scala | 39 +++++++++++++++++++ .../expressions/ExpressionEvalHelper.scala | 2 +- .../expressions/LiteralExpressionSuite.scala | 22 ++++++++++- 3 files changed, 61 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala index a597a17aadd99..bdade6a8ed0c0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala @@ -25,6 +25,7 @@ import javax.xml.bind.DatatypeConverter import org.json4s.JsonAST._ +import org.apache.spark.sql.AnalysisException import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow} import org.apache.spark.sql.catalyst.expressions.codegen._ import org.apache.spark.sql.catalyst.util.DateTimeUtils @@ -52,6 +53,11 @@ object Literal { case t: Timestamp => Literal(DateTimeUtils.fromJavaTimestamp(t), TimestampType) case d: Date => Literal(DateTimeUtils.fromJavaDate(d), DateType) case a: Array[Byte] => Literal(a, BinaryType) + case a: Array[_] => + val elementType = componentTypeToDataType(a.getClass.getComponentType()) + val dataType = ArrayType(elementType) + val convert = CatalystTypeConverters.createToCatalystConverter(dataType) + Literal(convert(a), dataType) case i: CalendarInterval => Literal(i, CalendarIntervalType) case null => Literal(null, NullType) case v: Literal => v @@ -59,6 +65,39 @@ object Literal { throw new RuntimeException("Unsupported literal type " + v.getClass + " " + v) } + private def componentTypeToDataType(clz: Class[_]): DataType = clz match { + // primitive type + case c: Class[_] if c == java.lang.Short.TYPE => ShortType + case c: Class[_] if c == java.lang.Integer.TYPE => IntegerType + case c: Class[_] if c == java.lang.Long.TYPE => LongType + case c: Class[_] if c == java.lang.Double.TYPE => DoubleType + case c: Class[_] if c == java.lang.Byte.TYPE => ByteType + case c: Class[_] if c == java.lang.Float.TYPE => FloatType + case c: Class[_] if c == java.lang.Boolean.TYPE => BooleanType + + // java class + case c: Class[_] if c == classOf[java.lang.String] => StringType + case c: Class[_] if c == classOf[java.sql.Date] => DateType + case c: Class[_] if c == classOf[java.sql.Timestamp] => TimestampType + case c: Class[_] if c == classOf[java.math.BigDecimal] => DecimalType.SYSTEM_DEFAULT + case c: Class[_] if c == classOf[Array[Byte]] => BinaryType + case c: Class[_] if c == classOf[java.lang.Short] => ShortType + case c: Class[_] if c == classOf[java.lang.Integer] => IntegerType + case c: Class[_] if c == classOf[java.lang.Long] => LongType + case c: Class[_] if c == classOf[java.lang.Double] => DoubleType + case c: Class[_] if c == classOf[java.lang.Byte] => ByteType + case c: Class[_] if c == classOf[java.lang.Float] => FloatType + case c: Class[_] if c == classOf[java.lang.Boolean] => BooleanType + + // scala class + case c: Class[_] if c == classOf[scala.math.BigInt] => DecimalType.SYSTEM_DEFAULT + case c: Class[_] if c == classOf[scala.math.BigDecimal] => DecimalType.SYSTEM_DEFAULT + + case c: Class[_] if c.isArray => ArrayType(componentTypeToDataType(c.getComponentType)) + + case c => throw new AnalysisException(s"Unsupported component type $c in arrays") + } + /** * Constructs a [[Literal]] of [[ObjectType]], for example when you need to pass an object * into code generation. diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala index f0c149c02b9aa..23ff6ead12c43 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow} import org.apache.spark.sql.catalyst.expressions.codegen._ import org.apache.spark.sql.catalyst.optimizer.SimpleTestOptimizer import org.apache.spark.sql.catalyst.plans.logical.{OneRowRelation, Project} -import org.apache.spark.sql.catalyst.util.MapData +import org.apache.spark.sql.catalyst.util.{ArrayData, MapData, GenericArrayData} import org.apache.spark.sql.types.DataType import org.apache.spark.util.Utils diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala index 450222d8cbba3..06c6f019753e0 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala @@ -21,6 +21,7 @@ import java.nio.charset.StandardCharsets import org.apache.spark.SparkFunSuite import org.apache.spark.sql.Row +import org.apache.spark.sql.catalyst.CatalystTypeConverters import org.apache.spark.sql.catalyst.util.DateTimeUtils import org.apache.spark.sql.types._ import org.apache.spark.unsafe.types.CalendarInterval @@ -43,6 +44,7 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper { checkEvaluation(Literal.create(null, TimestampType), null) checkEvaluation(Literal.create(null, CalendarIntervalType), null) checkEvaluation(Literal.create(null, ArrayType(ByteType, true)), null) + checkEvaluation(Literal.create(null, ArrayType(StringType, true)), null) checkEvaluation(Literal.create(null, MapType(StringType, IntegerType)), null) checkEvaluation(Literal.create(null, StructType(Seq.empty)), null) } @@ -122,5 +124,23 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper { } } - // TODO(davies): add tests for ArrayType, MapType and StructType + test("array") { + def toCatalyst(a: Array[_], elementType: DataType): Any = { + CatalystTypeConverters.createToCatalystConverter(ArrayType(elementType))(a) + } + checkEvaluation(Literal(Array(1, 2, 3)), toCatalyst(Array(1, 2, 3), IntegerType)) + checkEvaluation(Literal(Array("a", "b", "c")), toCatalyst(Array("a", "b", "c"), StringType)) + checkEvaluation(Literal(Array(1.0, 4.0)), toCatalyst(Array(1.0, 4.0), DoubleType)) + } + + test("unsupported types (map and struct) in literals") { + def checkUnsupportedTypeInLiteral(v: Any) = { + val errMsgMap = intercept[RuntimeException] { + Literal(v) + } + assert(errMsgMap.getMessage.startsWith("Unsupported literal type")) + } + checkUnsupportedTypeInLiteral(Map("key1" -> 1, "key2" ->2)) + checkUnsupportedTypeInLiteral(("mike", 29, 1.0)) + } } From e495ff580a18709485d834ac1457e9617f2b1412 Mon Sep 17 00:00:00 2001 From: Takeshi YAMAMURO Date: Tue, 27 Sep 2016 17:46:29 +0900 Subject: [PATCH 2/7] fix syntax errors --- .../spark/sql/catalyst/expressions/ExpressionEvalHelper.scala | 2 +- .../sql/catalyst/expressions/LiteralExpressionSuite.scala | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala index 23ff6ead12c43..f0c149c02b9aa 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala @@ -27,7 +27,7 @@ import org.apache.spark.sql.catalyst.{CatalystTypeConverters, InternalRow} import org.apache.spark.sql.catalyst.expressions.codegen._ import org.apache.spark.sql.catalyst.optimizer.SimpleTestOptimizer import org.apache.spark.sql.catalyst.plans.logical.{OneRowRelation, Project} -import org.apache.spark.sql.catalyst.util.{ArrayData, MapData, GenericArrayData} +import org.apache.spark.sql.catalyst.util.MapData import org.apache.spark.sql.types.DataType import org.apache.spark.util.Utils diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala index 06c6f019753e0..91a4692d0b1a0 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala @@ -134,13 +134,13 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper { } test("unsupported types (map and struct) in literals") { - def checkUnsupportedTypeInLiteral(v: Any) = { + def checkUnsupportedTypeInLiteral(v: Any): Unit = { val errMsgMap = intercept[RuntimeException] { Literal(v) } assert(errMsgMap.getMessage.startsWith("Unsupported literal type")) } - checkUnsupportedTypeInLiteral(Map("key1" -> 1, "key2" ->2)) + checkUnsupportedTypeInLiteral(Map("key1" -> 1, "key2" -> 2)) checkUnsupportedTypeInLiteral(("mike", 29, 1.0)) } } From c026826277b5c64f13caae205d10e62a933737b2 Mon Sep 17 00:00:00 2001 From: Takeshi YAMAMURO Date: Wed, 28 Sep 2016 10:09:07 +0900 Subject: [PATCH 3/7] Add comments --- .../apache/spark/sql/catalyst/expressions/literals.scala | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala index bdade6a8ed0c0..9fa0cf723f4ae 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala @@ -65,7 +65,12 @@ object Literal { throw new RuntimeException("Unsupported literal type " + v.getClass + " " + v) } - private def componentTypeToDataType(clz: Class[_]): DataType = clz match { + /** + * Returns the Spark SQL DataType for a given class object. Since this type needs to be resolved + * in runtime, we use match-case idioms for class objects here. However, there are similar + * functions in other files (e.g., HiveInspectors), so these functions need to merged into one. + */ + private[this] def componentTypeToDataType(clz: Class[_]): DataType = clz match { // primitive type case c: Class[_] if c == java.lang.Short.TYPE => ShortType case c: Class[_] if c == java.lang.Integer.TYPE => IntegerType From c9cdd29424c27c600a9e22c0b8ddf1a5998b6194 Mon Sep 17 00:00:00 2001 From: Takeshi YAMAMURO Date: Wed, 28 Sep 2016 14:26:57 +0900 Subject: [PATCH 4/7] Suport CalendarInterval --- .../sql/catalyst/expressions/literals.scala | 57 ++++++++++--------- .../expressions/LiteralExpressionSuite.scala | 15 +++-- 2 files changed, 41 insertions(+), 31 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala index 9fa0cf723f4ae..2da0655373119 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala @@ -17,12 +17,16 @@ package org.apache.spark.sql.catalyst.expressions +import java.lang.{Boolean => jBoolean, Byte => jByte, Double => jDouble, Float => jFloat, Integer => jInteger, Long => jLong, Short => jShort} +import java.math.{BigDecimal => jBigDecimal} import java.nio.charset.StandardCharsets import java.sql.{Date, Timestamp} import java.util import java.util.Objects import javax.xml.bind.DatatypeConverter +import scala.math.{BigDecimal, BigInt} + import org.json4s.JsonAST._ import org.apache.spark.sql.AnalysisException @@ -47,7 +51,7 @@ object Literal { case s: String => Literal(UTF8String.fromString(s), StringType) case b: Boolean => Literal(b, BooleanType) case d: BigDecimal => Literal(Decimal(d), DecimalType(Math.max(d.precision, d.scale), d.scale)) - case d: java.math.BigDecimal => + case d: jBigDecimal => Literal(Decimal(d), DecimalType(Math.max(d.precision, d.scale), d.scale())) case d: Decimal => Literal(d, DecimalType(Math.max(d.precision, d.scale), d.scale)) case t: Timestamp => Literal(DateTimeUtils.fromJavaTimestamp(t), TimestampType) @@ -71,32 +75,33 @@ object Literal { * functions in other files (e.g., HiveInspectors), so these functions need to merged into one. */ private[this] def componentTypeToDataType(clz: Class[_]): DataType = clz match { - // primitive type - case c: Class[_] if c == java.lang.Short.TYPE => ShortType - case c: Class[_] if c == java.lang.Integer.TYPE => IntegerType - case c: Class[_] if c == java.lang.Long.TYPE => LongType - case c: Class[_] if c == java.lang.Double.TYPE => DoubleType - case c: Class[_] if c == java.lang.Byte.TYPE => ByteType - case c: Class[_] if c == java.lang.Float.TYPE => FloatType - case c: Class[_] if c == java.lang.Boolean.TYPE => BooleanType - - // java class - case c: Class[_] if c == classOf[java.lang.String] => StringType - case c: Class[_] if c == classOf[java.sql.Date] => DateType - case c: Class[_] if c == classOf[java.sql.Timestamp] => TimestampType - case c: Class[_] if c == classOf[java.math.BigDecimal] => DecimalType.SYSTEM_DEFAULT + // primitive types + case c: Class[_] if c == jShort.TYPE => ShortType + case c: Class[_] if c == jInteger.TYPE => IntegerType + case c: Class[_] if c == jLong.TYPE => LongType + case c: Class[_] if c == jDouble.TYPE => DoubleType + case c: Class[_] if c == jByte.TYPE => ByteType + case c: Class[_] if c == jFloat.TYPE => FloatType + case c: Class[_] if c == jBoolean.TYPE => BooleanType + + // java classes + case c: Class[_] if c == classOf[Date] => DateType + case c: Class[_] if c == classOf[Timestamp] => TimestampType + case c: Class[_] if c == classOf[jBigDecimal] => DecimalType.SYSTEM_DEFAULT case c: Class[_] if c == classOf[Array[Byte]] => BinaryType - case c: Class[_] if c == classOf[java.lang.Short] => ShortType - case c: Class[_] if c == classOf[java.lang.Integer] => IntegerType - case c: Class[_] if c == classOf[java.lang.Long] => LongType - case c: Class[_] if c == classOf[java.lang.Double] => DoubleType - case c: Class[_] if c == classOf[java.lang.Byte] => ByteType - case c: Class[_] if c == classOf[java.lang.Float] => FloatType - case c: Class[_] if c == classOf[java.lang.Boolean] => BooleanType - - // scala class - case c: Class[_] if c == classOf[scala.math.BigInt] => DecimalType.SYSTEM_DEFAULT - case c: Class[_] if c == classOf[scala.math.BigDecimal] => DecimalType.SYSTEM_DEFAULT + case c: Class[_] if c == classOf[jShort] => ShortType + case c: Class[_] if c == classOf[jInteger] => IntegerType + case c: Class[_] if c == classOf[jLong] => LongType + case c: Class[_] if c == classOf[jDouble] => DoubleType + case c: Class[_] if c == classOf[jByte] => ByteType + case c: Class[_] if c == classOf[jFloat] => FloatType + case c: Class[_] if c == classOf[jBoolean] => BooleanType + + // other scala classes + case c: Class[_] if c == classOf[String] => StringType + case c: Class[_] if c == classOf[BigInt] => DecimalType.SYSTEM_DEFAULT + case c: Class[_] if c == classOf[BigDecimal] => DecimalType.SYSTEM_DEFAULT + case c: Class[_] if c == classOf[CalendarInterval] => CalendarIntervalType case c: Class[_] if c.isArray => ArrayType(componentTypeToDataType(c.getComponentType)) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala index 91a4692d0b1a0..4af4da8a9f0c2 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/LiteralExpressionSuite.scala @@ -125,12 +125,17 @@ class LiteralExpressionSuite extends SparkFunSuite with ExpressionEvalHelper { } test("array") { - def toCatalyst(a: Array[_], elementType: DataType): Any = { - CatalystTypeConverters.createToCatalystConverter(ArrayType(elementType))(a) + def checkArrayLiteral(a: Array[_], elementType: DataType): Unit = { + val toCatalyst = (a: Array[_], elementType: DataType) => { + CatalystTypeConverters.createToCatalystConverter(ArrayType(elementType))(a) + } + checkEvaluation(Literal(a), toCatalyst(a, elementType)) } - checkEvaluation(Literal(Array(1, 2, 3)), toCatalyst(Array(1, 2, 3), IntegerType)) - checkEvaluation(Literal(Array("a", "b", "c")), toCatalyst(Array("a", "b", "c"), StringType)) - checkEvaluation(Literal(Array(1.0, 4.0)), toCatalyst(Array(1.0, 4.0), DoubleType)) + checkArrayLiteral(Array(1, 2, 3), IntegerType) + checkArrayLiteral(Array("a", "b", "c"), StringType) + checkArrayLiteral(Array(1.0, 4.0), DoubleType) + checkArrayLiteral(Array(CalendarInterval.MICROS_PER_DAY, CalendarInterval.MICROS_PER_HOUR), + CalendarIntervalType) } test("unsupported types (map and struct) in literals") { From c843840741d235350f5c1ee613be76beb20897b8 Mon Sep 17 00:00:00 2001 From: Takeshi YAMAMURO Date: Thu, 29 Sep 2016 08:57:32 +0900 Subject: [PATCH 5/7] Rename aliased java class names --- .../sql/catalyst/expressions/literals.scala | 42 +++++++++++-------- 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala index 2da0655373119..a7fe911cad644 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala @@ -17,8 +17,14 @@ package org.apache.spark.sql.catalyst.expressions -import java.lang.{Boolean => jBoolean, Byte => jByte, Double => jDouble, Float => jFloat, Integer => jInteger, Long => jLong, Short => jShort} -import java.math.{BigDecimal => jBigDecimal} +import java.lang.{Boolean => JavaBoolean} +import java.lang.{Byte => JavaByte} +import java.lang.{Double => JavaDouble} +import java.lang.{Float => JavaFloat} +import java.lang.{Integer => JavaInteger} +import java.lang.{Long => JavaLong} +import java.lang.{Short => JavaShort} +import java.math.{BigDecimal => JavaBigDecimal} import java.nio.charset.StandardCharsets import java.sql.{Date, Timestamp} import java.util @@ -51,7 +57,7 @@ object Literal { case s: String => Literal(UTF8String.fromString(s), StringType) case b: Boolean => Literal(b, BooleanType) case d: BigDecimal => Literal(Decimal(d), DecimalType(Math.max(d.precision, d.scale), d.scale)) - case d: jBigDecimal => + case d: JavaBigDecimal => Literal(Decimal(d), DecimalType(Math.max(d.precision, d.scale), d.scale())) case d: Decimal => Literal(d, DecimalType(Math.max(d.precision, d.scale), d.scale)) case t: Timestamp => Literal(DateTimeUtils.fromJavaTimestamp(t), TimestampType) @@ -76,26 +82,26 @@ object Literal { */ private[this] def componentTypeToDataType(clz: Class[_]): DataType = clz match { // primitive types - case c: Class[_] if c == jShort.TYPE => ShortType - case c: Class[_] if c == jInteger.TYPE => IntegerType - case c: Class[_] if c == jLong.TYPE => LongType - case c: Class[_] if c == jDouble.TYPE => DoubleType - case c: Class[_] if c == jByte.TYPE => ByteType - case c: Class[_] if c == jFloat.TYPE => FloatType - case c: Class[_] if c == jBoolean.TYPE => BooleanType + case c: Class[_] if c == JavaShort.TYPE => ShortType + case c: Class[_] if c == JavaInteger.TYPE => IntegerType + case c: Class[_] if c == JavaLong.TYPE => LongType + case c: Class[_] if c == JavaDouble.TYPE => DoubleType + case c: Class[_] if c == JavaByte.TYPE => ByteType + case c: Class[_] if c == JavaFloat.TYPE => FloatType + case c: Class[_] if c == JavaBoolean.TYPE => BooleanType // java classes case c: Class[_] if c == classOf[Date] => DateType case c: Class[_] if c == classOf[Timestamp] => TimestampType - case c: Class[_] if c == classOf[jBigDecimal] => DecimalType.SYSTEM_DEFAULT + case c: Class[_] if c == classOf[JavaBigDecimal] => DecimalType.SYSTEM_DEFAULT case c: Class[_] if c == classOf[Array[Byte]] => BinaryType - case c: Class[_] if c == classOf[jShort] => ShortType - case c: Class[_] if c == classOf[jInteger] => IntegerType - case c: Class[_] if c == classOf[jLong] => LongType - case c: Class[_] if c == classOf[jDouble] => DoubleType - case c: Class[_] if c == classOf[jByte] => ByteType - case c: Class[_] if c == classOf[jFloat] => FloatType - case c: Class[_] if c == classOf[jBoolean] => BooleanType + case c: Class[_] if c == classOf[JavaShort] => ShortType + case c: Class[_] if c == classOf[JavaInteger] => IntegerType + case c: Class[_] if c == classOf[JavaLong] => LongType + case c: Class[_] if c == classOf[JavaDouble] => DoubleType + case c: Class[_] if c == classOf[JavaByte] => ByteType + case c: Class[_] if c == classOf[JavaFloat] => FloatType + case c: Class[_] if c == classOf[JavaBoolean] => BooleanType // other scala classes case c: Class[_] if c == classOf[String] => StringType From 41b92764b0f8b6de685b985913bafcf664f90f0c Mon Sep 17 00:00:00 2001 From: Takeshi YAMAMURO Date: Fri, 30 Sep 2016 08:39:53 +0900 Subject: [PATCH 6/7] Simplify entries --- .../spark/sql/catalyst/expressions/literals.scala | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala index a7fe911cad644..8b4972745ebbd 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala @@ -82,13 +82,13 @@ object Literal { */ private[this] def componentTypeToDataType(clz: Class[_]): DataType = clz match { // primitive types - case c: Class[_] if c == JavaShort.TYPE => ShortType - case c: Class[_] if c == JavaInteger.TYPE => IntegerType - case c: Class[_] if c == JavaLong.TYPE => LongType - case c: Class[_] if c == JavaDouble.TYPE => DoubleType - case c: Class[_] if c == JavaByte.TYPE => ByteType - case c: Class[_] if c == JavaFloat.TYPE => FloatType - case c: Class[_] if c == JavaBoolean.TYPE => BooleanType + case JavaShort.TYPE => ShortType + case JavaInteger.TYPE => IntegerType + case JavaLong.TYPE => LongType + case JavaDouble.TYPE => DoubleType + case JavaByte.TYPE => ByteType + case JavaFloat.TYPE => FloatType + case JavaBoolean.TYPE => BooleanType // java classes case c: Class[_] if c == classOf[Date] => DateType From ebee7e4c72e4bc415aee907b56f43cef1c2f0ae7 Mon Sep 17 00:00:00 2001 From: Takeshi YAMAMURO Date: Fri, 30 Sep 2016 09:54:50 +0900 Subject: [PATCH 7/7] Simplify patten matching --- .../sql/catalyst/expressions/literals.scala | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala index 8b4972745ebbd..1985e68c94e2d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/literals.scala @@ -91,27 +91,27 @@ object Literal { case JavaBoolean.TYPE => BooleanType // java classes - case c: Class[_] if c == classOf[Date] => DateType - case c: Class[_] if c == classOf[Timestamp] => TimestampType - case c: Class[_] if c == classOf[JavaBigDecimal] => DecimalType.SYSTEM_DEFAULT - case c: Class[_] if c == classOf[Array[Byte]] => BinaryType - case c: Class[_] if c == classOf[JavaShort] => ShortType - case c: Class[_] if c == classOf[JavaInteger] => IntegerType - case c: Class[_] if c == classOf[JavaLong] => LongType - case c: Class[_] if c == classOf[JavaDouble] => DoubleType - case c: Class[_] if c == classOf[JavaByte] => ByteType - case c: Class[_] if c == classOf[JavaFloat] => FloatType - case c: Class[_] if c == classOf[JavaBoolean] => BooleanType + case _ if clz == classOf[Date] => DateType + case _ if clz == classOf[Timestamp] => TimestampType + case _ if clz == classOf[JavaBigDecimal] => DecimalType.SYSTEM_DEFAULT + case _ if clz == classOf[Array[Byte]] => BinaryType + case _ if clz == classOf[JavaShort] => ShortType + case _ if clz == classOf[JavaInteger] => IntegerType + case _ if clz == classOf[JavaLong] => LongType + case _ if clz == classOf[JavaDouble] => DoubleType + case _ if clz == classOf[JavaByte] => ByteType + case _ if clz == classOf[JavaFloat] => FloatType + case _ if clz == classOf[JavaBoolean] => BooleanType // other scala classes - case c: Class[_] if c == classOf[String] => StringType - case c: Class[_] if c == classOf[BigInt] => DecimalType.SYSTEM_DEFAULT - case c: Class[_] if c == classOf[BigDecimal] => DecimalType.SYSTEM_DEFAULT - case c: Class[_] if c == classOf[CalendarInterval] => CalendarIntervalType + case _ if clz == classOf[String] => StringType + case _ if clz == classOf[BigInt] => DecimalType.SYSTEM_DEFAULT + case _ if clz == classOf[BigDecimal] => DecimalType.SYSTEM_DEFAULT + case _ if clz == classOf[CalendarInterval] => CalendarIntervalType - case c: Class[_] if c.isArray => ArrayType(componentTypeToDataType(c.getComponentType)) + case _ if clz.isArray => ArrayType(componentTypeToDataType(clz.getComponentType)) - case c => throw new AnalysisException(s"Unsupported component type $c in arrays") + case _ => throw new AnalysisException(s"Unsupported component type $clz in arrays") } /**