change the MAX_SCALE for decimal

Author: chenghao-intel

sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala

@@ -106,8 +106,8 @@ object DecimalType extends AbstractDataType {
   import scala.math.min
 
   val MAX_PRECISION = 38
-  val MAX_SCALE = 38
-  val SYSTEM_DEFAULT: DecimalType = DecimalType(MAX_PRECISION, 18)
+  val MAX_SCALE = 18
+  val SYSTEM_DEFAULT: DecimalType = DecimalType(MAX_PRECISION, MAX_SCALE)
   val USER_DEFAULT: DecimalType = DecimalType(10, 0)
 
   @deprecated("Does not support unlimited precision, please specify the precision and scale", "1.5")

sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala

@@ -35,6 +35,7 @@ class MyDialect extends DefaultParserDialect
 
 class SQLQuerySuite extends QueryTest with SharedSQLContext {
   import testImplicits._
+  import BigDecimal.RoundingMode._
 
   setupTestData()
 
@@ -1614,17 +1615,23 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext {
     checkAnswer(sql("select 10.3000 * 3.0"), Row(BigDecimal("30.90000")))
     checkAnswer(sql("select 10.30000 * 30.0"), Row(BigDecimal("309.000000")))
     checkAnswer(sql("select 10.300000000000000000 * 3.000000000000000000"),
-      Row(BigDecimal("30.900000000000000000000000000000000000", new MathContext(38))))
-    checkAnswer(sql("select 10.300000000000000000 * 3.0000000000000000000"),
-      Row(null))
+      Row(BigDecimal("30.9", new MathContext(38))))
+    checkAnswer(sql("select 10.30000000000000000000 * 3.000000000000000000000"),
+      Row(BigDecimal("30.9", new MathContext(38))))
 
     checkAnswer(sql("select 10.3 / 3.0"), Row(BigDecimal("3.433333")))
     checkAnswer(sql("select 10.3000 / 3.0"), Row(BigDecimal("3.4333333")))
     checkAnswer(sql("select 10.30000 / 30.0"), Row(BigDecimal("0.343333333")))
     checkAnswer(sql("select 10.300000000000000000 / 3.00000000000000000"),
-      Row(BigDecimal("3.4333333333333333333333333333333333333", new MathContext(38))))
-    checkAnswer(sql("select 10.3000000000000000000 / 3.00000000000000000"),
-      Row(null))
+      Row(BigDecimal("3.4333333333333333333333333").setScale(DecimalType.MAX_SCALE, HALF_UP)))
+    checkAnswer(sql("select 10.300000000000000000000 / 3.0000000000000000000"),
+      Row(BigDecimal("3.4333333333333333333333333").setScale(DecimalType.MAX_SCALE, HALF_UP)))
+    checkAnswer(sql("select 1030000.0000000000000000 / 3.0000000000000000000"),
+      Row(BigDecimal("343333.333333333333333333").setScale(DecimalType.MAX_SCALE, HALF_UP)))
+    checkAnswer(sql("select 10300000000000000000 / 3.0000000000000000000"),
+      Row(BigDecimal("3433333333333333333.333333333333333333", new MathContext(38))))
+    checkAnswer(sql("select 1030000000000000000000000 / 0.1"),
+      Row(BigDecimal("10300000000000000000000000", new MathContext(38))))
   }
 
   test("external sorting updates peak execution memory") {
@@ -1686,4 +1693,25 @@ class SQLQuerySuite extends QueryTest with SharedSQLContext {
     checkAnswer(
       sql("SELECT IF(a > 0, a, 0) FROM (SELECT key a FROM src) temp"), Seq(Row(1), Row(0)))
   }
+
+  test("SPARK-10215 Div of Decimal returns null") {
+    val d = Decimal(1.12321)
+    val df = Seq((d, 1)).toDF("a", "b")
+
+    checkAnswer(
+      df.selectExpr("b * a / b"),
+      Seq(Row(d.toBigDecimal.setScale(DecimalType.MAX_SCALE, HALF_UP))))
+    checkAnswer(
+      df.selectExpr("b * a / b / b"),
+      Seq(Row(d.toBigDecimal.setScale(DecimalType.MAX_SCALE, HALF_UP))))
+    checkAnswer(
+      df.selectExpr("b * a + b"),
+      Seq(Row(BigDecimal(2.12321).setScale(DecimalType.MAX_SCALE, HALF_UP))))
+    checkAnswer(
+      df.selectExpr("b * a - b"),
+      Seq(Row(BigDecimal(0.12321).setScale(DecimalType.MAX_SCALE, HALF_UP))))
+    checkAnswer(
+      df.selectExpr("b * a * b"),
+      Seq(Row(d.toBigDecimal.setScale(DecimalType.MAX_SCALE, HALF_UP))))
+  }
 }