Skip to content

Commit 8e96acf

Browse files
committed
[SPARK-20211][SQL] Fix the Precision and Scale of Decimal Values when the Input is BigDecimal between -1.0 and 1.0
### What changes were proposed in this pull request? The precision and scale of decimal values are wrong when the input is BigDecimal between -1.0 and 1.0. The BigDecimal's precision is the digit count starts from the leftmost nonzero digit based on the [JAVA's BigDecimal definition](https://docs.oracle.com/javase/7/docs/api/java/math/BigDecimal.html). However, our Decimal decision follows the database decimal standard, which is the total number of digits, including both to the left and the right of the decimal point. Thus, this PR is to fix the issue by doing the conversion. Before this PR, the following queries failed: ```SQL select 1 > 0.0001 select floor(0.0001) select ceil(0.0001) ``` ### How was this patch tested? Added test cases. Author: Xiao Li <[email protected]> Closes #18244 from gatorsmile/bigdecimal.
1 parent b78e384 commit 8e96acf

File tree

4 files changed

+75
-10
lines changed

4 files changed

+75
-10
lines changed

sql/catalyst/src/main/scala/org/apache/spark/sql/types/Decimal.scala

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,15 @@ final class Decimal extends Ordered[Decimal] with Serializable {
126126
def set(decimal: BigDecimal): Decimal = {
127127
this.decimalVal = decimal
128128
this.longVal = 0L
129-
this._precision = decimal.precision
129+
if (decimal.precision <= decimal.scale) {
130+
// For Decimal, we expect the precision is equal to or large than the scale, however,
131+
// in BigDecimal, the digit count starts from the leftmost nonzero digit of the exact
132+
// result. For example, the precision of 0.01 equals to 1 based on the definition, but
133+
// the scale is 2. The expected precision should be 3.
134+
this._precision = decimal.scale + 1
135+
} else {
136+
this._precision = decimal.precision
137+
}
130138
this._scale = decimal.scale
131139
this
132140
}

sql/catalyst/src/test/scala/org/apache/spark/sql/types/DecimalSuite.scala

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,16 @@ class DecimalSuite extends SparkFunSuite with PrivateMethodTester {
3232

3333
test("creating decimals") {
3434
checkDecimal(new Decimal(), "0", 1, 0)
35+
checkDecimal(Decimal(BigDecimal("0.09")), "0.09", 3, 2)
36+
checkDecimal(Decimal(BigDecimal("0.9")), "0.9", 2, 1)
37+
checkDecimal(Decimal(BigDecimal("0.90")), "0.90", 3, 2)
38+
checkDecimal(Decimal(BigDecimal("0.0")), "0.0", 2, 1)
39+
checkDecimal(Decimal(BigDecimal("0")), "0", 1, 0)
40+
checkDecimal(Decimal(BigDecimal("1.0")), "1.0", 2, 1)
41+
checkDecimal(Decimal(BigDecimal("-0.09")), "-0.09", 3, 2)
42+
checkDecimal(Decimal(BigDecimal("-0.9")), "-0.9", 2, 1)
43+
checkDecimal(Decimal(BigDecimal("-0.90")), "-0.90", 3, 2)
44+
checkDecimal(Decimal(BigDecimal("-1.0")), "-1.0", 2, 1)
3545
checkDecimal(Decimal(BigDecimal("10.030")), "10.030", 5, 3)
3646
checkDecimal(Decimal(BigDecimal("10.030"), 4, 1), "10.0", 4, 1)
3747
checkDecimal(Decimal(BigDecimal("-9.95"), 4, 1), "-10.0", 4, 1)

sql/core/src/test/resources/sql-tests/inputs/operators.sql

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,8 +65,15 @@ select ceiling(0);
6565
select ceiling(1);
6666
select ceil(1234567890123456);
6767
select ceiling(1234567890123456);
68+
select ceil(0.01);
69+
select ceiling(-0.10);
6870

6971
-- floor
7072
select floor(0);
7173
select floor(1);
7274
select floor(1234567890123456);
75+
select floor(0.01);
76+
select floor(-0.10);
77+
78+
-- comparison operator
79+
select 1 > 0.00001

sql/core/src/test/resources/sql-tests/results/operators.sql.out

Lines changed: 49 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
-- Automatically generated by SQLQueryTestSuite
2-
-- Number of queries: 45
2+
-- Number of queries: 50
33

44

55
-- !query 0
@@ -351,24 +351,64 @@ struct<CEIL(1234567890123456):bigint>
351351

352352

353353
-- !query 42
354-
select floor(0)
354+
select ceil(0.01)
355355
-- !query 42 schema
356-
struct<FLOOR(CAST(0 AS DOUBLE)):bigint>
356+
struct<CEIL(0.01):decimal(1,0)>
357357
-- !query 42 output
358-
0
358+
1
359359

360360

361361
-- !query 43
362-
select floor(1)
362+
select ceiling(-0.10)
363363
-- !query 43 schema
364-
struct<FLOOR(CAST(1 AS DOUBLE)):bigint>
364+
struct<CEIL(-0.10):decimal(1,0)>
365365
-- !query 43 output
366-
1
366+
0
367367

368368

369369
-- !query 44
370-
select floor(1234567890123456)
370+
select floor(0)
371371
-- !query 44 schema
372-
struct<FLOOR(1234567890123456):bigint>
372+
struct<FLOOR(CAST(0 AS DOUBLE)):bigint>
373373
-- !query 44 output
374+
0
375+
376+
377+
-- !query 45
378+
select floor(1)
379+
-- !query 45 schema
380+
struct<FLOOR(CAST(1 AS DOUBLE)):bigint>
381+
-- !query 45 output
382+
1
383+
384+
385+
-- !query 46
386+
select floor(1234567890123456)
387+
-- !query 46 schema
388+
struct<FLOOR(1234567890123456):bigint>
389+
-- !query 46 output
374390
1234567890123456
391+
392+
393+
-- !query 47
394+
select floor(0.01)
395+
-- !query 47 schema
396+
struct<FLOOR(0.01):decimal(1,0)>
397+
-- !query 47 output
398+
0
399+
400+
401+
-- !query 48
402+
select floor(-0.10)
403+
-- !query 48 schema
404+
struct<FLOOR(-0.10):decimal(1,0)>
405+
-- !query 48 output
406+
-1
407+
408+
409+
-- !query 49
410+
select 1 > 0.00001
411+
-- !query 49 schema
412+
struct<(CAST(1 AS BIGINT) > 0):boolean>
413+
-- !query 49 output
414+
true

0 commit comments

Comments
 (0)