Skip to content

AL-6084 in Cast for method of canCast, when DecimalType cast to DoubleType add transformable logic #542

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
songzhxlh-max merged 2 commits into from
Sep 22, 2022
Merged

AL-6084 in Cast for method of canCast, when DecimalType cast to DoubleType add transformable logic #542

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
230c758
AL-6084 in Cast for method of canCast, when DecimalType cast DecimalT…
songzhxlh-max Jun 28, 2022
5630cc1
fix ci
songzhxlh-max Sep 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions sql/catalyst/src/main/scala/org/apache/spark/sql/types/DecimalType.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,8 @@ case class DecimalType(precision: Int, scale: Int) extends FractionalType {
(precision - scale) <= (dt.precision - dt.scale) && scale <= dt.scale
case dt: IntegralType =>
isTighterThan(DecimalType.forType(dt))
case dt: DoubleType =>
isTighterThan(DecimalType(precision, scale))
case _ => false
}

Expand Down
6 changes: 6 additions & 0 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/types/DataTypeWriteCompatibilitySuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,12 @@ class StrictDataTypeWriteCompatibilitySuite extends DataTypeWriteCompatibilityBa
}
}

test("Check cast: decimalType to doubleType allowed") {
assert(canCast(DecimalType(30, 15).defaultConcreteType, DoubleType))
assert(canCast(DecimalType(38, 18).defaultConcreteType, DoubleType))
assert(canCast(DecimalType(38, 6).defaultConcreteType, DoubleType))
}

test("Check array types: unsafe casts are not allowed") {
val arrayOfLong = ArrayType(LongType)
val arrayOfInt = ArrayType(IntegerType)
Expand Down
13 changes: 5 additions & 8 deletions sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCV2Suite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1255,13 +1255,9 @@ class JDBCV2Suite extends QueryTest with SharedSparkSession with ExplainSuiteHel

val df4 = spark.table("h2.test.employee")
.filter(($"salary" > 1000d).and($"salary" < 12000d))
checkFiltersRemoved(df4, ansiMode)
val expectedPlanFragment4 = if (ansiMode) {
"PushedFilters: [SALARY IS NOT NULL, " +
"CAST(SALARY AS double) > 1000.0, CAST(SALARY AS double) < 12000.0], "
} else {
"PushedFilters: [SALARY IS NOT NULL], "
}
checkFiltersRemoved(df4)
val expectedPlanFragment4 = "PushedFilters: [SALARY IS NOT NULL, " +
"SALARY > 1000.00, SALARY < 12000.00], "
checkPushedInfo(df4, expectedPlanFragment4)
checkAnswer(df4, Seq(Row(1, "amy", 10000, 1000, true),
Row(1, "cathy", 9000, 1200, false), Row(2, "david", 10000, 1300, true)))
Expand All @@ -1274,7 +1270,8 @@ class JDBCV2Suite extends QueryTest with SharedSparkSession with ExplainSuiteHel
"PushedFilters: [DEPT IS NOT NULL, ABS(DEPT - 3) > 1, " +
"(COALESCE(CAST(SALARY AS double), BONUS)) > 2000.0]"
} else {
"PushedFilters: [DEPT IS NOT NULL]"
"PushedFilters: [DEPT IS NOT NULL, " +
"(COALESCE(CAST(SALARY AS double), BONUS)) > 2000.0]"
}
checkPushedInfo(df5, expectedPlanFragment5)
checkAnswer(df5, Seq(Row(1, "amy", 10000, 1000, true),
Expand Down
5 changes: 0 additions & 5 deletions sql/core/src/test/scala/org/apache/spark/sql/sources/InsertSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -625,11 +625,6 @@ class InsertSuite extends DataSourceTest with SharedSparkSession {
}.getMessage
assert(msg.contains("Cannot safely cast 'i': bigint to int"))

msg = intercept[AnalysisException] {
sql("insert into t select 1, 2.0")
}.getMessage
assert(msg.contains("Cannot safely cast 'd': decimal(2,1) to double"))

msg = intercept[AnalysisException] {
sql("insert into t select 1, 2.0D, 3")
}.getMessage
Expand Down