Skip to content

Commit dfde7d3

Browse files
viiryajeanlyn
authored andcommitted
[SPARK-7277] [SQL] Throw exception if the property mapred.reduce.tasks is set to -1
JIRA: https://issues.apache.org/jira/browse/SPARK-7277 As automatically determining the number of reducers is not supported (`mapred.reduce.tasks` is set to `-1`), we should throw exception to users. Author: Liang-Chi Hsieh <[email protected]> Closes apache#5811 from viirya/no_neg_reduce_tasks and squashes the following commits: e518f96 [Liang-Chi Hsieh] Consider other wrong setting values. fd9c817 [Liang-Chi Hsieh] Merge remote-tracking branch 'upstream/master' into no_neg_reduce_tasks 4ede705 [Liang-Chi Hsieh] Throw exception instead of warning message. 68a1c70 [Liang-Chi Hsieh] Show warning message if mapred.reduce.tasks is set to -1.
1 parent bcceeda commit dfde7d3

File tree

2 files changed

+18
-2
lines changed

2 files changed

+18
-2
lines changed

sql/core/src/main/scala/org/apache/spark/sql/execution/commands.scala

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -84,8 +84,14 @@ case class SetCommand(
8484
logWarning(
8585
s"Property ${SQLConf.Deprecated.MAPRED_REDUCE_TASKS} is deprecated, " +
8686
s"automatically converted to ${SQLConf.SHUFFLE_PARTITIONS} instead.")
87-
sqlContext.setConf(SQLConf.SHUFFLE_PARTITIONS, value)
88-
Seq(Row(s"${SQLConf.SHUFFLE_PARTITIONS}=$value"))
87+
if (value.toInt < 1) {
88+
val msg = s"Setting negative ${SQLConf.Deprecated.MAPRED_REDUCE_TASKS} for automatically " +
89+
"determining the number of reducers is not supported."
90+
throw new IllegalArgumentException(msg)
91+
} else {
92+
sqlContext.setConf(SQLConf.SHUFFLE_PARTITIONS, value)
93+
Seq(Row(s"${SQLConf.SHUFFLE_PARTITIONS}=$value"))
94+
}
8995

9096
// Configures a single property.
9197
case Some((key, Some(value))) =>

sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -871,6 +871,16 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll {
871871
conf.clear()
872872
}
873873

874+
test("SET commands with illegal or inappropriate argument") {
875+
conf.clear()
876+
// Set negative mapred.reduce.tasks for automatically determing
877+
// the number of reducers is not supported
878+
intercept[IllegalArgumentException](sql(s"SET mapred.reduce.tasks=-1"))
879+
intercept[IllegalArgumentException](sql(s"SET mapred.reduce.tasks=-01"))
880+
intercept[IllegalArgumentException](sql(s"SET mapred.reduce.tasks=-2"))
881+
conf.clear()
882+
}
883+
874884
test("apply schema") {
875885
val schema1 = StructType(
876886
StructField("f1", IntegerType, false) ::

0 commit comments

Comments
 (0)