Skip to content
Closed
3 changes: 3 additions & 0 deletions core/src/main/resources/error/error-classes.json
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,9 @@
"UNSUPPORTED_OPERATION" : {
"message" : [ "The operation is not supported: <operation>" ]
},
"UNSUPPORTED_SAVE_MODE" : {
"message" : [ "The saveMode is not supported: <saveMode> (<pathExists>)" ]
Comment thread
panbingkun marked this conversation as resolved.
Outdated
},
"UNTYPED_SCALA_UDF" : {
"message" : [ "You're using untyped Scala UDF, which does not have the input type information. Spark may blindly pass null to the Scala closure with primitive-type argument, and the closure will see the default value of the Java type for the null argument, e.g. `udf((x: Int) => x, IntegerType)`, the result is 0 for null input. To get rid of this error, you could:\n1. use typed Scala UDF APIs(without return type parameter), e.g. `udf((x: Int) => x)`\n2. use Java UDF APIs, e.g. `udf(new UDF1[String, Integer] { override def call(s: String): Integer = s.length() }, IntegerType)`, if input types are all non primitive\n3. set \"spark.sql.legacy.allowUntypedScalaUDF\" to true and use this API with caution" ]
},
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -582,8 +582,9 @@ object QueryExecutionErrors extends QueryErrorsBase {
""".stripMargin)
}

def unsupportedSaveModeError(saveMode: String, pathExists: Boolean): Throwable = {
new IllegalStateException(s"unsupported save mode $saveMode ($pathExists)")
def saveModeUnsupportedError(saveMode: Any, pathExists: Boolean): Throwable = {
new SparkIllegalStateException(errorClass = "UNSUPPORTED_SAVE_MODE",
messageParameters = Array(toSQLValue(saveMode), toSQLValue(pathExists)))
}

def cannotClearOutputDirectoryError(staticPrefixPath: Path): Throwable = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -133,7 +133,7 @@ case class InsertIntoHadoopFsRelationCommand(
case (SaveMode.Ignore, exists) =>
!exists
case (s, exists) =>
throw QueryExecutionErrors.unsupportedSaveModeError(s.toString, exists)
throw QueryExecutionErrors.saveModeUnsupportedError(s, exists)
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import java.util.Locale
import test.org.apache.spark.sql.connector.JavaSimpleWritableDataSource

import org.apache.spark.{SparkArithmeticException, SparkException, SparkIllegalStateException, SparkRuntimeException, SparkUnsupportedOperationException, SparkUpgradeException}
import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest}
import org.apache.spark.sql.{AnalysisException, DataFrame, QueryTest, SaveMode}
import org.apache.spark.sql.catalyst.util.BadRecordException
import org.apache.spark.sql.connector.SimpleWritableDataSource
import org.apache.spark.sql.execution.QueryExecutionException
Expand Down Expand Up @@ -424,4 +424,17 @@ class QueryExecutionErrorsSuite
matchMsg = true
)
}

test("UNSUPPORTED_SAVE_MODE: unsupported null saveMode") {
withTempPath { path =>
val e = intercept[SparkIllegalStateException] {
val saveMode: SaveMode = null
Comment thread
panbingkun marked this conversation as resolved.
Seq(1, 2).toDS().write.mode(saveMode).parquet(path.getAbsolutePath)
}
checkErrorClass(
exception = e,
errorClass = "UNSUPPORTED_SAVE_MODE",
msg = "The saveMode is not supported: NULL (false)")
}
}
}