Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ import scala.math.{max, min}

import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult, TypeCoercion}
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch
import org.apache.spark.sql.catalyst.expressions.Cast.{toSQLId, toSQLType}
import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.expressions.codegen.Block._
import org.apache.spark.sql.catalyst.trees.SQLQueryContext
Expand Down Expand Up @@ -1198,12 +1200,17 @@ case class Least(children: Seq[Expression]) extends ComplexTypeMergingExpression

override def checkInputDataTypes(): TypeCheckResult = {
if (children.length <= 1) {
TypeCheckResult.TypeCheckFailure(
s"input to function $prettyName requires at least two arguments")
DataTypeMismatch(
errorSubClass = "WRONG_NUM_PARAMS",
messageParameters = Map("actualNum" -> children.length.toString))
} else if (!TypeCoercion.haveSameType(inputTypesForMerging)) {
TypeCheckResult.TypeCheckFailure(
s"The expressions should all have the same type," +
s" got LEAST(${children.map(_.dataType.catalogString).mkString(", ")}).")
DataTypeMismatch(
errorSubClass = "DATA_DIFF_TYPES",
messageParameters = Map(
"functionName" -> toSQLId(prettyName),
"dataType" -> children.map(_.dataType).map(toSQLType).mkString("[", ", ", "]")
)
)
} else {
TypeUtils.checkForOrderingExpr(dataType, s"function $prettyName")
}
Expand Down Expand Up @@ -1281,12 +1288,17 @@ case class Greatest(children: Seq[Expression]) extends ComplexTypeMergingExpress

override def checkInputDataTypes(): TypeCheckResult = {
if (children.length <= 1) {
TypeCheckResult.TypeCheckFailure(
s"input to function $prettyName requires at least two arguments")
DataTypeMismatch(
errorSubClass = "WRONG_NUM_PARAMS",
messageParameters = Map("actualNum" -> children.length.toString))
} else if (!TypeCoercion.haveSameType(inputTypesForMerging)) {
TypeCheckResult.TypeCheckFailure(
s"The expressions should all have the same type," +
s" got GREATEST(${children.map(_.dataType.catalogString).mkString(", ")}).")
DataTypeMismatch(
errorSubClass = "DATA_DIFF_TYPES",
messageParameters = Map(
"functionName" -> toSQLId(prettyName),
"dataType" -> children.map(_.dataType).map(toSQLType).mkString("[", ", ", "]")
)
)
} else {
TypeUtils.checkForOrderingExpr(dataType, s"function $prettyName")
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,16 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer
parameters = messageParameters)
}

def assertErrorForWrongNumParameters(
expr: Expression, messageParameters: Map[String, String]): Unit = {
checkError(
exception = intercept[AnalysisException] {
assertSuccess(expr)
},
errorClass = "DATATYPE_MISMATCH.WRONG_NUM_PARAMS",
parameters = messageParameters)
}

def assertForWrongType(expr: Expression, messageParameters: Map[String, String]): Unit = {
checkError(
exception = intercept[AnalysisException] {
Expand Down Expand Up @@ -526,9 +536,24 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite with SQLHelper with Quer

test("check types for Greatest/Least") {
for (operator <- Seq[(Seq[Expression] => Expression)](Greatest, Least)) {
assertError(operator(Seq($"booleanField")), "requires at least two arguments")
assertError(operator(Seq($"intField", $"stringField")),
"should all have the same type")
val expr1 = operator(Seq($"booleanField"))
assertErrorForWrongNumParameters(
expr = expr1,
messageParameters = Map(
"sqlExpr" -> toSQLExpr(expr1),
"actualNum" -> "1")
)

val expr2 = operator(Seq($"intField", $"stringField"))
assertErrorForDataDifferingTypes(
expr = expr2,
messageParameters = Map(
"sqlExpr" -> toSQLExpr(expr2),
"functionName" -> toSQLId(expr2.prettyName),
"dataType" -> "[\"INT\", \"STRING\"]"
)
)

val expr3 = operator(Seq($"mapField", $"mapField"))
assertErrorForOrderingTypes(
expr = expr3,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import java.time.temporal.ChronoUnit

import org.apache.spark.{SparkArithmeticException, SparkFunSuite}
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.TypeCheckFailure
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
import org.apache.spark.sql.catalyst.dsl.expressions._
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
import org.apache.spark.sql.catalyst.trees.CurrentOrigin.withOrigin
Expand Down Expand Up @@ -503,10 +503,13 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
Timestamp.valueOf("2015-07-01 08:00:00"), InternalRow.empty)

// Type checking error
assert(
Least(Seq(Literal(1), Literal("1"))).checkInputDataTypes() ==
TypeCheckFailure("The expressions should all have the same type, " +
"got LEAST(int, string)."))
Least(Seq(Literal(1), Literal("1"))).checkInputDataTypes() match {
case TypeCheckResult.DataTypeMismatch(errorSubClass, messageParameters) =>
assert(errorSubClass == "DATA_DIFF_TYPES")
assert(messageParameters === Map(
"functionName" -> "`least`",
"dataType" -> "[\"INT\", \"STRING\"]"))
}

DataTypeTestUtils.ordered.foreach { dt =>
checkConsistencyBetweenInterpretedAndCodegen(Least, dt, 2)
Expand Down Expand Up @@ -561,10 +564,13 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
Timestamp.valueOf("2015-07-01 10:00:00"), InternalRow.empty)

// Type checking error
assert(
Greatest(Seq(Literal(1), Literal("1"))).checkInputDataTypes() ==
TypeCheckFailure("The expressions should all have the same type, " +
"got GREATEST(int, string)."))
Greatest(Seq(Literal(1), Literal("1"))).checkInputDataTypes() match {
case TypeCheckResult.DataTypeMismatch(errorSubClass, messageParameters) =>
assert(errorSubClass == "DATA_DIFF_TYPES")
assert(messageParameters === Map(
"functionName" -> "`greatest`",
"dataType" -> "[\"INT\", \"STRING\"]"))
}

DataTypeTestUtils.ordered.foreach { dt =>
checkConsistencyBetweenInterpretedAndCodegen(Greatest, dt, 2)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4229,15 +4229,57 @@ class DataFrameFunctionsSuite extends QueryTest with SharedSparkSession {
assert(errMsg.contains(s"input to function $name requires at least one argument"))
}

val funcsMustHaveAtLeastTwoArgs =
("greatest", (df: DataFrame) => df.select(greatest())) ::
("greatest", (df: DataFrame) => df.selectExpr("greatest()")) ::
("least", (df: DataFrame) => df.select(least())) ::
("least", (df: DataFrame) => df.selectExpr("least()")) :: Nil
funcsMustHaveAtLeastTwoArgs.foreach { case (name, func) =>
val errMsg = intercept[AnalysisException] { func(df) }.getMessage
assert(errMsg.contains(s"input to function $name requires at least two arguments"))
}
checkError(
exception = intercept[AnalysisException] {
df.select(greatest())
},
errorClass = "DATATYPE_MISMATCH.WRONG_NUM_PARAMS",
sqlState = None,
parameters = Map(
"sqlExpr" -> "\"greatest()\"",
"actualNum" -> "0")
)

checkError(
exception = intercept[AnalysisException] {
df.selectExpr("greatest()")
},
errorClass = "DATATYPE_MISMATCH.WRONG_NUM_PARAMS",
sqlState = None,
parameters = Map(
"sqlExpr" -> "\"greatest()\"",
"actualNum" -> "0"),
context = ExpectedContext(
fragment = "greatest()",
start = 0,
stop = 9)
)

checkError(
exception = intercept[AnalysisException] {
df.select(least())
},
errorClass = "DATATYPE_MISMATCH.WRONG_NUM_PARAMS",
sqlState = None,
parameters = Map(
"sqlExpr" -> "\"least()\"",
"actualNum" -> "0")
)

checkError(
exception = intercept[AnalysisException] {
df.selectExpr("least()")
},
errorClass = "DATATYPE_MISMATCH.WRONG_NUM_PARAMS",
sqlState = None,
parameters = Map(
"sqlExpr" -> "\"least()\"",
"actualNum" -> "0"),
context = ExpectedContext(
fragment = "least()",
start = 0,
stop = 6)
)
}

test("SPARK-24734: Fix containsNull of Concat for array type") {
Expand Down