Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ import java.lang.reflect.{Method, Modifier}
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.{FunctionRegistry, TypeCheckResult}
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.{DataTypeMismatch, TypeCheckSuccess}
import org.apache.spark.sql.catalyst.expressions.Cast.{toSQLExpr, toSQLType}
import org.apache.spark.sql.catalyst.expressions.codegen.CodegenFallback
import org.apache.spark.sql.errors.QueryErrorsBase
import org.apache.spark.sql.types._
import org.apache.spark.unsafe.types.UTF8String
import org.apache.spark.util.Utils
Expand Down Expand Up @@ -56,7 +56,9 @@ import org.apache.spark.util.Utils
since = "2.0.0",
group = "misc_funcs")
case class CallMethodViaReflection(children: Seq[Expression])
extends Nondeterministic with CodegenFallback {
extends Nondeterministic
with CodegenFallback
with QueryErrorsBase {

override def prettyName: String = getTagValue(FunctionRegistry.FUNC_ALIAS).getOrElse("reflect")

Expand All @@ -65,7 +67,7 @@ case class CallMethodViaReflection(children: Seq[Expression])
DataTypeMismatch(
errorSubClass = "WRONG_NUM_PARAMS",
messageParameters = Map(
"functionName" -> prettyName,
"functionName" -> toSQLId(prettyName),
"expectedNum" -> "> 1",
"actualNum" -> children.length.toString))
} else {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ case class Max(child: Expression) extends DeclarativeAggregate with UnaryLike[Ex
override def dataType: DataType = child.dataType

override def checkInputDataTypes(): TypeCheckResult =
TypeUtils.checkForOrderingExpr(child.dataType, "function max")
TypeUtils.checkForOrderingExpr(child.dataType, prettyName)

private lazy val max = AttributeReference("max", child.dataType)()

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ abstract class MaxMinBy extends DeclarativeAggregate with BinaryLike[Expression]
override def dataType: DataType = valueExpr.dataType

override def checkInputDataTypes(): TypeCheckResult =
TypeUtils.checkForOrderingExpr(orderingExpr.dataType, s"function $prettyName")
TypeUtils.checkForOrderingExpr(orderingExpr.dataType, prettyName)

// The attributes used to keep extremum (max or min) and associated aggregated values.
private lazy val extremumOrdering =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ case class Min(child: Expression) extends DeclarativeAggregate with UnaryLike[Ex
override def dataType: DataType = child.dataType

override def checkInputDataTypes(): TypeCheckResult =
TypeUtils.checkForOrderingExpr(child.dataType, "function min")
TypeUtils.checkForOrderingExpr(child.dataType, prettyName)

private lazy val min = AttributeReference("min", child.dataType)()

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1203,7 +1203,7 @@ case class Least(children: Seq[Expression]) extends ComplexTypeMergingExpression
DataTypeMismatch(
errorSubClass = "WRONG_NUM_PARAMS",
messageParameters = Map(
"functionName" -> prettyName,
"functionName" -> toSQLId(prettyName),
"expectedNum" -> "> 1",
"actualNum" -> children.length.toString))
} else if (!TypeCoercion.haveSameType(inputTypesForMerging)) {
Expand All @@ -1215,7 +1215,7 @@ case class Least(children: Seq[Expression]) extends ComplexTypeMergingExpression
)
)
} else {
TypeUtils.checkForOrderingExpr(dataType, s"function $prettyName")
TypeUtils.checkForOrderingExpr(dataType, prettyName)
}
}

Expand Down Expand Up @@ -1294,7 +1294,7 @@ case class Greatest(children: Seq[Expression]) extends ComplexTypeMergingExpress
DataTypeMismatch(
errorSubClass = "WRONG_NUM_PARAMS",
messageParameters = Map(
"functionName" -> prettyName,
"functionName" -> toSQLId(prettyName),
"expectedNum" -> "> 1",
"actualNum" -> children.length.toString))
} else if (!TypeCoercion.haveSameType(inputTypesForMerging)) {
Expand All @@ -1306,7 +1306,7 @@ case class Greatest(children: Seq[Expression]) extends ComplexTypeMergingExpress
)
)
} else {
TypeUtils.checkForOrderingExpr(dataType, s"function $prettyName")
TypeUtils.checkForOrderingExpr(dataType, prettyName)
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,15 +26,14 @@ import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, TypeCoercion, UnresolvedAttribute, UnresolvedSeed}
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch
import org.apache.spark.sql.catalyst.expressions.ArraySortLike.NullOrder
import org.apache.spark.sql.catalyst.expressions.Cast.{toSQLExpr, toSQLType}
import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.catalyst.expressions.codegen.Block._
import org.apache.spark.sql.catalyst.trees.{BinaryLike, SQLQueryContext, UnaryLike}
import org.apache.spark.sql.catalyst.trees.TreePattern.{ARRAYS_ZIP, CONCAT, TreePattern}
import org.apache.spark.sql.catalyst.util._
import org.apache.spark.sql.catalyst.util.DateTimeConstants._
import org.apache.spark.sql.catalyst.util.DateTimeUtils._
import org.apache.spark.sql.errors.QueryExecutionErrors
import org.apache.spark.sql.errors.{QueryErrorsBase, QueryExecutionErrors}
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
import org.apache.spark.sql.util.SQLOpenHashSet
Expand All @@ -47,8 +46,10 @@ import org.apache.spark.unsafe.types.{ByteArray, CalendarInterval, UTF8String}
* Base trait for [[BinaryExpression]]s with two arrays of the same element type and implicit
* casting.
*/
trait BinaryArrayExpressionWithImplicitCast extends BinaryExpression
with ImplicitCastInputTypes {
trait BinaryArrayExpressionWithImplicitCast
extends BinaryExpression
with ImplicitCastInputTypes
with QueryErrorsBase {

@transient protected lazy val elementType: DataType =
inputTypes.head.asInstanceOf[ArrayType].elementType
Expand All @@ -72,7 +73,7 @@ trait BinaryArrayExpressionWithImplicitCast extends BinaryExpression
DataTypeMismatch(
errorSubClass = "BINARY_ARRAY_DIFF_TYPES",
messageParameters = Map(
"functionName" -> prettyName,
"functionName" -> toSQLId(prettyName),
"arrayType" -> toSQLType(ArrayType),
"leftType" -> toSQLType(left.dataType),
"rightType" -> toSQLType(right.dataType)
Expand Down Expand Up @@ -219,7 +220,10 @@ case class MapKeys(child: Expression)
group = "map_funcs",
since = "3.3.0")
case class MapContainsKey(left: Expression, right: Expression)
extends RuntimeReplaceable with BinaryLike[Expression] with ImplicitCastInputTypes {
extends RuntimeReplaceable
with BinaryLike[Expression]
with ImplicitCastInputTypes
with QueryErrorsBase {

override lazy val replacement: Expression = ArrayContains(MapKeys(left), right)

Expand All @@ -240,14 +244,14 @@ case class MapContainsKey(left: Expression, right: Expression)
case (_, NullType) =>
DataTypeMismatch(
errorSubClass = "NULL_TYPE",
Map("functionName" -> prettyName))
Map("functionName" -> toSQLId(prettyName)))
case (MapType(kt, _, _), dt) if kt.sameType(dt) =>
TypeUtils.checkForOrderingExpr(kt, s"function $prettyName")
TypeUtils.checkForOrderingExpr(kt, prettyName)
case _ =>
DataTypeMismatch(
errorSubClass = "MAP_CONTAINS_KEY_DIFF_TYPES",
messageParameters = Map(
"functionName" -> prettyName,
"functionName" -> toSQLId(prettyName),
"dataType" -> toSQLType(MapType),
"leftType" -> toSQLType(left.dataType),
"rightType" -> toSQLType(right.dataType)
Expand Down Expand Up @@ -676,20 +680,21 @@ case class MapEntries(child: Expression)
""",
group = "map_funcs",
since = "2.4.0")
case class MapConcat(children: Seq[Expression]) extends ComplexTypeMergingExpression {
case class MapConcat(children: Seq[Expression])
extends ComplexTypeMergingExpression
with QueryErrorsBase {

override def checkInputDataTypes(): TypeCheckResult = {
val funcName = s"function $prettyName"
if (children.exists(!_.dataType.isInstanceOf[MapType])) {
DataTypeMismatch(
errorSubClass = "MAP_CONCAT_DIFF_TYPES",
messageParameters = Map(
"functionName" -> funcName,
"functionName" -> toSQLId(prettyName),
"dataType" -> children.map(_.dataType).map(toSQLType).mkString("[", ", ", "]")
)
)
} else {
val sameTypeCheck = TypeUtils.checkForSameTypeInputExpr(children.map(_.dataType), funcName)
val sameTypeCheck = TypeUtils.checkForSameTypeInputExpr(children.map(_.dataType), prettyName)
if (sameTypeCheck.isFailure) {
sameTypeCheck
} else {
Expand Down Expand Up @@ -802,7 +807,10 @@ case class MapConcat(children: Seq[Expression]) extends ComplexTypeMergingExpres
""",
group = "map_funcs",
since = "2.4.0")
case class MapFromEntries(child: Expression) extends UnaryExpression with NullIntolerant {
case class MapFromEntries(child: Expression)
extends UnaryExpression
with NullIntolerant
with QueryErrorsBase {

@transient
private lazy val dataTypeDetails: Option[(MapType, Boolean, Boolean)] = child.dataType match {
Expand All @@ -827,7 +835,7 @@ case class MapFromEntries(child: Expression) extends UnaryExpression with NullIn
DataTypeMismatch(
errorSubClass = "MAP_FROM_ENTRIES_WRONG_TYPE",
messageParameters = Map(
"functionName" -> prettyName,
"functionName" -> toSQLId(prettyName),
"childExpr" -> toSQLExpr(child),
"childType" -> toSQLType(child.dataType)
)
Expand Down Expand Up @@ -1290,7 +1298,7 @@ case class ArrayContains(left: Expression, right: Expression)
case (_, NullType) =>
TypeCheckResult.TypeCheckFailure("Null typed values cannot be used as arguments")
case (ArrayType(e1, _), e2) if e1.sameType(e2) =>
TypeUtils.checkForOrderingExpr(e2, s"function $prettyName")
TypeUtils.checkForOrderingExpr(e2, prettyName)
case _ => TypeCheckResult.TypeCheckFailure(s"Input to function $prettyName should have " +
s"been ${ArrayType.simpleString} followed by a value with same element type, but it's " +
s"[${left.dataType.catalogString}, ${right.dataType.catalogString}].")
Expand Down Expand Up @@ -1373,7 +1381,7 @@ case class ArraysOverlap(left: Expression, right: Expression)

override def checkInputDataTypes(): TypeCheckResult = super.checkInputDataTypes() match {
case TypeCheckResult.TypeCheckSuccess =>
TypeUtils.checkForOrderingExpr(elementType, s"function $prettyName")
TypeUtils.checkForOrderingExpr(elementType, prettyName)
case failure => failure
}

Expand Down Expand Up @@ -1901,7 +1909,7 @@ case class ArrayMin(child: Expression)
override def checkInputDataTypes(): TypeCheckResult = {
val typeCheckResult = super.checkInputDataTypes()
if (typeCheckResult.isSuccess) {
TypeUtils.checkForOrderingExpr(dataType, s"function $prettyName")
TypeUtils.checkForOrderingExpr(dataType, prettyName)
} else {
typeCheckResult
}
Expand Down Expand Up @@ -1974,7 +1982,7 @@ case class ArrayMax(child: Expression)
override def checkInputDataTypes(): TypeCheckResult = {
val typeCheckResult = super.checkInputDataTypes()
if (typeCheckResult.isSuccess) {
TypeUtils.checkForOrderingExpr(dataType, s"function $prettyName")
TypeUtils.checkForOrderingExpr(dataType, prettyName)
} else {
typeCheckResult
}
Expand Down Expand Up @@ -2063,7 +2071,7 @@ case class ArrayPosition(left: Expression, right: Expression)
override def checkInputDataTypes(): TypeCheckResult = {
(left.dataType, right.dataType) match {
case (ArrayType(e1, _), e2) if e1.sameType(e2) =>
TypeUtils.checkForOrderingExpr(e2, s"function $prettyName")
TypeUtils.checkForOrderingExpr(e2, prettyName)
case _ => TypeCheckResult.TypeCheckFailure(s"Input to function $prettyName should have " +
s"been ${ArrayType.simpleString} followed by a value with same element type, but it's " +
s"[${left.dataType.catalogString}, ${right.dataType.catalogString}].")
Expand Down Expand Up @@ -2419,7 +2427,7 @@ case class Concat(children: Seq[Expression]) extends ComplexTypeMergingExpressio
s" ${BinaryType.simpleString} or ${ArrayType.simpleString}, but it's " +
childTypes.map(_.catalogString).mkString("[", ", ", "]"))
}
TypeUtils.checkForSameTypeInputExpr(childTypes, s"function $prettyName")
TypeUtils.checkForSameTypeInputExpr(childTypes, prettyName)
}
}

Expand Down Expand Up @@ -3473,7 +3481,7 @@ case class ArrayRemove(left: Expression, right: Expression)
override def checkInputDataTypes(): TypeCheckResult = {
(left.dataType, right.dataType) match {
case (ArrayType(e1, _), e2) if e1.sameType(e2) =>
TypeUtils.checkForOrderingExpr(e2, s"function $prettyName")
TypeUtils.checkForOrderingExpr(e2, prettyName)
case _ => TypeCheckResult.TypeCheckFailure(s"Input to function $prettyName should have " +
s"been ${ArrayType.simpleString} followed by a value with same element type, but it's " +
s"[${left.dataType.catalogString}, ${right.dataType.catalogString}].")
Expand Down Expand Up @@ -3673,7 +3681,7 @@ case class ArrayDistinct(child: Expression)
super.checkInputDataTypes() match {
case f if f.isFailure => f
case TypeCheckResult.TypeCheckSuccess =>
TypeUtils.checkForOrderingExpr(elementType, s"function $prettyName")
TypeUtils.checkForOrderingExpr(elementType, prettyName)
}
}

Expand Down Expand Up @@ -3828,8 +3836,7 @@ trait ArrayBinaryLike
override def checkInputDataTypes(): TypeCheckResult = {
val typeCheckResult = super.checkInputDataTypes()
if (typeCheckResult.isSuccess) {
TypeUtils.checkForOrderingExpr(dataType.asInstanceOf[ArrayType].elementType,
s"function $prettyName")
TypeUtils.checkForOrderingExpr(dataType.asInstanceOf[ArrayType].elementType, prettyName)
} else {
typeCheckResult
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,7 @@ case class CreateArray(children: Seq[Expression], useStringTypeWhenEmpty: Boolea
override def stringArgs: Iterator[Any] = super.stringArgs.take(1)

override def checkInputDataTypes(): TypeCheckResult = {
TypeUtils.checkForSameTypeInputExpr(children.map(_.dataType), s"function $prettyName")
TypeUtils.checkForSameTypeInputExpr(children.map(_.dataType), prettyName)
}

private val defaultElementType: DataType = {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -444,7 +444,7 @@ case class GetMapValue(child: Expression, key: Expression)
super.checkInputDataTypes() match {
case f if f.isFailure => f
case TypeCheckResult.TypeCheckSuccess =>
TypeUtils.checkForOrderingExpr(keyType, s"function $prettyName")
TypeUtils.checkForOrderingExpr(keyType, prettyName)
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1023,7 +1023,7 @@ case class MapZipWith(left: Expression, right: Expression, function: Expression)
super.checkArgumentDataTypes() match {
case TypeCheckResult.TypeCheckSuccess =>
if (leftKeyType.sameType(rightKeyType)) {
TypeUtils.checkForOrderingExpr(leftKeyType, s"function $prettyName")
TypeUtils.checkForOrderingExpr(leftKeyType, prettyName)
} else {
TypeCheckResult.TypeCheckFailure(s"The input to function $prettyName should have " +
s"been two ${MapType.simpleString}s with compatible key types, but the key types are " +
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -355,7 +355,9 @@ case class GetJsonObject(json: Expression, path: Expression)
since = "1.6.0")
// scalastyle:on line.size.limit line.contains.tab
case class JsonTuple(children: Seq[Expression])
extends Generator with CodegenFallback {
extends Generator
with CodegenFallback
with QueryErrorsBase {

import SharedFactory._

Expand Down Expand Up @@ -396,15 +398,15 @@ case class JsonTuple(children: Seq[Expression])
DataTypeMismatch(
errorSubClass = "WRONG_NUM_PARAMS",
messageParameters = Map(
"functionName" -> prettyName,
"functionName" -> toSQLId(prettyName),
"expectedNum" -> "> 1",
"actualNum" -> children.length.toString))
} else if (children.forall(child => StringType.acceptsType(child.dataType))) {
TypeCheckResult.TypeCheckSuccess
} else {
DataTypeMismatch(
errorSubClass = "NON_STRING_TYPE",
messageParameters = Map("funcName" -> prettyName))
messageParameters = Map("funcName" -> toSQLId(prettyName)))
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1794,7 +1794,7 @@ case class WidthBucket(
TypeCheckSuccess
case _ =>
val types = Seq(value.dataType, minValue.dataType, maxValue.dataType)
TypeUtils.checkForSameTypeInputExpr(types, s"function $prettyName")
TypeUtils.checkForSameTypeInputExpr(types, prettyName)
}
case f => f
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ case class Coalesce(children: Seq[Expression])
TypeCheckResult.TypeCheckFailure(
s"input to function $prettyName requires at least one argument")
} else {
TypeUtils.checkForSameTypeInputExpr(children.map(_.dataType), s"function $prettyName")
TypeUtils.checkForSameTypeInputExpr(children.map(_.dataType), prettyName)
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -404,7 +404,7 @@ case class InSubquery(values: Seq[Expression], query: ListQuery)
|Right side:
|[${query.childOutputs.map(_.dataType.catalogString).mkString(", ")}].""".stripMargin)
} else {
TypeUtils.checkForOrderingExpr(value.dataType, s"function $prettyName")
TypeUtils.checkForOrderingExpr(value.dataType, prettyName)
}
}

Expand Down Expand Up @@ -453,7 +453,7 @@ case class In(value: Expression, list: Seq[Expression]) extends Predicate {
TypeCheckResult.TypeCheckFailure(s"Arguments must be same type but were: " +
s"${value.dataType.catalogString} != ${mismatchOpt.get.dataType.catalogString}")
} else {
TypeUtils.checkForOrderingExpr(value.dataType, s"function $prettyName")
TypeUtils.checkForOrderingExpr(value.dataType, prettyName)
}
}

Expand Down Expand Up @@ -934,7 +934,7 @@ abstract class BinaryComparison extends BinaryOperator with Predicate {

override def checkInputDataTypes(): TypeCheckResult = super.checkInputDataTypes() match {
case TypeCheckResult.TypeCheckSuccess =>
TypeUtils.checkForOrderingExpr(left.dataType, this.getClass.getSimpleName)
TypeUtils.checkForOrderingExpr(left.dataType, symbol)
case failure => failure
}

Expand Down
Loading