diff --git a/core/src/main/resources/error/error-classes.json b/core/src/main/resources/error/error-classes.json index d4c0910c5ad7..f9257b6c21be 100644 --- a/core/src/main/resources/error/error-classes.json +++ b/core/src/main/resources/error/error-classes.json @@ -352,6 +352,12 @@ ], "sqlState" : "42000" }, + "UNRESOLVED_MAP_KEY" : { + "message" : [ + "Cannot resolve column as a map key. If the key is a string literal, please add single quotes around it. Otherwise, did you mean one of the following column(s)? []" + ], + "sqlState" : "42000" + }, "UNSUPPORTED_DATATYPE" : { "message" : [ "Unsupported data type " @@ -556,4 +562,4 @@ ], "sqlState" : "40000" } -} \ No newline at end of file +} diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala index 931a0fcf77f0..4d2dd1752609 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala @@ -3420,8 +3420,8 @@ class Analyzer(override val catalogManager: CatalogManager) i.userSpecifiedCols.map { col => i.table.resolve(Seq(col), resolver).getOrElse( - throw QueryCompilationErrors.unresolvedColumnError( - col, i.table.output.map(_.name), i.origin)) + throw QueryCompilationErrors.unresolvedAttributeError( + "UNRESOLVED_COLUMN", col, i.table.output.map(_.name), i.origin)) } } diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala index f9f8b590a311..759683b8c001 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/CheckAnalysis.scala @@ -91,6 +91,26 @@ trait CheckAnalysis extends PredicateHelper with LookupCatalog { } } + private def isMapWithStringKey(e: Expression): Boolean = if (e.resolved) { + e.dataType match { + case m: MapType => m.keyType.isInstanceOf[StringType] + case _ => false + } + } else { + false + } + + private def failUnresolvedAttribute( + operator: LogicalPlan, + a: Attribute, + errorClass: String): Nothing = { + val missingCol = a.sql + val candidates = operator.inputSet.toSeq.map(_.qualifiedName) + val orderedCandidates = StringUtils.orderStringsBySimilarity(missingCol, candidates) + throw QueryCompilationErrors.unresolvedAttributeError( + errorClass, missingCol, orderedCandidates, a.origin) + } + def checkAnalysis(plan: LogicalPlan): Unit = { // We transform up and order the rules so as to catch the first possible failure instead // of the result of cascading resolution failures. Inline all CTEs in the plan to help check @@ -160,11 +180,11 @@ trait CheckAnalysis extends PredicateHelper with LookupCatalog { throw QueryCompilationErrors.commandUnsupportedInV2TableError("SHOW TABLE EXTENDED") case operator: LogicalPlan => - // Check argument data types of higher-order functions downwards first. - // If the arguments of the higher-order functions are resolved but the type check fails, - // the argument functions will not get resolved, but we should report the argument type - // check failure instead of claiming the argument functions are unresolved. operator transformExpressionsDown { + // Check argument data types of higher-order functions downwards first. + // If the arguments of the higher-order functions are resolved but the type check fails, + // the argument functions will not get resolved, but we should report the argument type + // check failure instead of claiming the argument functions are unresolved. case hof: HigherOrderFunction if hof.argumentsResolved && hof.checkArgumentDataTypes().isFailure => hof.checkArgumentDataTypes() match { @@ -172,15 +192,16 @@ trait CheckAnalysis extends PredicateHelper with LookupCatalog { hof.failAnalysis( s"cannot resolve '${hof.sql}' due to argument data type mismatch: $message") } + + // If an attribute can't be resolved as a map key of string type, either the key should be + // surrounded with single quotes, or there is a typo in the attribute name. + case GetMapValue(map, key: Attribute, _) if isMapWithStringKey(map) && !key.resolved => + failUnresolvedAttribute(operator, key, "UNRESOLVED_MAP_KEY") } getAllExpressions(operator).foreach(_.foreachUp { case a: Attribute if !a.resolved => - val missingCol = a.sql - val candidates = operator.inputSet.toSeq.map(_.qualifiedName) - val orderedCandidates = StringUtils.orderStringsBySimilarity(missingCol, candidates) - throw QueryCompilationErrors.unresolvedColumnError( - missingCol, orderedCandidates, a.origin) + failUnresolvedAttribute(operator, a, "UNRESOLVED_COLUMN") case s: Star => withPosition(s) { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala index b2db00cd2b4e..198fd0cd1f2f 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala @@ -461,7 +461,7 @@ case class GetMapValue( @transient private lazy val ordering: Ordering[Any] = TypeUtils.getInterpretedOrdering(keyType) - private def keyType = child.dataType.asInstanceOf[MapType].keyType + private[catalyst] def keyType = child.dataType.asInstanceOf[MapType].keyType override def checkInputDataTypes(): TypeCheckResult = { super.checkInputDataTypes() match { diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala index 4ee53c56f69e..7ed5c7857711 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/errors/QueryCompilationErrors.scala @@ -144,11 +144,14 @@ private[sql] object QueryCompilationErrors extends QueryErrorsBase { s"side of the join. The $side-side columns: [${plan.output.map(_.name).mkString(", ")}]") } - def unresolvedColumnError( - colName: String, candidates: Seq[String], origin: Origin): Throwable = { + def unresolvedAttributeError( + errorClass: String, + colName: String, + candidates: Seq[String], + origin: Origin): Throwable = { val candidateIds = candidates.map(candidate => toSQLId(candidate)) new AnalysisException( - errorClass = "UNRESOLVED_COLUMN", + errorClass = errorClass, messageParameters = Array(toSQLId(colName), candidateIds.mkString(", ")), origin = origin) } diff --git a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala index 06e6bec3fd15..bab5a1068288 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/errors/QueryCompilationErrorsSuite.scala @@ -401,6 +401,18 @@ class QueryCompilationErrorsSuite ) } + test("UNRESOLVED_MAP_KEY: string type literal should be quoted") { + checkAnswer(sql("select m['a'] from (select map('a', 'b') as m, 'aa' as aa)"), Row("b")) + checkError( + exception = intercept[AnalysisException] { + sql("select m[a] from (select map('a', 'b') as m, 'aa' as aa)") + }, + errorClass = "UNRESOLVED_MAP_KEY", + parameters = Map("columnName" -> "`a`", + "proposal" -> + "`__auto_generated_subquery_name`.`m`, `__auto_generated_subquery_name`.`aa`")) + } + test("UNRESOLVED_COLUMN: SELECT distinct does not work correctly " + "if order by missing attribute") { checkAnswer(