Skip to content

[SPARK-40936][SQL][TESTS] Refactor AnalysisTest#assertAnalysisErrorClass by reusing the SparkFunSuite#checkError #38413

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
LuciferYang wants to merge 3 commits into from
Closed

[SPARK-40936][SQL][TESTS] Refactor AnalysisTest#assertAnalysisErrorClass by reusing the SparkFunSuite#checkError #38413

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5ac5fe7
simplify
LuciferYang Oct 27, 2022
96cf8f2
refactor
LuciferYang Oct 28, 2022
cdb677a
Merge branch 'upmaster' into simplify-assertAnalysisErrorClass
LuciferYang Oct 28, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 3 additions & 5 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisErrorSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -119,8 +119,7 @@ class AnalysisErrorSuite extends AnalysisTest {
messageParameters: Map[String, String],
caseSensitive: Boolean = true): Unit = {
test(name) {
assertAnalysisErrorClass(plan, errorClass, messageParameters,
caseSensitive = true, line = -1, pos = -1)
assertAnalysisErrorClass(plan, errorClass, messageParameters, caseSensitive = caseSensitive)
}
}

Expand Down Expand Up @@ -899,9 +898,8 @@ class AnalysisErrorSuite extends AnalysisTest {
"inputSql" -> inputSql,
"inputType" -> inputType,
"requiredType" -> "(\"INT\" or \"BIGINT\")"),
caseSensitive = false,
line = -1,
pos = -1)
caseSensitive = false
)
}
}

Expand Down
12 changes: 6 additions & 6 deletions ...rc/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisExceptionPositionSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,8 @@ class AnalysisExceptionPositionSuite extends AnalysisTest {
parsePlan("SHOW COLUMNS FROM unknown IN db"),
"TABLE_OR_VIEW_NOT_FOUND",
Map("relationName" -> "`db`.`unknown`"),
line = 1,
pos = 18)
Array(ExpectedContext("unknown", 18, 24))
)
verifyTableOrViewPosition("ALTER TABLE unknown RENAME TO t", "unknown")
verifyTableOrViewPosition("ALTER VIEW unknown RENAME TO v", "unknown")
}
Expand Down Expand Up @@ -92,13 +92,13 @@ class AnalysisExceptionPositionSuite extends AnalysisTest {
}

private def verifyPosition(sql: String, table: String): Unit = {
val expectedPos = sql.indexOf(table)
assert(expectedPos != -1)
val startPos = sql.indexOf(table)
assert(startPos != -1)
assertAnalysisErrorClass(
parsePlan(sql),
"TABLE_OR_VIEW_NOT_FOUND",
Map("relationName" -> s"`$table`"),
line = 1,
pos = expectedPos)
Array(ExpectedContext(table, startPos, startPos + table.length - 1))
)
}
}
39 changes: 20 additions & 19 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -104,10 +104,8 @@ class AnalysisSuite extends AnalysisTest with Matchers {
Project(Seq(UnresolvedAttribute("tBl.a")),
SubqueryAlias("TbL", UnresolvedRelation(TableIdentifier("TaBlE")))),
"UNRESOLVED_COLUMN.WITH_SUGGESTION",
Map("objectName" -> "`tBl`.`a`", "proposal" -> "`TbL`.`a`"),
caseSensitive = true,
line = -1,
pos = -1)
Map("objectName" -> "`tBl`.`a`", "proposal" -> "`TbL`.`a`")
)

checkAnalysisWithoutViewWrapper(
Project(Seq(UnresolvedAttribute("TbL.a")),
Expand Down Expand Up @@ -716,9 +714,8 @@ class AnalysisSuite extends AnalysisTest with Matchers {
assertAnalysisErrorClass(parsePlan("WITH t(x) AS (SELECT 1) SELECT * FROM t WHERE y = 1"),
"UNRESOLVED_COLUMN.WITH_SUGGESTION",
Map("objectName" -> "`y`", "proposal" -> "`t`.`x`"),
caseSensitive = true,
line = -1,
pos = -1)
Array(ExpectedContext("y", 46, 46))
Copy link
Contributor Author

@LuciferYang LuciferYang Oct 28, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

spark/core/src/test/scala/org/apache/spark/SparkFunSuite.scala

Lines 319 to 332 in cf086b1

val actualQueryContext = exception.getQueryContext()
assert(actualQueryContext.length === queryContext.length, "Invalid length of the query context")
actualQueryContext.zip(queryContext).foreach { case (actual, expected) =>
assert(actual.objectType() === expected.objectType(),
"Invalid objectType of a query context Actual:" + actual.toString)
assert(actual.objectName() === expected.objectName(),
"Invalid objectName of a query context. Actual:" + actual.toString)
assert(actual.startIndex() === expected.startIndex(),
"Invalid startIndex of a query context. Actual:" + actual.toString)
assert(actual.stopIndex() === expected.stopIndex(),
"Invalid stopIndex of a query context. Actual:" + actual.toString)
assert(actual.fragment() === expected.fragment(),
"Invalid fragment of a query context. Actual:" + actual.toString)
}

This change is due to the checkError method will perform a forced check when actualQueryContext is not empty. If we can to relax some check conditions, can add a precondition queryContext.nonEmpty for the queryContext check.

)
}

test("CTE with non-matching column alias") {
Expand All @@ -729,7 +726,8 @@ class AnalysisSuite extends AnalysisTest with Matchers {

test("SPARK-28251: Insert into non-existing table error message is user friendly") {
assertAnalysisErrorClass(parsePlan("INSERT INTO test VALUES (1)"),
"TABLE_OR_VIEW_NOT_FOUND", Map("relationName" -> "`test`"))
"TABLE_OR_VIEW_NOT_FOUND", Map("relationName" -> "`test`"),
Array(ExpectedContext("test", 12, 15)))
}

test("check CollectMetrics resolved") {
Expand Down Expand Up @@ -1157,9 +1155,8 @@ class AnalysisSuite extends AnalysisTest with Matchers {
|""".stripMargin),
"UNRESOLVED_COLUMN.WITH_SUGGESTION",
Map("objectName" -> "`c`.`y`", "proposal" -> "`x`"),
caseSensitive = true,
line = -1,
pos = -1)
Array(ExpectedContext("c.y", 123, 125))
)
}

test("SPARK-38118: Func(wrong_type) in the HAVING clause should throw data mismatch error") {
Expand All @@ -1178,7 +1175,9 @@ class AnalysisSuite extends AnalysisTest with Matchers {
"inputSql" -> "\"c\"",
"inputType" -> "\"BOOLEAN\"",
"requiredType" -> "\"NUMERIC\" or \"ANSI INTERVAL\""),
caseSensitive = false)
queryContext = Array(ExpectedContext("mean(t.c)", 65, 73)),
caseSensitive = false
)

assertAnalysisErrorClass(
inputPlan = parsePlan(
Expand All @@ -1195,6 +1194,7 @@ class AnalysisSuite extends AnalysisTest with Matchers {
"inputSql" -> "\"c\"",
"inputType" -> "\"BOOLEAN\"",
"requiredType" -> "\"NUMERIC\" or \"ANSI INTERVAL\""),
queryContext = Array(ExpectedContext("mean(c)", 91, 97)),
caseSensitive = false)

assertAnalysisErrorClass(
Expand All @@ -1213,9 +1213,9 @@ class AnalysisSuite extends AnalysisTest with Matchers {
"inputType" -> "\"BOOLEAN\"",
"requiredType" ->
"(\"NUMERIC\" or \"INTERVAL DAY TO SECOND\" or \"INTERVAL YEAR TO MONTH\")"),
caseSensitive = false,
line = -1,
pos = -1)
queryContext = Array(ExpectedContext("abs(t.c)", 65, 72)),
caseSensitive = false
)

assertAnalysisErrorClass(
inputPlan = parsePlan(
Expand All @@ -1233,9 +1233,9 @@ class AnalysisSuite extends AnalysisTest with Matchers {
"inputType" -> "\"BOOLEAN\"",
"requiredType" ->
"(\"NUMERIC\" or \"INTERVAL DAY TO SECOND\" or \"INTERVAL YEAR TO MONTH\")"),
caseSensitive = false,
line = -1,
pos = -1)
queryContext = Array(ExpectedContext("abs(c)", 91, 96)),
caseSensitive = false
)
}

test("SPARK-39354: should be [TABLE_OR_VIEW_NOT_FOUND]") {
Expand All @@ -1246,7 +1246,8 @@ class AnalysisSuite extends AnalysisTest with Matchers {
|FROM t1
|JOIN t2 ON t1.user_id = t2.user_id
|WHERE t1.dt >= DATE_SUB('2020-12-27', 90)""".stripMargin),
"TABLE_OR_VIEW_NOT_FOUND", Map("relationName" -> "`t2`"))
"TABLE_OR_VIEW_NOT_FOUND", Map("relationName" -> "`t2`"),
Array(ExpectedContext("t2", 84, 85)))
}

test("SPARK-39144: nested subquery expressions deduplicate relations should be done bottom up") {
Expand Down
39 changes: 10 additions & 29 deletions sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/AnalysisTest.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ import org.apache.spark.sql.types.StructType

trait AnalysisTest extends PlanTest {

import org.apache.spark.QueryContext

protected def extendedAnalysisRules: Seq[Rule[LogicalPlan]] = Nil

protected def createTempView(
Expand Down Expand Up @@ -174,40 +176,19 @@ trait AnalysisTest extends PlanTest {
inputPlan: LogicalPlan,
expectedErrorClass: String,
expectedMessageParameters: Map[String, String],
caseSensitive: Boolean = true,
line: Int = -1,
pos: Int = -1): Unit = {
queryContext: Array[QueryContext] = Array.empty,
caseSensitive: Boolean = true): Unit = {
withSQLConf(SQLConf.CASE_SENSITIVE.key -> caseSensitive.toString) {
val analyzer = getAnalyzer
val e = intercept[AnalysisException] {
analyzer.checkAnalysis(analyzer.execute(inputPlan))
}

if (e.getErrorClass != expectedErrorClass ||
e.messageParameters != expectedMessageParameters ||
(line >= 0 && e.line.getOrElse(-1) != line) ||
(pos >= 0) && e.startPosition.getOrElse(-1) != pos) {
var failMsg = ""
if (e.getErrorClass != expectedErrorClass) {
failMsg +=
s"""Error class should be: ${expectedErrorClass}
|Actual error class: ${e.getErrorClass}
""".stripMargin
}
if (e.messageParameters != expectedMessageParameters) {
failMsg +=
s"""Message parameters should be: ${expectedMessageParameters.mkString("\n ")}
|Actual message parameters: ${e.messageParameters.mkString("\n ")}
""".stripMargin
}
if (e.line.getOrElse(-1) != line || e.startPosition.getOrElse(-1) != pos) {
failMsg +=
s"""Line/position should be: $line, $pos
|Actual line/position: ${e.line.getOrElse(-1)}, ${e.startPosition.getOrElse(-1)}
""".stripMargin
}
fail(failMsg)
}
checkError(
exception = e,
errorClass = expectedErrorClass,
parameters = expectedMessageParameters,
queryContext = queryContext
)
}
}

Expand Down
30 changes: 10 additions & 20 deletions ...catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ResolveSubquerySuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -134,45 +134,35 @@ class ResolveSubquerySuite extends AnalysisTest {
assertAnalysisErrorClass(
lateralJoin(t1, lateralJoin(t2, t0.select($"a", $"b", $"c"))),
"UNRESOLVED_COLUMN.WITHOUT_SUGGESTION",
Map("objectName" -> "`a`"),
caseSensitive = true,
line = -1,
pos = -1)
Map("objectName" -> "`a`")
)
}

test("lateral subquery with unresolvable attributes") {
// SELECT * FROM t1, LATERAL (SELECT a, c)
assertAnalysisErrorClass(
lateralJoin(t1, t0.select($"a", $"c")),
"UNRESOLVED_COLUMN.WITHOUT_SUGGESTION",
Map("objectName" -> "`c`"),
caseSensitive = true,
line = -1,
pos = -1)
Map("objectName" -> "`c`")
)
// SELECT * FROM t1, LATERAL (SELECT a, b, c, d FROM t2)
assertAnalysisErrorClass(
lateralJoin(t1, t2.select($"a", $"b", $"c", $"d")),
"UNRESOLVED_COLUMN.WITH_SUGGESTION",
Map("objectName" -> "`d`", "proposal" -> "`b`, `c`"),
caseSensitive = true,
line = -1,
pos = -1)
Map("objectName" -> "`d`", "proposal" -> "`b`, `c`")
)
// SELECT * FROM t1, LATERAL (SELECT * FROM t2, LATERAL (SELECT t1.a))
assertAnalysisErrorClass(
lateralJoin(t1, lateralJoin(t2, t0.select($"t1.a"))),
"UNRESOLVED_COLUMN.WITHOUT_SUGGESTION",
Map("objectName" -> "`t1`.`a`"),
caseSensitive = true,
line = -1,
pos = -1)
Map("objectName" -> "`t1`.`a`")
)
// SELECT * FROM t1, LATERAL (SELECT * FROM t2, LATERAL (SELECT a, b))
assertAnalysisErrorClass(
lateralJoin(t1, lateralJoin(t2, t0.select($"a", $"b"))),
"UNRESOLVED_COLUMN.WITHOUT_SUGGESTION",
Map("objectName" -> "`a`"),
caseSensitive = true,
line = -1,
pos = -1)
Map("objectName" -> "`a`")
)
}

test("lateral subquery with struct type") {
Expand Down
12 changes: 4 additions & 8 deletions ...catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/V2WriteAnalysisSuite.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -691,10 +691,8 @@ abstract class V2WriteAnalysisSuiteBase extends AnalysisTest {
assertAnalysisErrorClass(
parsedPlan,
"UNRESOLVED_COLUMN.WITH_SUGGESTION",
Map("objectName" -> "`a`", "proposal" -> "`x`, `y`"),
caseSensitive = true,
line = -1,
pos = -1)
Map("objectName" -> "`a`", "proposal" -> "`x`, `y`")
)

val tableAcceptAnySchema = TestRelationAcceptAnySchema(StructType(Seq(
StructField("x", DoubleType, nullable = false),
Expand All @@ -706,10 +704,8 @@ abstract class V2WriteAnalysisSuiteBase extends AnalysisTest {
assertAnalysisErrorClass(
parsedPlan2,
"UNRESOLVED_COLUMN.WITH_SUGGESTION",
Map("objectName" -> "`a`", "proposal" -> "`x`, `y`"),
caseSensitive = true,
line = -1,
pos = -1)
Map("objectName" -> "`a`", "proposal" -> "`x`, `y`")
)
}

test("SPARK-36498: reorder inner fields with byName mode") {
Expand Down