-
Notifications
You must be signed in to change notification settings - Fork 29k
[SPARK-31334][SQL] Don't ResolveReference/ResolveMissingReference when Filter condition with aggregate expression #28107
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
11eb148
1cabd08
df257df
54df9d3
24e9d4d
0d81b4d
dfda7ff
a5cf877
4a799fe
5f6eef1
cc0b018
b2855bd
20febce
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -1391,11 +1391,39 @@ class Analyzer( | |
| notMatchedActions = newNotMatchedActions) | ||
| } | ||
|
|
||
| // When filter condition is havingConditions, columns haven't been handled by | ||
| // TypeCoercion, this make a situation that cond isn't resolved because of aggregate | ||
| // functions's checkInputDataType method. Then it can't be handled by | ||
| // ResolveAggregateFunctions, finally cause column resolve error. | ||
| // For this situation, we don't resolve cond's reference here | ||
| case f @ Filter(cond, agg @ Aggregate(_, _, _)) if containsAggregate(cond) => f | ||
|
|
||
| case q: LogicalPlan => | ||
| logTrace(s"Attempting to resolve ${q.simpleString(SQLConf.get.maxToStringFields)}") | ||
| q.mapExpressions(resolveExpressionTopDown(_, q)) | ||
| } | ||
|
|
||
| def containsAggregate(e: Expression): Boolean = { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. why can't we reuse
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Since here function is still UnresolvedFunction, we can't just reuse this. |
||
| e.find { | ||
| // In current loop, functions maybe unresolved, | ||
| // we should judge if it is aggregate function now | ||
| case func: UnresolvedFunction => | ||
| try { | ||
| v1SessionCatalog.lookupFunction(func.name, func.arguments) | ||
| .isInstanceOf[AggregateFunction] | ||
| } catch { | ||
| // When UnresolvedFunction is a UDF function, we can't lookup function since | ||
| // it's arguments is unresolved. If throw exception when lookup functions, | ||
| // let's assume that we don't deal with this situation right now, | ||
| // after next loop, this function's arguments will be resolved | ||
| // then we can judge if this function is aggregate function next time. | ||
| case _: Exception => true | ||
| } | ||
| case _ => | ||
| false | ||
| }.isDefined || e.find(_.isInstanceOf[AggregateExpression]).isDefined | ||
| } | ||
|
|
||
| def resolveAssignments( | ||
| assignments: Seq[Assignment], | ||
| mergeInto: MergeIntoTable, | ||
|
|
@@ -1679,7 +1707,13 @@ class Analyzer( | |
| Project(child.output, newSort) | ||
| } | ||
|
|
||
| case f @ Filter(cond, child) if (!f.resolved || f.missingInput.nonEmpty) && child.resolved => | ||
| // When filter condition is havingConditions, columns haven't been handled by | ||
| // TypeCoercion, this make a situation that cond isn't resolved because of aggregate | ||
| // functions's checkInputDataType method. Then it can't be handled by | ||
| // ResolveAggregateFunctions, finally cause column resolve error. | ||
| // For this situation, we don't resolve cond's reference here | ||
| case f @ Filter(cond, child) if (!f.resolved || f.missingInput.nonEmpty) && child.resolved | ||
| && (!child.isInstanceOf[Aggregate] || !ResolveReferences.containsAggregate(cond)) => | ||
| val (newCond, newChild) = resolveExprsAndAddMissingAttrs(Seq(cond), child) | ||
| if (child.output == newChild.output) { | ||
| f.copy(condition = newCond.head) | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.