From 53ed33d91608737e0ff648816dcedb6410ec09e9 Mon Sep 17 00:00:00 2001 From: Franck Thang Date: Mon, 17 Jan 2022 17:44:14 +0100 Subject: [PATCH 1/4] [SPARK-37290][SQL] Exponential planning time in case of non-deterministic function --- .../apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala index 49634a2a0eb8..d14d41620e53 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala @@ -185,7 +185,7 @@ trait UnaryNode extends LogicalPlan with UnaryLike[LogicalPlan] { allConstraints += EqualNullSafe(a.toAttribute, l) case a @ Alias(e, _) => // For every alias in `projectList`, replace the reference in constraints by its attribute. - allConstraints ++= allConstraints.map(_ transform { + allConstraints ++= allConstraints.filter(_.deterministic).map(_ transform { case expr: Expression if expr.semanticEquals(e) => a.toAttribute }) From ab0684e518f32c2b96e4b7ac80c561c004a064c3 Mon Sep 17 00:00:00 2001 From: Franck Thang Date: Mon, 17 Jan 2022 20:57:17 +0100 Subject: [PATCH 2/4] [SPARK-37290][SQL] Exponential planning time in case of non-deterministic function --- .../apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala index d14d41620e53..49634a2a0eb8 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala @@ -185,7 +185,7 @@ trait UnaryNode extends LogicalPlan with UnaryLike[LogicalPlan] { allConstraints += EqualNullSafe(a.toAttribute, l) case a @ Alias(e, _) => // For every alias in `projectList`, replace the reference in constraints by its attribute. - allConstraints ++= allConstraints.filter(_.deterministic).map(_ transform { + allConstraints ++= allConstraints.map(_ transform { case expr: Expression if expr.semanticEquals(e) => a.toAttribute }) From ec3a2823a0009513d29e104d34e4227b03fb2faa Mon Sep 17 00:00:00 2001 From: Franck Thang Date: Mon, 17 Jan 2022 20:58:40 +0100 Subject: [PATCH 3/4] [SPARK-37290][SQL] Exponential planning time in case of non-deterministic function --- .../apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala index 49634a2a0eb8..d14d41620e53 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala @@ -185,7 +185,7 @@ trait UnaryNode extends LogicalPlan with UnaryLike[LogicalPlan] { allConstraints += EqualNullSafe(a.toAttribute, l) case a @ Alias(e, _) => // For every alias in `projectList`, replace the reference in constraints by its attribute. - allConstraints ++= allConstraints.map(_ transform { + allConstraints ++= allConstraints.filter(_.deterministic).map(_ transform { case expr: Expression if expr.semanticEquals(e) => a.toAttribute }) From bc452a70c43fa01acb4394972deb07608ace6334 Mon Sep 17 00:00:00 2001 From: Franck Thang Date: Tue, 22 Feb 2022 03:36:46 +0100 Subject: [PATCH 4/4] [SPARK-37290][SQL] Exponential planning time in case of non-deterministic function --- .../apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala index d14d41620e53..5ae2a7da826a 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/LogicalPlan.scala @@ -183,9 +183,9 @@ trait UnaryNode extends LogicalPlan with UnaryLike[LogicalPlan] { projectList.foreach { case a @ Alias(l: Literal, _) => allConstraints += EqualNullSafe(a.toAttribute, l) - case a @ Alias(e, _) => + case a @ Alias(e, _) if e.deterministic => // For every alias in `projectList`, replace the reference in constraints by its attribute. - allConstraints ++= allConstraints.filter(_.deterministic).map(_ transform { + allConstraints ++= allConstraints.map(_ transform { case expr: Expression if expr.semanticEquals(e) => a.toAttribute })