From b2cbe32a01a936deccc6d8941c0df45a38ad7f69 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Sat, 8 Feb 2020 22:49:53 +0300 Subject: [PATCH 1/3] Fix cache initialization for foldable expressions --- .../spark/sql/catalyst/expressions/regexpExpressions.scala | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala index c9ddc70bf5bc6..6a4af7c16c591 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala @@ -43,8 +43,10 @@ trait StringRegexExpression extends Expression // try cache the pattern for Literal private lazy val cache: Pattern = pattern match { - case Literal(value: String, StringType) => compile(value) - case _ => null + case p: Expression if p.foldable => + compile(p.eval().asInstanceOf[UTF8String].toString) + case _ => + null } protected def compile(str: String): Pattern = if (str == null) { From 0986e5cbfbc395ed92f2ca695253741ac9e168fb Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Sat, 8 Feb 2020 23:11:45 +0300 Subject: [PATCH 2/3] Fix comment --- .../spark/sql/catalyst/expressions/regexpExpressions.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala index 6a4af7c16c591..60086e449135c 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala @@ -41,7 +41,7 @@ trait StringRegexExpression extends Expression override def dataType: DataType = BooleanType - // try cache the pattern for Literal + // try cache foldable pattern private lazy val cache: Pattern = pattern match { case p: Expression if p.foldable => compile(p.eval().asInstanceOf[UTF8String].toString) From 7bc5d9e4bb2a35add0e4f6dc2c4571b9c02a9319 Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Mon, 10 Feb 2020 10:46:37 +0300 Subject: [PATCH 3/3] Avoid unnecessary changes --- .../spark/sql/catalyst/expressions/regexpExpressions.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala index 60086e449135c..9660dbd6baba4 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala @@ -45,8 +45,7 @@ trait StringRegexExpression extends Expression private lazy val cache: Pattern = pattern match { case p: Expression if p.foldable => compile(p.eval().asInstanceOf[UTF8String].toString) - case _ => - null + case _ => null } protected def compile(str: String): Pattern = if (str == null) {