From f86eb846101eaf04311ba5ad9b1a40e2a905b97d Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Fri, 30 Aug 2024 14:45:41 +0000 Subject: [PATCH 1/3] Fix nullability of base64 --- .../spark/sql/catalyst/expressions/stringExpressions.scala | 3 ++- .../sql/catalyst/expressions/StringExpressionsSuite.scala | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala index 5a30bbdf846a2..6ccd5a451eafc 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala @@ -2714,7 +2714,8 @@ case class Base64(child: Expression, chunkBase64: Boolean) dataType, "encode", Seq(child, Literal(chunkBase64, BooleanType)), - Seq(BinaryType, BooleanType)) + Seq(BinaryType, BooleanType), + returnNullable = false) override def toString: String = s"$prettyName($child)" diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala index 7210979f08462..fcd87d3567e60 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala @@ -467,6 +467,12 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { val b = $"b".binary.at(0) val bytes = Array[Byte](1, 2, 3, 4) + assert(!Base64(Literal(bytes)).nullable) + assert(Base64(Literal.create(null, BinaryType)).nullable) + assert(!UnBase64(Literal("AQIDBA==")).nullable) + assert(UnBase64(Literal.create(null, StringType)).nullable) + + checkEvaluation(Base64(Literal(bytes)), "AQIDBA==", create_row("abdef")) checkEvaluation(Base64(UnBase64(Literal("AQIDBA=="))), "AQIDBA==", create_row("abdef")) checkEvaluation(Base64(UnBase64(Literal(""))), "", create_row("abdef")) From 1659d323c5afaa6a6adafc4e5371979209c1f98e Mon Sep 17 00:00:00 2001 From: Adam Binford Date: Sun, 1 Sep 2024 17:43:34 -0400 Subject: [PATCH 2/3] Add more nullable tests --- .../spark/sql/catalyst/expressions/StringExpressionsSuite.scala | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala index fcd87d3567e60..0250119a2efc7 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala @@ -469,8 +469,10 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { assert(!Base64(Literal(bytes)).nullable) assert(Base64(Literal.create(null, BinaryType)).nullable) + assert(Base64(Literal(bytes).castNullable()).nullable) assert(!UnBase64(Literal("AQIDBA==")).nullable) assert(UnBase64(Literal.create(null, StringType)).nullable) + assert(UnBase64(Literal("AQIDBA==").castNullable()).nullable) checkEvaluation(Base64(Literal(bytes)), "AQIDBA==", create_row("abdef")) From 640e04a909c7c89b8abda23983100e2819fd761f Mon Sep 17 00:00:00 2001 From: Maxim Gekk Date: Mon, 2 Sep 2024 08:03:40 +0200 Subject: [PATCH 3/3] Update sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala --- .../spark/sql/catalyst/expressions/StringExpressionsSuite.scala | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala index 0250119a2efc7..b39820f0d317d 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala @@ -474,7 +474,6 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { assert(UnBase64(Literal.create(null, StringType)).nullable) assert(UnBase64(Literal("AQIDBA==").castNullable()).nullable) - checkEvaluation(Base64(Literal(bytes)), "AQIDBA==", create_row("abdef")) checkEvaluation(Base64(UnBase64(Literal("AQIDBA=="))), "AQIDBA==", create_row("abdef")) checkEvaluation(Base64(UnBase64(Literal(""))), "", create_row("abdef"))