[SPARK-40194][SQL] SPLIT function on empty regex should truncate trailing empty string.

common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java

@@ -1000,6 +1000,17 @@ public static UTF8String concatWs(UTF8String separator, UTF8String... inputs) {
   }
 
   public UTF8String[] split(UTF8String pattern, int limit) {
+    // This is a special case for converting string into array of symbols without a trailing empty
+    // string. E.g. `"hello".split("", 0) => ["h", "e", "l", "l", "o"].
+    // Note that negative limit will preserve a trailing empty string.
+    if (limit == 0 && numBytes() != 0 && pattern.numBytes() == 0) {
+      byte[] input = getBytes();
+      UTF8String[] result = new UTF8String[numBytes];
+      for (int i = 0; i < numBytes; i++) {
+        result[i] = UTF8String.fromBytes(input, i, 1);
+      }
+      return result;
+    }
     return split(pattern.toString(), limit);
   }
 

common/unsafe/src/test/java/org/apache/spark/unsafe/types/UTF8StringSuite.java

@@ -409,6 +409,15 @@ public void split() {
     assertArrayEquals(
       new UTF8String[]{fromString("ab"), fromString("def,ghi,")},
       fromString("ab,def,ghi,").split(fromString(","), 2));
+    assertArrayEquals(
+      new UTF8String[]{fromString("a"), fromString("b")},
+      fromString("ab").split(fromString(""), 0));
+    assertArrayEquals(
+      new UTF8String[]{fromString("a"), fromString("b"), fromString("")},
+      fromString("ab").split(fromString(""), -1));
+    assertArrayEquals(
+      new UTF8String[]{fromString("")},
+      fromString("").split(fromString(""), 0));
   }
 
   @Test

docs/sql-migration-guide.md

@@ -27,6 +27,7 @@ license: |
   - Since Spark 3.4, Number or Number(\*) from Teradata will be treated as Decimal(38,18). In Spark 3.3 or earlier, Number or Number(\*) from Teradata will be treated as Decimal(38, 0), in which case the fractional part will be removed.
   - Since Spark 3.4, v1 database, table, permanent view and function identifier will include 'spark_catalog' as the catalog name if database is defined, e.g. a table identifier will be: `spark_catalog.default.t`. To restore the legacy behavior, set `spark.sql.legacy.v1IdentifierNoCatalog` to `true`.
   - Since Spark 3.4, when ANSI SQL mode(configuration `spark.sql.ansi.enabled`) is on, Spark SQL always returns NULL result on getting a map value with a non-existing key. In Spark 3.3 or earlier, there will be an error.
+  - Since Spark 3.4, `split` function ignores trailing empty strings when `regex` parameter is empty and `limit` parameter is not specified or set to 0.
 
 ## Upgrading from Spark SQL 3.2 to 3.3
 

sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/regexpExpressions.scala

@@ -527,7 +527,7 @@ case class StringSplit(str: Expression, regex: Expression, limit: Expression)
   override def second: Expression = regex
   override def third: Expression = limit
 
-  def this(exp: Expression, regex: Expression) = this(exp, regex, Literal(-1));
+  def this(exp: Expression, regex: Expression) = this(exp, regex, Literal(0))
 
   override def nullSafeEval(string: Any, regex: Any, limit: Any): Any = {
     val strings = string.asInstanceOf[UTF8String].split(

sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/RegexpExpressionsSuite.scala

@@ -461,6 +461,15 @@ class RegexpExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
       StringSplit(s1, s2, -1), Seq("aa", "bb", "cc"), row1)
     checkEvaluation(StringSplit(s1, s2, -1), null, row2)
     checkEvaluation(StringSplit(s1, s2, -1), null, row3)
+    // Empty regex
+    checkEvaluation(
+      StringSplit(Literal("hello"), Literal(""), 0), Seq("h", "e", "l", "l", "o"), row1)
+    checkEvaluation(
+      StringSplit(Literal("hello"), Literal(""), -1), Seq("h", "e", "l", "l", "o", ""), row1)
+    checkEvaluation(
+      StringSplit(Literal(""), Literal(""), -1), Seq(""), row1)
+    checkEvaluation(
+      StringSplit(Literal(""), Literal(""), 0), Seq(""), row1)
 
     // Test escaping of arguments
     GenerateUnsafeProjection.generate(

sql/core/src/test/resources/sql-functions/sql-expression-schema.md

@@ -287,7 +287,7 @@
 | org.apache.spark.sql.catalyst.expressions.StringRepeat | repeat | SELECT repeat('123', 2) | struct<repeat(123, 2):string> |
 | org.apache.spark.sql.catalyst.expressions.StringReplace | replace | SELECT replace('ABCabc', 'abc', 'DEF') | struct<replace(ABCabc, abc, DEF):string> |
 | org.apache.spark.sql.catalyst.expressions.StringSpace | space | SELECT concat(space(2), '1') | struct<concat(space(2), 1):string> |
-| org.apache.spark.sql.catalyst.expressions.StringSplit | split | SELECT split('oneAtwoBthreeC', '[ABC]') | struct<split(oneAtwoBthreeC, [ABC], -1):array<string>> |
+| org.apache.spark.sql.catalyst.expressions.StringSplit | split | SELECT split('oneAtwoBthreeC', '[ABC]') | struct<split(oneAtwoBthreeC, [ABC], 0):array<string>> |
 | org.apache.spark.sql.catalyst.expressions.StringToMap | str_to_map | SELECT str_to_map('a:1,b:2,c:3', ',', ':') | struct<str_to_map(a:1,b:2,c:3, ,, :):map<string,string>> |
 | org.apache.spark.sql.catalyst.expressions.StringTranslate | translate | SELECT translate('AaBbCc', 'abc', '123') | struct<translate(AaBbCc, abc, 123):string> |
 | org.apache.spark.sql.catalyst.expressions.StringTrim | trim | SELECT trim('    SparkSQL   ') | struct<trim(    SparkSQL   ):string> |

sql/core/src/test/resources/sql-tests/inputs/string-functions.sql

@@ -26,6 +26,11 @@ select right("abcd", -2), right("abcd", 0), right("abcd", 'a');
 -- split function
 SELECT split('aa1cc2ee3', '[1-9]+');
 SELECT split('aa1cc2ee3', '[1-9]+', 2);
+SELECT split('hello', '');
+SELECT split('hello', '', -1);
+SELECT split('', '');
+SELECT split('abc', null);
+SELECT split(null, 'b');
 
 -- split_part function
 SELECT split_part('11.12.13', '.', 2);

sql/core/src/test/resources/sql-tests/results/ansi/higher-order-functions.sql.out

@@ -272,6 +272,6 @@ struct<v:array<int>>
 -- !query
 select aggregate(split('abcdefgh',''), array(array('')), (acc, x) -> array(array(x)))
 -- !query schema
-struct<aggregate(split(abcdefgh, , -1), array(array()), lambdafunction(array(array(namedlambdavariable())), namedlambdavariable(), namedlambdavariable()), lambdafunction(namedlambdavariable(), namedlambdavariable())):array<array<string>>>
+struct<aggregate(split(abcdefgh, , 0), array(array()), lambdafunction(array(array(namedlambdavariable())), namedlambdavariable(), namedlambdavariable()), lambdafunction(namedlambdavariable(), namedlambdavariable())):array<array<string>>>
 -- !query output
-[[""]]
+[["h"]]

sql/core/src/test/resources/sql-tests/results/ansi/string-functions.sql.out

@@ -142,7 +142,7 @@ org.apache.spark.SparkNumberFormatException
 -- !query
 SELECT split('aa1cc2ee3', '[1-9]+')
 -- !query schema
-struct<split(aa1cc2ee3, [1-9]+, -1):array<string>>
+struct<split(aa1cc2ee3, [1-9]+, 0):array<string>>
 -- !query output
 ["aa","cc","ee",""]
 
@@ -155,6 +155,46 @@ struct<split(aa1cc2ee3, [1-9]+, 2):array<string>>
 ["aa","cc2ee3"]
 
 
+-- !query
+SELECT split('hello', '')
+-- !query schema
+struct<split(hello, , 0):array<string>>
+-- !query output
+["h","e","l","l","o"]
+
+
+-- !query
+SELECT split('hello', '', -1)
+-- !query schema
+struct<split(hello, , -1):array<string>>
+-- !query output
+["h","e","l","l","o",""]
+
+
+-- !query
+SELECT split('', '')
+-- !query schema
+struct<split(, , 0):array<string>>
+-- !query output
+[""]
+
+
+-- !query
+SELECT split('abc', null)
+-- !query schema
+struct<split(abc, NULL, 0):array<string>>
+-- !query output
+NULL
+
+
+-- !query
+SELECT split(null, 'b')
+-- !query schema
+struct<split(NULL, b, 0):array<string>>
+-- !query output
+NULL
+
+
 -- !query
 SELECT split_part('11.12.13', '.', 2)
 -- !query schema

sql/core/src/test/resources/sql-tests/results/charvarchar.sql.out

@@ -1054,7 +1054,7 @@ NetEase	NetEase	Spark-	NetEase
 -- !query
 select split(c7, 'e'), split(c8, 'e'), split(v, 'a'), split(s, 'e') from char_tbl4
 -- !query schema
-struct<split(c7, e, -1):array<string>,split(c8, e, -1):array<string>,split(v, a, -1):array<string>,split(s, e, -1):array<string>>
+struct<split(c7, e, 0):array<string>,split(c8, e, 0):array<string>,split(v, a, 0):array<string>,split(s, e, 0):array<string>>
 -- !query output
 NULL	NULL	NULL	NULL
 NULL	NULL	["S"]	NULL

sql/core/src/test/resources/sql-tests/results/higher-order-functions.sql.out

@@ -272,6 +272,6 @@ struct<v:array<int>>
 -- !query
 select aggregate(split('abcdefgh',''), array(array('')), (acc, x) -> array(array(x)))
 -- !query schema
-struct<aggregate(split(abcdefgh, , -1), array(array()), lambdafunction(array(array(namedlambdavariable())), namedlambdavariable(), namedlambdavariable()), lambdafunction(namedlambdavariable(), namedlambdavariable())):array<array<string>>>
+struct<aggregate(split(abcdefgh, , 0), array(array()), lambdafunction(array(array(namedlambdavariable())), namedlambdavariable(), namedlambdavariable()), lambdafunction(namedlambdavariable(), namedlambdavariable())):array<array<string>>>
 -- !query output
-[[""]]
+[["h"]]

sql/core/src/test/resources/sql-tests/results/string-functions.sql.out

@@ -108,7 +108,7 @@ struct<right(abcd, -2):string,right(abcd, 0):string,right(abcd, a):string>
 -- !query
 SELECT split('aa1cc2ee3', '[1-9]+')
 -- !query schema
-struct<split(aa1cc2ee3, [1-9]+, -1):array<string>>
+struct<split(aa1cc2ee3, [1-9]+, 0):array<string>>
 -- !query output
 ["aa","cc","ee",""]
 
@@ -121,6 +121,46 @@ struct<split(aa1cc2ee3, [1-9]+, 2):array<string>>
 ["aa","cc2ee3"]
 
 
+-- !query
+SELECT split('hello', '')
+-- !query schema
+struct<split(hello, , 0):array<string>>
+-- !query output
+["h","e","l","l","o"]
+
+
+-- !query
+SELECT split('hello', '', -1)
+-- !query schema
+struct<split(hello, , -1):array<string>>
+-- !query output
+["h","e","l","l","o",""]
+
+
+-- !query
+SELECT split('', '')
+-- !query schema
+struct<split(, , 0):array<string>>
+-- !query output
+[""]
+
+
+-- !query
+SELECT split('abc', null)
+-- !query schema
+struct<split(abc, NULL, 0):array<string>>
+-- !query output
+NULL
+
+
+-- !query
+SELECT split(null, 'b')
+-- !query schema
+struct<split(NULL, b, 0):array<string>>
+-- !query output
+NULL
+
+
 -- !query
 SELECT split_part('11.12.13', '.', 2)
 -- !query schema