fix(text): unicode support and word splitting according to case

text/_util.ts

@@ -1,9 +1,22 @@
 // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
 
+const RE_ACRONYM = /\p{Lu}{2,}/gu; // e.g. ID, URL
+const RE_CAPITALIZED_WORD = /\p{Lu}\p{Ll}+/gu; // e.g. Apple
+const RE_LOWERCASED_WORD = /\P{Lu}+/gu; // e.g. apple, this will also match a sequence of unicode letters in languages without a concept of upper/lower case
+const RE_DIGITS = /\p{N}+/gu; // e.g. 123
+const RE_MATCH_WORD_BY_CASE = new RegExp(
+  `${RE_ACRONYM.source}|${RE_CAPITALIZED_WORD.source}|${RE_DIGITS.source}|${RE_LOWERCASED_WORD.source}`,
+  "gu",
+);
+
+function splitByCase(input: string): string[] {
+  const matches = input.match(RE_MATCH_WORD_BY_CASE);
+
+  return matches || [input];
+}
+
 export function splitToWords(input: string) {
-  input = input.replaceAll(/[^a-zA-Z0-9\s-_]/g, "");
-  if (/[\s-_]+/.test(input)) return input.split(/[\s-_]+/);
-  return input.split(/(?=[A-Z])+/);
+  return input.split(/[^\p{L}\p{N}]+/gu).filter(Boolean).flatMap(splitByCase);
 }
 
 export function capitalizeWord(word: string): string {

text/_util_test.ts

@@ -21,6 +21,15 @@ Deno.test({
   },
 });
 
+Deno.test({
+  name: "split() handles a delimiter sequence",
+  fn() {
+    const result = splitToWords("I am   -> thirsty!");
+    const expected = ["I", "am", "thirsty"];
+    assertEquals(result, expected);
+  },
+});
+
 Deno.test({
   name: "split() handles upper case delimiter",
   fn() {
@@ -39,6 +48,42 @@ Deno.test({
   },
 });
 
+Deno.test({
+  name: "split() handles casing",
+  fn() {
+    const result = splitToWords("denoIsAwesome");
+    const expected = ["deno", "Is", "Awesome"];
+    assertEquals(result, expected);
+  },
+});
+
+Deno.test({
+  name: "split() handles unicode",
+  fn() {
+    const result = splitToWords("шруберри IsAwesome");
+    const expected = ["шруберри", "Is", "Awesome"];
+    assertEquals(result, expected);
+  },
+});
+
+Deno.test({
+  name: "split() handles unicode casing",
+  fn() {
+    const result = splitToWords("шруберриШруберри");
+    const expected = ["шруберри", "Шруберри"];
+    assertEquals(result, expected);
+  },
+});
+
+Deno.test({
+  name: "split() handles languages without casing",
+  fn() {
+    const result = splitToWords("אין_על דינו");
+    const expected = ["אין", "על", "דינו"];
+    assertEquals(result, expected);
+  },
+});
+
 Deno.test({
   name: "split() handles screaming snake case",
   fn() {

text/case_test.ts

@@ -87,6 +87,12 @@ Deno.test("toPascalCase() trims whitespace", () => {
   assertEquals(result, expected);
 });
 
+Deno.test("toPascalCase() converts a single word with Cyrillic letters", () => {
+  const input = "шруберри";
+  const expected = "Шруберри";
+  assertEquals(toPascalCase(input), expected);
+});
+
 Deno.test("toSnakeCase() handles an empty string", () => {
   assertEquals(toSnakeCase(""), "");
 });