From 72f14e5a7815f47aa5367a2d7d8db4bbd3616cf3 Mon Sep 17 00:00:00 2001
From: mahmoudhas <mahmoudhas@outlook.com>
Date: Thu, 15 Oct 2026 08:52:41 -0700
Subject: [PATCH] fix: keep JSON string patterns on the JSON-safe FSM path

---
 cpp/json_schema_converter.cc | 25 ++++++++++++-------------
 tests/cpp/test_json.cc       | 30 ++++++++++++++++++++++++++++++
 2 files changed, 42 insertions(+), 13 deletions(-)

diff --git a/cpp/json_schema_converter.cc b/cpp/json_schema_converter.cc
index 5ca329f94..0808a2281 100644
--- a/cpp/json_schema_converter.cc
+++ b/cpp/json_schema_converter.cc
@@ -2208,21 +2208,16 @@ int32_t JSONSchemaConverter::GenerateFromSpec(
  * json_string=true so the pattern is compiled into a single automaton by GrammarFSMBuilder;
  * json_string=true excludes the characters that must be escaped in a JSON string ('"', '\\'
  * and the control characters) from every character match, so classes like \S cannot emit an
- * unescaped quote. Fall back to the CFG expansion when the FSM regex engine does not support
- * the pattern, or when the exclusion makes the pattern unmatchable (e.g. a pattern requiring
- * a literal '"').
+ * unescaped quote. JSON-string patterns must never fall back to the raw regex CFG: that
+ * path does not escape quotes or control characters. If the safe canonical language is
+ * empty or unsupported, reject the schema rather than generate malformed JSON.
  */
 int32_t JSONSchemaConverter::RegexExpression(
     const std::string& regex, bool json_string, bool force_cfg_expansion
 ) {
-  bool can_use_fsm = !force_cfg_expansion;
-  if (json_string) {
-    can_use_fsm =
-        can_use_fsm && std::all_of(regex.begin(), regex.end(), [](unsigned char character) {
-          return character >= 0x20 && character <= 0x7e;
-        });
-  }
-  if (can_use_fsm) {
+  // The FSM handles UTF-8 patterns too; routing those through the raw CFG loses
+  // JSON-string exclusions (for example, in a Unicode pattern containing \\S).
+  if (!force_cfg_expansion) {
     auto fsm_result = GrammarFSMBuilder::Regex(regex, json_string);
     if (fsm_result.IsOk()) {
       auto fsm = std::move(fsm_result).Unwrap();
@@ -2238,8 +2233,12 @@ int32_t JSONSchemaConverter::RegexExpression(
     }
   }
 
-  // Keep regex conversion independent. Only the uncommon fallback path converts its existing
-  // EBNF result to a subgrammar; the JSON Schema rule graph itself is still built directly.
+  XGRAMMAR_CHECK(!json_string)
+      << "JSON string pattern has no JSON-safe canonical representation or uses unsupported regex "
+         "syntax; patterns requiring escaped quotes, backslashes or control characters are not "
+         "supported";
+
+  // Raw regex/numeric conversion is independent and may still use its CFG fallback.
   return AddSubGrammar(Grammar::FromEBNF(RegexToEBNF(regex)));
 }
 
diff --git a/tests/cpp/test_json.cc b/tests/cpp/test_json.cc
index d48e07b3c..d0f7f7049 100644
--- a/tests/cpp/test_json.cc
+++ b/tests/cpp/test_json.cc
@@ -52,5 +52,35 @@ TEST(JSONTest, JSONSchemaCarriageReturnWhitespaceModes) {
   EXPECT_FALSE(MatchesEntireString(capped, "{\r\n\r\"value\": 3}"));
 }
 
+TEST(JSONTest, UnicodePatternsRetainJSONStringExclusions) {
+  GrammarCompiler compiler(TokenizerInfo(std::vector<std::string>{}), 1, false);
+  auto grammar = compiler.CompileJSONSchema(
+      R"({"type":"string","pattern":"^[é\\s\\S]{1,64}$"})"
+  );
+  EXPECT_TRUE(MatchesEntireString(grammar, "\"hello é 🙂 world\""));
+  EXPECT_FALSE(MatchesEntireString(grammar, "\"hello\"world\""));
+  for (char control = 0; control < 0x20; ++control) {
+    EXPECT_FALSE(MatchesEntireString(grammar, std::string("\"hello") + control + "world\""));
+  }
+  auto literal = compiler.CompileJSONSchema(R"({"type":"string","pattern":"^é🙂+$"})");
+  EXPECT_TRUE(MatchesEntireString(literal, "\"é🙂🙂\""));
+  EXPECT_FALSE(MatchesEntireString(literal, "\"éé\""));
+}
+
+TEST(JSONTest, EscapedOnlyPatternsFailBeforeGenerationInsteadOfUsingRawCFG) {
+  GrammarCompiler compiler(TokenizerInfo(std::vector<std::string>{}), 1, false);
+  for (const auto* schema : {
+           R"({"type":"string","pattern":"^\\\"$"})",
+           R"({"type":"string","pattern":"^\\n$"})",
+           R"({"type":"string","pattern":"^\\\\$"})",
+           R"({"type":"object","properties":{"text":{"type":"string","pattern":"^\\\"$"}}})",
+       }) {
+    EXPECT_ANY_THROW(compiler.CompileJSONSchema(schema));
+  }
+  // The independent raw-regex API is not JSON and must keep accepting these bytes.
+  auto raw = compiler.CompileRegex("^\"$");
+  EXPECT_TRUE(MatchesEntireString(raw, "\""));
+}
+
 }  // namespace
 }  // namespace xgrammar