fix: preserve Unicode character class ranges

cpp/fsm_builder.cc

@@ -136,7 +136,7 @@ static void AddSameLengthCharacterRange(FSM& fsm, int from, int to, uint32_t min
     if ((max & 0x00FFFF) != 0xBFBF) {
       int tmp_state_max = fsm.AddState();
       fsm.AddEdge(from, tmp_state_max, byte_max[2], byte_max[2]);
-      AddSameLengthCharacterRange(fsm, tmp_state_max, to, 0x0080, (max & 0x00FFFF));
+      AddSameLengthCharacterRange(fsm, tmp_state_max, to, 0x008080, (max & 0x00FFFF));
     } else {
       byte_max[2]++;
     }

cpp/grammar_functor.cc

@@ -1420,6 +1420,7 @@ class GrammarFSMBuilderImpl {
   void BuildNegativeCharacterClass(const GrammarExpr& expr, int start_state, int end_state);
   void AppendFSM(FSMWithStartEnd fsm, int start_state, std::vector<int32_t>* end_states);
   void AddCharacterRange(int from, int to, uint32_t min, uint32_t max);
+  void AddCodepointRange(int from, int to, uint32_t low, uint32_t high);
 
   FSM& target_fsm_;
   const std::string* rule_name_;
@@ -1432,45 +1433,47 @@ void GrammarFSMBuilderImpl::AddCharacterRange(int from, int to, uint32_t min, ui
   AddPackedUTF8RangeEdges(target_fsm_, from, to, min, max);
 }
 
+void GrammarFSMBuilderImpl::AddCodepointRange(int from, int to, uint32_t low, uint32_t high) {
+  // Do not bridge UTF-8 widths with the packed-byte range helper: its historical minimum
+  // byte sequences include overlong encodings (e.g. C0 80), which are not codepoints.
+  constexpr uint32_t width_ends[] = {0x7F, 0x7FF, 0xFFFF, 0x10FFFF};
+  for (uint32_t width_end : width_ends) {
+    if (low <= high && low <= width_end) {
+      auto end = std::min(high, width_end);
+      AddCharacterRange(from, to, CodepointToPackedUTF8(low), CodepointToPackedUTF8(end));
+      low = end + 1;
+    }
+  }
+}
+
 void GrammarFSMBuilderImpl::BuildNegativeCharacterClass(
     const GrammarExpr& expr, int start_state, int end_state
 ) {
   XGRAMMAR_DCHECK(
       expr.type == ExprType::kCharacterClass || expr.type == ExprType::kCharacterClassStar
   );
   XGRAMMAR_DCHECK(expr[0]);  // Negative character class should be true.
-  std::bitset<128> char_set;
+  // Complement codepoint ranges before encoding them. Truncating their endpoints to bytes
+  // incorrectly made, for example, [^\x00-\U0010ffff] accept every multi-byte character.
+  constexpr uint32_t kMaxCodepoint = 0x10FFFF;
+  std::vector<std::pair<uint32_t, uint32_t>> excluded_ranges;
   for (int i = 1; i < static_cast<int>(expr.size()); i += 2) {
-    uint8_t byte_min = static_cast<uint8_t>(expr[i]);
-    uint8_t byte_max = static_cast<uint8_t>(expr[i + 1]);
-    if (byte_max > 128) {
-      XGRAMMAR_LOG(WARNING) << "Negative Character class contains byte greater than 127, "
-                            << "clamping to 127.";
-      byte_max = 127;
+    excluded_ranges.emplace_back(expr[i], expr[i + 1]);
+  }
+  std::sort(excluded_ranges.begin(), excluded_ranges.end());
+  uint32_t next = 0;
+  for (const auto& [low, high] : excluded_ranges) {
+    if (low > next) {
+      AddCodepointRange(start_state, end_state, next, std::min(low - 1, kMaxCodepoint));
     }
-    for (uint8_t j = byte_min; j <= byte_max; ++j) {
-      char_set.set(j);
+    if (high >= kMaxCodepoint) {
+      return;
     }
+    next = std::max(next, high + 1);
   }
-
-  int left_bound = -1;
-  for (int i = 0; i < 128; ++i) {
-    if (!char_set[i]) {
-      left_bound = i;
-      int right_bound = i + 1;
-      while (right_bound < 128 && !char_set[right_bound]) {
-        right_bound++;
-      }
-      target_fsm_.AddEdge(
-          start_state,
-          end_state,
-          static_cast<uint8_t>(left_bound),
-          static_cast<uint8_t>(right_bound - 1)
-      );
-      i = right_bound;
-    }
+  if (next <= kMaxCodepoint) {
+    AddCodepointRange(start_state, end_state, next, kMaxCodepoint);
   }
-  AddCharacterRange(start_state, end_state, kMin2BytesUnicode, kMax4BytesUnicode);
 }
 
 void GrammarFSMBuilderImpl::AddCharacterClassTransitions(
@@ -1486,10 +1489,7 @@ void GrammarFSMBuilderImpl::AddCharacterClassTransitions(
     for (int i = 1; i < static_cast<int>(expr.size()); i += 2) {
       uint32_t codepoint_min = static_cast<uint32_t>(expr[i]);
       uint32_t codepoint_max = static_cast<uint32_t>(expr[i + 1]);
-      // Convert Unicode codepoints to packed UTF-8 format for AddCharacterRange
-      uint32_t packed_min = CodepointToPackedUTF8(codepoint_min);
-      uint32_t packed_max = CodepointToPackedUTF8(codepoint_max);
-      AddCharacterRange(start_state, end_state, packed_min, packed_max);
+      AddCodepointRange(start_state, end_state, codepoint_min, codepoint_max);
     }
   }
 }

tests/cpp/test_fsm_builder.cc

@@ -266,15 +266,23 @@ TEST(XGrammarFSMBuilderTest, TestCharacterClassFSMBuilder3) {
   auto fsm = GrammarFSMBuilder::CharacterClass(grammar_expr);
   auto fsm_printed = fsm.ToString();
   std::string expected_fsm_printed =
-      R"(FSM(num_states=8, start=0, end=[1], edges=[
-0: [[\0-@]->1, [[-`]->1, [{-\x7f]->1, [\xc0-\xdf]->2, [\xe0-\xef]->3, [\xf0-\xf7]->5]
+      R"(FSM(num_states=16, start=0, end=[1], edges=[
+0: [[\0-@]->1, [[-`]->1, [{-\x7f]->1, [\xc2-\xdf]->2, '\xe0'->3, [\xe1-\xef]->5, '\xf0'->7, '\xf4'->10, [\xf1-\xf3]->13]
 1: []
 2: [[\x80-\xbf]->1]
-3: [[\x80-\xbf]->4]
+3: [[\xa0-\xbf]->4]
 4: [[\x80-\xbf]->1]
 5: [[\x80-\xbf]->6]
-6: [[\x80-\xbf]->7]
-7: [[\x80-\xbf]->1]
+6: [[\x80-\xbf]->1]
+7: [[\x90-\xbf]->8]
+8: [[\x80-\xbf]->9]
+9: [[\x80-\xbf]->1]
+10: [[\x80-\x8f]->11]
+11: [[\x80-\xbf]->12]
+12: [[\x80-\xbf]->1]
+13: [[\x80-\xbf]->14]
+14: [[\x80-\xbf]->15]
+15: [[\x80-\xbf]->1]
 ]))";
   EXPECT_EQ(fsm_printed, expected_fsm_printed);
 }
@@ -287,14 +295,22 @@ TEST(XGrammarFSMBuilderTest, TestCharacterClassFSMBuilder4) {
   auto fsm = GrammarFSMBuilder::CharacterClass(grammar_expr);
   auto fsm_printed = fsm.ToString();
   std::string expected_fsm_printed =
-      R"(FSM(num_states=7, start=0, end=[0], edges=[
-0: [[\0-@]->0, [[-`]->0, [{-\x7f]->0, [\xc0-\xdf]->1, [\xe0-\xef]->2, [\xf0-\xf7]->4]
+      R"(FSM(num_states=15, start=0, end=[0], edges=[
+0: [[\0-@]->0, [[-`]->0, [{-\x7f]->0, [\xc2-\xdf]->1, '\xe0'->2, [\xe1-\xef]->4, '\xf0'->6, '\xf4'->9, [\xf1-\xf3]->12]
 1: [[\x80-\xbf]->0]
-2: [[\x80-\xbf]->3]
+2: [[\xa0-\xbf]->3]
 3: [[\x80-\xbf]->0]
 4: [[\x80-\xbf]->5]
-5: [[\x80-\xbf]->6]
-6: [[\x80-\xbf]->0]
+5: [[\x80-\xbf]->0]
+6: [[\x90-\xbf]->7]
+7: [[\x80-\xbf]->8]
+8: [[\x80-\xbf]->0]
+9: [[\x80-\x8f]->10]
+10: [[\x80-\xbf]->11]
+11: [[\x80-\xbf]->0]
+12: [[\x80-\xbf]->13]
+13: [[\x80-\xbf]->14]
+14: [[\x80-\xbf]->0]
 ]))";
   EXPECT_EQ(fsm_printed, expected_fsm_printed);
 }

tests/python/test_grammar_fsm_builder.py

@@ -134,6 +134,88 @@ def test_sequence_with_utf8_byte_strings():
     assert not _matcher_accepts(matcher, "你好")
 
 
+def test_negated_classes_preserve_unicode_and_reject_overlong_utf8():
+    matcher = _make_string_matcher('root ::= "x" [^0-9] "y" [^a-z]* "z"')
+    for value in ("xéyz", "x你y😀z", "xAy你好éz"):
+        assert _matcher_accepts(matcher, value)
+    for value in ("x1yz", "xéyaz"):
+        assert not _matcher_accepts(matcher, value)
+    for value in (
+        b"x\xc0\x80yz",
+        b"x\xc1\xbfyz",
+        b"x\xe0\x80\x80yz",
+        b"xAy\xf0\x80\x80\x80z",
+        b"xAy\xf4\x90\x80\x80z",
+    ):
+        matcher.reset()
+        assert not matcher.accept_string(value)
+
+
+def test_nested_negated_class_preserves_calls_and_unicode():
+    matcher = _make_string_matcher(
+        'root ::= a b c\na ::= "x" [0-9]*\nb ::= a "y" | [^xyz]\nc ::= b{1,3} "end"'
+    )
+    for value in ("xé你end", "x12x3y😀end", "xAé🙂好end", "xéx42yend"):
+        assert _matcher_accepts(matcher, value)
+    for value in ("xéend", "xé你好😀Aend", "xéyend"):
+        assert not _matcher_accepts(matcher, value)
+    for value in (b"x\xc0\x80aend", b"xA\xc0\x80end", b"xA\xf4\x90\x80\x80end"):
+        matcher.reset()
+        assert not matcher.accept_string(value)
+
+
+@pytest.mark.parametrize(
+    "suffix,valid", [("", []), ("*", [""]), ('* "end"', ["end"]), (' | "ok"', ["ok"])]
+)
+def test_empty_unicode_complement(suffix, valid):
+    matcher = _make_string_matcher(r"root ::= [^\u0000-\U0010FFFF]" + suffix)
+    for value in [*valid, "a", "\0", "é", "你", "😀", "\U0010ffff", "你end"]:
+        assert _matcher_accepts(matcher, value) == (value in valid)
+
+
+def test_three_byte_upper_tail_range():
+    matcher = _make_string_matcher(r"root ::= [\u0000-\u574e]")
+    assert _matcher_accepts(matcher, "候")
+    assert not _matcher_accepts(matcher, "坏")
+
+
+def test_unicode_complement_ranges():
+    matcher = _make_string_matcher(r"root ::= [^😀-🙏é你A-C]")
+    for value in ("D", "ê", "好", "😁", "🙏", "😀", "é", "你", "A", "C", "🚀"):
+        expected = value not in "é你ABC" and not ("😀" <= value <= "🙏")
+        assert _matcher_accepts(matcher, value) == expected
+
+
+@pytest.mark.parametrize("expression", [r"[^a]", r"[\u0000-\U0010ffff]", r"[\x5d-\U0010ffff]"])
+def test_character_class_utf8_boundaries_and_mask(expression):
+    # Positive ranges emitted after intersection must not regain invalid UTF-8 paths.
+    invalid = [
+        b"\xc0\x80",
+        b"\xc1\xbf",
+        b"\xe0\x80\x80",
+        b"\xf0\x80\x80\x80",
+        b"\xf4\x90\x80\x80",
+        b"\xc0",
+        b"\xc1",
+    ]
+    valid = ["]", "\x7f", "\x80", "\u07ff", "\u0800", "\uffff", "\U00010000", "\U0010ffff"]
+    vocabulary = invalid + [value.encode() for value in valid] + [b"[EOS]"]
+    info = xgr.TokenizerInfo(vocabulary, stop_token_ids=[len(vocabulary) - 1])
+    matcher = xgr.GrammarMatcher(
+        xgr.GrammarCompiler(info).compile_grammar("root ::= " + expression)
+    )
+    mask = xgr.allocate_token_bitmask(1, len(vocabulary))
+    matcher.fill_next_token_bitmask(mask)
+    for index, token in enumerate(vocabulary):
+        expected = len(invalid) <= index < len(vocabulary) - 1
+        assert bool((int(mask[0, index // 32]) >> (index % 32)) & 1) == expected
+        matcher.reset()
+        assert matcher.accept_token(index) == expected
+    for token in invalid:
+        matcher.reset()
+        assert not matcher.accept_string(token)
+
+
 # --- Long mixed sequences stress the streaming concatenation loop ---
 
 
@@ -343,18 +425,24 @@ def _fsm_snapshot(snapshot: str) -> str:
         'root ::= "x" [^0-9] "y" [^a-z]* "z"',
         _fsm_snapshot(
             r"""
-            Rule 0: root, FSM: CompactFSM(num_states=11, start=7, end=[10], edges=[
+            Rule 0: root, FSM: CompactFSM(num_states=17, start=7, end=[16], edges=[
             0: [[\x80-\xbf]->2]
             1: [[\x80-\xbf]->3]
             2: [[\x80-\xbf]->5]
             3: [[\x80-\xbf]->6]
-            4: [[\0-/]->5, [:-\x7f]->5, [\xc0-\xdf]->2, [\xe0-\xef]->0, [\xf0-\xf7]->8]
+            4: [[\0-/]->5, [:-\x7f]->5, [\xc2-\xdf]->2, '\xe0'->8, [\xe1-\xef]->0, '\xf0'->9, [\xf1-\xf3]->11, '\xf4'->10]
             5: ['y'->6]
-            6: [[\0-`]->6, 'z'->10, [{-\x7f]->6, [\xc0-\xdf]->3, [\xe0-\xef]->1, [\xf0-\xf7]->9]
+            6: [[\0-`]->6, 'z'->16, [{-\x7f]->6, [\xc2-\xdf]->3, '\xe0'->12, [\xe1-\xef]->1, '\xf0'->13, [\xf1-\xf3]->15, '\xf4'->14]
             7: ['x'->4]
-            8: [[\x80-\xbf]->0]
-            9: [[\x80-\xbf]->1]
-            10: []
+            8: [[\xa0-\xbf]->2]
+            9: [[\x90-\xbf]->0]
+            10: [[\x80-\x8f]->0]
+            11: [[\x80-\xbf]->0]
+            12: [[\xa0-\xbf]->3]
+            13: [[\x90-\xbf]->1]
+            14: [[\x80-\x8f]->1]
+            15: [[\x80-\xbf]->1]
+            16: []
             ])
             """
         ),
@@ -597,41 +685,44 @@ def _fsm_snapshot(snapshot: str) -> str:
         'root ::= a b c\na ::= "x" [0-9]*\nb ::= a "y" | [^xyz]\nc ::= b{1,3} "end"',
         _fsm_snapshot(
             r"""
-            Rule 0: root, FSM: CompactFSM(num_states=23, start=2, end=[3], edges=[
+            Rule 0: root, FSM: CompactFSM(num_states=26, start=2, end=[3], edges=[
             0: [Rule(1)->1, [0-9]->0]
             1: [Rule(2)->3]
             2: ['x'->0]
             3: []
             ])
-            Rule 1: b, FSM: CompactFSM(num_states=23, start=7, end=[5], edges=[
+            Rule 1: b, FSM: CompactFSM(num_states=26, start=7, end=[5], edges=[
             4: [[\x80-\xbf]->6]
             5: []
             6: [[\x80-\xbf]->5]
-            7: [[\0-w]->5, 'x'->8, [{-\x7f]->5, [\xc0-\xdf]->6, [\xe0-\xef]->4, [\xf0-\xf7]->9]
+            7: [[\0-w]->5, 'x'->8, [{-\x7f]->5, [\xc2-\xdf]->6, '\xe0'->9, [\xe1-\xef]->4, '\xf0'->10, [\xf1-\xf3]->12, '\xf4'->11]
             8: [[0-9]->8, 'y'->5]
-            9: [[\x80-\xbf]->4]
-            ])
-            Rule 2: c, FSM: CompactFSM(num_states=23, start=11, end=[14], edges=[
-            10: ['e'->12]
-            11: [Rule(5)->10]
-            12: ['n'->13]
-            13: ['d'->14]
-            14: []
+            9: [[\xa0-\xbf]->6]
+            10: [[\x90-\xbf]->4]
+            11: [[\x80-\x8f]->4]
+            12: [[\x80-\xbf]->4]
             ])
-            Rule 3: c_1, FSM: CompactFSM(num_states=23, start=15, end=[15, 17], edges=[
-            15: [Rule(1)->16]
-            16: [Rule(4)->17]
+            Rule 2: c, FSM: CompactFSM(num_states=26, start=14, end=[17], edges=[
+            13: ['e'->15]
+            14: [Rule(5)->13]
+            15: ['n'->16]
+            16: ['d'->17]
             17: []
             ])
-            Rule 4: c_2, FSM: CompactFSM(num_states=23, start=18, end=[18, 19], edges=[
+            Rule 3: c_1, FSM: CompactFSM(num_states=26, start=18, end=[18, 20], edges=[
             18: [Rule(1)->19]
-            19: []
+            19: [Rule(4)->20]
+            20: []
             ])
-            Rule 5: c_3, FSM: CompactFSM(num_states=23, start=21, end=[22], edges=[
-            20: [Rule(3)->22]
-            21: [Rule(1)->20]
+            Rule 4: c_2, FSM: CompactFSM(num_states=26, start=21, end=[21, 22], edges=[
+            21: [Rule(1)->22]
             22: []
             ])
+            Rule 5: c_3, FSM: CompactFSM(num_states=26, start=24, end=[25], edges=[
+            23: [Rule(3)->25]
+            24: [Rule(1)->23]
+            25: []
+            ])
             """
         ),
         id="nested-rule-references",