fix: ignore a string pattern whose regex syntax is not supported

cpp/json_schema_converter.cc

@@ -2388,6 +2388,36 @@ int32_t JSONSchemaConverter::RegexExpression(
   return AddSubGrammar(Grammar::FromEBNF(RegexToEBNF(regex)));
 }
 
+std::optional<int32_t> JSONSchemaConverter::PatternExpression(
+    const std::string& pattern, bool json_string, bool force_cfg_expansion
+) {
+  // Every syntax the regex parser cannot handle is reported through RegexConverter::RaiseError,
+  // whose message starts with this prefix. The FSM engine instead returns an error, and
+  // RegexExpression then falls back to that parser, so this is exactly the case where neither
+  // engine on this path can compile the pattern. Other errors (e.g. a null character) still
+  // propagate. Nothing is added to the grammar before the parser throws.
+  static const std::string kRegexParsingError = "Regex parsing error";
+  try {
+    return RegexExpression(pattern, json_string, force_cfg_expansion);
+  } catch (const LogFatalError& error) {
+    std::string message = error.what();
+    auto reason_begin = message.find(kRegexParsingError);
+    if (reason_begin == std::string::npos) {
+      throw;
+    }
+    // what() carries the time and source location; keep only the parser's own message.
+    std::string reason = message.substr(reason_begin);
+    while (!reason.empty() && (reason.back() == '\n' || reason.back() == ' ')) {
+      reason.pop_back();
+    }
+    // Ignoring the pattern only makes the grammar accept more strings, as for the lookahead
+    // assertions the FSM engine already skips and for an unsupported format or multipleOf.
+    XGRAMMAR_LOG(WARNING) << "pattern \"" << pattern << "\" is not supported (" << reason
+                          << "); ignoring pattern, minLength/maxLength still apply";
+    return std::nullopt;
+  }
+}
+
 // ==================== Generate Methods ====================
 
 void JSONSchemaConverter::WarnDroppedLengthConstraints(
@@ -2758,11 +2788,12 @@ int32_t JSONSchemaConverter::GenerateString(const StringSpec& spec, const std::s
       return Sequence({ByteString("\""), RegexExpression(*regex, false, true), ByteString("\"")});
     }
   }
-  // Check for pattern
+  // Check for pattern. A pattern whose syntax is not supported is ignored with a warning, and the
+  // string falls through to the plain or length-bounded branches below.
   if (spec.pattern.has_value()) {
-    return Sequence(
-        {ByteString("\""), RegexExpression(*spec.pattern, /*json_string=*/true), ByteString("\"")}
-    );
+    if (auto pattern_expr = PatternExpression(*spec.pattern, /*json_string=*/true)) {
+      return Sequence({ByteString("\""), *pattern_expr, ByteString("\"")});
+    }
   }
   // Check for length constraints. They are dropped when there are exclusions: intersecting the
   // unrolled bound with the exclusion automaton emits one rule per position and automaton state
@@ -4001,7 +4032,10 @@ int32_t XMLToolCallingConverter::GenerateString(
       }
     }
     if (spec.pattern.has_value()) {
-      return RegexExpression(*spec.pattern, false, /*force_cfg_expansion=*/true);
+      if (auto pattern_expr =
+              PatternExpression(*spec.pattern, false, /*force_cfg_expansion=*/true)) {
+        return *pattern_expr;
+      }
     }
     const bool bounded = spec.min_length != 0 || spec.max_length != -1;
     // Without exclusions, a length bound or an unrecognized format keeps the plain repetition.

cpp/json_schema_converter.h

@@ -442,6 +442,15 @@ class JSONSchemaConverter {
   int32_t RegexExpression(
       const std::string& regex, bool json_string = false, bool force_cfg_expansion = false
   );
+  /*!
+   * \brief RegexExpression for a string schema's "pattern", or std::nullopt when the regex
+   * engine that compiles it cannot parse the pattern's syntax (lookbehind, backreferences, \b,
+   * \p{...}, ...). In that case a warning is logged and the caller compiles the string without
+   * the pattern, so one unsupported pattern no longer fails the whole schema.
+   */
+  std::optional<int32_t> PatternExpression(
+      const std::string& pattern, bool json_string, bool force_cfg_expansion = false
+  );
   /*!
    * \brief Rules matching the regex (one of the converter's own ASCII string bodies) minus
    * every string containing one of excludes_, and minus excluded_keys as whole strings. With

tests/python/test_json_schema_converter.py

@@ -2875,6 +2875,92 @@ def test_string_pattern_with_length_still_ignores_the_bound(capfd):
     assert _is_grammar_accept_string(grammar, '"abcd"')
 
 
+# --- string patterns whose regex syntax is not supported (issue #1001) -------------------------
+#
+# A pattern that the regex engine compiling it cannot parse used to fail the whole schema, so one
+# such pattern in one tool broke a request with many tools. It is now ignored with a warning, like
+# the lookahead assertions the FSM engine already skips: the grammar only accepts more strings.
+
+_UNSUPPORTED_PATTERNS = [
+    pytest.param(r"(?<=a)b", id="lookbehind"),
+    pytest.param(r"(?<!a)b", id="negative-lookbehind"),
+    pytest.param(r"^(a)\1$", id="backreference"),
+    pytest.param(r"\bword\b", id="word-boundary"),
+    pytest.param(r"^\p{L}+$", id="unicode-property-class"),
+    pytest.param(r"(?s)a.b", id="dotall-flag"),
+    pytest.param(r"(?>ab)c", id="atomic-group"),
+]
+
+
+@pytest.mark.parametrize("json_format", ["json", "qwen_xml"])
+@pytest.mark.parametrize("pattern", _UNSUPPORTED_PATTERNS)
+def test_string_unsupported_pattern_is_ignored_with_a_warning(capfd, pattern, json_format):
+    schema = {
+        "type": "object",
+        "properties": {"k": {"type": "string", "pattern": pattern}},
+        "required": ["k"],
+    }
+    output = _compile_and_capture(capfd, schema, json_format=json_format)
+    assert f'pattern "{pattern}" is not supported' in output, output
+    assert "ignoring pattern" in output, output
+
+
+def test_string_lookahead_pattern_compiles_for_xml_tool_calls(capfd):
+    # The issue's example, from a real tool schema. The JSON converter's FSM engine already skips
+    # the lookahead; the XML tool-calling converter compiles patterns through the CFG parser,
+    # which rejected it and failed the whole schema.
+    schema = {
+        "type": "object",
+        "properties": {"path": {"type": "string", "pattern": r"^(?!\.\.?(?:\/|$))[^\s]+$"}},
+        "required": ["path"],
+    }
+    output = _compile_and_capture(capfd, schema, json_format="qwen_xml")
+    assert "is not supported" in output, output
+
+
+@pytest.mark.parametrize("pattern", _UNSUPPORTED_PATTERNS)
+def test_string_unsupported_pattern_compiles_as_a_plain_string(capfd, pattern):
+    grammar = xgr.Grammar.from_json_schema(json.dumps({"type": "string", "pattern": pattern}))
+    plain = xgr.Grammar.from_json_schema(json.dumps({"type": "string"}))
+    capfd.readouterr()
+    for instance in ['"abc"', '""', '"a \\"quoted\\" word"', "1"]:
+        assert _is_grammar_accept_string(grammar, instance) == _is_grammar_accept_string(
+            plain, instance
+        ), instance
+
+
+def test_string_unsupported_pattern_keeps_length_bounds(capfd):
+    # With the pattern ignored, the length-bounded branch applies again.
+    schema = {"type": "string", "pattern": r"(?<=a)b", "minLength": 2, "maxLength": 4}
+    grammar = xgr.Grammar.from_json_schema(json.dumps(schema))
+    capfd.readouterr()
+    assert not _is_grammar_accept_string(grammar, '"a"')
+    assert _is_grammar_accept_string(grammar, '"abcd"')
+    assert not _is_grammar_accept_string(grammar, '"abcde"')
+
+
+def test_string_supported_pattern_is_still_enforced_next_to_an_unsupported_one(capfd):
+    # Ignoring is per pattern: a sibling property's supported pattern keeps constraining it.
+    schema = {
+        "type": "object",
+        "properties": {
+            "a": {"type": "string", "pattern": r"\bx"},
+            "b": {"type": "string", "pattern": "^[0-9]+$"},
+        },
+        "required": ["a", "b"],
+    }
+    grammar = xgr.Grammar.from_json_schema(json.dumps(schema))
+    capfd.readouterr()
+    assert _is_grammar_accept_string(grammar, '{"a": "anything", "b": "123"}')
+    assert not _is_grammar_accept_string(grammar, '{"a": "anything", "b": "x"}')
+
+
+def test_string_pattern_with_a_null_character_still_raises():
+    # Only unsupported regex syntax is relaxed; an invalid pattern is still an error.
+    with pytest.raises(RuntimeError, match="null characters"):
+        xgr.Grammar.from_json_schema(json.dumps({"type": "string", "pattern": "a\u0000b"}))
+
+
 def test_type_array():
     schema = {
         "type": ["integer", "string"],