diff --git a/cpp/grammar_functor.cc b/cpp/grammar_functor.cc index c87dbe7fb..d2cca09c6 100644 --- a/cpp/grammar_functor.cc +++ b/cpp/grammar_functor.cc @@ -1981,6 +1981,182 @@ Result GrammarFSMBuilderImpl::Regex(const std::string& regex, b return ResultOk(std::move(result)); } +/*! + * \brief Lower the regular subset used by RegexToEBNF without changing terminal semantics. + * + * A shared callee entry/exit would let one call return to another call's continuation. Inline + * each finite call instead, and connect only direct tail self-references to that call's own + * entry. The entry is private even when the caller's start also has an optional exit. + */ +class RegularGrammarFSMBuilder { + public: + RegularGrammarFSMBuilder(const Grammar& grammar, int max_num_states) + : grammar_(grammar), max_num_states_(max_num_states), active_(grammar->NumRules(), false) {} + + Result Build() { + if (max_num_states_ < 2) { + return ResultErr("Regular grammar FSM state limit must be at least 2"); + } + int start = fsm_.AddState(); + int end = fsm_.AddState(); + if (!BuildRule(grammar_->GetRootRuleId(), start, end)) { + return ResultErr(error_); + } + auto result = FSMWithStartEnd(fsm_, start, {end}).SimplifyEpsilon(); + return ResultOk(result.MergeEquivalentStates()); + } + + private: + bool Fail(const std::string& message) { + error_ = "Cannot flatten regular grammar: " + message; + return false; + } + + int AddState() { + if (fsm_.NumStates() >= max_num_states_) { + Fail("intermediate FSM exceeds " + std::to_string(max_num_states_) + " states"); + return -1; + } + return fsm_.AddState(); + } + + bool Append(const FSMWithStartEnd& leaf, int start, int end) { + if (leaf.GetFsm().NumStates() > max_num_states_ - fsm_.NumStates()) { + return Fail("intermediate FSM exceeds " + std::to_string(max_num_states_) + " states"); + } + std::vector mapping; + fsm_.AddFSM(leaf.GetFsm(), &mapping); + fsm_.AddEpsilonEdge(start, mapping[leaf.GetStart()]); + for (int leaf_end : leaf.GetEnds()) { + fsm_.AddEpsilonEdge(mapping[leaf_end], end); + } + return true; + } + + bool BuildRule(int rule_id, int start, int end) { + const auto& rule = grammar_->GetRule(rule_id); + if (active_[rule_id]) { + return Fail("mutual or non-tail recursion involving rule " + rule.name); + } + if (depth_ >= 256) { + return Fail("rule call depth exceeds 256"); + } + if (rule.lookahead_assertion_id >= 0 || rule.max_tokens >= 0 || rule.max_chars >= 0 || + !rule.capture_name.empty() || rule.is_lazy || rule.temperature.has_value() || + grammar_->GetSuffixStopInfo(rule_id) != nullptr) { + return Fail("annotated rule " + rule.name + " is not a plain regular rule"); + } + int entry = AddState(); + if (entry < 0) { + return false; + } + fsm_.AddEpsilonEdge(start, entry); + active_[rule_id] = true; + ++depth_; + bool ok = BuildExpression( + grammar_->GetGrammarExpr(rule.body_expr_id), entry, end, rule_id, entry, true + ); + --depth_; + active_[rule_id] = false; + return ok; + } + + bool BuildExpression( + const GrammarExpr& expr, int start, int end, int rule_id, int rule_start, bool tail + ) { + switch (expr.type) { + case ExprType::kEmptyStr: + fsm_.AddEpsilonEdge(start, end); + return true; + case ExprType::kByteString: + if (expr.size() >= max_num_states_ - fsm_.NumStates()) { + return Fail("byte string exceeds the intermediate FSM state limit"); + } + return Append(GrammarFSMBuilderImpl::ByteString(expr), start, end); + case ExprType::kCharacterClass: + case ExprType::kCharacterClassStar: + return Append(GrammarFSMBuilderImpl::CharacterClass(expr), start, end); + case ExprType::kRuleRef: + if (expr[0] == rule_id) { + if (!tail) { + return Fail("non-tail self-recursion in rule " + grammar_->GetRule(rule_id).name); + } + fsm_.AddEpsilonEdge(start, rule_start); + return true; + } + return BuildRule(expr[0], start, end); + case ExprType::kChoices: + for (int child : expr) { + if (!BuildExpression( + grammar_->GetGrammarExpr(child), start, end, rule_id, rule_start, tail + )) { + return false; + } + } + return true; + case ExprType::kSequence: { + int current = start; + for (int i = 0; i < expr.size(); ++i) { + int next = i + 1 == expr.size() ? end : AddState(); + if (next < 0 || !BuildExpression( + grammar_->GetGrammarExpr(expr[i]), + current, + next, + rule_id, + rule_start, + tail && i + 1 == expr.size() + )) { + return false; + } + current = next; + } + if (expr.size() == 0) { + fsm_.AddEpsilonEdge(start, end); + } + return true; + } + case ExprType::kRepeat: { + int64_t lower = expr[1], upper = expr[2]; + if (lower < 0 || (upper != -1 && upper < lower)) { + return Fail("invalid repetition bounds"); + } + // Even empty callees should not make an enormous explicit repetition consume + // unbounded compilation time. Large bounded patterns need a different representation. + int64_t count = upper == -1 ? lower : upper; + if (count > max_num_states_) { + return Fail("repetition exceeds the intermediate FSM state limit"); + } + int current = start; + for (int64_t i = 0; i <= count; ++i) { + if (i >= lower) { + fsm_.AddEpsilonEdge(current, end); + } + if (i == count) { + break; + } + int next = AddState(); + if (next < 0 || !BuildRule(expr[0], current, next)) { + return false; + } + current = next; + } + return upper != -1 || BuildRule(expr[0], current, current); + } + default: + return Fail( + "unsupported terminal or expression in rule " + grammar_->GetRule(rule_id).name + ); + } + } + + const Grammar& grammar_; + int max_num_states_; + FSM fsm_; + std::vector active_; + int depth_ = 0; + std::string error_; +}; + class RepetitionRangeExpanderImpl : public GrammarMutator { public: using GrammarMutator::Apply; @@ -3782,6 +3958,12 @@ Result GrammarFSMBuilder::Regex(const std::string& regex, bool return GrammarFSMBuilderImpl::Regex(regex, json_string); } +Result GrammarFSMBuilder::FromRegularGrammar( + const Grammar& grammar, int max_num_states +) { + return RegularGrammarFSMBuilder(grammar, max_num_states).Build(); +} + const std::bitset<256>& GrammarFSMBuilder::JSONStringForbiddenChars() { static const std::bitset<256> forbidden_chars = [] { std::bitset<256> chars; diff --git a/cpp/grammar_functor.h b/cpp/grammar_functor.h index 0b4da505b..e7871cab9 100644 --- a/cpp/grammar_functor.h +++ b/cpp/grammar_functor.h @@ -406,6 +406,16 @@ class GrammarFSMBuilder { * the characters in JSONStringForbiddenChars() are excluded from every character match. */ static Result Regex(const std::string& regex, bool json_string = false); + /*! + * \brief Flatten a regular EBNF grammar without reinterpreting its terminal expressions. + * Supports finite rule calls, direct tail self-recursion, and repetition. Each call gets + * separate return states. Other recursion, non-byte terminals, and rule annotations return + * an error. Intended for the existing RegexToEBNF fallback, not arbitrary CFGs. + * \param max_num_states Bound on the intermediate automaton size. + */ + static Result FromRegularGrammar( + const Grammar& grammar, int max_num_states = 100000 + ); /*! \brief The characters that must be escaped inside a JSON string literal: the control * characters 0x00-0x1F, the quote '"' and the backslash '\\'. */ static const std::bitset<256>& JSONStringForbiddenChars(); diff --git a/tests/cpp/test_regular_grammar_fsm.cc b/tests/cpp/test_regular_grammar_fsm.cc new file mode 100644 index 000000000..71311a1ea --- /dev/null +++ b/tests/cpp/test_regular_grammar_fsm.cc @@ -0,0 +1,104 @@ +/*! + * \file tests/cpp/test_regular_grammar_fsm.cc + * \brief Preserve EBNF terminal and call semantics when flattening regular grammars. + */ + +#include +#include + +#include "grammar_functor.h" + +using namespace xgrammar; + +TEST(RegularGrammarFSM, RepeatedCallsHaveSeparateReturns) { + auto grammar = Grammar::FromEBNF(R"( +root ::= part "x" part "y" +part ::= "a" part | "b" +)"); + auto result = GrammarFSMBuilder::FromRegularGrammar(grammar); + ASSERT_TRUE(result.IsOk()); + auto fsm = std::move(result).Unwrap(); + for (const auto* text : {"bxby", "abxby", "bxaaby", "aaabxaaaby"}) { + EXPECT_TRUE(fsm.AcceptString(text)) << text; + } + for (const auto* text : {"", "by", "bxy", "bxay", "abxay", "bybx", "bxbyy"}) { + EXPECT_FALSE(fsm.AcceptString(text)) << text; + } +} + +TEST(RegularGrammarFSM, RecursiveCalleeCannotUseCallersOptionalExit) { + auto grammar = Grammar::FromEBNF(R"( +root ::= part? "z" part* +part ::= "a" part | "b" +)"); + auto result = GrammarFSMBuilder::FromRegularGrammar(grammar); + ASSERT_TRUE(result.IsOk()); + auto fsm = std::move(result).Unwrap(); + for (const auto* text : {"z", "bz", "abz", "zb", "zaab", "bzab", "abzbaab"}) { + EXPECT_TRUE(fsm.AcceptString(text)) << text; + } + for (const auto* text : {"az", "za", "bza", "aaz", "zba"}) { + EXPECT_FALSE(fsm.AcceptString(text)) << text; + } +} + +TEST(RegularGrammarFSM, RegexFallbackMatchesOriginalGrammar) { + GrammarCompiler compiler(TokenizerInfo(std::vector{}), 1, false); + const std::vector patterns = { + R"((ab|c){1,3}d?)", + R"((a*b)+c{2,})", + R"((ab|){0,3}z)", + R"([é你]+|\u0061\s\D)", + R"([^你])", + R"(^[a-zA-Z0-9\.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9-]+\.[a-zA-Z]{2,}$)" + }; + const std::vector samples = { + "", + "ab", + "abc", + "abccd", + "d", + "cccc", + "bcc", + "aabbccc", + "abcc", + "acccc", + "z", + "abz", + "ababz", + "abababz", + "ababababz", + "é你é", + "你", + "a b", + "a 1", + "a\tb", + "name@example.com", + "name@example", + "name@.com", + "<|close|>" + }; + for (const auto& pattern : patterns) { + auto grammar = Grammar::FromRegex(pattern); + auto result = GrammarFSMBuilder::FromRegularGrammar(grammar); + ASSERT_TRUE(result.IsOk()) << pattern; + auto fsm = std::move(result).Unwrap(); + GrammarMatcher matcher(compiler.CompileGrammar(grammar), std::nullopt, true); + for (const auto& sample : samples) { + matcher.Reset(); + bool expected = matcher.AcceptString(sample) && matcher.IsTerminated(); + EXPECT_EQ(fsm.AcceptString(sample), expected) << pattern << " input=" << sample; + } + } +} + +TEST(RegularGrammarFSM, RejectsUnsupportedRecursionAndBounds) { + for (const auto* source : + {"root ::= \"a\" root \"b\" | \"\"", + "root ::= \"a\" child | \"\"\nchild ::= \"b\" root | \"c\"", + "root ::= Token(1)"}) { + EXPECT_TRUE(GrammarFSMBuilder::FromRegularGrammar(Grammar::FromEBNF(source)).IsErr()) << source; + } + auto grammar = Grammar::FromRegex("(ab){100}"); + EXPECT_TRUE(GrammarFSMBuilder::FromRegularGrammar(grammar, 10).IsErr()); +}