From 29f936cd3c120e417af78250c4549e80d42bd822 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Tue, 29 Sep 2026 05:29:39 +0000 Subject: [PATCH 1/5] [Rust Frontend] Replay roundtrip output grammars through XGrammar Export the output grammar built by the initialized roundtrip parser, with the rendered completion as its generation, as checked-in replay cases. A PEP 723 uv script compiles every case with the real XGrammar and replays token-zero cases through the matcher, and a nextest wrapper runs it in the Rust CI. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Bugen Zhao --- rust/src/chat/Cargo.toml | 1 + rust/src/chat/tests/grammar_replay.py | 111 ++++++++ rust/src/chat/tests/grammar_replay.rs | 24 ++ .../DeepSeek-V3.2-Exp.tool_call_mix.json | 108 +++++++ .../DeepSeek-V4-Flash.tool_call_mix.json | 108 +++++++ .../DeepSeek-V4.1-Flash.tool_call_mix.json | 108 +++++++ .../GLM-4.7-Flash.tool_call_mix.json | 100 +++++++ .../GLM-5.2-FP8.tool_call_mix.json | 100 +++++++ .../grammar_replay/Hy3.tool_call_mix.json | 266 ++++++++++++++++++ .../Kimi-K2.5.tool_call_mix.json | 135 +++++++++ .../grammar_replay/Kimi-K3.tool_call_mix.json | 231 +++++++++++++++ .../MiMo-V2.6-Flash-RL.tool_call_mix.json | 98 +++++++ .../MiniMax-M2.5.tool_call_mix.json | 117 ++++++++ .../Qwen3-0.6B.tool_call_mix.json | 107 +++++++ .../Qwen3.5-4B.tool_call_mix.json | 104 +++++++ rust/src/chat/tests/roundtrip.rs | 224 ++++++++++++--- rust/src/parser/src/output_grammar.rs | 6 +- 17 files changed, 1911 insertions(+), 37 deletions(-) create mode 100755 rust/src/chat/tests/grammar_replay.py create mode 100644 rust/src/chat/tests/grammar_replay.rs create mode 100644 rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.tool_call_mix.json create mode 100644 rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.tool_call_mix.json create mode 100644 rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.tool_call_mix.json create mode 100644 rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.tool_call_mix.json create mode 100644 rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.tool_call_mix.json create mode 100644 rust/src/chat/tests/grammar_replay/Hy3.tool_call_mix.json create mode 100644 rust/src/chat/tests/grammar_replay/Kimi-K2.5.tool_call_mix.json create mode 100644 rust/src/chat/tests/grammar_replay/Kimi-K3.tool_call_mix.json create mode 100644 rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.tool_call_mix.json create mode 100644 rust/src/chat/tests/grammar_replay/MiniMax-M2.5.tool_call_mix.json create mode 100644 rust/src/chat/tests/grammar_replay/Qwen3-0.6B.tool_call_mix.json create mode 100644 rust/src/chat/tests/grammar_replay/Qwen3.5-4B.tool_call_mix.json diff --git a/rust/src/chat/Cargo.toml b/rust/src/chat/Cargo.toml index a110020ec110..d145fc237aee 100644 --- a/rust/src/chat/Cargo.toml +++ b/rust/src/chat/Cargo.toml @@ -46,6 +46,7 @@ clap.workspace = true expect-test.workspace = true paste.workspace = true rmp-serde.workspace = true +serde_json = { workspace = true, features = ["raw_value"] } serial_test.workspace = true tempfile.workspace = true tokio.workspace = true diff --git a/rust/src/chat/tests/grammar_replay.py b/rust/src/chat/tests/grammar_replay.py new file mode 100755 index 000000000000..8a50b12725ec --- /dev/null +++ b/rust/src/chat/tests/grammar_replay.py @@ -0,0 +1,111 @@ +#!/usr/bin/env -S uv run --script +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +# /// script +# requires-python = ">=3.11" +# dependencies = [ +# "tiktoken>=0.6.0", +# # xgrammar imports torch unconditionally; only the CPU build is needed. +# "torch", +# "transformers>=5.10.4", +# # Keep in sync with requirements/common.txt. +# "xgrammar==0.2.7", +# ] +# +# [[tool.uv.index]] +# name = "pytorch-cpu" +# url = "https://download.pytorch.org/whl/cpu" +# explicit = true +# +# [tool.uv.sources] +# torch = { index = "pytorch-cpu" } +# /// +"""Replay the grammar cases exported by the roundtrip tests through XGrammar. + +Every case is compiled against the model tokenizer the same way as +`vllm/v1/structured_output/backend_xgrammar.py`. Token-zero cases are then +replayed from the first generated token: each token must be allowed by the +bitmask and accepted by the matcher, and the final stop token must terminate it. +""" + +import json +import sys +from functools import cache +from pathlib import Path + +import xgrammar as xgr +from transformers import AutoTokenizer + +CASES_DIR = Path(__file__).parent / "grammar_replay" + + +@cache +def load_tokenizer(model_id: str, vocab_size: int): + tokenizer = AutoTokenizer.from_pretrained(model_id, trust_remote_code=True) + tokenizer_info = xgr.TokenizerInfo.from_huggingface( + tokenizer, vocab_size=vocab_size + ) + return tokenizer, xgr.GrammarCompiler(tokenizer_info) + + +def replay(case: dict) -> str: + """Check one case and return a summary of what was checked.""" + tokenizer, compiler = load_tokenizer(case["model_id"], case["vocab_size"]) + generation = case["generation_token_ids"] + + # The case token IDs come from the Rust tokenizer; the runner must load the + # same vocabulary for the grammar to mean the same thing. + encoded = tokenizer.encode(case["completion"], add_special_tokens=False) + if encoded != generation[:-1]: + raise AssertionError( + "tokenizer mismatch: the completion encodes to " + f"{encoded}, but the case records {generation[:-1]}" + ) + + grammar = case["grammar"] + structural_tag = {"type": "structural_tag", "format": grammar["format"]} + compiled = compiler.compile_structural_tag(json.dumps(structural_tag)) + if grammar["coverage"] != "from_token_zero": + return f"compiled only ({grammar['coverage']} is gated by the engine)" + + matcher = xgr.GrammarMatcher( + compiled, override_stop_tokens=case["all_stop_token_ids"] + ) + bitmask = xgr.allocate_token_bitmask(1, case["vocab_size"]) + for index, token_id in enumerate(generation): + matcher.fill_next_token_bitmask(bitmask) + allowed = (bitmask[0, token_id // 32].item() >> (token_id % 32)) & 1 + if not allowed or not matcher.accept_token(token_id): + prefix = tokenizer.decode(generation[:index]) + raise AssertionError( + f"token {index} ({token_id}, {tokenizer.decode([token_id])!r}) " + f"rejected after {prefix!r}" + ) + if not matcher.is_terminated(): + raise AssertionError("matcher not terminated after the stop token") + return f"replayed {len(generation)} tokens from token zero" + + +def main() -> int: + paths = sorted(CASES_DIR.glob("*.json")) + if not paths: + print(f"no grammar replay cases in {CASES_DIR}", file=sys.stderr) + return 1 + + failures = 0 + for path in paths: + case = json.loads(path.read_text()) + try: + summary = replay(case) + except Exception as error: + failures += 1 + coverage = case["grammar"]["coverage"] + print(f"FAIL {path.name} [{coverage}]: {error}", file=sys.stderr) + else: + print(f"PASS {path.name}: {summary}") + + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/rust/src/chat/tests/grammar_replay.rs b/rust/src/chat/tests/grammar_replay.rs new file mode 100644 index 000000000000..4ddfa6bf247c --- /dev/null +++ b/rust/src/chat/tests/grammar_replay.rs @@ -0,0 +1,24 @@ +// SPDX-License-Identifier: Apache-2.0 +// SPDX-FileCopyrightText: Copyright contributors to the vLLM project + +//! Replays the grammar cases exported by the roundtrip tests through the real +//! XGrammar, so that parser-built output grammars are checked against the +//! engine's compiler and matcher. + +use std::path::PathBuf; +use std::process::Command; + +#[test] +fn xgrammar_accepts_roundtrip_generations() { + let script = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/grammar_replay.py"); + let output = Command::new(&script) + .output() + .unwrap_or_else(|error| panic!("failed to execute {:?}: {error}", script)); + assert!( + output.status.success(), + "grammar replay failed: status={:?}\nstdout:\n{}\nstderr:\n{}", + output.status.code(), + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); +} diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.tool_call_mix.json new file mode 100644 index 000000000000..64909a8355e0 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.tool_call_mix.json @@ -0,0 +1,108 @@ +{ + "model_id": "deepseek-ai/DeepSeek-V3.2-Exp", + "vocab_size": 129280, + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "<｜DSML｜function_calls>" + ], + "tags": [ + { + "type": "tag", + "begin": "<｜DSML｜function_calls>\n", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<｜DSML｜invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<｜DSML｜invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.\n\n<｜DSML｜function_calls>\n<｜DSML｜invoke name=\"get_weather\">\n<｜DSML｜parameter name=\"location\" string=\"true\">Shanghai\n\n<｜DSML｜invoke name=\"add\">\n<｜DSML｜parameter name=\"y\" string=\"false\">1.0\n<｜DSML｜parameter name=\"x\" string=\"false\">2\n<｜DSML｜parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 43, 759, 1926, 270, 6704, 339, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 3327, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1], + "all_stop_token_ids": [1] +} diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.tool_call_mix.json new file mode 100644 index 000000000000..4eddceb5813c --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.tool_call_mix.json @@ -0,0 +1,108 @@ +{ + "model_id": "deepseek-ai/DeepSeek-V4-Flash", + "vocab_size": 129280, + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "<｜DSML｜tool_calls>" + ], + "tags": [ + { + "type": "tag", + "begin": "<｜DSML｜tool_calls>\n", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<｜DSML｜invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<｜DSML｜invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.\n\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"get_weather\">\n<｜DSML｜parameter name=\"location\" string=\"true\">Shanghai\n\n<｜DSML｜invoke name=\"add\">\n<｜DSML｜parameter name=\"y\" string=\"false\">1.0\n<｜DSML｜parameter name=\"x\" string=\"false\">2\n<｜DSML｜parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 43, 759, 1926, 270, 6704, 339, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 30, 128825, 40148, 5406, 2329, 1281, 3327, 3816, 30, 128825, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1], + "all_stop_token_ids": [1] +} diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.tool_call_mix.json new file mode 100644 index 000000000000..c3ac215bc0b6 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.tool_call_mix.json @@ -0,0 +1,108 @@ +{ + "model_id": "deepseek-ai/DeepSeek-V4.1-Flash", + "vocab_size": 129280, + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "<｜DSML｜ calls>" + ], + "tags": [ + { + "type": "tag", + "begin": "<｜DSML｜ calls>\n", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<｜DSML｜ invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_v4_1_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<｜DSML｜ invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_v4_1_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name=\"get_weather\">\n<｜DSML｜ parameter name=\"location\" string=\"true\">Shanghai\n\n<｜DSML｜ invoke name=\"add\">\n<｜DSML｜ parameter name=\"y\" string=\"false\">1.0\n<｜DSML｜ parameter name=\"x\" string=\"false\">2\n<｜DSML｜ parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 43, 759, 1926, 270, 6704, 339, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 30, 128825, 34756, 2329, 1281, 3327, 3816, 30, 128825, 10767, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1], + "all_stop_token_ids": [1] +} diff --git a/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.tool_call_mix.json new file mode 100644 index 000000000000..7a5f07df6b11 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.tool_call_mix.json @@ -0,0 +1,100 @@ +{ + "model_id": "zai-org/GLM-4.7-Flash", + "vocab_size": 154880, + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "get_weather", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "" + }, + { + "type": "tag", + "begin": "add", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "", + "", + "", + "", + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 40, 686, 1618, 279, 7375, 13, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820], + "all_stop_token_ids": [154820, 154827, 154829] +} diff --git a/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.tool_call_mix.json new file mode 100644 index 000000000000..d2465d4a536c --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.tool_call_mix.json @@ -0,0 +1,100 @@ +{ + "model_id": "zai-org/GLM-5.2-FP8", + "vocab_size": 154880, + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "get_weather", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "" + }, + { + "type": "tag", + "begin": "add", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "", + "", + "", + "", + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 40, 686, 1618, 279, 7375, 13, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820], + "all_stop_token_ids": [154820, 154827, 154829] +} diff --git a/rust/src/chat/tests/grammar_replay/Hy3.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/Hy3.tool_call_mix.json new file mode 100644 index 000000000000..ac50c02787dc --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Hy3.tool_call_mix.json @@ -0,0 +1,266 @@ +{ + "model_id": "tencent/Hy3", + "vocab_size": 120832, + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "\n", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "get_weather\n", + "content": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "location" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "end": "" + }, + { + "type": "tag", + "begin": "add\n", + "content": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "y" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "x" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "items" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "end": "" + } + ], + "separator": "\n", + "at_least_one": true, + "stop_after_first": false + }, + "end": "\n" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.\nget_weather\nlocation\nShanghai\n\nadd\ny\n1.0\nx\n2\nitems\n[\"left\", \"right\"]\n\n", + "generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 40, 823, 1609, 252, 7106, 13, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120033, 2176, 120043, 185, 120066, 88, 120067, 185, 120068, 16, 13, 15, 120069, 185, 120066, 87, 120067, 185, 120068, 17, 120069, 185, 120066, 12792, 120067, 185, 120068, 5831, 1639, 1221, 478, 1331, 7029, 120069, 185, 120034, 185, 120036, 120025], + "all_stop_token_ids": [120025] +} diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K2.5.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/Kimi-K2.5.tool_call_mix.json new file mode 100644 index 000000000000..410850ac66a3 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Kimi-K2.5.tool_call_mix.json @@ -0,0 +1,135 @@ +{ + "model_id": "moonshotai/Kimi-K2.5", + "vocab_size": 163840, + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "<|tool_calls_section_begin|>" + ], + "tags": [ + { + "type": "tag", + "begin": "<|tool_calls_section_begin|>", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|tool_call_begin|>functions.get_weather:", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "\\d+" + }, + { + "type": "const_string", + "value": "<|tool_call_argument_begin|>" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + } + ] + }, + "end": "<|tool_call_end|>" + }, + { + "type": "tag", + "begin": "<|tool_call_begin|>functions.add:", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "\\d+" + }, + { + "type": "const_string", + "value": "<|tool_call_argument_begin|>" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + } + ] + }, + "end": "<|tool_call_end|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "<|tool_calls_section_end|>" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "", + "<|tool_call_begin|>" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_call_begin|>functions.add:1<|tool_call_argument_begin|>{\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}<|tool_call_end|><|tool_calls_section_end|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 40, 748, 2390, 276, 7697, 13, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163597, 41937, 3149, 25, 16, 163598, 8264, 88, 1289, 220, 16, 13, 15, 11, 414, 87, 1289, 220, 17, 11, 414, 7989, 1289, 8681, 3794, 665, 414, 2277, 3956, 92, 163599, 163596, 163586], + "all_stop_token_ids": [163585, 163586] +} diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K3.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/Kimi-K3.tool_call_mix.json new file mode 100644 index 000000000000..86b0ec6896d9 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Kimi-K3.tool_call_mix.json @@ -0,0 +1,231 @@ +{ + "model_id": "moonshotai/Kimi-K3", + "vocab_size": 163840, + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|open|>response<|sep|>" + } + }, + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [ + "<|open|>", + "<|close|>", + "<|end_of_msg|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>response<|sep|>" + }, + { + "type": "optional", + "content": { + "type": "tag", + "begin": "<|open|>tools<|sep|>", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|open|>call tool=\"get_weather\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "plus", + "content": { + "type": "tag", + "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", + "content": { + "type": "any_text", + "excludes": [ + "<|close|>argument<|sep|>", + "<|close|>call<|sep|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>argument<|sep|>" + } + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>call tool=\"add\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "plus", + "content": { + "type": "or", + "elements": [ + { + "type": "tag", + "begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "array", + "items": { + "type": "string" + } + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "<|close|>argument<|sep|>" + } + ] + } + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "<|close|>tools<|sep|>" + } + }, + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|close|>message<|sep|>" + } + } + ] + }, + "coverage": "final_output_only" + }, + "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|>I will call the tools.<|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool=\"add\" index=\"2\"<|sep|><|open|>argument key=\"y\" type=\"number\"<|sep|>1.0<|close|>argument<|sep|><|open|>argument key=\"x\" type=\"number\"<|sep|>2<|close|>argument<|sep|><|open|>argument key=\"items\" type=\"array\"<|sep|>[\"left\",\"right\"]<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 40, 748, 2390, 276, 7697, 13, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163587, 10257, 4453, 878, 1210, 1, 4002, 878, 17, 1, 163589, 163587, 47185, 2355, 878, 88, 1, 1798, 878, 10006, 1, 163589, 16, 13, 15, 163588, 47185, 163589, 163587, 47185, 2355, 878, 87, 1, 1798, 878, 10006, 1, 163589, 17, 163588, 47185, 163589, 163587, 47185, 2355, 878, 7989, 1, 1798, 878, 4392, 1, 163589, 3530, 3794, 3923, 2277, 3956, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586], + "all_stop_token_ids": [163585, 163586] +} diff --git a/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.tool_call_mix.json new file mode 100644 index 000000000000..7c1bfa948eee --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.tool_call_mix.json @@ -0,0 +1,98 @@ +{ + "model_id": "XiaomiMiMo/MiMo-V2.6-Flash-RL", + "vocab_size": 152576, + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + } + }, + { + "type": "triggered_tags", + "triggers": [ + "\n\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n\n" + }, + { + "type": "tag", + "begin": "\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n\n" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.Shanghai1.02[\"left\", \"right\"]", + "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 40, 686, 1618, 279, 7375, 13, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151657, 27, 1688, 92784, 1784, 16181, 29368, 29, 16, 13, 15, 522, 16181, 1784, 16181, 25387, 29, 17, 522, 16181, 1784, 16181, 28, 3615, 29, 1183, 2359, 497, 330, 1291, 1341, 522, 16181, 1472, 1688, 29, 151658, 151645], + "all_stop_token_ids": [151643, 151645, 151672] +} diff --git a/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.tool_call_mix.json new file mode 100644 index 000000000000..2dfaabd0e505 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.tool_call_mix.json @@ -0,0 +1,117 @@ +{ + "model_id": "MiniMaxAI/MiniMax-M2.5", + "vocab_size": 200064, + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "\n", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "minimax_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "minimax_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n\nI will call the tools.\n\n\nShanghai\n\n\n1.0\n2\n[\"left\", \"right\"]\n\n", + "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 367, 73, 715, 1728, 275, 6629, 320, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 60, 88638, 1925, 1139, 2132, 6143, 60, 24662, 1925, 1139, 121, 3361, 49, 46, 48, 1579, 24662, 1100, 60, 24662, 1925, 1139, 120, 3361, 50, 1579, 24662, 1100, 60, 24662, 1925, 1139, 12405, 3361, 7795, 2038, 1164, 494, 1581, 8760, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020], + "all_stop_token_ids": [200020] +} diff --git a/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.tool_call_mix.json new file mode 100644 index 000000000000..7a1d3a785114 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.tool_call_mix.json @@ -0,0 +1,107 @@ +{ + "model_id": "Qwen/Qwen3-0.6B", + "vocab_size": 151936, + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + } + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "\n{\"name\": \"get_weather\", \"arguments\": ", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "}\n" + }, + { + "type": "tag", + "begin": "\n{\"name\": \"add\", \"arguments\": ", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "}\n" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "\nNeed call the weather and add tools.\n\n\nI will call the tools.\n\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n\n\n{\"name\": \"add\", \"arguments\": {\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}}\n", + "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 40, 686, 1618, 279, 7375, 624, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 198, 151657, 198, 4913, 606, 788, 330, 718, 497, 330, 16370, 788, 5212, 88, 788, 220, 16, 13, 15, 11, 330, 87, 788, 220, 17, 11, 330, 3615, 788, 4383, 2359, 497, 330, 1291, 1341, 11248, 151658, 151645], + "all_stop_token_ids": [151643, 151645] +} diff --git a/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.tool_call_mix.json new file mode 100644 index 000000000000..ad62176e4e40 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.tool_call_mix.json @@ -0,0 +1,104 @@ +{ + "model_id": "Qwen/Qwen3.5-4B", + "vocab_size": 248320, + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "triggered_tags", + "triggers": [ + "\n\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n\n" + }, + { + "type": "tag", + "begin": "\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n\n" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n\nI will call the tools.\n\n\n\n\nShanghai\n\n\n\n\n\n\n1.0\n\n\n2\n\n\n[\"left\", \"right\"]\n\n\n", + "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 40, 668, 1562, 279, 7141, 13, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 198, 248058, 198, 27, 1628, 89587, 29, 198, 27, 15704, 28408, 29, 198, 16, 13, 15, 198, 510, 15704, 29, 198, 27, 15704, 24582, 29, 198, 17, 198, 510, 15704, 29, 198, 27, 15704, 28, 3494, 29, 198, 1143, 2282, 487, 328, 1246, 1293, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046], + "all_stop_token_ids": [248044, 248046] +} diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index a0ccbe59abfb..59f96a430724 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -6,20 +6,29 @@ //! The invariant under test is that a structured assistant message rendered as history can be //! parsed from the generated assistant completion and then rendered back to the exact same //! assistant completion. +//! +//! The tool-call fixture also snapshots the output grammar built by the initialized parser, with +//! the rendered completion as its generation, under `tests/grammar_replay/`. `grammar_replay.rs` +//! replays these cases through the real XGrammar. +use std::collections::BTreeSet; +use std::path::PathBuf; use std::pin::Pin; use std::sync::Arc; use anyhow::{Context as _, Result, bail, ensure}; use futures::{Stream, StreamExt as _, stream}; +use serde::ser::Error as _; +use serde::{Serialize, Serializer}; use serde_json_fmt::JsonFormat as JsonFmt; use serial_test::file_serial; use vllm_chat::{ AssistantContentBlock, AssistantMessage, AssistantMessageExt as _, AssistantToolCall, ChatEvent, ChatMessage, ChatRequest, ChatRole, ChatTool, FinishReason, GenerationPromptMode, LoadModelBackendsOptions, NewChatOutputProcessorOptions, ParserSelection, RendererSelection, - load_model_backends, + ToolStrictLevel, load_model_backends, }; +use vllm_parser::output_grammar::BuiltOutputGrammar; use vllm_text::{DecodedTextEvent, Finished, Prompt}; use vllm_tokenizer::Tokenizer; @@ -568,6 +577,8 @@ async fn run_roundtrip_tool_call_mix( result.closed_completion ); + check_grammar_replay_case(case, backends, "tool_call_mix", &result)?; + Ok(()) } @@ -645,6 +656,9 @@ struct RoundtripResult { /// Assistant-completion suffix cut after rendering the parsed assistant /// back as history. rerendered_closed_completion: Prompt, + /// Output grammar built by the same initialized parser that parsed the + /// completion. + output_grammar: Option, } /// Render, parse, and rerender one assistant turn through the production @@ -658,13 +672,17 @@ async fn run_roundtrip( let renderer = backends.chat_backend.chat_renderer(); let rendered = render_closed_completion(renderer.as_ref(), request, &assistant)?; - let parsed_message = parse_completion(case, backends, request, &rendered).await?; + let ParsedCompletion { + message: parsed_message, + output_grammar, + } = parse_completion(case, backends, request, &rendered).await?; let rerendered = render_closed_completion(renderer.as_ref(), request, &parsed_message)?; Ok(RoundtripResult { parsed_message, closed_completion: rendered.completion, rerendered_closed_completion: rerendered.completion, + output_grammar, }) } @@ -711,14 +729,23 @@ fn render_closed_completion( Ok(RenderedTurn { prompt, completion }) } +/// Output of one production output processor over a rendered completion. +struct ParsedCompletion { + /// Terminal assistant message. + message: AssistantMessage, + /// Output grammar built by the processor after initialization. + output_grammar: Option, +} + /// Feed one rendered assistant completion body into the real output processor -/// and collect its terminal assistant message. +/// and collect its terminal assistant message, along with the output grammar +/// the initialized processor builds for the request. async fn parse_completion( case: &RoundtripCase, backends: &vllm_chat::LoadedModelBackends, base_request: &ChatRequest, rendered: &RenderedTurn, -) -> Result { +) -> Result { let tokenizer = backends.text_backend.tokenizer(); let prompt_token_ids = match &rendered.prompt { Prompt::Text(prompt) => tokenizer @@ -731,12 +758,15 @@ async fn parse_completion( let mut processor = backends.chat_backend.new_chat_output_processor( &mut request, NewChatOutputProcessorOptions { - tool_strict_level: vllm_chat::ToolStrictLevel::Auto, + // Pin every tool schema so that builder-backed parsers build an + // output grammar for replay. The strict level does not affect parsing. + tool_strict_level: ToolStrictLevel::Parameter, tool_call_parser: &case.tool_call_parser, reasoning_parser: &case.reasoning_parser, }, )?; processor.initialize(&prompt_token_ids)?; + let output_grammar = processor.build_output_grammar()?; let decoded = decoded_completion_stream( tokenizer.as_ref(), @@ -748,7 +778,10 @@ async fn parse_completion( while let Some(event) = events.next().await { if let ChatEvent::Done { message, .. } = event? { - return Ok(message); + return Ok(ParsedCompletion { + message, + output_grammar, + }); } } @@ -772,14 +805,8 @@ fn decoded_completion_stream( prompt_logprobs: None, }]; - let chunks = match completion { - Prompt::Text(text) => { - let body = text.strip_suffix(assistant_stop_suffix).with_context(|| { - format!( - "closed assistant completion did not end with {:?}: {:?}", - assistant_stop_suffix, text - ) - })?; + let chunks = match completion_body(tokenizer, completion, assistant_stop_suffix)? { + CompletionBody::Text(body) => { // The engine returns token ids, so attribute the rendered text // through the production decoder, then split it into small char // chunks to exercise marker splits. @@ -807,27 +834,12 @@ fn decoded_completion_stream( }) .collect() } - Prompt::TokenIds(token_ids) => { - let body = if assistant_stop_suffix.is_empty() { - token_ids.as_slice() - } else { - let stop_token_ids = tokenizer - .encode(assistant_stop_suffix, false) - .context("failed to encode token-id completion stop suffix")?; - token_ids.strip_suffix(stop_token_ids.as_slice()).with_context(|| { - format!( - "token-id completion did not end with {:?}: {:?}", - assistant_stop_suffix, token_ids - ) - })? - }; - incremental_decode_chunks( - tokenizer, - &prompt_token_ids, - body, - TOKEN_COMPLETION_CHUNK_TOKENS, - )? - } + CompletionBody::TokenIds(body) => incremental_decode_chunks( + tokenizer, + &prompt_token_ids, + body, + TOKEN_COMPLETION_CHUNK_TOKENS, + )?, }; if chunks.is_empty() { @@ -868,6 +880,48 @@ fn decoded_completion_stream( Ok(stream::iter(events).map(Ok).boxed()) } +/// Generated part of a closed assistant completion, without the rendered stop +/// suffix. +enum CompletionBody<'a> { + Text(&'a str), + TokenIds(&'a [u32]), +} + +/// Strip the rendered stop suffix from a closed assistant completion. +fn completion_body<'a>( + tokenizer: &dyn Tokenizer, + completion: &'a Prompt, + assistant_stop_suffix: &str, +) -> Result> { + match completion { + Prompt::Text(text) => { + let body = text.strip_suffix(assistant_stop_suffix).with_context(|| { + format!( + "closed assistant completion did not end with {:?}: {:?}", + assistant_stop_suffix, text + ) + })?; + Ok(CompletionBody::Text(body)) + } + Prompt::TokenIds(token_ids) => { + let body = if assistant_stop_suffix.is_empty() { + token_ids.as_slice() + } else { + let stop_token_ids = tokenizer + .encode(assistant_stop_suffix, false) + .context("failed to encode token-id completion stop suffix")?; + token_ids.strip_suffix(stop_token_ids.as_slice()).with_context(|| { + format!( + "token-id completion did not end with {:?}: {:?}", + assistant_stop_suffix, token_ids + ) + })? + }; + Ok(CompletionBody::TokenIds(body)) + } + } +} + /// One decoded completion chunk fed into the output processor. struct DecodedCompletionChunk { /// Attributed for token-id completions, as the production decoder emits it. @@ -941,6 +995,106 @@ fn incremental_decode_chunks( Ok(chunks) } +/// One parser-built output grammar with the generation it must accept, replayed +/// through the real XGrammar by `grammar_replay.py`. +#[derive(Serialize)] +struct GrammarReplayCase<'a> { + /// Hugging Face model id whose tokenizer the replay loads. + model_id: &'static str, + /// Model vocabulary size, which sizes the XGrammar token mask. + vocab_size: usize, + /// Parser-built grammar; only token-zero grammars are replayed from the + /// first generated token. + grammar: &'a BuiltOutputGrammar, + /// Generated completion text, without the stop token. + completion: String, + /// Generated token IDs presented to the matcher, ending with a stop token. + #[serde(serialize_with = "serialize_one_line")] + generation_token_ids: Vec, + /// Request stop set, passed to the matcher as `override_stop_tokens`. + #[serde(serialize_with = "serialize_one_line")] + all_stop_token_ids: BTreeSet, +} + +/// Serialize a token ID list on one line inside the pretty-printed case. +fn serialize_one_line( + token_ids: &impl Serialize, + serializer: S, +) -> std::result::Result { + let json = spaced_json_fmt().format_to_string(token_ids).map_err(S::Error::custom)?; + serde_json::value::RawValue::from_string(json) + .map_err(S::Error::custom)? + .serialize(serializer) +} + +/// Directory of checked-in grammar replay cases. +fn grammar_replay_dir() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/grammar_replay") +} + +/// Snapshot the output grammar of one roundtrip as a grammar replay case, or +/// check that no stale case remains when the parser builds none. +fn check_grammar_replay_case( + case: &RoundtripCase, + backends: &vllm_chat::LoadedModelBackends, + fixture: &str, + result: &RoundtripResult, +) -> Result<()> { + let model_name = case.model_id.rsplit_once('/').map_or(case.model_id, |(_, name)| name); + let path = grammar_replay_dir().join(format!("{model_name}.{fixture}.json")); + let Some(grammar) = &result.output_grammar else { + ensure!( + !path.exists(), + "{} builds no output grammar, but grammar replay case {path:?} exists", + case.model_id + ); + return Ok(()); + }; + + let tokenizer = backends.text_backend.tokenizer(); + let hints = backends.text_backend.sampling_hints()?; + // Lowered `_all_stop_token_ids` of a request without user stop tokens. + let mut all_stop_token_ids = hints.extra_eos_token_ids; + all_stop_token_ids.extend(hints.primary_eos_token_id); + + // The generation ends with the first token of the rendered stop suffix, or + // with the primary EOS for templates that render none. + let stop_token_id = if case.assistant_stop_suffix.is_empty() { + hints.primary_eos_token_id.context("model has no primary EOS token")? + } else { + tokenizer.encode(case.assistant_stop_suffix, false)?[0] + }; + ensure!( + all_stop_token_ids.contains(&stop_token_id), + "stop token {stop_token_id} of {:?} is not in the stop set {all_stop_token_ids:?}", + case.assistant_stop_suffix + ); + + let (completion, mut generation_token_ids) = match completion_body( + tokenizer.as_ref(), + &result.closed_completion, + case.assistant_stop_suffix, + )? { + CompletionBody::Text(body) => (body.to_string(), tokenizer.encode(body, false)?), + CompletionBody::TokenIds(body) => (tokenizer.decode(body, false)?, body.to_vec()), + }; + generation_token_ids.push(stop_token_id); + + let replay_case = GrammarReplayCase { + model_id: case.model_id, + vocab_size: backends.text_backend.model_vocab_size(), + grammar, + completion, + generation_token_ids, + all_stop_token_ids, + }; + let mut json = serde_json::to_string_pretty(&replay_case)?; + json.push('\n'); + expect_test::expect_file![path].assert_eq(&json); + + Ok(()) +} + /// Build a chat request fixture with parser-enabling tool-choice semantics. fn roundtrip_request( request_id: impl Into, diff --git a/rust/src/parser/src/output_grammar.rs b/rust/src/parser/src/output_grammar.rs index f19254d36482..db56f97708d8 100644 --- a/rust/src/parser/src/output_grammar.rs +++ b/rust/src/parser/src/output_grammar.rs @@ -5,6 +5,7 @@ use std::{fmt, str::FromStr}; +use serde::Serialize; use serde_with::{DeserializeFromStr, SerializeDisplay}; use thiserror::Error; use xgrammar_structural_tag::builders::{StructuralTagBuilder, StructuralTagOptions}; @@ -20,7 +21,8 @@ use crate::tool::Tool; pub type Result = std::result::Result; /// How much of the generated stream an output grammar covers. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "snake_case")] pub enum GrammarCoverage { /// The grammar covers reasoning and advances from the first generated token. FromTokenZero, @@ -29,7 +31,7 @@ pub enum GrammarCoverage { } /// One parser-built output grammar and its generated-stream coverage. -#[derive(Debug, Clone, PartialEq)] +#[derive(Debug, Clone, PartialEq, Serialize)] pub struct BuiltOutputGrammar { /// Typed XGrammar structural-tag format. pub format: Format, From 1c299a8093302f8c4da077d6fde3c92bc2c10791 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Tue, 29 Sep 2026 06:04:44 +0000 Subject: [PATCH 2/5] [Rust Frontend] Replay required tool calls in grammar replay Run the roundtrip tool-call fixture once per tool-choice variant: `auto` with content before the calls, and `required` with tool calls only. Each model's variants share one replay file with model-level vocab size and stop set. Kimi K3 and MiMo V2.6 run `auto` only: the K3 renderer appends a tool-choice system message after the history, and MiMo renders compact tool calls that its Qwen3-Coder builder tags do not match. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Bugen Zhao --- rust/src/chat/tests/grammar_replay.py | 32 +- .../grammar_replay/DeepSeek-V3.2-Exp.json | 211 +++++++ .../DeepSeek-V3.2-Exp.tool_call_mix.json | 108 ---- .../grammar_replay/DeepSeek-V4-Flash.json | 211 +++++++ .../DeepSeek-V4-Flash.tool_call_mix.json | 108 ---- .../grammar_replay/DeepSeek-V4.1-Flash.json | 211 +++++++ .../DeepSeek-V4.1-Flash.tool_call_mix.json | 108 ---- .../tests/grammar_replay/GLM-4.7-Flash.json | 201 +++++++ .../GLM-4.7-Flash.tool_call_mix.json | 100 ---- .../tests/grammar_replay/GLM-5.2-FP8.json | 201 +++++++ .../GLM-5.2-FP8.tool_call_mix.json | 100 ---- rust/src/chat/tests/grammar_replay/Hy3.json | 530 ++++++++++++++++++ .../grammar_replay/Hy3.tool_call_mix.json | 266 --------- .../chat/tests/grammar_replay/Kimi-K2.5.json | 264 +++++++++ .../Kimi-K2.5.tool_call_mix.json | 135 ----- .../chat/tests/grammar_replay/Kimi-K3.json | 235 ++++++++ .../grammar_replay/Kimi-K3.tool_call_mix.json | 231 -------- .../grammar_replay/MiMo-V2.6-Flash-RL.json | 102 ++++ .../MiMo-V2.6-Flash-RL.tool_call_mix.json | 98 ---- .../tests/grammar_replay/MiniMax-M2.5.json | 229 ++++++++ .../MiniMax-M2.5.tool_call_mix.json | 117 ---- .../chat/tests/grammar_replay/Qwen3-0.6B.json | 215 +++++++ .../Qwen3-0.6B.tool_call_mix.json | 107 ---- .../chat/tests/grammar_replay/Qwen3.5-4B.json | 209 +++++++ .../Qwen3.5-4B.tool_call_mix.json | 104 ---- rust/src/chat/tests/roundtrip.rs | 238 +++++--- 26 files changed, 2998 insertions(+), 1673 deletions(-) create mode 100644 rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json delete mode 100644 rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.tool_call_mix.json create mode 100644 rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json delete mode 100644 rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.tool_call_mix.json create mode 100644 rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json delete mode 100644 rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.tool_call_mix.json create mode 100644 rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json delete mode 100644 rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.tool_call_mix.json create mode 100644 rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json delete mode 100644 rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.tool_call_mix.json create mode 100644 rust/src/chat/tests/grammar_replay/Hy3.json delete mode 100644 rust/src/chat/tests/grammar_replay/Hy3.tool_call_mix.json create mode 100644 rust/src/chat/tests/grammar_replay/Kimi-K2.5.json delete mode 100644 rust/src/chat/tests/grammar_replay/Kimi-K2.5.tool_call_mix.json create mode 100644 rust/src/chat/tests/grammar_replay/Kimi-K3.json delete mode 100644 rust/src/chat/tests/grammar_replay/Kimi-K3.tool_call_mix.json create mode 100644 rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.json delete mode 100644 rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.tool_call_mix.json create mode 100644 rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json delete mode 100644 rust/src/chat/tests/grammar_replay/MiniMax-M2.5.tool_call_mix.json create mode 100644 rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json delete mode 100644 rust/src/chat/tests/grammar_replay/Qwen3-0.6B.tool_call_mix.json create mode 100644 rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json delete mode 100644 rust/src/chat/tests/grammar_replay/Qwen3.5-4B.tool_call_mix.json diff --git a/rust/src/chat/tests/grammar_replay.py b/rust/src/chat/tests/grammar_replay.py index 8a50b12725ec..d34df627bae0 100755 --- a/rust/src/chat/tests/grammar_replay.py +++ b/rust/src/chat/tests/grammar_replay.py @@ -22,7 +22,8 @@ # /// """Replay the grammar cases exported by the roundtrip tests through XGrammar. -Every case is compiled against the model tokenizer the same way as +Each file holds one model's cases, keyed by tool-choice variant. Every case is +compiled against the model tokenizer the same way as `vllm/v1/structured_output/backend_xgrammar.py`. Token-zero cases are then replayed from the first generated token: each token must be allowed by the bitmask and accepted by the matcher, and the final stop token must terminate it. @@ -48,9 +49,9 @@ def load_tokenizer(model_id: str, vocab_size: int): return tokenizer, xgr.GrammarCompiler(tokenizer_info) -def replay(case: dict) -> str: - """Check one case and return a summary of what was checked.""" - tokenizer, compiler = load_tokenizer(case["model_id"], case["vocab_size"]) +def replay(model: dict, case: dict) -> str: + """Check one case of a model and return a summary of what was checked.""" + tokenizer, compiler = load_tokenizer(model["model_id"], model["vocab_size"]) generation = case["generation_token_ids"] # The case token IDs come from the Rust tokenizer; the runner must load the @@ -69,9 +70,9 @@ def replay(case: dict) -> str: return f"compiled only ({grammar['coverage']} is gated by the engine)" matcher = xgr.GrammarMatcher( - compiled, override_stop_tokens=case["all_stop_token_ids"] + compiled, override_stop_tokens=model["all_stop_token_ids"] ) - bitmask = xgr.allocate_token_bitmask(1, case["vocab_size"]) + bitmask = xgr.allocate_token_bitmask(1, model["vocab_size"]) for index, token_id in enumerate(generation): matcher.fill_next_token_bitmask(bitmask) allowed = (bitmask[0, token_id // 32].item() >> (token_id % 32)) & 1 @@ -94,15 +95,16 @@ def main() -> int: failures = 0 for path in paths: - case = json.loads(path.read_text()) - try: - summary = replay(case) - except Exception as error: - failures += 1 - coverage = case["grammar"]["coverage"] - print(f"FAIL {path.name} [{coverage}]: {error}", file=sys.stderr) - else: - print(f"PASS {path.name}: {summary}") + model = json.loads(path.read_text()) + for name, case in model["cases"].items(): + try: + summary = replay(model, case) + except Exception as error: + failures += 1 + coverage = case["grammar"]["coverage"] + print(f"FAIL {path.name} {name} [{coverage}]: {error}", file=sys.stderr) + else: + print(f"PASS {path.name} {name}: {summary}") return 1 if failures else 0 diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json new file mode 100644 index 000000000000..462fad9d0ecb --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json @@ -0,0 +1,211 @@ +{ + "model_id": "deepseek-ai/DeepSeek-V3.2-Exp", + "vocab_size": 129280, + "all_stop_token_ids": [1], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "<｜DSML｜function_calls>" + ], + "tags": [ + { + "type": "tag", + "begin": "<｜DSML｜function_calls>\n", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<｜DSML｜invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<｜DSML｜invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.\n\n<｜DSML｜function_calls>\n<｜DSML｜invoke name=\"get_weather\">\n<｜DSML｜parameter name=\"location\" string=\"true\">Shanghai\n\n<｜DSML｜invoke name=\"add\">\n<｜DSML｜parameter name=\"y\" string=\"false\">1.0\n<｜DSML｜parameter name=\"x\" string=\"false\">2\n<｜DSML｜parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 43, 759, 1926, 270, 6704, 339, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 3327, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n<｜DSML｜function_calls>\n" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<｜DSML｜invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<｜DSML｜invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n<｜DSML｜function_calls>\n<｜DSML｜invoke name=\"get_weather\">\n<｜DSML｜parameter name=\"location\" string=\"true\">Shanghai\n\n<｜DSML｜invoke name=\"add\">\n<｜DSML｜parameter name=\"y\" string=\"false\">1.0\n<｜DSML｜parameter name=\"x\" string=\"false\">2\n<｜DSML｜parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 271, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 3327, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.tool_call_mix.json deleted file mode 100644 index 64909a8355e0..000000000000 --- a/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.tool_call_mix.json +++ /dev/null @@ -1,108 +0,0 @@ -{ - "model_id": "deepseek-ai/DeepSeek-V3.2-Exp", - "vocab_size": 129280, - "grammar": { - "format": { - "type": "sequence", - "elements": [ - { - "type": "tag", - "begin": "", - "content": { - "type": "any_text", - "excludes": [], - "max_tokens": null, - "max_chars": null - }, - "end": "" - }, - { - "type": "triggered_tags", - "triggers": [ - "<｜DSML｜function_calls>" - ], - "tags": [ - { - "type": "tag", - "begin": "<｜DSML｜function_calls>\n", - "content": { - "type": "tags_with_separator", - "tags": [ - { - "type": "tag", - "begin": "<｜DSML｜invoke name=\"get_weather\">\n", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "location": { - "type": "string" - } - }, - "required": [ - "location" - ] - }, - "style": "deepseek_xml", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "\n" - }, - { - "type": "tag", - "begin": "<｜DSML｜invoke name=\"add\">\n", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "y": { - "type": "number" - }, - "x": { - "type": "number" - }, - "items": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "y", - "x", - "items" - ] - }, - "style": "deepseek_xml", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "\n" - } - ], - "separator": "", - "at_least_one": true, - "stop_after_first": false - }, - "end": "" - } - ], - "at_least_one": false, - "stop_after_first": false, - "excludes": [ - "", - "" - ] - } - ] - }, - "coverage": "from_token_zero" - }, - "completion": "Need call the weather and add tools.I will call the tools.\n\n<｜DSML｜function_calls>\n<｜DSML｜invoke name=\"get_weather\">\n<｜DSML｜parameter name=\"location\" string=\"true\">Shanghai\n\n<｜DSML｜invoke name=\"add\">\n<｜DSML｜parameter name=\"y\" string=\"false\">1.0\n<｜DSML｜parameter name=\"x\" string=\"false\">2\n<｜DSML｜parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", - "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 43, 759, 1926, 270, 6704, 339, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 3327, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1], - "all_stop_token_ids": [1] -} diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json new file mode 100644 index 000000000000..1c3d9e6640ae --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json @@ -0,0 +1,211 @@ +{ + "model_id": "deepseek-ai/DeepSeek-V4-Flash", + "vocab_size": 129280, + "all_stop_token_ids": [1], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "<｜DSML｜tool_calls>" + ], + "tags": [ + { + "type": "tag", + "begin": "<｜DSML｜tool_calls>\n", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<｜DSML｜invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<｜DSML｜invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.\n\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"get_weather\">\n<｜DSML｜parameter name=\"location\" string=\"true\">Shanghai\n\n<｜DSML｜invoke name=\"add\">\n<｜DSML｜parameter name=\"y\" string=\"false\">1.0\n<｜DSML｜parameter name=\"x\" string=\"false\">2\n<｜DSML｜parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 43, 759, 1926, 270, 6704, 339, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 30, 128825, 40148, 5406, 2329, 1281, 3327, 3816, 30, 128825, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n<｜DSML｜tool_calls>\n" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<｜DSML｜invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<｜DSML｜invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"get_weather\">\n<｜DSML｜parameter name=\"location\" string=\"true\">Shanghai\n\n<｜DSML｜invoke name=\"add\">\n<｜DSML｜parameter name=\"y\" string=\"false\">1.0\n<｜DSML｜parameter name=\"x\" string=\"false\">2\n<｜DSML｜parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 30, 128825, 40148, 5406, 2329, 1281, 3327, 3816, 30, 128825, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.tool_call_mix.json deleted file mode 100644 index 4eddceb5813c..000000000000 --- a/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.tool_call_mix.json +++ /dev/null @@ -1,108 +0,0 @@ -{ - "model_id": "deepseek-ai/DeepSeek-V4-Flash", - "vocab_size": 129280, - "grammar": { - "format": { - "type": "sequence", - "elements": [ - { - "type": "tag", - "begin": "", - "content": { - "type": "any_text", - "excludes": [], - "max_tokens": null, - "max_chars": null - }, - "end": "" - }, - { - "type": "triggered_tags", - "triggers": [ - "<｜DSML｜tool_calls>" - ], - "tags": [ - { - "type": "tag", - "begin": "<｜DSML｜tool_calls>\n", - "content": { - "type": "tags_with_separator", - "tags": [ - { - "type": "tag", - "begin": "<｜DSML｜invoke name=\"get_weather\">\n", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "location": { - "type": "string" - } - }, - "required": [ - "location" - ] - }, - "style": "deepseek_xml", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "\n" - }, - { - "type": "tag", - "begin": "<｜DSML｜invoke name=\"add\">\n", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "y": { - "type": "number" - }, - "x": { - "type": "number" - }, - "items": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "y", - "x", - "items" - ] - }, - "style": "deepseek_xml", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "\n" - } - ], - "separator": "", - "at_least_one": true, - "stop_after_first": false - }, - "end": "" - } - ], - "at_least_one": false, - "stop_after_first": false, - "excludes": [ - "", - "" - ] - } - ] - }, - "coverage": "from_token_zero" - }, - "completion": "Need call the weather and add tools.I will call the tools.\n\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"get_weather\">\n<｜DSML｜parameter name=\"location\" string=\"true\">Shanghai\n\n<｜DSML｜invoke name=\"add\">\n<｜DSML｜parameter name=\"y\" string=\"false\">1.0\n<｜DSML｜parameter name=\"x\" string=\"false\">2\n<｜DSML｜parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", - "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 43, 759, 1926, 270, 6704, 339, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 30, 128825, 40148, 5406, 2329, 1281, 3327, 3816, 30, 128825, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1], - "all_stop_token_ids": [1] -} diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json new file mode 100644 index 000000000000..0a5635abd411 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json @@ -0,0 +1,211 @@ +{ + "model_id": "deepseek-ai/DeepSeek-V4.1-Flash", + "vocab_size": 129280, + "all_stop_token_ids": [1], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "<｜DSML｜ calls>" + ], + "tags": [ + { + "type": "tag", + "begin": "<｜DSML｜ calls>\n", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<｜DSML｜ invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_v4_1_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<｜DSML｜ invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_v4_1_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name=\"get_weather\">\n<｜DSML｜ parameter name=\"location\" string=\"true\">Shanghai\n\n<｜DSML｜ invoke name=\"add\">\n<｜DSML｜ parameter name=\"y\" string=\"false\">1.0\n<｜DSML｜ parameter name=\"x\" string=\"false\">2\n<｜DSML｜ parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 43, 759, 1926, 270, 6704, 339, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 30, 128825, 34756, 2329, 1281, 3327, 3816, 30, 128825, 10767, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n<｜DSML｜ calls>\n" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<｜DSML｜ invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_v4_1_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "<｜DSML｜ invoke name=\"add\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "deepseek_v4_1_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name=\"get_weather\">\n<｜DSML｜ parameter name=\"location\" string=\"true\">Shanghai\n\n<｜DSML｜ invoke name=\"add\">\n<｜DSML｜ parameter name=\"y\" string=\"false\">1.0\n<｜DSML｜ parameter name=\"x\" string=\"false\">2\n<｜DSML｜ parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 30, 128825, 34756, 2329, 1281, 3327, 3816, 30, 128825, 10767, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.tool_call_mix.json deleted file mode 100644 index c3ac215bc0b6..000000000000 --- a/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.tool_call_mix.json +++ /dev/null @@ -1,108 +0,0 @@ -{ - "model_id": "deepseek-ai/DeepSeek-V4.1-Flash", - "vocab_size": 129280, - "grammar": { - "format": { - "type": "sequence", - "elements": [ - { - "type": "tag", - "begin": "", - "content": { - "type": "any_text", - "excludes": [], - "max_tokens": null, - "max_chars": null - }, - "end": "" - }, - { - "type": "triggered_tags", - "triggers": [ - "<｜DSML｜ calls>" - ], - "tags": [ - { - "type": "tag", - "begin": "<｜DSML｜ calls>\n", - "content": { - "type": "tags_with_separator", - "tags": [ - { - "type": "tag", - "begin": "<｜DSML｜ invoke name=\"get_weather\">\n", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "location": { - "type": "string" - } - }, - "required": [ - "location" - ] - }, - "style": "deepseek_v4_1_xml", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "\n" - }, - { - "type": "tag", - "begin": "<｜DSML｜ invoke name=\"add\">\n", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "y": { - "type": "number" - }, - "x": { - "type": "number" - }, - "items": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "y", - "x", - "items" - ] - }, - "style": "deepseek_v4_1_xml", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "\n" - } - ], - "separator": "", - "at_least_one": true, - "stop_after_first": false - }, - "end": "" - } - ], - "at_least_one": false, - "stop_after_first": false, - "excludes": [ - "", - "" - ] - } - ] - }, - "coverage": "from_token_zero" - }, - "completion": "Need call the weather and add tools.I will call the tools.\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name=\"get_weather\">\n<｜DSML｜ parameter name=\"location\" string=\"true\">Shanghai\n\n<｜DSML｜ invoke name=\"add\">\n<｜DSML｜ parameter name=\"y\" string=\"false\">1.0\n<｜DSML｜ parameter name=\"x\" string=\"false\">2\n<｜DSML｜ parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", - "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 43, 759, 1926, 270, 6704, 339, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 30, 128825, 34756, 2329, 1281, 3327, 3816, 30, 128825, 10767, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1], - "all_stop_token_ids": [1] -} diff --git a/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json b/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json new file mode 100644 index 000000000000..ac3b2fdacef9 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json @@ -0,0 +1,201 @@ +{ + "model_id": "zai-org/GLM-4.7-Flash", + "vocab_size": 154880, + "all_stop_token_ids": [154820, 154827, 154829], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "get_weather", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "" + }, + { + "type": "tag", + "begin": "add", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "", + "", + "", + "", + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 40, 686, 1618, 279, 7375, 13, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "get_weather", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "" + }, + { + "type": "tag", + "begin": "add", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "" + } + ], + "at_least_one": true, + "stop_after_first": false, + "excludes": [ + "", + "", + "", + "", + "", + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.tool_call_mix.json deleted file mode 100644 index 7a5f07df6b11..000000000000 --- a/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.tool_call_mix.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "model_id": "zai-org/GLM-4.7-Flash", - "vocab_size": 154880, - "grammar": { - "format": { - "type": "sequence", - "elements": [ - { - "type": "tag", - "begin": "", - "content": { - "type": "any_text", - "excludes": [], - "max_tokens": null, - "max_chars": null - }, - "end": "" - }, - { - "type": "triggered_tags", - "triggers": [ - "" - ], - "tags": [ - { - "type": "tag", - "begin": "get_weather", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "location": { - "type": "string" - } - }, - "required": [ - "location" - ] - }, - "style": "glm_xml", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "" - }, - { - "type": "tag", - "begin": "add", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "y": { - "type": "number" - }, - "x": { - "type": "number" - }, - "items": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "y", - "x", - "items" - ] - }, - "style": "glm_xml", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "" - } - ], - "at_least_one": false, - "stop_after_first": false, - "excludes": [ - "", - "", - "", - "", - "", - "", - "" - ] - } - ] - }, - "coverage": "from_token_zero" - }, - "completion": "Need call the weather and add tools.I will call the tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", - "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 40, 686, 1618, 279, 7375, 13, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820], - "all_stop_token_ids": [154820, 154827, 154829] -} diff --git a/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json b/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json new file mode 100644 index 000000000000..eb8355b5f4f7 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json @@ -0,0 +1,201 @@ +{ + "model_id": "zai-org/GLM-5.2-FP8", + "vocab_size": 154880, + "all_stop_token_ids": [154820, 154827, 154829], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "get_weather", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "" + }, + { + "type": "tag", + "begin": "add", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "", + "", + "", + "", + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 40, 686, 1618, 279, 7375, 13, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "get_weather", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "" + }, + { + "type": "tag", + "begin": "add", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "" + } + ], + "at_least_one": true, + "stop_after_first": false, + "excludes": [ + "", + "", + "", + "", + "", + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.tool_call_mix.json deleted file mode 100644 index d2465d4a536c..000000000000 --- a/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.tool_call_mix.json +++ /dev/null @@ -1,100 +0,0 @@ -{ - "model_id": "zai-org/GLM-5.2-FP8", - "vocab_size": 154880, - "grammar": { - "format": { - "type": "sequence", - "elements": [ - { - "type": "tag", - "begin": "", - "content": { - "type": "any_text", - "excludes": [], - "max_tokens": null, - "max_chars": null - }, - "end": "" - }, - { - "type": "triggered_tags", - "triggers": [ - "" - ], - "tags": [ - { - "type": "tag", - "begin": "get_weather", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "location": { - "type": "string" - } - }, - "required": [ - "location" - ] - }, - "style": "glm_xml", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "" - }, - { - "type": "tag", - "begin": "add", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "y": { - "type": "number" - }, - "x": { - "type": "number" - }, - "items": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "y", - "x", - "items" - ] - }, - "style": "glm_xml", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "" - } - ], - "at_least_one": false, - "stop_after_first": false, - "excludes": [ - "", - "", - "", - "", - "", - "", - "" - ] - } - ] - }, - "coverage": "from_token_zero" - }, - "completion": "Need call the weather and add tools.I will call the tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", - "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 40, 686, 1618, 279, 7375, 13, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820], - "all_stop_token_ids": [154820, 154827, 154829] -} diff --git a/rust/src/chat/tests/grammar_replay/Hy3.json b/rust/src/chat/tests/grammar_replay/Hy3.json new file mode 100644 index 000000000000..2b59aef801f7 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Hy3.json @@ -0,0 +1,530 @@ +{ + "model_id": "tencent/Hy3", + "vocab_size": 120832, + "all_stop_token_ids": [120025], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "\n", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "get_weather\n", + "content": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "location" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "end": "" + }, + { + "type": "tag", + "begin": "add\n", + "content": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "y" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "x" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "items" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "end": "" + } + ], + "separator": "\n", + "at_least_one": true, + "stop_after_first": false + }, + "end": "\n" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.\nget_weather\nlocation\nShanghai\n\nadd\ny\n1.0\nx\n2\nitems\n[\"left\", \"right\"]\n\n", + "generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 40, 823, 1609, 252, 7106, 13, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120033, 2176, 120043, 185, 120066, 88, 120067, 185, 120068, 16, 13, 15, 120069, 185, 120066, 87, 120067, 185, 120068, 17, 120069, 185, 120066, 12792, 120067, 185, 120068, 5831, 1639, 1221, 478, 1331, 7029, 120069, 185, 120034, 185, 120036, 120025] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "get_weather\n", + "content": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "location" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "end": "" + }, + { + "type": "tag", + "begin": "add\n", + "content": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "y" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "x" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "items" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "end": "" + } + ], + "separator": "\n", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\nget_weather\nlocation\nShanghai\n\nadd\ny\n1.0\nx\n2\nitems\n[\"left\", \"right\"]\n\n", + "generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120033, 2176, 120043, 185, 120066, 88, 120067, 185, 120068, 16, 13, 15, 120069, 185, 120066, 87, 120067, 185, 120068, 17, 120069, 185, 120066, 12792, 120067, 185, 120068, 5831, 1639, 1221, 478, 1331, 7029, 120069, 185, 120034, 185, 120036, 120025] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/Hy3.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/Hy3.tool_call_mix.json deleted file mode 100644 index ac50c02787dc..000000000000 --- a/rust/src/chat/tests/grammar_replay/Hy3.tool_call_mix.json +++ /dev/null @@ -1,266 +0,0 @@ -{ - "model_id": "tencent/Hy3", - "vocab_size": 120832, - "grammar": { - "format": { - "type": "sequence", - "elements": [ - { - "type": "tag", - "begin": "", - "content": { - "type": "any_text", - "excludes": [], - "max_tokens": null, - "max_chars": null - }, - "end": "" - }, - { - "type": "triggered_tags", - "triggers": [ - "" - ], - "tags": [ - { - "type": "tag", - "begin": "\n", - "content": { - "type": "tags_with_separator", - "tags": [ - { - "type": "tag", - "begin": "get_weather\n", - "content": { - "type": "sequence", - "elements": [ - { - "type": "sequence", - "elements": [ - { - "type": "const_string", - "value": "" - }, - { - "type": "const_string", - "value": "location" - }, - { - "type": "const_string", - "value": "" - }, - { - "type": "const_string", - "value": "\n" - }, - { - "type": "const_string", - "value": "" - }, - { - "type": "any_text", - "excludes": [ - "", - "", - "", - "", - "", - "", - "", - "", - "" - ], - "max_tokens": null, - "max_chars": null - }, - { - "type": "const_string", - "value": "" - }, - { - "type": "const_string", - "value": "\n" - } - ] - } - ] - }, - "end": "" - }, - { - "type": "tag", - "begin": "add\n", - "content": { - "type": "sequence", - "elements": [ - { - "type": "sequence", - "elements": [ - { - "type": "const_string", - "value": "" - }, - { - "type": "const_string", - "value": "y" - }, - { - "type": "const_string", - "value": "" - }, - { - "type": "const_string", - "value": "\n" - }, - { - "type": "const_string", - "value": "" - }, - { - "type": "any_text", - "excludes": [ - "", - "", - "", - "", - "", - "", - "", - "", - "" - ], - "max_tokens": null, - "max_chars": null - }, - { - "type": "const_string", - "value": "" - }, - { - "type": "const_string", - "value": "\n" - } - ] - }, - { - "type": "sequence", - "elements": [ - { - "type": "const_string", - "value": "" - }, - { - "type": "const_string", - "value": "x" - }, - { - "type": "const_string", - "value": "" - }, - { - "type": "const_string", - "value": "\n" - }, - { - "type": "const_string", - "value": "" - }, - { - "type": "any_text", - "excludes": [ - "", - "", - "", - "", - "", - "", - "", - "", - "" - ], - "max_tokens": null, - "max_chars": null - }, - { - "type": "const_string", - "value": "" - }, - { - "type": "const_string", - "value": "\n" - } - ] - }, - { - "type": "sequence", - "elements": [ - { - "type": "const_string", - "value": "" - }, - { - "type": "const_string", - "value": "items" - }, - { - "type": "const_string", - "value": "" - }, - { - "type": "const_string", - "value": "\n" - }, - { - "type": "const_string", - "value": "" - }, - { - "type": "any_text", - "excludes": [ - "", - "", - "", - "", - "", - "", - "", - "", - "" - ], - "max_tokens": null, - "max_chars": null - }, - { - "type": "const_string", - "value": "" - }, - { - "type": "const_string", - "value": "\n" - } - ] - } - ] - }, - "end": "" - } - ], - "separator": "\n", - "at_least_one": true, - "stop_after_first": false - }, - "end": "\n" - } - ], - "at_least_one": false, - "stop_after_first": false, - "excludes": [] - } - ] - }, - "coverage": "from_token_zero" - }, - "completion": "Need call the weather and add tools.I will call the tools.\nget_weather\nlocation\nShanghai\n\nadd\ny\n1.0\nx\n2\nitems\n[\"left\", \"right\"]\n\n", - "generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 40, 823, 1609, 252, 7106, 13, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120033, 2176, 120043, 185, 120066, 88, 120067, 185, 120068, 16, 13, 15, 120069, 185, 120066, 87, 120067, 185, 120068, 17, 120069, 185, 120066, 12792, 120067, 185, 120068, 5831, 1639, 1221, 478, 1331, 7029, 120069, 185, 120034, 185, 120036, 120025], - "all_stop_token_ids": [120025] -} diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K2.5.json b/rust/src/chat/tests/grammar_replay/Kimi-K2.5.json new file mode 100644 index 000000000000..f86299048685 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Kimi-K2.5.json @@ -0,0 +1,264 @@ +{ + "model_id": "moonshotai/Kimi-K2.5", + "vocab_size": 163840, + "all_stop_token_ids": [163585, 163586], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "triggered_tags", + "triggers": [ + "<|tool_calls_section_begin|>" + ], + "tags": [ + { + "type": "tag", + "begin": "<|tool_calls_section_begin|>", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|tool_call_begin|>functions.get_weather:", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "\\d+" + }, + { + "type": "const_string", + "value": "<|tool_call_argument_begin|>" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + } + ] + }, + "end": "<|tool_call_end|>" + }, + { + "type": "tag", + "begin": "<|tool_call_begin|>functions.add:", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "\\d+" + }, + { + "type": "const_string", + "value": "<|tool_call_argument_begin|>" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + } + ] + }, + "end": "<|tool_call_end|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "<|tool_calls_section_end|>" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "", + "<|tool_call_begin|>" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_call_begin|>functions.add:1<|tool_call_argument_begin|>{\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}<|tool_call_end|><|tool_calls_section_end|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 40, 748, 2390, 276, 7697, 13, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163597, 41937, 3149, 25, 16, 163598, 8264, 88, 1289, 220, 16, 13, 15, 11, 414, 87, 1289, 220, 17, 11, 414, 7989, 1289, 8681, 3794, 665, 414, 2277, 3956, 92, 163599, 163596, 163586] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "<|tool_calls_section_begin|>" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|tool_call_begin|>functions.get_weather:", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "\\d+" + }, + { + "type": "const_string", + "value": "<|tool_call_argument_begin|>" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + } + ] + }, + "end": "<|tool_call_end|>" + }, + { + "type": "tag", + "begin": "<|tool_call_begin|>functions.add:", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "\\d+" + }, + { + "type": "const_string", + "value": "<|tool_call_argument_begin|>" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + } + ] + }, + "end": "<|tool_call_end|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "<|tool_calls_section_end|>" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_call_begin|>functions.add:1<|tool_call_argument_begin|>{\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}<|tool_call_end|><|tool_calls_section_end|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163597, 41937, 3149, 25, 16, 163598, 8264, 88, 1289, 220, 16, 13, 15, 11, 414, 87, 1289, 220, 17, 11, 414, 7989, 1289, 8681, 3794, 665, 414, 2277, 3956, 92, 163599, 163596, 163586] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K2.5.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/Kimi-K2.5.tool_call_mix.json deleted file mode 100644 index 410850ac66a3..000000000000 --- a/rust/src/chat/tests/grammar_replay/Kimi-K2.5.tool_call_mix.json +++ /dev/null @@ -1,135 +0,0 @@ -{ - "model_id": "moonshotai/Kimi-K2.5", - "vocab_size": 163840, - "grammar": { - "format": { - "type": "sequence", - "elements": [ - { - "type": "tag", - "begin": "", - "content": { - "type": "any_text", - "excludes": [], - "max_tokens": null, - "max_chars": null - }, - "end": "" - }, - { - "type": "triggered_tags", - "triggers": [ - "<|tool_calls_section_begin|>" - ], - "tags": [ - { - "type": "tag", - "begin": "<|tool_calls_section_begin|>", - "content": { - "type": "tags_with_separator", - "tags": [ - { - "type": "tag", - "begin": "<|tool_call_begin|>functions.get_weather:", - "content": { - "type": "sequence", - "elements": [ - { - "type": "regex", - "pattern": "\\d+" - }, - { - "type": "const_string", - "value": "<|tool_call_argument_begin|>" - }, - { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "location": { - "type": "string" - } - }, - "required": [ - "location" - ] - }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null - } - ] - }, - "end": "<|tool_call_end|>" - }, - { - "type": "tag", - "begin": "<|tool_call_begin|>functions.add:", - "content": { - "type": "sequence", - "elements": [ - { - "type": "regex", - "pattern": "\\d+" - }, - { - "type": "const_string", - "value": "<|tool_call_argument_begin|>" - }, - { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "y": { - "type": "number" - }, - "x": { - "type": "number" - }, - "items": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "y", - "x", - "items" - ] - }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null - } - ] - }, - "end": "<|tool_call_end|>" - } - ], - "separator": "", - "at_least_one": true, - "stop_after_first": false - }, - "end": "<|tool_calls_section_end|>" - } - ], - "at_least_one": false, - "stop_after_first": false, - "excludes": [ - "", - "", - "<|tool_call_begin|>" - ] - } - ] - }, - "coverage": "from_token_zero" - }, - "completion": "Need call the weather and add tools.I will call the tools.<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_call_begin|>functions.add:1<|tool_call_argument_begin|>{\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}<|tool_call_end|><|tool_calls_section_end|>", - "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 40, 748, 2390, 276, 7697, 13, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163597, 41937, 3149, 25, 16, 163598, 8264, 88, 1289, 220, 16, 13, 15, 11, 414, 87, 1289, 220, 17, 11, 414, 7989, 1289, 8681, 3794, 665, 414, 2277, 3956, 92, 163599, 163596, 163586], - "all_stop_token_ids": [163585, 163586] -} diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K3.json b/rust/src/chat/tests/grammar_replay/Kimi-K3.json new file mode 100644 index 000000000000..48a69968a3dd --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Kimi-K3.json @@ -0,0 +1,235 @@ +{ + "model_id": "moonshotai/Kimi-K3", + "vocab_size": 163840, + "all_stop_token_ids": [163585, 163586], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|open|>response<|sep|>" + } + }, + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [ + "<|open|>", + "<|close|>", + "<|end_of_msg|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>response<|sep|>" + }, + { + "type": "optional", + "content": { + "type": "tag", + "begin": "<|open|>tools<|sep|>", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|open|>call tool=\"get_weather\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "plus", + "content": { + "type": "tag", + "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", + "content": { + "type": "any_text", + "excludes": [ + "<|close|>argument<|sep|>", + "<|close|>call<|sep|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>argument<|sep|>" + } + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>call tool=\"add\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "plus", + "content": { + "type": "or", + "elements": [ + { + "type": "tag", + "begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "array", + "items": { + "type": "string" + } + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "<|close|>argument<|sep|>" + } + ] + } + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "<|close|>tools<|sep|>" + } + }, + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|close|>message<|sep|>" + } + } + ] + }, + "coverage": "final_output_only" + }, + "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|>I will call the tools.<|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool=\"add\" index=\"2\"<|sep|><|open|>argument key=\"y\" type=\"number\"<|sep|>1.0<|close|>argument<|sep|><|open|>argument key=\"x\" type=\"number\"<|sep|>2<|close|>argument<|sep|><|open|>argument key=\"items\" type=\"array\"<|sep|>[\"left\",\"right\"]<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 40, 748, 2390, 276, 7697, 13, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163587, 10257, 4453, 878, 1210, 1, 4002, 878, 17, 1, 163589, 163587, 47185, 2355, 878, 88, 1, 1798, 878, 10006, 1, 163589, 16, 13, 15, 163588, 47185, 163589, 163587, 47185, 2355, 878, 87, 1, 1798, 878, 10006, 1, 163589, 17, 163588, 47185, 163589, 163587, 47185, 2355, 878, 7989, 1, 1798, 878, 4392, 1, 163589, 3530, 3794, 3923, 2277, 3956, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K3.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/Kimi-K3.tool_call_mix.json deleted file mode 100644 index 86b0ec6896d9..000000000000 --- a/rust/src/chat/tests/grammar_replay/Kimi-K3.tool_call_mix.json +++ /dev/null @@ -1,231 +0,0 @@ -{ - "model_id": "moonshotai/Kimi-K3", - "vocab_size": 163840, - "grammar": { - "format": { - "type": "sequence", - "elements": [ - { - "type": "optional", - "content": { - "type": "const_string", - "value": "<|open|>response<|sep|>" - } - }, - { - "type": "tag", - "begin": "", - "content": { - "type": "any_text", - "excludes": [ - "<|open|>", - "<|close|>", - "<|end_of_msg|>" - ], - "max_tokens": null, - "max_chars": null - }, - "end": "<|close|>response<|sep|>" - }, - { - "type": "optional", - "content": { - "type": "tag", - "begin": "<|open|>tools<|sep|>", - "content": { - "type": "tags_with_separator", - "tags": [ - { - "type": "tag", - "begin": "<|open|>call tool=\"get_weather\" index=\"", - "content": { - "type": "sequence", - "elements": [ - { - "type": "regex", - "pattern": "[1-9][0-9]*" - }, - { - "type": "const_string", - "value": "\"<|sep|>" - }, - { - "type": "or", - "elements": [ - { - "type": "plus", - "content": { - "type": "tag", - "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", - "content": { - "type": "any_text", - "excludes": [ - "<|close|>argument<|sep|>", - "<|close|>call<|sep|>" - ], - "max_tokens": null, - "max_chars": null - }, - "end": "<|close|>argument<|sep|>" - } - }, - { - "type": "tag", - "begin": "<|open|>json type=\"object\"<|sep|>", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "location": { - "type": "string" - } - }, - "required": [ - "location" - ] - }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "<|close|>json<|sep|>" - } - ] - } - ] - }, - "end": "<|close|>call<|sep|>" - }, - { - "type": "tag", - "begin": "<|open|>call tool=\"add\" index=\"", - "content": { - "type": "sequence", - "elements": [ - { - "type": "regex", - "pattern": "[1-9][0-9]*" - }, - { - "type": "const_string", - "value": "\"<|sep|>" - }, - { - "type": "or", - "elements": [ - { - "type": "plus", - "content": { - "type": "or", - "elements": [ - { - "type": "tag", - "begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>", - "content": { - "type": "json_schema", - "json_schema": { - "type": "number" - }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "<|close|>argument<|sep|>" - }, - { - "type": "tag", - "begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>", - "content": { - "type": "json_schema", - "json_schema": { - "type": "number" - }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "<|close|>argument<|sep|>" - }, - { - "type": "tag", - "begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>", - "content": { - "type": "json_schema", - "json_schema": { - "type": "array", - "items": { - "type": "string" - } - }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "<|close|>argument<|sep|>" - } - ] - } - }, - { - "type": "tag", - "begin": "<|open|>json type=\"object\"<|sep|>", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "y": { - "type": "number" - }, - "x": { - "type": "number" - }, - "items": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "y", - "x", - "items" - ] - }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "<|close|>json<|sep|>" - } - ] - } - ] - }, - "end": "<|close|>call<|sep|>" - } - ], - "separator": "", - "at_least_one": true, - "stop_after_first": false - }, - "end": "<|close|>tools<|sep|>" - } - }, - { - "type": "optional", - "content": { - "type": "const_string", - "value": "<|close|>message<|sep|>" - } - } - ] - }, - "coverage": "final_output_only" - }, - "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|>I will call the tools.<|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool=\"add\" index=\"2\"<|sep|><|open|>argument key=\"y\" type=\"number\"<|sep|>1.0<|close|>argument<|sep|><|open|>argument key=\"x\" type=\"number\"<|sep|>2<|close|>argument<|sep|><|open|>argument key=\"items\" type=\"array\"<|sep|>[\"left\",\"right\"]<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>", - "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 40, 748, 2390, 276, 7697, 13, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163587, 10257, 4453, 878, 1210, 1, 4002, 878, 17, 1, 163589, 163587, 47185, 2355, 878, 88, 1, 1798, 878, 10006, 1, 163589, 16, 13, 15, 163588, 47185, 163589, 163587, 47185, 2355, 878, 87, 1, 1798, 878, 10006, 1, 163589, 17, 163588, 47185, 163589, 163587, 47185, 2355, 878, 7989, 1, 1798, 878, 4392, 1, 163589, 3530, 3794, 3923, 2277, 3956, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586], - "all_stop_token_ids": [163585, 163586] -} diff --git a/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.json b/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.json new file mode 100644 index 000000000000..02ea28577032 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.json @@ -0,0 +1,102 @@ +{ + "model_id": "XiaomiMiMo/MiMo-V2.6-Flash-RL", + "vocab_size": 152576, + "all_stop_token_ids": [151643, 151645, 151672], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + } + }, + { + "type": "triggered_tags", + "triggers": [ + "\n\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n\n" + }, + { + "type": "tag", + "begin": "\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n\n" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.I will call the tools.Shanghai1.02[\"left\", \"right\"]", + "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 40, 686, 1618, 279, 7375, 13, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151657, 27, 1688, 92784, 1784, 16181, 29368, 29, 16, 13, 15, 522, 16181, 1784, 16181, 25387, 29, 17, 522, 16181, 1784, 16181, 28, 3615, 29, 1183, 2359, 497, 330, 1291, 1341, 522, 16181, 1472, 1688, 29, 151658, 151645] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.tool_call_mix.json deleted file mode 100644 index 7c1bfa948eee..000000000000 --- a/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.tool_call_mix.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "model_id": "XiaomiMiMo/MiMo-V2.6-Flash-RL", - "vocab_size": 152576, - "grammar": { - "format": { - "type": "sequence", - "elements": [ - { - "type": "optional", - "content": { - "type": "tag", - "begin": "", - "content": { - "type": "any_text", - "excludes": [], - "max_tokens": null, - "max_chars": null - }, - "end": "" - } - }, - { - "type": "triggered_tags", - "triggers": [ - "\n\n\n", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "location": { - "type": "string" - } - }, - "required": [ - "location" - ] - }, - "style": "qwen_xml", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "\n\n" - }, - { - "type": "tag", - "begin": "\n\n", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "y": { - "type": "number" - }, - "x": { - "type": "number" - }, - "items": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "y", - "x", - "items" - ] - }, - "style": "qwen_xml", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "\n\n" - } - ], - "at_least_one": false, - "stop_after_first": false, - "excludes": [ - "", - "" - ] - } - ] - }, - "coverage": "from_token_zero" - }, - "completion": "Need call the weather and add tools.I will call the tools.Shanghai1.02[\"left\", \"right\"]", - "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 40, 686, 1618, 279, 7375, 13, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151657, 27, 1688, 92784, 1784, 16181, 29368, 29, 16, 13, 15, 522, 16181, 1784, 16181, 25387, 29, 17, 522, 16181, 1784, 16181, 28, 3615, 29, 1183, 2359, 497, 330, 1291, 1341, 522, 16181, 1472, 1688, 29, 151658, 151645], - "all_stop_token_ids": [151643, 151645, 151672] -} diff --git a/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json b/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json new file mode 100644 index 000000000000..113c8639fba2 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json @@ -0,0 +1,229 @@ +{ + "model_id": "MiniMaxAI/MiniMax-M2.5", + "vocab_size": 200064, + "all_stop_token_ids": [200020], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "\n", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "minimax_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "minimax_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n\nI will call the tools.\n\n\nShanghai\n\n\n1.0\n2\n[\"left\", \"right\"]\n\n", + "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 367, 73, 715, 1728, 275, 6629, 320, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 60, 88638, 1925, 1139, 2132, 6143, 60, 24662, 1925, 1139, 121, 3361, 49, 46, 48, 1579, 24662, 1100, 60, 24662, 1925, 1139, 120, 3361, 50, 1579, 24662, 1100, 60, 24662, 1925, 1139, 12405, 3361, 7795, 2038, 1164, 494, 1581, 8760, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n" + }, + { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "minimax_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + }, + { + "type": "tag", + "begin": "\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "minimax_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n\n1.0\n2\n[\"left\", \"right\"]\n\n", + "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 4368, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 60, 88638, 1925, 1139, 2132, 6143, 60, 24662, 1925, 1139, 121, 3361, 49, 46, 48, 1579, 24662, 1100, 60, 24662, 1925, 1139, 120, 3361, 50, 1579, 24662, 1100, 60, 24662, 1925, 1139, 12405, 3361, 7795, 2038, 1164, 494, 1581, 8760, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.tool_call_mix.json deleted file mode 100644 index 2dfaabd0e505..000000000000 --- a/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.tool_call_mix.json +++ /dev/null @@ -1,117 +0,0 @@ -{ - "model_id": "MiniMaxAI/MiniMax-M2.5", - "vocab_size": 200064, - "grammar": { - "format": { - "type": "sequence", - "elements": [ - { - "type": "sequence", - "elements": [ - { - "type": "tag", - "begin": "", - "content": { - "type": "any_text", - "excludes": [], - "max_tokens": null, - "max_chars": null - }, - "end": "" - }, - { - "type": "const_string", - "value": "\n\n" - } - ] - }, - { - "type": "triggered_tags", - "triggers": [ - "" - ], - "tags": [ - { - "type": "tag", - "begin": "\n", - "content": { - "type": "tags_with_separator", - "tags": [ - { - "type": "tag", - "begin": "\n", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "location": { - "type": "string" - } - }, - "required": [ - "location" - ] - }, - "style": "minimax_xml", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "\n" - }, - { - "type": "tag", - "begin": "\n", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "y": { - "type": "number" - }, - "x": { - "type": "number" - }, - "items": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "y", - "x", - "items" - ] - }, - "style": "minimax_xml", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "\n" - } - ], - "separator": "", - "at_least_one": true, - "stop_after_first": false - }, - "end": "" - } - ], - "at_least_one": false, - "stop_after_first": false, - "excludes": [ - "", - "" - ] - } - ] - }, - "coverage": "from_token_zero" - }, - "completion": "Need call the weather and add tools.\n\n\nI will call the tools.\n\n\nShanghai\n\n\n1.0\n2\n[\"left\", \"right\"]\n\n", - "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 367, 73, 715, 1728, 275, 6629, 320, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 60, 88638, 1925, 1139, 2132, 6143, 60, 24662, 1925, 1139, 121, 3361, 49, 46, 48, 1579, 24662, 1100, 60, 24662, 1925, 1139, 120, 3361, 50, 1579, 24662, 1100, 60, 24662, 1925, 1139, 12405, 3361, 7795, 2038, 1164, 494, 1581, 8760, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020], - "all_stop_token_ids": [200020] -} diff --git a/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json b/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json new file mode 100644 index 000000000000..efd2fb09f9af --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json @@ -0,0 +1,215 @@ +{ + "model_id": "Qwen/Qwen3-0.6B", + "vocab_size": 151936, + "all_stop_token_ids": [151643, 151645], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + } + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "\n{\"name\": \"get_weather\", \"arguments\": ", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "}\n" + }, + { + "type": "tag", + "begin": "\n{\"name\": \"add\", \"arguments\": ", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "}\n" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "\nNeed call the weather and add tools.\n\n\nI will call the tools.\n\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n\n\n{\"name\": \"add\", \"arguments\": {\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}}\n", + "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 40, 686, 1618, 279, 7375, 624, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 198, 151657, 198, 4913, 606, 788, 330, 718, 497, 330, 16370, 788, 5212, 88, 788, 220, 16, 13, 15, 11, 330, 87, 788, 220, 17, 11, 330, 3615, 788, 4383, 2359, 497, 330, 1291, 1341, 11248, 151658, 151645] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + } + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "\n{\"name\": \"get_weather\", \"arguments\": ", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "}\n" + }, + { + "type": "tag", + "begin": "\n{\"name\": \"add\", \"arguments\": ", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "}\n" + } + ], + "at_least_one": true, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "\nNeed call the weather and add tools.\n\n\n\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n\n\n{\"name\": \"add\", \"arguments\": {\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}}\n", + "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 198, 151657, 198, 4913, 606, 788, 330, 718, 497, 330, 16370, 788, 5212, 88, 788, 220, 16, 13, 15, 11, 330, 87, 788, 220, 17, 11, 330, 3615, 788, 4383, 2359, 497, 330, 1291, 1341, 11248, 151658, 151645] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.tool_call_mix.json deleted file mode 100644 index 7a1d3a785114..000000000000 --- a/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.tool_call_mix.json +++ /dev/null @@ -1,107 +0,0 @@ -{ - "model_id": "Qwen/Qwen3-0.6B", - "vocab_size": 151936, - "grammar": { - "format": { - "type": "sequence", - "elements": [ - { - "type": "optional", - "content": { - "type": "sequence", - "elements": [ - { - "type": "tag", - "begin": "", - "content": { - "type": "any_text", - "excludes": [], - "max_tokens": null, - "max_chars": null - }, - "end": "" - }, - { - "type": "const_string", - "value": "\n\n" - } - ] - } - }, - { - "type": "triggered_tags", - "triggers": [ - "" - ], - "tags": [ - { - "type": "tag", - "begin": "\n{\"name\": \"get_weather\", \"arguments\": ", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "location": { - "type": "string" - } - }, - "required": [ - "location" - ] - }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "}\n" - }, - { - "type": "tag", - "begin": "\n{\"name\": \"add\", \"arguments\": ", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "y": { - "type": "number" - }, - "x": { - "type": "number" - }, - "items": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "y", - "x", - "items" - ] - }, - "style": "json", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "}\n" - } - ], - "at_least_one": false, - "stop_after_first": false, - "excludes": [ - "", - "" - ] - } - ] - }, - "coverage": "from_token_zero" - }, - "completion": "\nNeed call the weather and add tools.\n\n\nI will call the tools.\n\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n\n\n{\"name\": \"add\", \"arguments\": {\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}}\n", - "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 40, 686, 1618, 279, 7375, 624, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 198, 151657, 198, 4913, 606, 788, 330, 718, 497, 330, 16370, 788, 5212, 88, 788, 220, 16, 13, 15, 11, 330, 87, 788, 220, 17, 11, 330, 3615, 788, 4383, 2359, 497, 330, 1291, 1341, 11248, 151658, 151645], - "all_stop_token_ids": [151643, 151645] -} diff --git a/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json b/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json new file mode 100644 index 000000000000..d90efdcc57e0 --- /dev/null +++ b/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json @@ -0,0 +1,209 @@ +{ + "model_id": "Qwen/Qwen3.5-4B", + "vocab_size": 248320, + "all_stop_token_ids": [248044, 248046], + "cases": { + "auto": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "triggered_tags", + "triggers": [ + "\n\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n\n" + }, + { + "type": "tag", + "begin": "\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n\n" + } + ], + "at_least_one": false, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n\nI will call the tools.\n\n\n\n\nShanghai\n\n\n\n\n\n\n1.0\n\n\n2\n\n\n[\"left\", \"right\"]\n\n\n", + "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 40, 668, 1562, 279, 7141, 13, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 198, 248058, 198, 27, 1628, 89587, 29, 198, 27, 15704, 28408, 29, 198, 16, 13, 15, 198, 510, 15704, 29, 198, 27, 15704, 24582, 29, 198, 17, 198, 510, 15704, 29, 198, 27, 15704, 28, 3494, 29, 198, 1143, 2282, 487, 328, 1246, 1293, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "triggered_tags", + "triggers": [ + "\n\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n\n" + }, + { + "type": "tag", + "begin": "\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n\n" + } + ], + "at_least_one": true, + "stop_after_first": false, + "excludes": [ + "", + "" + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n\n\n\n\n\n1.0\n\n\n2\n\n\n[\"left\", \"right\"]\n\n\n", + "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 198, 248058, 198, 27, 1628, 89587, 29, 198, 27, 15704, 28408, 29, 198, 16, 13, 15, 198, 510, 15704, 29, 198, 27, 15704, 24582, 29, 198, 17, 198, 510, 15704, 29, 198, 27, 15704, 28, 3494, 29, 198, 1143, 2282, 487, 328, 1246, 1293, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046] + } + } +} diff --git a/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.tool_call_mix.json b/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.tool_call_mix.json deleted file mode 100644 index ad62176e4e40..000000000000 --- a/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.tool_call_mix.json +++ /dev/null @@ -1,104 +0,0 @@ -{ - "model_id": "Qwen/Qwen3.5-4B", - "vocab_size": 248320, - "grammar": { - "format": { - "type": "sequence", - "elements": [ - { - "type": "sequence", - "elements": [ - { - "type": "tag", - "begin": "", - "content": { - "type": "any_text", - "excludes": [], - "max_tokens": null, - "max_chars": null - }, - "end": "" - }, - { - "type": "const_string", - "value": "\n\n" - } - ] - }, - { - "type": "triggered_tags", - "triggers": [ - "\n\n\n", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "location": { - "type": "string" - } - }, - "required": [ - "location" - ] - }, - "style": "qwen_xml", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "\n\n" - }, - { - "type": "tag", - "begin": "\n\n", - "content": { - "type": "json_schema", - "json_schema": { - "type": "object", - "properties": { - "y": { - "type": "number" - }, - "x": { - "type": "number" - }, - "items": { - "type": "array", - "items": { - "type": "string" - } - } - }, - "required": [ - "y", - "x", - "items" - ] - }, - "style": "qwen_xml", - "any_order": false, - "max_whitespace_cnt": null - }, - "end": "\n\n" - } - ], - "at_least_one": false, - "stop_after_first": false, - "excludes": [ - "", - "" - ] - } - ] - }, - "coverage": "from_token_zero" - }, - "completion": "Need call the weather and add tools.\n\n\nI will call the tools.\n\n\n\n\nShanghai\n\n\n\n\n\n\n1.0\n\n\n2\n\n\n[\"left\", \"right\"]\n\n\n", - "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 40, 668, 1562, 279, 7141, 13, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 198, 248058, 198, 27, 1628, 89587, 29, 198, 27, 15704, 28408, 29, 198, 16, 13, 15, 198, 510, 15704, 29, 198, 27, 15704, 24582, 29, 198, 17, 198, 510, 15704, 29, 198, 27, 15704, 28, 3494, 29, 198, 1143, 2282, 487, 328, 1246, 1293, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046], - "all_stop_token_ids": [248044, 248046] -} diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index 59f96a430724..b7f0bac55f60 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -7,11 +7,11 @@ //! parsed from the generated assistant completion and then rendered back to the exact same //! assistant completion. //! -//! The tool-call fixture also snapshots the output grammar built by the initialized parser, with -//! the rendered completion as its generation, under `tests/grammar_replay/`. `grammar_replay.rs` -//! replays these cases through the real XGrammar. +//! The tool-call fixture also snapshots the output grammar built by the initialized parser for each +//! tool-choice variant, with the rendered completion as its generation, as one file per model under +//! `tests/grammar_replay/`. `grammar_replay.rs` replays these cases through the real XGrammar. -use std::collections::BTreeSet; +use std::collections::{BTreeMap, BTreeSet}; use std::path::PathBuf; use std::pin::Pin; use std::sync::Arc; @@ -24,9 +24,9 @@ use serde_json_fmt::JsonFormat as JsonFmt; use serial_test::file_serial; use vllm_chat::{ AssistantContentBlock, AssistantMessage, AssistantMessageExt as _, AssistantToolCall, - ChatEvent, ChatMessage, ChatRequest, ChatRole, ChatTool, FinishReason, GenerationPromptMode, - LoadModelBackendsOptions, NewChatOutputProcessorOptions, ParserSelection, RendererSelection, - ToolStrictLevel, load_model_backends, + ChatEvent, ChatMessage, ChatRequest, ChatRole, ChatTool, ChatToolChoice, FinishReason, + GenerationPromptMode, LoadModelBackendsOptions, NewChatOutputProcessorOptions, ParserSelection, + RendererSelection, ToolStrictLevel, load_model_backends, }; use vllm_parser::output_grammar::BuiltOutputGrammar; use vllm_text::{DecodedTextEvent, Finished, Prompt}; @@ -55,6 +55,8 @@ struct RoundtripCase { json_fmt: JsonFmt, /// Whether the template renders tool-call argument object keys in sorted order. sort_json_keys: bool, + /// Tool-choice variants run by the tool-call fixture. + tool_choice_variants: &'static [ToolChoiceVariant], } #[derive(Clone, Copy)] @@ -123,6 +125,29 @@ impl ThinkingBehavior { } } +/// Tool-choice variant of the tool-call fixture. +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Serialize)] +#[serde(rename_all = "snake_case")] +enum ToolChoiceVariant { + /// `auto`, with content before the tool calls. + Auto, + /// `required`, with tool calls only: builder grammars admit no content + /// before the calls. + Required, +} + +impl ToolChoiceVariant { + /// Every variant, for templates and parsers that support all of them. + const ALL: &[Self] = &[Self::Auto, Self::Required]; + + fn tool_choice(self) -> ChatToolChoice { + match self { + Self::Auto => ChatToolChoice::Auto, + Self::Required => ChatToolChoice::Required, + } + } +} + impl RoundtripCase { /// Qwen3 XML tool-call format with `qwen3` reasoning tags. fn qwen3() -> Self { @@ -134,6 +159,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: spaced_json_fmt(), sort_json_keys: false, + tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -147,6 +173,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -160,6 +187,10 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + // TODO: MiMo renders compact `` calls, but its + // tool parser reuses the Qwen3-Coder builder with newline-delimited tags: + // `required` rejects the calls, and the `auto` trigger never fires. + tool_choice_variants: &[ToolChoiceVariant::Auto], } } @@ -173,6 +204,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -186,6 +218,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -199,6 +232,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: false }, json_fmt: compact_json_fmt(), sort_json_keys: false, + tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -212,6 +246,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -225,6 +260,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: false }, json_fmt: compact_json_fmt(), sort_json_keys: false, + tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -238,6 +274,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -251,6 +288,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -264,6 +302,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -277,6 +316,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: true, + tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -291,6 +331,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: spaced_json_fmt(), sort_json_keys: false, + tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -308,6 +349,9 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + // The renderer appends a `tool_choice=required` system message after the + // history, so the completion is not a suffix of the full render. + tool_choice_variants: &[ToolChoiceVariant::Auto], } } @@ -325,6 +369,7 @@ impl RoundtripCase { }, json_fmt: compact_json_fmt(), sort_json_keys: false, + tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -338,6 +383,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -351,6 +397,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -364,6 +411,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -377,6 +425,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, + tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -390,6 +439,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: true, + tool_choice_variants: ToolChoiceVariant::ALL, } } } @@ -460,6 +510,7 @@ async fn run_roundtrip_reasoning_and_content_inner( "roundtrip-reasoning-content", vec![ChatMessage::text(ChatRole::User, "What is 2 + 2?")], Vec::new(), + None, thinking, case.thinking_behavior, ); @@ -498,11 +549,25 @@ async fn run_roundtrip_reasoning_and_content_inner( Ok(()) } -/// Run the fixed reasoning+multiple-tools fixture for one model/parser case. +/// Run the fixed reasoning+multiple-tools fixture for one model/parser case, +/// once per tool-choice variant. async fn run_roundtrip_tool_call_mix( case: &RoundtripCase, backends: &vllm_chat::LoadedModelBackends, ) -> Result<()> { + let mut results = Vec::new(); + for &variant in case.tool_choice_variants { + let result = run_roundtrip_tool_call_mix_inner(case, backends, variant).await?; + results.push((variant, result)); + } + check_grammar_replay_file(case, backends, results) +} + +async fn run_roundtrip_tool_call_mix_inner( + case: &RoundtripCase, + backends: &vllm_chat::LoadedModelBackends, + variant: ToolChoiceVariant, +) -> Result { let request = roundtrip_request( "roundtrip-reasoning-tools", vec![ChatMessage::text( @@ -510,41 +575,38 @@ async fn run_roundtrip_tool_call_mix( "Check Shanghai weather and add 1.0 plus 2.", )], test_tools(), + Some(variant.tool_choice()), Some(true), // always enable thinking in this fixture case.thinking_behavior, ); let expected_reasoning = "Need call the weather and add tools."; - let expected_text = "I will call the tools."; - - let result = run_roundtrip( - case, - backends, - &request, - AssistantMessage { - content: vec![ - AssistantContentBlock::Reasoning { - text: expected_reasoning.to_string(), - }, - AssistantContentBlock::Text { - text: expected_text.to_string(), - }, - AssistantContentBlock::ToolCall(AssistantToolCall { - id: "functions.get_weather:0".to_string(), - name: "get_weather".to_string(), - arguments: r#"{"location":"Shanghai"}"#.to_string(), - }), - AssistantContentBlock::ToolCall(AssistantToolCall { - id: "functions.add:1".to_string(), - name: "add".to_string(), - // Intentionally use a non-lexical order of keys to verify text-level - // fidelity of the roundtrip where JSON formatting remains stable. The - // `items` key also exercises templates that call `arguments.items()`. - arguments: r#"{"y":1.0,"x":2,"items":["left","right"]}"#.to_string(), - }), - ], - }, - ) - .await?; + let expected_text = match variant { + ToolChoiceVariant::Auto => Some("I will call the tools."), + ToolChoiceVariant::Required => None, + }; + + let mut content = vec![AssistantContentBlock::Reasoning { + text: expected_reasoning.to_string(), + }]; + content.extend(expected_text.map(|text| AssistantContentBlock::Text { + text: text.to_string(), + })); + content.extend([ + AssistantContentBlock::ToolCall(AssistantToolCall { + id: "functions.get_weather:0".to_string(), + name: "get_weather".to_string(), + arguments: r#"{"location":"Shanghai"}"#.to_string(), + }), + AssistantContentBlock::ToolCall(AssistantToolCall { + id: "functions.add:1".to_string(), + name: "add".to_string(), + // Intentionally use a non-lexical order of keys to verify text-level + // fidelity of the roundtrip where JSON formatting remains stable. The + // `items` key also exercises templates that call `arguments.items()`. + arguments: r#"{"y":1.0,"x":2,"items":["left","right"]}"#.to_string(), + }), + ]); + let result = run_roundtrip(case, backends, &request, AssistantMessage { content }).await?; assert_eq!( result.parsed_message.reasoning().as_deref(), @@ -552,7 +614,10 @@ async fn run_roundtrip_tool_call_mix( "parsed message: {:#?}", result.parsed_message ); - assert_eq!(result.parsed_message.text(), expected_text); + assert_eq!( + result.parsed_message.text(), + expected_text.unwrap_or_default() + ); let tool_calls = result.parsed_message.tool_calls().collect::>(); assert_eq!( @@ -577,9 +642,7 @@ async fn run_roundtrip_tool_call_mix( result.closed_completion ); - check_grammar_replay_case(case, backends, "tool_call_mix", &result)?; - - Ok(()) + Ok(result) } /// Compact JSON argument formatting used by JSON-native parsers/renderers. @@ -995,28 +1058,35 @@ fn incremental_decode_chunks( Ok(chunks) } -/// One parser-built output grammar with the generation it must accept, replayed -/// through the real XGrammar by `grammar_replay.py`. +/// Grammar replay cases of one model, replayed through the real XGrammar by +/// `grammar_replay.py`. #[derive(Serialize)] -struct GrammarReplayCase<'a> { +struct GrammarReplayFile { /// Hugging Face model id whose tokenizer the replay loads. model_id: &'static str, /// Model vocabulary size, which sizes the XGrammar token mask. vocab_size: usize, + /// Request stop set, passed to the matcher as `override_stop_tokens`. + #[serde(serialize_with = "serialize_one_line")] + all_stop_token_ids: BTreeSet, + /// Cases keyed by tool-choice variant. + cases: BTreeMap, +} + +/// One parser-built output grammar with the generation it must accept. +#[derive(Serialize)] +struct GrammarReplayCase { /// Parser-built grammar; only token-zero grammars are replayed from the /// first generated token. - grammar: &'a BuiltOutputGrammar, + grammar: BuiltOutputGrammar, /// Generated completion text, without the stop token. completion: String, /// Generated token IDs presented to the matcher, ending with a stop token. #[serde(serialize_with = "serialize_one_line")] generation_token_ids: Vec, - /// Request stop set, passed to the matcher as `override_stop_tokens`. - #[serde(serialize_with = "serialize_one_line")] - all_stop_token_ids: BTreeSet, } -/// Serialize a token ID list on one line inside the pretty-printed case. +/// Serialize a token ID list on one line inside the pretty-printed file. fn serialize_one_line( token_ids: &impl Serialize, serializer: S, @@ -1027,29 +1097,35 @@ fn serialize_one_line( .serialize(serializer) } -/// Directory of checked-in grammar replay cases. +/// Directory of checked-in grammar replay files. fn grammar_replay_dir() -> PathBuf { PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/grammar_replay") } -/// Snapshot the output grammar of one roundtrip as a grammar replay case, or -/// check that no stale case remains when the parser builds none. -fn check_grammar_replay_case( +/// Snapshot the output grammars of one model's roundtrips, keyed by variant, +/// as its grammar replay file, or check that no stale file remains when the +/// parser builds none. +fn check_grammar_replay_file( case: &RoundtripCase, backends: &vllm_chat::LoadedModelBackends, - fixture: &str, - result: &RoundtripResult, + results: Vec<(ToolChoiceVariant, RoundtripResult)>, ) -> Result<()> { let model_name = case.model_id.rsplit_once('/').map_or(case.model_id, |(_, name)| name); - let path = grammar_replay_dir().join(format!("{model_name}.{fixture}.json")); - let Some(grammar) = &result.output_grammar else { + let path = grammar_replay_dir().join(format!("{model_name}.json")); + let results = results + .into_iter() + .filter_map(|(variant, result)| { + Some((variant, result.output_grammar?, result.closed_completion)) + }) + .collect::>(); + if results.is_empty() { ensure!( !path.exists(), - "{} builds no output grammar, but grammar replay case {path:?} exists", + "{} builds no output grammar, but grammar replay file {path:?} exists", case.model_id ); return Ok(()); - }; + } let tokenizer = backends.text_backend.tokenizer(); let hints = backends.text_backend.sampling_hints()?; @@ -1070,25 +1146,34 @@ fn check_grammar_replay_case( case.assistant_stop_suffix ); - let (completion, mut generation_token_ids) = match completion_body( - tokenizer.as_ref(), - &result.closed_completion, - case.assistant_stop_suffix, - )? { - CompletionBody::Text(body) => (body.to_string(), tokenizer.encode(body, false)?), - CompletionBody::TokenIds(body) => (tokenizer.decode(body, false)?, body.to_vec()), - }; - generation_token_ids.push(stop_token_id); + let mut cases = BTreeMap::new(); + for (variant, grammar, closed_completion) in results { + let (completion, mut generation_token_ids) = match completion_body( + tokenizer.as_ref(), + &closed_completion, + case.assistant_stop_suffix, + )? { + CompletionBody::Text(body) => (body.to_string(), tokenizer.encode(body, false)?), + CompletionBody::TokenIds(body) => (tokenizer.decode(body, false)?, body.to_vec()), + }; + generation_token_ids.push(stop_token_id); + cases.insert( + variant, + GrammarReplayCase { + grammar, + completion, + generation_token_ids, + }, + ); + } - let replay_case = GrammarReplayCase { + let file = GrammarReplayFile { model_id: case.model_id, vocab_size: backends.text_backend.model_vocab_size(), - grammar, - completion, - generation_token_ids, all_stop_token_ids, + cases, }; - let mut json = serde_json::to_string_pretty(&replay_case)?; + let mut json = serde_json::to_string_pretty(&file)?; json.push('\n'); expect_test::expect_file![path].assert_eq(&json); @@ -1100,10 +1185,11 @@ fn roundtrip_request( request_id: impl Into, messages: Vec, tools: Vec, + tool_choice: Option, thinking: Option, thinking_behavior: ThinkingBehavior, ) -> ChatRequest { - let tool_context = vllm_chat::ResolvedToolContext::new(&messages, tools, None, true) + let tool_context = vllm_chat::ResolvedToolContext::new(&messages, tools, tool_choice, true) .expect("tool context should resolve"); let mut request = ChatRequest { request_id: request_id.into(), From 3635b5a848b0df91a45d8d39ce7c941a04c9dc2e Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Tue, 29 Sep 2026 06:12:21 +0000 Subject: [PATCH 3/5] [Rust Frontend] Replay named tool choice in grammar replay Add a named `get_weather` variant to the roundtrip tool-call fixture, with a single call and no content, and replay its grammar alongside the `auto` and `required` variants. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Bugen Zhao --- .../grammar_replay/DeepSeek-V3.2-Exp.json | 58 +++++++++++ .../grammar_replay/DeepSeek-V4-Flash.json | 58 +++++++++++ .../grammar_replay/DeepSeek-V4.1-Flash.json | 58 +++++++++++ .../tests/grammar_replay/GLM-4.7-Flash.json | 45 +++++++++ .../tests/grammar_replay/GLM-5.2-FP8.json | 45 +++++++++ rust/src/chat/tests/grammar_replay/Hy3.json | 95 +++++++++++++++++++ .../chat/tests/grammar_replay/Kimi-K2.5.json | 71 ++++++++++++++ .../tests/grammar_replay/MiniMax-M2.5.json | 67 +++++++++++++ .../chat/tests/grammar_replay/Qwen3-0.6B.json | 57 +++++++++++ .../chat/tests/grammar_replay/Qwen3.5-4B.json | 54 +++++++++++ rust/src/chat/tests/roundtrip.rs | 33 ++++--- 11 files changed, 629 insertions(+), 12 deletions(-) diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json index 462fad9d0ecb..0a8181857133 100644 --- a/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json @@ -206,6 +206,64 @@ }, "completion": "Need call the weather and add tools.\n\n<｜DSML｜function_calls>\n<｜DSML｜invoke name=\"get_weather\">\n<｜DSML｜parameter name=\"location\" string=\"true\">Shanghai\n\n<｜DSML｜invoke name=\"add\">\n<｜DSML｜parameter name=\"y\" string=\"false\">1.0\n<｜DSML｜parameter name=\"x\" string=\"false\">2\n<｜DSML｜parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 271, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 3327, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n<｜DSML｜function_calls>\n" + }, + { + "type": "tag", + "begin": "<｜DSML｜invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n<｜DSML｜function_calls>\n<｜DSML｜invoke name=\"get_weather\">\n<｜DSML｜parameter name=\"location\" string=\"true\">Shanghai\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128799, 271, 30, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 1018, 30, 28217, 10525, 7398, 28217, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 28217, 10525, 7398, 28217, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 28217, 10525, 7398, 28217, 41523, 1018, 1718, 28217, 10525, 7398, 28217, 40148, 5406, 1018, 1718, 28217, 10525, 7398, 28217, 8701, 4941, 12548, 32, 1] } } } diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json index 1c3d9e6640ae..afece45d1b19 100644 --- a/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json @@ -206,6 +206,64 @@ }, "completion": "Need call the weather and add tools.\n\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"get_weather\">\n<｜DSML｜parameter name=\"location\" string=\"true\">Shanghai\n\n<｜DSML｜invoke name=\"add\">\n<｜DSML｜parameter name=\"y\" string=\"false\">1.0\n<｜DSML｜parameter name=\"x\" string=\"false\">2\n<｜DSML｜parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 30, 128825, 40148, 5406, 2329, 1281, 3327, 3816, 30, 128825, 41523, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 41523, 1018, 30, 128825, 41523, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n<｜DSML｜tool_calls>\n" + }, + { + "type": "tag", + "begin": "<｜DSML｜invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n<｜DSML｜tool_calls>\n<｜DSML｜invoke name=\"get_weather\">\n<｜DSML｜parameter name=\"location\" string=\"true\">Shanghai\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 72461, 4941, 12548, 1018, 30, 128825, 40148, 5406, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 41523, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 41523, 1018, 1718, 128825, 40148, 5406, 1018, 1718, 128825, 72461, 4941, 12548, 32, 1] } } } diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json index 0a5635abd411..bccbb0e3da5c 100644 --- a/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json @@ -206,6 +206,64 @@ }, "completion": "Need call the weather and add tools.\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name=\"get_weather\">\n<｜DSML｜ parameter name=\"location\" string=\"true\">Shanghai\n\n<｜DSML｜ invoke name=\"add\">\n<｜DSML｜ parameter name=\"y\" string=\"false\">1.0\n<｜DSML｜ parameter name=\"x\" string=\"false\">2\n<｜DSML｜ parameter name=\"items\" string=\"false\">[\"left\", \"right\"]\n\n", "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 30, 128825, 34756, 2329, 1281, 3327, 3816, 30, 128825, 10767, 2329, 1281, 91, 4, 3418, 1281, 19836, 3320, 19, 16, 18, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 90, 4, 3418, 1281, 19836, 3320, 20, 1718, 128825, 10767, 1018, 30, 128825, 10767, 2329, 1281, 17173, 4, 3418, 1281, 19836, 3320, 10931, 4589, 1760, 582, 2459, 13747, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n<｜DSML｜ calls>\n" + }, + { + "type": "tag", + "begin": "<｜DSML｜ invoke name=\"get_weather\">\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "deepseek_v4_1_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n<｜DSML｜ calls>\n<｜DSML｜ invoke name=\"get_weather\">\n<｜DSML｜ parameter name=\"location\" string=\"true\">Shanghai\n\n", + "generation_token_ids": [50249, 1926, 270, 9670, 305, 1258, 6704, 16, 128822, 271, 30, 128825, 10699, 1018, 30, 128825, 34756, 2329, 1281, 1133, 65, 50219, 3816, 30, 128825, 10767, 2329, 1281, 33182, 4, 3418, 1281, 11476, 3320, 119606, 28011, 1718, 128825, 10767, 1018, 1718, 128825, 34756, 1018, 1718, 128825, 10699, 32, 1] } } } diff --git a/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json b/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json index ac3b2fdacef9..4cede08af96e 100644 --- a/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json +++ b/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json @@ -196,6 +196,51 @@ }, "completion": "Need call the weather and add tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "tag", + "begin": "get_weather", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "" + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.get_weatherlocationShanghai", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154820] } } } diff --git a/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json b/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json index eb8355b5f4f7..4beb103b355c 100644 --- a/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json +++ b/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json @@ -196,6 +196,51 @@ }, "completion": "Need call the weather and add tools.get_weatherlocationShanghaiaddy1.0x2items[\"left\", \"right\"]", "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154843, 718, 154847, 88, 154848, 154849, 16, 13, 15, 154850, 154847, 87, 154848, 154849, 17, 154850, 154847, 3615, 154848, 154849, 1183, 2359, 497, 330, 1291, 1341, 154850, 154844, 154820] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "tag", + "begin": "get_weather", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "glm_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "" + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.get_weatherlocationShanghai", + "generation_token_ids": [23591, 1618, 279, 9101, 323, 912, 7375, 13, 154842, 154843, 455, 68852, 154847, 2527, 154848, 154849, 2016, 29953, 154850, 154844, 154820] } } } diff --git a/rust/src/chat/tests/grammar_replay/Hy3.json b/rust/src/chat/tests/grammar_replay/Hy3.json index 2b59aef801f7..f6ae7b0969fd 100644 --- a/rust/src/chat/tests/grammar_replay/Hy3.json +++ b/rust/src/chat/tests/grammar_replay/Hy3.json @@ -525,6 +525,101 @@ }, "completion": "Need call the weather and add tools.\nget_weather\nlocation\nShanghai\n\nadd\ny\n1.0\nx\n2\nitems\n[\"left\", \"right\"]\n\n", "generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120033, 2176, 120043, 185, 120066, 88, 120067, 185, 120068, 16, 13, 15, 120069, 185, 120066, 87, 120067, 185, 120068, 17, 120069, 185, 120066, 12792, 120067, 185, 120068, 5831, 1639, 1221, 478, 1331, 7029, 120069, 185, 120034, 185, 120036, 120025] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n" + }, + { + "type": "tag", + "begin": "get_weather\n", + "content": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "location" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "any_text", + "excludes": [ + "", + "", + "", + "", + "", + "", + "", + "", + "" + ], + "max_tokens": null, + "max_chars": null + }, + { + "type": "const_string", + "value": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\nget_weather\nlocation\nShanghai\n\n", + "generation_token_ids": [47893, 1609, 252, 9969, 289, 1189, 7106, 13, 120030, 120035, 185, 120033, 874, 62, 39899, 120043, 185, 120066, 24806, 120067, 185, 120068, 4169, 768, 32381, 120069, 185, 120034, 185, 120036, 120025] } } } diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K2.5.json b/rust/src/chat/tests/grammar_replay/Kimi-K2.5.json index f86299048685..466e8ae899fd 100644 --- a/rust/src/chat/tests/grammar_replay/Kimi-K2.5.json +++ b/rust/src/chat/tests/grammar_replay/Kimi-K2.5.json @@ -259,6 +259,77 @@ }, "completion": "Need call the weather and add tools.<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_call_begin|>functions.add:1<|tool_call_argument_begin|>{\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}<|tool_call_end|><|tool_calls_section_end|>", "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163597, 41937, 3149, 25, 16, 163598, 8264, 88, 1289, 220, 16, 13, 15, 11, 414, 87, 1289, 220, 17, 11, 414, 7989, 1289, 8681, 3794, 665, 414, 2277, 3956, 92, 163599, 163596, 163586] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "<|tool_calls_section_begin|>" + }, + { + "type": "tag", + "begin": "<|tool_call_begin|>functions.get_weather:", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "\\d+" + }, + { + "type": "const_string", + "value": "<|tool_call_argument_begin|>" + }, + { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + } + ] + }, + "end": "<|tool_call_end|>" + }, + { + "type": "const_string", + "value": "<|tool_calls_section_end|>" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.<|tool_calls_section_begin|><|tool_call_begin|>functions.get_weather:0<|tool_call_argument_begin|>{\"location\": \"Shanghai\"}<|tool_call_end|><|tool_calls_section_end|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163607, 163595, 163597, 41937, 1150, 21055, 2800, 25, 15, 163598, 8264, 5791, 1289, 414, 2797, 31231, 16934, 163599, 163596, 163586] } } } diff --git a/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json b/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json index 113c8639fba2..7fb4be4a7561 100644 --- a/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json +++ b/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json @@ -224,6 +224,73 @@ }, "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n\n1.0\n2\n[\"left\", \"right\"]\n\n", "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 4368, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 60, 88638, 1925, 1139, 2132, 6143, 60, 24662, 1925, 1139, 121, 3361, 49, 46, 48, 1579, 24662, 1100, 60, 24662, 1925, 1139, 120, 3361, 50, 1579, 24662, 1100, 60, 24662, 1925, 1139, 12405, 3361, 7795, 2038, 1164, 494, 1581, 8760, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "sequence", + "elements": [ + { + "type": "const_string", + "value": "\n\n" + }, + { + "type": "tag", + "begin": "\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "minimax_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n" + }, + { + "type": "const_string", + "value": "" + } + ] + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n", + "generation_token_ids": [42478, 1728, 275, 12562, 306, 1187, 6629, 320, 200051, 4368, 200052, 10, 60, 88638, 1925, 1139, 848, 95, 51872, 6143, 60, 24662, 1925, 1139, 28415, 3361, 165806, 1579, 24662, 1100, 1579, 88638, 1100, 200053, 200020] } } } diff --git a/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json b/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json index efd2fb09f9af..579afdbd7bf7 100644 --- a/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json +++ b/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json @@ -210,6 +210,63 @@ }, "completion": "\nNeed call the weather and add tools.\n\n\n\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n\n\n{\"name\": \"add\", \"arguments\": {\"y\": 1.0, \"x\": 2, \"items\": [\"left\", \"right\"]}}\n", "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 198, 151657, 198, 4913, 606, 788, 330, 718, 497, 330, 16370, 788, 5212, 88, 788, 220, 16, 13, 15, 11, 330, 87, 788, 220, 17, 11, 330, 3615, 788, 4383, 2359, 497, 330, 1291, 1341, 11248, 151658, 151645] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + } + }, + { + "type": "tag", + "begin": "\n{\"name\": \"get_weather\", \"arguments\": ", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "}\n" + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "\nNeed call the weather and add tools.\n\n\n\n{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Shanghai\"}}\n", + "generation_token_ids": [151667, 198, 23657, 1618, 279, 9104, 323, 912, 7375, 624, 151668, 271, 151657, 198, 4913, 606, 788, 330, 455, 69364, 497, 330, 16370, 788, 5212, 2527, 788, 330, 2016, 30070, 95642, 151658, 151645] } } } diff --git a/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json b/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json index d90efdcc57e0..d28c6a98f3e5 100644 --- a/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json +++ b/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json @@ -204,6 +204,60 @@ }, "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n\n\n\n\n\n1.0\n\n\n2\n\n\n[\"left\", \"right\"]\n\n\n", "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 198, 248058, 198, 27, 1628, 89587, 29, 198, 27, 15704, 28408, 29, 198, 16, 13, 15, 198, 510, 15704, 29, 198, 27, 15704, 24582, 29, 198, 17, 198, 510, 15704, 29, 198, 27, 15704, 28, 3494, 29, 198, 1143, 2282, 487, 328, 1246, 1293, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "sequence", + "elements": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + }, + { + "type": "const_string", + "value": "\n\n" + } + ] + }, + { + "type": "tag", + "begin": "\n\n", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null + }, + "end": "\n\n" + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.\n\n\n\n\n\nShanghai\n\n\n", + "generation_token_ids": [22916, 1562, 279, 8831, 321, 884, 7141, 13, 198, 248069, 271, 248058, 198, 27, 1628, 27362, 67017, 29, 198, 27, 15704, 28, 2447, 29, 198, 1948, 29086, 198, 510, 15704, 29, 198, 510, 1628, 29, 198, 248059, 248046] } } } diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index b7f0bac55f60..8889025cf5f9 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -134,16 +134,21 @@ enum ToolChoiceVariant { /// `required`, with tool calls only: builder grammars admit no content /// before the calls. Required, + /// Named `get_weather`, with a single call to it and no content. + Named, } impl ToolChoiceVariant { /// Every variant, for templates and parsers that support all of them. - const ALL: &[Self] = &[Self::Auto, Self::Required]; + const ALL: &[Self] = &[Self::Auto, Self::Required, Self::Named]; fn tool_choice(self) -> ChatToolChoice { match self { Self::Auto => ChatToolChoice::Auto, Self::Required => ChatToolChoice::Required, + Self::Named => ChatToolChoice::Function { + name: "get_weather".to_string(), + }, } } } @@ -580,9 +585,10 @@ async fn run_roundtrip_tool_call_mix_inner( case.thinking_behavior, ); let expected_reasoning = "Need call the weather and add tools."; - let expected_text = match variant { - ToolChoiceVariant::Auto => Some("I will call the tools."), - ToolChoiceVariant::Required => None, + let (expected_text, tool_call_count) = match variant { + ToolChoiceVariant::Auto => (Some("I will call the tools."), 2), + ToolChoiceVariant::Required => (None, 2), + ToolChoiceVariant::Named => (None, 1), }; let mut content = vec![AssistantContentBlock::Reasoning { @@ -591,7 +597,7 @@ async fn run_roundtrip_tool_call_mix_inner( content.extend(expected_text.map(|text| AssistantContentBlock::Text { text: text.to_string(), })); - content.extend([ + let tool_calls = [ AssistantContentBlock::ToolCall(AssistantToolCall { id: "functions.get_weather:0".to_string(), name: "get_weather".to_string(), @@ -605,7 +611,8 @@ async fn run_roundtrip_tool_call_mix_inner( // `items` key also exercises templates that call `arguments.items()`. arguments: r#"{"y":1.0,"x":2,"items":["left","right"]}"#.to_string(), }), - ]); + ]; + content.extend(tool_calls.into_iter().take(tool_call_count)); let result = run_roundtrip(case, backends, &request, AssistantMessage { content }).await?; assert_eq!( @@ -622,7 +629,7 @@ async fn run_roundtrip_tool_call_mix_inner( let tool_calls = result.parsed_message.tool_calls().collect::>(); assert_eq!( tool_calls.len(), - 2, + tool_call_count, "parsed message: {:#?}", result.parsed_message ); @@ -631,11 +638,13 @@ async fn run_roundtrip_tool_call_mix_inner( tool_calls[0].arguments, expected_arguments(case, r#"{"location": "Shanghai"}"#)?, ); - assert_eq!(tool_calls[1].name, "add"); - assert_eq!( - tool_calls[1].arguments, - expected_arguments(case, r#"{"y": 1.0, "x": 2, "items": ["left", "right"]}"#)?, - ); + if let Some(add) = tool_calls.get(1) { + assert_eq!(add.name, "add"); + assert_eq!( + add.arguments, + expected_arguments(case, r#"{"y": 1.0, "x": 2, "items": ["left", "right"]}"#)?, + ); + } assert_eq!( result.rerendered_closed_completion, From 55e68907261c0f1a9d90a398c8e70842d1bec3b9 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Tue, 29 Sep 2026 07:25:58 +0000 Subject: [PATCH 4/5] [Rust Frontend] Replay MiMo required and named tool choice With the MiMo structural-tag builder, MiMo's grammar tags match the compact `` calls its template renders, so run every tool-choice variant for MiMo. Regenerate the replay files for the `excludes` field that `xgrammar-structural-tag` 0.4.0 serializes on JSON schema formats. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Bugen Zhao --- .../grammar_replay/DeepSeek-V3.2-Exp.json | 15 +- .../grammar_replay/DeepSeek-V4-Flash.json | 15 +- .../grammar_replay/DeepSeek-V4.1-Flash.json | 15 +- .../tests/grammar_replay/GLM-4.7-Flash.json | 15 +- .../tests/grammar_replay/GLM-5.2-FP8.json | 15 +- .../chat/tests/grammar_replay/Kimi-K2.5.json | 15 +- .../chat/tests/grammar_replay/Kimi-K3.json | 15 +- .../grammar_replay/MiMo-V2.6-Flash-RL.json | 168 +++++++++++++++++- .../tests/grammar_replay/MiniMax-M2.5.json | 15 +- .../chat/tests/grammar_replay/Qwen3-0.6B.json | 15 +- .../chat/tests/grammar_replay/Qwen3.5-4B.json | 15 +- rust/src/chat/tests/roundtrip.rs | 5 +- 12 files changed, 261 insertions(+), 62 deletions(-) diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json index 0a8181857133..4903b5900c74 100644 --- a/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V3.2-Exp.json @@ -49,7 +49,8 @@ }, "style": "deepseek_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n" }, @@ -82,7 +83,8 @@ }, "style": "deepseek_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n" } @@ -152,7 +154,8 @@ }, "style": "deepseek_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n" }, @@ -185,7 +188,8 @@ }, "style": "deepseek_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n" } @@ -248,7 +252,8 @@ }, "style": "deepseek_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n" }, diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json index afece45d1b19..1a286d8acda9 100644 --- a/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V4-Flash.json @@ -49,7 +49,8 @@ }, "style": "deepseek_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n" }, @@ -82,7 +83,8 @@ }, "style": "deepseek_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n" } @@ -152,7 +154,8 @@ }, "style": "deepseek_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n" }, @@ -185,7 +188,8 @@ }, "style": "deepseek_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n" } @@ -248,7 +252,8 @@ }, "style": "deepseek_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n" }, diff --git a/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json b/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json index bccbb0e3da5c..a21d8b90e5af 100644 --- a/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json +++ b/rust/src/chat/tests/grammar_replay/DeepSeek-V4.1-Flash.json @@ -49,7 +49,8 @@ }, "style": "deepseek_v4_1_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n" }, @@ -82,7 +83,8 @@ }, "style": "deepseek_v4_1_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n" } @@ -152,7 +154,8 @@ }, "style": "deepseek_v4_1_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n" }, @@ -185,7 +188,8 @@ }, "style": "deepseek_v4_1_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n" } @@ -248,7 +252,8 @@ }, "style": "deepseek_v4_1_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n" }, diff --git a/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json b/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json index 4cede08af96e..b289ec62e62b 100644 --- a/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json +++ b/rust/src/chat/tests/grammar_replay/GLM-4.7-Flash.json @@ -43,7 +43,8 @@ }, "style": "glm_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "" }, @@ -76,7 +77,8 @@ }, "style": "glm_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "" } @@ -140,7 +142,8 @@ }, "style": "glm_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "" }, @@ -173,7 +176,8 @@ }, "style": "glm_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "" } @@ -231,7 +235,8 @@ }, "style": "glm_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "" } diff --git a/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json b/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json index 4beb103b355c..404667c30a7b 100644 --- a/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json +++ b/rust/src/chat/tests/grammar_replay/GLM-5.2-FP8.json @@ -43,7 +43,8 @@ }, "style": "glm_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "" }, @@ -76,7 +77,8 @@ }, "style": "glm_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "" } @@ -140,7 +142,8 @@ }, "style": "glm_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "" }, @@ -173,7 +176,8 @@ }, "style": "glm_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "" } @@ -231,7 +235,8 @@ }, "style": "glm_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "" } diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K2.5.json b/rust/src/chat/tests/grammar_replay/Kimi-K2.5.json index 466e8ae899fd..8ac6e689f78b 100644 --- a/rust/src/chat/tests/grammar_replay/Kimi-K2.5.json +++ b/rust/src/chat/tests/grammar_replay/Kimi-K2.5.json @@ -60,7 +60,8 @@ }, "style": "json", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] } ] }, @@ -106,7 +107,8 @@ }, "style": "json", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] } ] }, @@ -190,7 +192,8 @@ }, "style": "json", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] } ] }, @@ -236,7 +239,8 @@ }, "style": "json", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] } ] }, @@ -312,7 +316,8 @@ }, "style": "json", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] } ] }, diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K3.json b/rust/src/chat/tests/grammar_replay/Kimi-K3.json index 48a69968a3dd..5a62e2596887 100644 --- a/rust/src/chat/tests/grammar_replay/Kimi-K3.json +++ b/rust/src/chat/tests/grammar_replay/Kimi-K3.json @@ -90,7 +90,8 @@ }, "style": "json", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "<|close|>json<|sep|>" } @@ -132,7 +133,8 @@ }, "style": "json", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "<|close|>argument<|sep|>" }, @@ -146,7 +148,8 @@ }, "style": "json", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "<|close|>argument<|sep|>" }, @@ -163,7 +166,8 @@ }, "style": "json", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "<|close|>argument<|sep|>" } @@ -199,7 +203,8 @@ }, "style": "json", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "<|close|>json<|sep|>" } diff --git a/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.json b/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.json index 02ea28577032..582f29a1b943 100644 --- a/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.json +++ b/rust/src/chat/tests/grammar_replay/MiMo-V2.6-Flash-RL.json @@ -25,12 +25,12 @@ { "type": "triggered_tags", "triggers": [ - "\n" ], "tags": [ { "type": "tag", - "begin": "\n\n", + "begin": "", "content": { "type": "json_schema", "json_schema": { @@ -46,13 +46,14 @@ }, "style": "qwen_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, - "end": "\n\n" + "end": "" }, { "type": "tag", - "begin": "\n\n", + "begin": "", "content": { "type": "json_schema", "json_schema": { @@ -79,16 +80,19 @@ }, "style": "qwen_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, - "end": "\n\n" + "end": "" } ], "at_least_one": false, "stop_after_first": false, "excludes": [ "", - "" + "", + "", + "Need call the weather and add tools.I will call the tools.Shanghai1.02[\"left\", \"right\"]", "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 40, 686, 1618, 279, 7375, 13, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151657, 27, 1688, 92784, 1784, 16181, 29368, 29, 16, 13, 15, 522, 16181, 1784, 16181, 25387, 29, 17, 522, 16181, 1784, 16181, 28, 3615, 29, 1183, 2359, 497, 330, 1291, 1341, 522, 16181, 1472, 1688, 29, 151658, 151645] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + } + }, + { + "type": "triggered_tags", + "triggers": [ + "" + ], + "tags": [ + { + "type": "tag", + "begin": "", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + }, + { + "type": "tag", + "begin": "", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ], + "at_least_one": true, + "stop_after_first": false, + "excludes": [ + "", + "", + "", + "Need call the weather and add tools.Shanghai1.02[\"left\", \"right\"]", + "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151657, 27, 1688, 92784, 1784, 16181, 29368, 29, 16, 13, 15, 522, 16181, 1784, 16181, 25387, 29, 17, 522, 16181, 1784, 16181, 28, 3615, 29, 1183, 2359, 497, 330, 1291, 1341, 522, 16181, 1472, 1688, 29, 151658, 151645] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [], + "max_tokens": null, + "max_chars": null + }, + "end": "" + } + }, + { + "type": "tag", + "begin": "", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "qwen_xml", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "" + } + ] + }, + "coverage": "from_token_zero" + }, + "completion": "Need call the weather and add tools.Shanghai", + "generation_token_ids": [151667, 23657, 1618, 279, 9104, 323, 912, 7375, 13, 151668, 151657, 27, 1688, 28280, 69364, 1784, 16181, 28, 2527, 29, 2016, 30070, 522, 16181, 1472, 1688, 29, 151658, 151645] } } } diff --git a/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json b/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json index 7fb4be4a7561..64f65e2f5a32 100644 --- a/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json +++ b/rust/src/chat/tests/grammar_replay/MiniMax-M2.5.json @@ -58,7 +58,8 @@ }, "style": "minimax_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n" }, @@ -91,7 +92,8 @@ }, "style": "minimax_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n" } @@ -170,7 +172,8 @@ }, "style": "minimax_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n" }, @@ -203,7 +206,8 @@ }, "style": "minimax_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n" } @@ -275,7 +279,8 @@ }, "style": "minimax_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n" }, diff --git a/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json b/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json index 579afdbd7bf7..5d171fc5f080 100644 --- a/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json +++ b/rust/src/chat/tests/grammar_replay/Qwen3-0.6B.json @@ -55,7 +55,8 @@ }, "style": "json", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "}\n" }, @@ -88,7 +89,8 @@ }, "style": "json", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "}\n" } @@ -159,7 +161,8 @@ }, "style": "json", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "}\n" }, @@ -192,7 +195,8 @@ }, "style": "json", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "}\n" } @@ -257,7 +261,8 @@ }, "style": "json", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "}\n" } diff --git a/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json b/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json index d28c6a98f3e5..8bd4c901e541 100644 --- a/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json +++ b/rust/src/chat/tests/grammar_replay/Qwen3.5-4B.json @@ -52,7 +52,8 @@ }, "style": "qwen_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n\n" }, @@ -85,7 +86,8 @@ }, "style": "qwen_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n\n" } @@ -153,7 +155,8 @@ }, "style": "qwen_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n\n" }, @@ -186,7 +189,8 @@ }, "style": "qwen_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n\n" } @@ -248,7 +252,8 @@ }, "style": "qwen_xml", "any_order": false, - "max_whitespace_cnt": null + "max_whitespace_cnt": null, + "excludes": [] }, "end": "\n\n" } diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index 8889025cf5f9..c45815439505 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -192,10 +192,7 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, - // TODO: MiMo renders compact `` calls, but its - // tool parser reuses the Qwen3-Coder builder with newline-delimited tags: - // `required` rejects the calls, and the `auto` trigger never fires. - tool_choice_variants: &[ToolChoiceVariant::Auto], + tool_choice_variants: ToolChoiceVariant::ALL, } } From e6ce31dce2ec1a561fbf6a56a3785fb1282dd9b3 Mon Sep 17 00:00:00 2001 From: Bugen Zhao Date: Tue, 29 Sep 2026 08:43:01 +0000 Subject: [PATCH 5/5] [Rust Frontend] Split roundtrip completions around trailing control messages Kimi K3 renders request-scoped control messages, such as the `tool_choice=required` instruction, after the whole history. The generation prompt is then `A + C + G` while the closed history is `A + G + B + C`, so the prompt is no longer a prefix of the full render. Locate the completion `B` by finding the unique split of `C` and `G`, falling back to the prefix cut when `C` is empty. This lets Kimi K3 run the `required` variant, and every model now runs every tool-choice variant. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Bugen Zhao --- .../chat/tests/grammar_replay/Kimi-K3.json | 347 ++++++++++++++++++ rust/src/chat/tests/roundtrip.rs | 108 ++++-- 2 files changed, 417 insertions(+), 38 deletions(-) diff --git a/rust/src/chat/tests/grammar_replay/Kimi-K3.json b/rust/src/chat/tests/grammar_replay/Kimi-K3.json index 5a62e2596887..ffe5061192fd 100644 --- a/rust/src/chat/tests/grammar_replay/Kimi-K3.json +++ b/rust/src/chat/tests/grammar_replay/Kimi-K3.json @@ -235,6 +235,353 @@ }, "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|>I will call the tools.<|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool=\"add\" index=\"2\"<|sep|><|open|>argument key=\"y\" type=\"number\"<|sep|>1.0<|close|>argument<|sep|><|open|>argument key=\"x\" type=\"number\"<|sep|>2<|close|>argument<|sep|><|open|>argument key=\"items\" type=\"array\"<|sep|>[\"left\",\"right\"]<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>", "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 40, 748, 2390, 276, 7697, 13, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163587, 10257, 4453, 878, 1210, 1, 4002, 878, 17, 1, 163589, 163587, 47185, 2355, 878, 88, 1, 1798, 878, 10006, 1, 163589, 16, 13, 15, 163588, 47185, 163589, 163587, 47185, 2355, 878, 87, 1, 1798, 878, 10006, 1, 163589, 17, 163588, 47185, 163589, 163587, 47185, 2355, 878, 7989, 1, 1798, 878, 4392, 1, 163589, 3530, 3794, 3923, 2277, 3956, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586] + }, + "required": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|open|>response<|sep|>" + } + }, + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [ + "<|open|>", + "<|close|>", + "<|end_of_msg|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>response<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>tools<|sep|>", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|open|>call tool=\"get_weather\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "plus", + "content": { + "type": "tag", + "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", + "content": { + "type": "any_text", + "excludes": [ + "<|close|>argument<|sep|>", + "<|close|>call<|sep|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>argument<|sep|>" + } + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>call tool=\"add\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "plus", + "content": { + "type": "or", + "elements": [ + { + "type": "tag", + "begin": "<|open|>argument key=\"y\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"x\" type=\"number\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "number" + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>argument key=\"items\" type=\"array\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "array", + "items": { + "type": "string" + } + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>argument<|sep|>" + } + ] + } + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "y": { + "type": "number" + }, + "x": { + "type": "number" + }, + "items": { + "type": "array", + "items": { + "type": "string" + } + } + }, + "required": [ + "y", + "x", + "items" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "<|close|>tools<|sep|>" + }, + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|close|>message<|sep|>" + } + } + ] + }, + "coverage": "final_output_only" + }, + "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|><|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|open|>call tool=\"add\" index=\"2\"<|sep|><|open|>argument key=\"y\" type=\"number\"<|sep|>1.0<|close|>argument<|sep|><|open|>argument key=\"x\" type=\"number\"<|sep|>2<|close|>argument<|sep|><|open|>argument key=\"items\" type=\"array\"<|sep|>[\"left\",\"right\"]<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163587, 10257, 4453, 878, 1210, 1, 4002, 878, 17, 1, 163589, 163587, 47185, 2355, 878, 88, 1, 1798, 878, 10006, 1, 163589, 16, 13, 15, 163588, 47185, 163589, 163587, 47185, 2355, 878, 87, 1, 1798, 878, 10006, 1, 163589, 17, 163588, 47185, 163589, 163587, 47185, 2355, 878, 7989, 1, 1798, 878, 4392, 1, 163589, 3530, 3794, 3923, 2277, 3956, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586] + }, + "named": { + "grammar": { + "format": { + "type": "sequence", + "elements": [ + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|open|>response<|sep|>" + } + }, + { + "type": "tag", + "begin": "", + "content": { + "type": "any_text", + "excludes": [ + "<|open|>", + "<|close|>", + "<|end_of_msg|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>response<|sep|>" + }, + { + "type": "tag", + "begin": "<|open|>tools<|sep|>", + "content": { + "type": "tags_with_separator", + "tags": [ + { + "type": "tag", + "begin": "<|open|>call tool=\"get_weather\" index=\"", + "content": { + "type": "sequence", + "elements": [ + { + "type": "regex", + "pattern": "[1-9][0-9]*" + }, + { + "type": "const_string", + "value": "\"<|sep|>" + }, + { + "type": "or", + "elements": [ + { + "type": "plus", + "content": { + "type": "tag", + "begin": "<|open|>argument key=\"location\" type=\"string\"<|sep|>", + "content": { + "type": "any_text", + "excludes": [ + "<|close|>argument<|sep|>", + "<|close|>call<|sep|>" + ], + "max_tokens": null, + "max_chars": null + }, + "end": "<|close|>argument<|sep|>" + } + }, + { + "type": "tag", + "begin": "<|open|>json type=\"object\"<|sep|>", + "content": { + "type": "json_schema", + "json_schema": { + "type": "object", + "properties": { + "location": { + "type": "string" + } + }, + "required": [ + "location" + ] + }, + "style": "json", + "any_order": false, + "max_whitespace_cnt": null, + "excludes": [] + }, + "end": "<|close|>json<|sep|>" + } + ] + } + ] + }, + "end": "<|close|>call<|sep|>" + } + ], + "separator": "", + "at_least_one": true, + "stop_after_first": false + }, + "end": "<|close|>tools<|sep|>" + }, + { + "type": "optional", + "content": { + "type": "const_string", + "value": "<|close|>message<|sep|>" + } + } + ] + }, + "coverage": "final_output_only" + }, + "completion": "Need call the weather and add tools.<|close|>think<|sep|><|open|>response<|sep|><|close|>response<|sep|><|open|>tools<|sep|><|open|>call tool=\"get_weather\" index=\"1\"<|sep|><|open|>argument key=\"location\" type=\"string\"<|sep|>Shanghai<|close|>argument<|sep|><|close|>call<|sep|><|close|>tools<|sep|><|close|>message<|sep|>", + "generation_token_ids": [20082, 2390, 276, 10666, 316, 1239, 7697, 13, 163588, 39964, 163589, 163587, 12092, 163589, 163588, 12092, 163589, 163587, 25385, 163589, 163587, 10257, 4453, 878, 618, 21055, 2800, 1, 4002, 878, 16, 1, 163589, 163587, 47185, 2355, 878, 5791, 1, 1798, 878, 2033, 1, 163589, 2797, 31231, 163588, 47185, 163589, 163588, 10257, 163589, 163588, 25385, 163589, 163588, 2778, 163589, 163586] } } } diff --git a/rust/src/chat/tests/roundtrip.rs b/rust/src/chat/tests/roundtrip.rs index c45815439505..6925ae1b0223 100644 --- a/rust/src/chat/tests/roundtrip.rs +++ b/rust/src/chat/tests/roundtrip.rs @@ -12,6 +12,8 @@ //! `tests/grammar_replay/`. `grammar_replay.rs` replays these cases through the real XGrammar. use std::collections::{BTreeMap, BTreeSet}; +use std::fmt::Debug; +use std::ops::Range; use std::path::PathBuf; use std::pin::Pin; use std::sync::Arc; @@ -55,8 +57,6 @@ struct RoundtripCase { json_fmt: JsonFmt, /// Whether the template renders tool-call argument object keys in sorted order. sort_json_keys: bool, - /// Tool-choice variants run by the tool-call fixture. - tool_choice_variants: &'static [ToolChoiceVariant], } #[derive(Clone, Copy)] @@ -139,8 +139,7 @@ enum ToolChoiceVariant { } impl ToolChoiceVariant { - /// Every variant, for templates and parsers that support all of them. - const ALL: &[Self] = &[Self::Auto, Self::Required, Self::Named]; + const ALL: [Self; 3] = [Self::Auto, Self::Required, Self::Named]; fn tool_choice(self) -> ChatToolChoice { match self { @@ -164,7 +163,6 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: spaced_json_fmt(), sort_json_keys: false, - tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -178,7 +176,6 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, - tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -192,7 +189,6 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, - tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -206,7 +202,6 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, - tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -220,7 +215,6 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, - tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -234,7 +228,6 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: false }, json_fmt: compact_json_fmt(), sort_json_keys: false, - tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -248,7 +241,6 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, - tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -262,7 +254,6 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: false }, json_fmt: compact_json_fmt(), sort_json_keys: false, - tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -276,7 +267,6 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, - tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -290,7 +280,6 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, - tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -304,7 +293,6 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, - tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -318,7 +306,6 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: true, - tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -333,7 +320,6 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: spaced_json_fmt(), sort_json_keys: false, - tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -351,9 +337,6 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Toggleable { default: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, - // The renderer appends a `tool_choice=required` system message after the - // history, so the completion is not a suffix of the full render. - tool_choice_variants: &[ToolChoiceVariant::Auto], } } @@ -371,7 +354,6 @@ impl RoundtripCase { }, json_fmt: compact_json_fmt(), sort_json_keys: false, - tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -385,7 +367,6 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, - tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -399,7 +380,6 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, - tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -413,7 +393,6 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, - tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -427,7 +406,6 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: false, - tool_choice_variants: ToolChoiceVariant::ALL, } } @@ -441,7 +419,6 @@ impl RoundtripCase { thinking_behavior: ThinkingBehavior::Always { value: true }, json_fmt: compact_json_fmt(), sort_json_keys: true, - tool_choice_variants: ToolChoiceVariant::ALL, } } } @@ -558,7 +535,7 @@ async fn run_roundtrip_tool_call_mix( backends: &vllm_chat::LoadedModelBackends, ) -> Result<()> { let mut results = Vec::new(); - for &variant in case.tool_choice_variants { + for variant in ToolChoiceVariant::ALL { let result = run_roundtrip_tool_call_mix_inner(case, backends, variant).await?; results.push((variant, result)); } @@ -762,7 +739,7 @@ struct RenderedTurn { } /// Render `history` as a production prompt and `history + assistant` as closed -/// history, then return the production prompt and assistant-completion suffix. +/// history, then return the production prompt and the assistant completion. fn render_closed_completion( renderer: &dyn vllm_chat::ChatRenderer, base_request: &ChatRequest, @@ -779,18 +756,15 @@ fn render_closed_completion( let completion = match (&prompt, full) { (Prompt::Text(prompt), Prompt::Text(full)) => { - ensure!( - full.starts_with(prompt), - "full prompt must extend production prompt\nprompt: {prompt:?}\nfull: {full:?}" - ); - Prompt::Text(full[prompt.len()..].to_string()) + let range = completion_range(prompt.as_bytes(), full.as_bytes()) + .with_context(|| format!("prompt: {prompt:?}\nfull: {full:?}"))?; + let completion = full.get(range).context("completion splits a UTF-8 character")?; + Prompt::Text(completion.to_string()) } (Prompt::TokenIds(prompt), Prompt::TokenIds(full)) => { - ensure!( - full.starts_with(prompt), - "full prompt must extend production prompt\nprompt: {prompt:?}\nfull: {full:?}" - ); - Prompt::TokenIds(full[prompt.len()..].to_vec()) + let range = completion_range(prompt, &full) + .with_context(|| format!("prompt: {prompt:?}\nfull: {full:?}"))?; + Prompt::TokenIds(full[range].to_vec()) } (prompt, full) => bail!("prompt kind changed between renders: {prompt:?} vs {full:?}"), }; @@ -798,6 +772,64 @@ fn render_closed_completion( Ok(RenderedTurn { prompt, completion }) } +/// Locate the assistant completion in `full`, the history rendered with the +/// assistant message, given `prompt`, the same history rendered for generation. +/// +/// Some templates render request-scoped control messages after the whole +/// history, such as Kimi K3's `tool_choice=required` message. With history `A`, +/// control messages `C`, generation prompt `G`, and completion `B`, the renders +/// are `prompt = A + C + G` and `full = A + G + B + C`. `C` is empty for most +/// templates, which makes `prompt` a prefix of `full`. +fn completion_range(prompt: &[T], full: &[T]) -> Result> { + if full.starts_with(prompt) { + return Ok(prompt.len()..full.len()); + } + ensure!( + full.len() >= prompt.len(), + "full prompt is shorter than production prompt" + ); + + // `A` is a common prefix of both renders; try every split of the remaining + // `prompt` into `C + G` and require exactly one to fit `full`. + let common_prefix_len = prompt.iter().zip(full).take_while(|(p, f)| p == f).count(); + let mut control_lens = Vec::new(); + for history_len in 0..=common_prefix_len { + let rest = &prompt[history_len..]; + for control_len in 1..=rest.len() { + let (control, generation) = rest.split_at(control_len); + if full[history_len..].starts_with(generation) && full.ends_with(control) { + control_lens.push(control_len); + } + } + } + let [control_len] = control_lens[..] else { + bail!( + "full prompt must be `A + G + B + C` for production prompt `A + C + G`; found {} splits", + control_lens.len() + ); + }; + + Ok(prompt.len() - control_len..full.len() - control_len) +} + +#[test] +fn completion_range_skips_trailing_control_messages() { + let history = "hi"; + let control = "call tools"; + let generation = ""; + let completion = "call()"; + + let prompt = format!("{history}{generation}"); + let full = format!("{history}{generation}{completion}"); + let range = completion_range(prompt.as_bytes(), full.as_bytes()).unwrap(); + assert_eq!(&full[range], completion); + + let prompt = format!("{history}{control}{generation}"); + let full = format!("{history}{generation}{completion}{control}"); + let range = completion_range(prompt.as_bytes(), full.as_bytes()).unwrap(); + assert_eq!(&full[range], completion); +} + /// Output of one production output processor over a rendered completion. struct ParsedCompletion { /// Terminal assistant message.