Repository navigation
fix(tool_parser): serialize tool-call arguments with vLLM/json.dumps spacing #1836
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -394,6 +394,54 @@ pub(crate) fn handle_json_tool_streaming( | |||||||||||||||||
| Ok(result) | ||||||||||||||||||
| } | ||||||||||||||||||
|
|
||||||||||||||||||
| /// serde_json formatter with Python `json.dumps` separators (`", "` / `": "`). | ||||||||||||||||||
| struct SpacedFormatter; | ||||||||||||||||||
|
|
||||||||||||||||||
| impl serde_json::ser::Formatter for SpacedFormatter { | ||||||||||||||||||
| fn begin_array_value<W: ?Sized + std::io::Write>( | ||||||||||||||||||
| &mut self, | ||||||||||||||||||
| writer: &mut W, | ||||||||||||||||||
| first: bool, | ||||||||||||||||||
| ) -> std::io::Result<()> { | ||||||||||||||||||
| if first { | ||||||||||||||||||
| Ok(()) | ||||||||||||||||||
| } else { | ||||||||||||||||||
| writer.write_all(b", ") | ||||||||||||||||||
| } | ||||||||||||||||||
| } | ||||||||||||||||||
|
|
||||||||||||||||||
| fn begin_object_key<W: ?Sized + std::io::Write>( | ||||||||||||||||||
| &mut self, | ||||||||||||||||||
| writer: &mut W, | ||||||||||||||||||
| first: bool, | ||||||||||||||||||
| ) -> std::io::Result<()> { | ||||||||||||||||||
| if first { | ||||||||||||||||||
| Ok(()) | ||||||||||||||||||
| } else { | ||||||||||||||||||
| writer.write_all(b", ") | ||||||||||||||||||
| } | ||||||||||||||||||
| } | ||||||||||||||||||
|
|
||||||||||||||||||
| fn begin_object_value<W: ?Sized + std::io::Write>( | ||||||||||||||||||
| &mut self, | ||||||||||||||||||
| writer: &mut W, | ||||||||||||||||||
| ) -> std::io::Result<()> { | ||||||||||||||||||
| writer.write_all(b": ") | ||||||||||||||||||
| } | ||||||||||||||||||
| } | ||||||||||||||||||
|
|
||||||||||||||||||
| /// Serialize tool-call arguments like Python `json.dumps` (spaced separators), so | ||||||||||||||||||
| /// they match vLLM — chat templates emit `arguments` verbatim, so compact vs spaced | ||||||||||||||||||
| /// would diverge the prompt across multi-turn tool calls. | ||||||||||||||||||
| pub fn args_to_json_string<T: serde::Serialize>(value: &T) -> ParserResult<String> { | ||||||||||||||||||
| let mut buf = Vec::new(); | ||||||||||||||||||
| let mut ser = serde_json::Serializer::with_formatter(&mut buf, SpacedFormatter); | ||||||||||||||||||
| value | ||||||||||||||||||
| .serialize(&mut ser) | ||||||||||||||||||
| .map_err(|e| ParserError::ParsingFailed(e.to_string()))?; | ||||||||||||||||||
| Ok(String::from_utf8_lossy(&buf).into_owned()) | ||||||||||||||||||
|
Comment on lines
+439
to
+442
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Using
Suggested change
|
||||||||||||||||||
| } | ||||||||||||||||||
|
|
||||||||||||||||||
| #[cfg(test)] | ||||||||||||||||||
| mod tests { | ||||||||||||||||||
| use super::*; | ||||||||||||||||||
|
|
||||||||||||||||||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -160,7 +160,7 @@ impl MinimaxM2Parser { | |
| let param_types = helpers::param_types_for_function(tools, func_name); | ||
| let parameters = self.parse_parameters(params_text, ¶m_types); | ||
|
|
||
| match serde_json::to_string(¶meters) { | ||
| match helpers::args_to_json_string(¶meters) { | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. While using |
||
| Ok(arguments_str) => calls.push(ToolCall { | ||
| function: FunctionCall { | ||
| name: func_name.to_string(), | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -216,8 +216,7 @@ impl QwenXmlParser { | |
| } | ||
| } | ||
|
|
||
| let arguments = serde_json::to_string(¶meters) | ||
| .map_err(|e| ParserError::ParsingFailed(e.to_string()))?; | ||
| let arguments = helpers::args_to_json_string(¶meters)?; | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. While using |
||
|
|
||
| Ok(Some(ToolCall { | ||
| function: FunctionCall { | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -965,3 +965,23 @@ async fn test_minimax_invalid_json_in_parameters() { | |
|
|
||
| assert_eq!(normal_text, ""); | ||
| } | ||
|
|
||
| #[tokio::test] | ||
| async fn test_minimax_arguments_use_python_json_spacing() { | ||
| // Spaced separators (", " / ": "), matching vLLM's json.dumps. | ||
| let parser = MinimaxM2Parser::new(); | ||
| let input = r#"<minimax:tool_call> | ||
| <invoke name="get_weather"> | ||
| <parameter name="city">Tokyo</parameter> | ||
| <parameter name="units">celsius</parameter> | ||
| </invoke> | ||
| </minimax:tool_call>"#; | ||
| let (_n, tools) = parser.parse_complete(input).await.unwrap(); | ||
| assert_eq!(tools.len(), 1); | ||
| let a = &tools[0].function.arguments; | ||
| assert!( | ||
| a.contains("\": \""), | ||
| "expected spaced key/value separators, got {a}" | ||
| ); | ||
| assert!(!a.contains("\":\""), "must not be compact, got {a}"); | ||
|
Comment on lines
+982
to
+986
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 📐 Maintainability & Code Quality | 🟡 Minor | ⚡ Quick win Expand the spacing assertion to cover item separators too. The test currently checks Suggested test assertion update assert!(
a.contains("\": \""),
"expected spaced key/value separators, got {a}"
);
assert!(!a.contains("\":\""), "must not be compact, got {a}");
+ assert!(a.contains(", "), "expected spaced item separators, got {a}");
+ assert!(!a.contains(",\""), "must not use compact item separators, got {a}");🤖 Prompt for AI Agents |
||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
🟡 Nit: The JSON-based parsers (
llama.rs:82,json.rs:146,qwen.rs:93,deepseek.rs:123,deepseek31.rs:110,glm4_moe.rs:142,step3.rs:396,pythonic.rs:284,mistral.rs:191) also parse model output and re-serialize withserde_json::to_string(compact). If vLLM re-serializes those withjson.dumps(spaced) too, they'd have the same spacing divergence in multi-turn prompts.Intentionally out of scope for this PR, or worth a follow-up?