From 20aca5dbc9295f25eea84175609dade5d96ccaff Mon Sep 17 00:00:00 2001 From: ishandhanani Date: Thu, 23 Apr 2026 22:48:11 -0500 Subject: [PATCH 01/21] Add DeepSeek V4 parser support --- lib/parsers/src/reasoning/mod.rs | 42 +++++++++++ lib/parsers/src/tool_calling/config.rs | 16 +++++ lib/parsers/src/tool_calling/dsml/parser.rs | 78 +++++++++++++++++++++ lib/parsers/src/tool_calling/parsers.rs | 54 ++++++++++++++ 4 files changed, 190 insertions(+) diff --git a/lib/parsers/src/reasoning/mod.rs b/lib/parsers/src/reasoning/mod.rs index 3a8a568c2477..bbfb67e4d651 100644 --- a/lib/parsers/src/reasoning/mod.rs +++ b/lib/parsers/src/reasoning/mod.rs @@ -29,6 +29,9 @@ fn get_reasoning_parser_map() -> &'static HashMap<&'static str, ReasoningParserT map.insert("basic", ReasoningParserType::Basic); map.insert("gpt_oss", ReasoningParserType::GptOss); map.insert("qwen3", ReasoningParserType::Qwen); + map.insert("deepseek_v4", ReasoningParserType::Qwen); + map.insert("deepseek-v4", ReasoningParserType::Qwen); + map.insert("deepseekv4", ReasoningParserType::Qwen); map.insert("nemotron_deci", ReasoningParserType::NemotronDeci); map.insert("kimi", ReasoningParserType::Kimi); map.insert("kimi_k25", ReasoningParserType::KimiK25); @@ -246,6 +249,9 @@ mod tests { "basic", "gpt_oss", "qwen3", + "deepseek_v4", + "deepseek-v4", + "deepseekv4", "nemotron_deci", "kimi", "kimi_k25", @@ -262,6 +268,42 @@ mod tests { } } + #[test] + fn test_deepseek_v4_detect_and_parse() { + for parser_name in ["deepseek_v4", "deepseek-v4", "deepseekv4"] { + let mut parser = ReasoningParserType::get_reasoning_parser_from_name(parser_name); + let result = parser.detect_and_parse_reasoning("thinkinganswer", &[]); + assert_eq!(result.reasoning_text, "thinking"); + assert_eq!(result.normal_text, "answer"); + } + } + + #[test] + fn test_deepseek_v4_no_forced_reasoning_without_tags() { + let mut parser = ReasoningParserType::get_reasoning_parser_from_name("deepseek_v4"); + let result = parser.detect_and_parse_reasoning("answer only", &[]); + assert_eq!(result.reasoning_text, ""); + assert_eq!(result.normal_text, "answer only"); + } + + #[test] + fn test_deepseek_v4_streaming() { + let mut parser = ReasoningParserType::get_reasoning_parser_from_name("deepseek_v4"); + + let chunks = ["rea", "sonanswer"]; + let mut reasoning = String::new(); + let mut normal = String::new(); + + for chunk in chunks { + let result = parser.parse_reasoning_streaming_incremental(chunk, &[]); + reasoning.push_str(&result.reasoning_text); + normal.push_str(&result.normal_text); + } + + assert_eq!(reasoning, "reason"); + assert_eq!(normal, "answer"); + } + #[test] fn test_kimi_k25_detect_and_parse() { // (description, input, expected_reasoning, expected_normal) diff --git a/lib/parsers/src/tool_calling/config.rs b/lib/parsers/src/tool_calling/config.rs index c0a5a46448c2..35ab92208f7e 100644 --- a/lib/parsers/src/tool_calling/config.rs +++ b/lib/parsers/src/tool_calling/config.rs @@ -391,6 +391,22 @@ impl ToolCallConfig { } } + pub fn deepseek_v4() -> Self { + // DeepSeek V4 format (DSML): + // <｜DSML｜tool_calls> + // <｜DSML｜invoke name="function_name"> + // <｜DSML｜parameter name="param_name" string="true|false">value + // + // + Self { + parser_config: ParserConfig::Dsml(DsmlParserConfig { + function_calls_start: "<｜DSML｜tool_calls>".to_string(), + function_calls_end: "".to_string(), + ..Default::default() + }), + } + } + pub fn minimax_m2() -> Self { // MiniMax-M2.1 format: // diff --git a/lib/parsers/src/tool_calling/dsml/parser.rs b/lib/parsers/src/tool_calling/dsml/parser.rs index 3f23c9c0ac5f..5d9d72a628b0 100644 --- a/lib/parsers/src/tool_calling/dsml/parser.rs +++ b/lib/parsers/src/tool_calling/dsml/parser.rs @@ -224,6 +224,14 @@ mod tests { DsmlParserConfig::default() } + fn get_v4_test_config() -> DsmlParserConfig { + DsmlParserConfig { + function_calls_start: "<｜DSML｜tool_calls>".to_string(), + function_calls_end: "".to_string(), + ..Default::default() + } + } + #[test] fn test_detect_tool_call_start() { let config = get_test_config(); @@ -239,6 +247,22 @@ mod tests { assert!(!detect_tool_call_start_dsml("no tool call here", &config)); } + #[test] + fn test_detect_tool_call_start_v4() { + let config = get_v4_test_config(); + assert!(detect_tool_call_start_dsml("<｜DSML｜tool_calls>", &config)); + assert!(detect_tool_call_start_dsml( + "text <｜DSML｜tool_calls>", + &config + )); + assert!(detect_tool_call_start_dsml("<｜DSML｜tool_c", &config)); + assert!(!detect_tool_call_start_dsml( + "<｜DSML｜function_calls>", + &config + )); + assert!(!detect_tool_call_start_dsml("no tool call here", &config)); + } + #[test] fn test_find_tool_call_end_position() { let config = get_test_config(); @@ -247,6 +271,14 @@ mod tests { assert_eq!(&text[pos..], "more"); } + #[test] + fn test_find_tool_call_end_position_v4() { + let config = get_v4_test_config(); + let text = "<｜DSML｜tool_calls><｜DSML｜invoke name=\"test\">more"; + let pos = find_tool_call_end_position_dsml(text, &config); + assert_eq!(&text[pos..], "more"); + } + #[test] fn test_parse_single_tool_call_string_param() { let input = r#"<｜DSML｜function_calls> @@ -320,6 +352,52 @@ mod tests { assert_eq!(args2["location"], "Hangzhou"); } + #[test] + fn test_parse_deepseek_v4_multiple_tool_calls() { + let input = r#"Let's check this. <｜DSML｜tool_calls> +<｜DSML｜invoke name="get_favorite_tourist_spot"> +<｜DSML｜parameter name="city" string="true">Beijing + +<｜DSML｜invoke name="search"> +<｜DSML｜parameter name="query" string="true">search agent benchmark 2024 +<｜DSML｜parameter name="topn" string="false">10 +<｜DSML｜parameter name="source" string="true">web + +"#; + + let config = get_v4_test_config(); + let (calls, normal) = try_tool_call_parse_dsml(input, &config).unwrap(); + assert_eq!(calls.len(), 2); + assert_eq!(normal, Some("Let's check this.".to_string())); + + let (name1, args1) = extract_name_and_args(calls[0].clone()); + assert_eq!(name1, "get_favorite_tourist_spot"); + assert_eq!(args1["city"], "Beijing"); + + let (name2, args2) = extract_name_and_args(calls[1].clone()); + assert_eq!(name2, "search"); + assert_eq!(args2["query"], "search agent benchmark 2024"); + assert_eq!(args2["topn"], 10); + assert_eq!(args2["source"], "web"); + } + + #[test] + fn test_parse_deepseek_v4_no_parameters() { + let input = r#"<｜DSML｜tool_calls> +<｜DSML｜invoke name="get_current_time"> + +"#; + + let config = get_v4_test_config(); + let (calls, normal) = try_tool_call_parse_dsml(input, &config).unwrap(); + assert_eq!(calls.len(), 1); + assert_eq!(normal, Some("".to_string())); + + let (name, args) = extract_name_and_args(calls[0].clone()); + assert_eq!(name, "get_current_time"); + assert_eq!(args, serde_json::json!({})); + } + #[test] fn test_parse_with_normal_text() { let input = r#"Here's the result: <｜DSML｜function_calls> diff --git a/lib/parsers/src/tool_calling/parsers.rs b/lib/parsers/src/tool_calling/parsers.rs index 5f72bc4c17a4..96846d7be7e9 100644 --- a/lib/parsers/src/tool_calling/parsers.rs +++ b/lib/parsers/src/tool_calling/parsers.rs @@ -43,6 +43,9 @@ pub fn get_tool_parser_map() -> &'static HashMap<&'static str, ToolCallConfig> { map.insert("deepseek_v3", ToolCallConfig::deepseek_v3()); map.insert("deepseek_v3_1", ToolCallConfig::deepseek_v3_1()); map.insert("deepseek_v3_2", ToolCallConfig::deepseek_v3_2()); + map.insert("deepseek_v4", ToolCallConfig::deepseek_v4()); + map.insert("deepseek-v4", ToolCallConfig::deepseek_v4()); + map.insert("deepseekv4", ToolCallConfig::deepseek_v4()); map.insert("qwen3_coder", ToolCallConfig::qwen3_coder()); map.insert("jamba", ToolCallConfig::jamba()); map.insert("minimax_m2", ToolCallConfig::minimax_m2()); @@ -241,6 +244,9 @@ mod tests { "deepseek_v3", "deepseek_v3_1", "deepseek_v3_2", + "deepseek_v4", + "deepseek-v4", + "deepseekv4", "qwen3_coder", "jamba", "nemotron_nano", @@ -1702,6 +1708,54 @@ Remember, San Francisco weather can be quite unpredictable, particularly with it assert_eq!(args["source"], "web"); } + #[tokio::test] + async fn test_deepseek_v4_single_tool_call() { + let input = r#"<｜DSML｜tool_calls> +<｜DSML｜invoke name="get_datetime"> +<｜DSML｜parameter name="timezone" string="true">Asia/Shanghai + +"#; + + let (tool_calls, normal_text) = + detect_and_parse_tool_call(input, Some("deepseek_v4"), None) + .await + .expect("Failed to parse"); + + assert_eq!(tool_calls.len(), 1); + assert_eq!(tool_calls[0].function.name, "get_datetime"); + assert_eq!(normal_text, Some("".to_string())); + + let args: serde_json::Value = + serde_json::from_str(&tool_calls[0].function.arguments).unwrap(); + assert_eq!(args["timezone"], "Asia/Shanghai"); + } + + #[tokio::test] + async fn test_deepseek_v4_compatibility_aliases() { + let input = r#"<｜DSML｜tool_calls> +<｜DSML｜invoke name="search"> +<｜DSML｜parameter name="query" string="true">search agent benchmark 2024 +<｜DSML｜parameter name="topn" string="false">10 +<｜DSML｜parameter name="source" string="true">web + +"#; + + for parser_name in ["deepseek_v4", "deepseek-v4", "deepseekv4"] { + let (tool_calls, _) = detect_and_parse_tool_call(input, Some(parser_name), None) + .await + .expect("Failed to parse"); + + assert_eq!(tool_calls.len(), 1); + assert_eq!(tool_calls[0].function.name, "search"); + + let args: serde_json::Value = + serde_json::from_str(&tool_calls[0].function.arguments).unwrap(); + assert_eq!(args["query"], "search agent benchmark 2024"); + assert_eq!(args["topn"], 10); + assert_eq!(args["source"], "web"); + } + } + #[tokio::test] async fn test_hermes_parser_without_new_line() { let input = r#"{"name": "get_weather", "arguments": {"location": "San Francisco, CA", "unit": "celsius"}}" From fcb30aa285c5144e08623b33f61df985a4b3fa06 Mon Sep 17 00:00:00 2001 From: ayushag Date: Thu, 23 Apr 2026 21:46:53 -0700 Subject: [PATCH 02/21] chore: deepseek-v4 reasoning and tool calling streamning tests Signed-off-by: ayushag --- ...completion_stream_content_before_tool.json | 182 + ...t_completion_stream_fragmented_tokens.json | 3422 +++++++++++++++++ ...t_completion_stream_mixed_param_types.json | 218 ++ .../chat_completion_stream_multi_tool.json | 244 ++ .../chat_completion_stream_no_params.json | 146 + .../chat_completion_stream_no_tool.json | 14 + .../chat_completion_stream_special_chars.json | 201 + .../chat_completion_stream_tool.json | 164 + lib/llm/tests/test_streaming_tool_parsers.rs | 164 + 9 files changed, 4755 insertions(+) create mode 100644 lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_content_before_tool.json create mode 100644 lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_fragmented_tokens.json create mode 100644 lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_mixed_param_types.json create mode 100644 lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_multi_tool.json create mode 100644 lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_no_params.json create mode 100644 lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_no_tool.json create mode 100644 lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_special_chars.json create mode 100644 lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_tool.json diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_content_before_tool.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_content_before_tool.json new file mode 100644 index 000000000000..5fddf247c4fe --- /dev/null +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_content_before_tool.json @@ -0,0 +1,182 @@ +{ + "request_id": "deepseek-v4-content-before-tool-test", + "expected_output": { + "normal_content": "Let me check the forecast for Tokyo right now.", + "reasoning_content": "The user wants today's weather in Tokyo. I'll acknowledge the request, then call get_weather.", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_weather", + "arguments": "{\"location\": \"Tokyo\", \"unit\": \"celsius\"}" + } + } + ] + }, + "input_stream": [ + { + "data": { + "id": "chatcmpl-deepseek-v4-content-before-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "The user wants today's weather in Tokyo. I'll acknowledge the request, then call get_weather.", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "The user wants today's weather in Tokyo. I'll acknowledge the request, then call get_weather." + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-content-before-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "Let me check the forecast for Tokyo right now.", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-content-before-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜tool_calls>\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-content-before-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜invoke name=\"get_weather\">\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-content-before-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜parameter name=\"location\" string=\"true\">Tokyo\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-content-before-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜parameter name=\"unit\" string=\"true\">celsius\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-content-before-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-content-before-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-content-before-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": null, + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + }, + "finish_reason": "tool_calls" + } + ] + } + } + ] +} diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_fragmented_tokens.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_fragmented_tokens.json new file mode 100644 index 000000000000..8f9ccff8445a --- /dev/null +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_fragmented_tokens.json @@ -0,0 +1,3422 @@ +{ + "request_id": "deepseek-v4-fragmented-tokens-test", + "expected_output": { + "normal_content": "", + "reasoning_content": "Break tokens apart aggressively.", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "ping", + "arguments": "{\"host\": \"example.com\"}" + } + } + ] + }, + "input_stream": [ + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "B", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "B" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "r", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "r" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "e", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "e" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "a", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "a" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "k", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "k" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": " ", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": " " + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "t", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "t" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "o", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "o" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "k", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "k" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "e", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "e" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "n" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "s", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "s" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": " ", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": " " + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "a", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "a" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "p", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "p" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "a", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "a" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "r", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "r" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "t", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "t" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": " ", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": " " + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "a", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "a" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "g", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "g" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "g", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "g" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "r", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "r" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "e", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "e" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "s", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "s" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "s", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "s" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "i", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "i" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "v", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "v" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "e", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "e" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "l", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "l" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "y", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "y" + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": ".", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "." + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "<", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "｜", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "D", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "S", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "M", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "L", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "｜", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "t", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "o", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "o", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "l", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "_", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "c", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "a", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "l", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "l", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "s", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": ">", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "<", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "｜", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "D", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "S", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "M", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "L", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "｜", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "i", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "v", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "o", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "k", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "e", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": " ", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "a", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "m", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "e", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "=", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "\"", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "p", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "i", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "g", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "\"", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": ">", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "<", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "｜", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "D", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "S", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "M", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "L", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "｜", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "p", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "a", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "r", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "a", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "m", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "e", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "t", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "e", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "r", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": " ", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "a", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "m", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "e", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "=", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "\"", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "h", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "o", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "s", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "t", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "\"", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": " ", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "s", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "t", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "r", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "i", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "g", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "=", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "\"", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "t", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "r", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "u", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "e", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "\"", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": ">", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "e", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "x", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "a", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "m", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "p", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "l", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "e", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": ".", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "c", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "o", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "m", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "<", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "/", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "｜", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "D", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "S", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "M", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "L", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "｜", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "p", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "a", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "r", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "a", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "m", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "e", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "t", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "e", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "r", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": ">", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "<", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "/", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "｜", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "D", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "S", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "M", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "L", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "｜", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "i", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "v", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "o", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "k", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "e", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": ">", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "<", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "/", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "｜", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "D", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "S", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "M", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "L", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "｜", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "t", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "o", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "o", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "l", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "_", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "c", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "a", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "l", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "l", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": "s", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": ">", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-fragmented-tokens", + "choices": [ + { + "index": 0, + "delta": { + "content": null, + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + }, + "finish_reason": "tool_calls" + } + ] + } + } + ] +} diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_mixed_param_types.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_mixed_param_types.json new file mode 100644 index 000000000000..711950300227 --- /dev/null +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_mixed_param_types.json @@ -0,0 +1,218 @@ +{ + "request_id": "deepseek-v4-mixed-param-types-test", + "expected_output": { + "normal_content": "", + "reasoning_content": "The user asked me to send a high-priority overdue-billing notification. I'll call send_notification with the appropriate parameters.", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "send_notification", + "arguments": "{\"recipient\": \"user@example.com\", \"priority\": 3, \"urgent\": true, \"tags\": [\"billing\", \"overdue\"], \"metadata\": {\"ticket\": \"T-42\", \"retries\": 2}}" + } + } + ] + }, + "input_stream": [ + { + "data": { + "id": "chatcmpl-deepseek-v4-mixed", + "choices": [ + { + "index": 0, + "delta": { + "content": "The user asked me to send a high-priority overdue-billing notification. I'll call send_notification with the appropriate parameters.", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "The user asked me to send a high-priority overdue-billing notification. I'll call send_notification with the appropriate parameters." + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-mixed", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜tool_calls>\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-mixed", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜invoke name=\"send_notification\">\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-mixed", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜parameter name=\"recipient\" string=\"true\">user@example.com\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-mixed", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜parameter name=\"priority\" string=\"false\">3\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-mixed", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜parameter name=\"urgent\" string=\"false\">true\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-mixed", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜parameter name=\"tags\" string=\"false\">[\"billing\", \"overdue\"]\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-mixed", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜parameter name=\"metadata\" string=\"false\">{\"ticket\": \"T-42\", \"retries\": 2}\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-mixed", + "choices": [ + { + "index": 0, + "delta": { + "content": "\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-mixed", + "choices": [ + { + "index": 0, + "delta": { + "content": "", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-mixed", + "choices": [ + { + "index": 0, + "delta": { + "content": null, + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + }, + "finish_reason": "tool_calls" + } + ] + } + } + ] +} diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_multi_tool.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_multi_tool.json new file mode 100644 index 000000000000..fca42aa16b75 --- /dev/null +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_multi_tool.json @@ -0,0 +1,244 @@ +{ + "request_id": "deepseek-v4-multi-tool-test", + "expected_output": { + "normal_content": "", + "reasoning_content": "The user wants to check the weather in Beijing and Shanghai, I need to call the get_current_weather tool to get this information.", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": "{\"location\": \"Beijing\", \"format\": \"celsius\"}" + } + }, + { + "id": "call_2", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": "{\"location\": \"Shanghai\", \"format\": \"celsius\"}" + } + } + ] + }, + "input_stream": [ + { + "data": { + "id": "chatcmpl-deepseek-v4-multi-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "The user wants to check the weather in Beijing and Shanghai, I need to call the get_current_weather tool to get this information.", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "The user wants to check the weather in Beijing and Shanghai, I need to call the get_current_weather tool to get this information." + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-multi-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜tool_calls>\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-multi-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜invoke name=\"get_current_weather\">\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-multi-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜parameter name=\"location\" string=\"true\">Beijing\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-multi-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜parameter name=\"format\" string=\"true\">celsius\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-multi-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-multi-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜invoke name=\"get_current_weather\">\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-multi-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜parameter name=\"location\" string=\"true\">Shanghai\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-multi-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜parameter name=\"format\" string=\"true\">celsius\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-multi-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-multi-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-multi-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": null, + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + }, + "finish_reason": "tool_calls" + } + ] + } + } + ] +} diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_no_params.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_no_params.json new file mode 100644 index 000000000000..68668ac932dd --- /dev/null +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_no_params.json @@ -0,0 +1,146 @@ +{ + "request_id": "deepseek-v4-no-params-test", + "expected_output": { + "normal_content": "", + "reasoning_content": "The user asked for the current time. I'll call get_current_time which takes no parameters.", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_current_time", + "arguments": "{}" + } + } + ] + }, + "input_stream": [ + { + "data": { + "id": "chatcmpl-deepseek-v4-no-params", + "choices": [ + { + "index": 0, + "delta": { + "content": "The user asked for the current time. I'll call get_current_time which takes no parameters.", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "The user asked for the current time. I'll call get_current_time which takes no parameters." + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-no-params", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜tool_calls>\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-no-params", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜invoke name=\"get_current_time\">\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-no-params", + "choices": [ + { + "index": 0, + "delta": { + "content": "\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-no-params", + "choices": [ + { + "index": 0, + "delta": { + "content": "\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-no-params", + "choices": [ + { + "index": 0, + "delta": { + "content": "", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-no-params", + "choices": [ + { + "index": 0, + "delta": { + "content": null, + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + }, + "finish_reason": "tool_calls" + } + ] + } + } + ] +} diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_no_tool.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_no_tool.json new file mode 100644 index 000000000000..a19ee9084343 --- /dev/null +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_no_tool.json @@ -0,0 +1,14 @@ +{ + "request_id": "deepseek-v4-no-tool-test", + "expected_output": { + "normal_content": "Hi! I'm here to help — what would you like to work on today?", + "reasoning_content": "User greeted me politely. A short friendly reply is appropriate; no tools needed.", + "tool_calls": [] + }, + "input_stream": [ + {"data":{"id":"chatcmpl-deepseek-v4-no-tool","choices":[{"index":0,"delta":{"content":"User greeted me politely. A short friendly reply is appropriate; no tools needed.","function_call":null,"tool_calls":null,"role":"assistant","refusal":null,"reasoning_content":"User greeted me politely. A short friendly reply is appropriate; no tools needed."}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-no-tool","choices":[{"index":0,"delta":{"content":"Hi! I'm here to help — ","function_call":null,"tool_calls":null,"role":"assistant","refusal":null,"reasoning_content":null}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-no-tool","choices":[{"index":0,"delta":{"content":"what would you like to work on today?","function_call":null,"tool_calls":null,"role":"assistant","refusal":null,"reasoning_content":null}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-no-tool","choices":[{"index":0,"delta":{"content":null,"function_call":null,"tool_calls":null,"role":"assistant","refusal":null,"reasoning_content":null},"finish_reason":"stop"}]}} + ] +} diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_special_chars.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_special_chars.json new file mode 100644 index 000000000000..082ee79820e3 --- /dev/null +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_special_chars.json @@ -0,0 +1,201 @@ +{ + "request_id": "deepseek-v4-special-chars-test", + "expected_output": { + "normal_content": "", + "reasoning_content": "The user wants me to save a multiline note with special characters, quotes, unicode, and emoji. I'll call save_note.", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "save_note", + "arguments": "{\"note\": \"He said \\\"hello\\\".\\n\\t'world' `backtick` — 中文测试 — 🚀✨ & .\"}" + } + } + ], + "finish_reason": "tool_calls" + }, + "input_stream": [ + { + "data": { + "id": "chatcmpl-deepseek-v4-special", + "choices": [ + { + "index": 0, + "delta": { + "content": "The user wants me to save a multiline note with special characters, quotes, unicode, and emoji. I'll call save_note.", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "The user wants me to save a multiline note with special characters, quotes, unicode, and emoji. I'll call save_note." + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-special", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜tool_calls>\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-special", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜invoke name=\"save_note\">\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-special", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜parameter name=\"note\" string=\"true\">", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-special", + "choices": [ + { + "index": 0, + "delta": { + "content": "He said \"hello\".\n\t'world' `backtick` ", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-special", + "choices": [ + { + "index": 0, + "delta": { + "content": "— 中文测试 — 🚀✨ ", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-special", + "choices": [ + { + "index": 0, + "delta": { + "content": " & .\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-special", + "choices": [ + { + "index": 0, + "delta": { + "content": "\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-special", + "choices": [ + { + "index": 0, + "delta": { + "content": "", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-special", + "choices": [ + { + "index": 0, + "delta": { + "content": null, + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + }, + "finish_reason": "tool_calls" + } + ] + } + } + ] +} diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_tool.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_tool.json new file mode 100644 index 000000000000..85fddc391375 --- /dev/null +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_tool.json @@ -0,0 +1,164 @@ +{ + "request_id": "deepseek-v4-tool-call-test", + "expected_output": { + "normal_content": "", + "reasoning_content": "User wants the current weather in Beijing. I'll call get_current_weather with celsius units.", + "tool_calls": [ + { + "id": "call_1", + "type": "function", + "function": { + "name": "get_current_weather", + "arguments": "{\"location\": \"Beijing\", \"format\": \"celsius\"}" + } + } + ] + }, + "input_stream": [ + { + "data": { + "id": "chatcmpl-deepseek-v4-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "User wants the current weather in Beijing. I'll call get_current_weather with celsius units.", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": "User wants the current weather in Beijing. I'll call get_current_weather with celsius units." + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜tool_calls>\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜invoke name=\"get_current_weather\">\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜parameter name=\"location\" string=\"true\">Beijing\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "<｜DSML｜parameter name=\"format\" string=\"true\">celsius\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "\n", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": "", + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + } + } + ] + } + }, + { + "data": { + "id": "chatcmpl-deepseek-v4-tool", + "choices": [ + { + "index": 0, + "delta": { + "content": null, + "function_call": null, + "tool_calls": null, + "role": "assistant", + "refusal": null, + "reasoning_content": null + }, + "finish_reason": "tool_calls" + } + ] + } + } + ] +} diff --git a/lib/llm/tests/test_streaming_tool_parsers.rs b/lib/llm/tests/test_streaming_tool_parsers.rs index ceb3b1a6c8cf..28beb5e3dce7 100644 --- a/lib/llm/tests/test_streaming_tool_parsers.rs +++ b/lib/llm/tests/test_streaming_tool_parsers.rs @@ -1106,6 +1106,170 @@ mod tests { ); } + // ---- DeepSeek V4 (DSML format) streaming parser tests ---- + // + // V4 emits tool calls inside a DSML block: + // <｜DSML｜tool_calls> + // <｜DSML｜invoke name="fn"> + // <｜DSML｜parameter name="k" string="true|false">v + // + // + // Fixtures live under tests/data/vllm/deepseek-v4/. + + /// Shared harness for DeepSeek V4 e2e fixtures that end in a tool call. + async fn run_deepseek_v4_tool_call_fixture(file_path: &str) { + let test_data = load_test_data(file_path); + let input_stream = stream::iter(test_data.stream_chunks); + + let output_chunks = parse_response_stream( + input_stream, + true, + true, + Some("deepseek_v4".to_string()), + Some("deepseek_v4".to_string()), + ) + .await; + + assert!(!output_chunks.is_empty(), "Should have output chunks"); + + let aggregated = aggregate_content_from_chunks(&output_chunks); + + assert_eq!( + aggregated.reasoning_content, test_data.expected_reasoning_content, + "Should have extracted reasoning content.", + ); + assert_eq!( + aggregated.normal_content, test_data.expected_normal_content, + "Normal content should match expected value.", + ); + + let expected_has_tool_calls = !test_data.expected_tool_calls.is_empty(); + assert_eq!( + aggregated.has_tool_calls, expected_has_tool_calls, + "Tool calls presence should match expected value" + ); + assert_tool_calls(&aggregated.tool_calls, &test_data.expected_tool_calls); + + assert!( + validate_finish_reason(&output_chunks, FinishReason::ToolCalls), + "finish_reason validation failed for tool call case" + ); + } + + /// Single tool call, thinking mode (direct V4 analog of the V3 tool fixture). + #[tokio::test] + async fn test_deepseek_v4_e2e_with_tools_vllm() { + let file_path = format!( + "{}/vllm/deepseek-v4/chat_completion_stream_tool.json", + DATA_ROOT_PATH + ); + run_deepseek_v4_tool_call_fixture(&file_path).await; + } + + /// No tool call — thinking + plain body; finish_reason=stop. + #[tokio::test] + async fn test_deepseek_v4_e2e_with_no_tools_vllm() { + let file_path = format!( + "{}/vllm/deepseek-v4/chat_completion_stream_no_tool.json", + DATA_ROOT_PATH + ); + let test_data = load_test_data(&file_path); + let input_stream = stream::iter(test_data.stream_chunks); + + let output_chunks = parse_response_stream( + input_stream, + true, + true, + Some("deepseek_v4".to_string()), + Some("deepseek_v4".to_string()), + ) + .await; + + assert!(!output_chunks.is_empty(), "Should have output chunks"); + + let aggregated = aggregate_content_from_chunks(&output_chunks); + + assert_eq!( + aggregated.reasoning_content, test_data.expected_reasoning_content, + "Should have extracted reasoning content.", + ); + assert_eq!( + aggregated.normal_content, test_data.expected_normal_content, + "Normal content should match expected value.", + ); + assert!(!aggregated.has_tool_calls, "Should not have any tool calls"); + + assert!( + validate_finish_reason(&output_chunks, FinishReason::Stop), + "finish_reason validation failed for non-tool call case" + ); + } + + /// Two parallel tool calls inside one DSML block. + #[tokio::test] + async fn test_deepseek_v4_e2e_multi_tool_vllm() { + let file_path = format!( + "{}/vllm/deepseek-v4/chat_completion_stream_multi_tool.json", + DATA_ROOT_PATH + ); + run_deepseek_v4_tool_call_fixture(&file_path).await; + } + + /// string="true" vs string="false" — numbers, booleans, arrays, objects must + /// round-trip as their proper JSON types inside arguments. + #[tokio::test] + async fn test_deepseek_v4_e2e_mixed_param_types_vllm() { + let file_path = format!( + "{}/vllm/deepseek-v4/chat_completion_stream_mixed_param_types.json", + DATA_ROOT_PATH + ); + run_deepseek_v4_tool_call_fixture(&file_path).await; + } + + /// Tool call with zero parameters — invoke body is empty; arguments = "{}". + #[tokio::test] + async fn test_deepseek_v4_e2e_no_params_vllm() { + let file_path = format!( + "{}/vllm/deepseek-v4/chat_completion_stream_no_params.json", + DATA_ROOT_PATH + ); + run_deepseek_v4_tool_call_fixture(&file_path).await; + } + + /// Body text emitted before the DSML block — parser must populate both + /// normal_content and tool_calls. + #[tokio::test] + async fn test_deepseek_v4_e2e_content_before_tool_vllm() { + let file_path = format!( + "{}/vllm/deepseek-v4/chat_completion_stream_content_before_tool.json", + DATA_ROOT_PATH + ); + run_deepseek_v4_tool_call_fixture(&file_path).await; + } + + /// Parameter value containing unicode, emoji, embedded quotes/newlines/tabs, + /// and fragments that look like sentinels but aren't — must not confuse the + /// parser, which anchors only on the exact token. + #[tokio::test] + async fn test_deepseek_v4_e2e_special_chars_vllm() { + let file_path = format!( + "{}/vllm/deepseek-v4/chat_completion_stream_special_chars.json", + DATA_ROOT_PATH + ); + run_deepseek_v4_tool_call_fixture(&file_path).await; + } + + /// Adversarial streaming: every DSML character is its own delta (~200 chunks). + /// Exercises buffer accumulation across chunk boundaries. + #[tokio::test] + async fn test_deepseek_v4_e2e_fragmented_tokens_vllm() { + let file_path = format!( + "{}/vllm/deepseek-v4/chat_completion_stream_fragmented_tokens.json", + DATA_ROOT_PATH + ); + run_deepseek_v4_tool_call_fixture(&file_path).await; + } + // ---- Kimi K2 streaming jail reproduction tests ---- // // These reproduce the customer-reported issue (DIS-1765): Kimi K2 agentic From 587d66893bc7fe206a8227bf40132743026b6352 Mon Sep 17 00:00:00 2001 From: ayushag Date: Thu, 23 Apr 2026 22:22:40 -0700 Subject: [PATCH 03/21] chore: port v4 formatter Signed-off-by: ayushag --- lib/llm/src/preprocessor/prompt.rs | 1 + .../src/preprocessor/prompt/deepseek_v4.rs | 988 +++++ .../tests/data/deepseek-v4/test_input_1.json | 1 + .../tests/data/deepseek-v4/test_input_2.json | 1 + .../tests/data/deepseek-v4/test_input_3.json | 1 + .../tests/data/deepseek-v4/test_input_4.json | 1 + .../tests/data/deepseek-v4/test_output_1.txt | 36 + .../tests/data/deepseek-v4/test_output_2.txt | 1 + .../tests/data/deepseek-v4/test_output_3.txt | 38 + .../tests/data/deepseek-v4/test_output_4.txt | 29 + ...completion_stream_content_before_tool.json | 187 +- ...t_completion_stream_fragmented_tokens.json | 3607 +---------------- ...t_completion_stream_mixed_param_types.json | 225 +- .../chat_completion_stream_multi_tool.json | 252 +- .../chat_completion_stream_no_params.json | 146 - .../chat_completion_stream_no_tool.json | 14 +- .../chat_completion_stream_special_chars.json | 207 +- .../chat_completion_stream_tool.json | 168 +- lib/llm/tests/deepseek_v4_encoding.rs | 100 + lib/llm/tests/test_streaming_tool_parsers.rs | 10 - 20 files changed, 1447 insertions(+), 4566 deletions(-) create mode 100644 lib/llm/src/preprocessor/prompt/deepseek_v4.rs create mode 100644 lib/llm/tests/data/deepseek-v4/test_input_1.json create mode 100644 lib/llm/tests/data/deepseek-v4/test_input_2.json create mode 100644 lib/llm/tests/data/deepseek-v4/test_input_3.json create mode 100644 lib/llm/tests/data/deepseek-v4/test_input_4.json create mode 100644 lib/llm/tests/data/deepseek-v4/test_output_1.txt create mode 100644 lib/llm/tests/data/deepseek-v4/test_output_2.txt create mode 100644 lib/llm/tests/data/deepseek-v4/test_output_3.txt create mode 100644 lib/llm/tests/data/deepseek-v4/test_output_4.txt delete mode 100644 lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_no_params.json create mode 100644 lib/llm/tests/deepseek_v4_encoding.rs diff --git a/lib/llm/src/preprocessor/prompt.rs b/lib/llm/src/preprocessor/prompt.rs index 105e67c6c0d8..8b1f5fde437e 100644 --- a/lib/llm/src/preprocessor/prompt.rs +++ b/lib/llm/src/preprocessor/prompt.rs @@ -26,6 +26,7 @@ use std::sync::Arc; use crate::preprocessor::media::MediaDecoder; pub mod deepseek_v32; +pub mod deepseek_v4; mod template; pub use template::{ChatTemplate, ContextMixins}; diff --git a/lib/llm/src/preprocessor/prompt/deepseek_v4.rs b/lib/llm/src/preprocessor/prompt/deepseek_v4.rs new file mode 100644 index 000000000000..5cdb7694a819 --- /dev/null +++ b/lib/llm/src/preprocessor/prompt/deepseek_v4.rs @@ -0,0 +1,988 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! DeepSeek V4 native prompt formatting +//! +//! Native Rust port of DeepSeek V4's chat encoding (encoding_dsv4.py). +//! +//! Reference: DeepSeek-V4-Pro/encoding/encoding_dsv4.py + +use anyhow::{Context, Result}; +use serde_json::Value as JsonValue; + +/// Special tokens for DeepSeek V4 +pub mod tokens { + pub const BOS: &str = "<｜begin▁of▁sentence｜>"; + pub const EOS: &str = "<｜end▁of▁sentence｜>"; + pub const THINKING_START: &str = ""; + pub const THINKING_END: &str = ""; + pub const DSML_TOKEN: &str = "｜DSML｜"; + pub const USER_START: &str = "<｜User｜>"; + pub const ASSISTANT_START: &str = "<｜Assistant｜>"; + pub const LATEST_REMINDER: &str = "<｜latest_reminder｜>"; + + // Quick-instruction task tokens + pub const TASK_ACTION: &str = "<｜action｜>"; + pub const TASK_QUERY: &str = "<｜query｜>"; + pub const TASK_AUTHORITY: &str = "<｜authority｜>"; + pub const TASK_DOMAIN: &str = "<｜domain｜>"; + pub const TASK_TITLE: &str = "<｜title｜>"; + pub const TASK_READ_URL: &str = "<｜read_url｜>"; +} + +const TOOL_CALLS_BLOCK_NAME: &str = "tool_calls"; + +const RESPONSE_FORMAT_TEMPLATE: &str = + "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{schema}"; + +const TOOLS_TEMPLATE: &str = r#"## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml_token}tool_calls>" block like the following: + +<{dsml_token}tool_calls> +<{dsml_token}invoke name="$TOOL_NAME"> +<{dsml_token}parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<{dsml_token}invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by {thinking_start_token}), you MUST output your complete reasoning inside {thinking_start_token}...{thinking_end_token} BEFORE any tool calls or final response. + +Otherwise, output directly after {thinking_end_token} with tool calls or final response. + +### Available Tool Schemas + +{tool_schemas} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +"#; + +const REASONING_EFFORT_MAX: &str = "Reasoning Effort: Absolute maximum with no shortcuts permitted.\nYou MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\nExplicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n"; + +/// Thinking mode for the model +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ThinkingMode { + Chat, + Thinking, +} + +impl ThinkingMode { + pub fn as_str(&self) -> &'static str { + match self { + ThinkingMode::Chat => "chat", + ThinkingMode::Thinking => "thinking", + } + } +} + +/// Reasoning effort level. `None` conveyed as `Option`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ReasoningEffort { + Max, + High, +} + +/// Serialize a JSON value to match Python's `json.dumps(ensure_ascii=False)` spacing. +/// Python inserts a space after every `:` and `,` outside of strings. +fn to_json(value: &JsonValue) -> String { + let compact = serde_json::to_string(value).unwrap_or_else(|_| "{}".to_string()); + + let mut result = String::with_capacity(compact.len() + compact.len() / 4); + let mut in_string = false; + let mut prev_char = '\0'; + + for ch in compact.chars() { + if ch == '"' && prev_char != '\\' { + in_string = !in_string; + } + + result.push(ch); + + if !in_string && (ch == ':' || ch == ',') { + result.push(' '); + } + + prev_char = ch; + } + + result +} + +/// Extract function definitions from OpenAI-format tool list. +fn tools_from_openai_format(tools: &[JsonValue]) -> Vec { + tools + .iter() + .filter_map(|tool| tool.get("function").cloned()) + .collect() +} + +/// Render tool schemas into the system prompt format. +fn render_tools(tools: &[JsonValue]) -> String { + let tools_json: Vec = tools_from_openai_format(tools) + .iter() + .map(to_json) + .collect(); + + TOOLS_TEMPLATE + .replace("{tool_schemas}", &tools_json.join("\n")) + .replace("{dsml_token}", tokens::DSML_TOKEN) + .replace("{thinking_start_token}", tokens::THINKING_START) + .replace("{thinking_end_token}", tokens::THINKING_END) +} + +/// Find the index of the last user/developer message. +fn find_last_user_index(messages: &[JsonValue]) -> Option { + messages + .iter() + .enumerate() + .rev() + .find(|(_, msg)| { + msg.get("role") + .and_then(|r| r.as_str()) + .map(|r| r == "user" || r == "developer") + .unwrap_or(false) + }) + .map(|(idx, _)| idx) +} + +/// Extract visible text from OpenAI-style message content. +fn extract_visible_text(content: &JsonValue) -> String { + match content { + JsonValue::String(text) => text.clone(), + JsonValue::Array(items) => items + .iter() + .filter_map(|item| { + if let Some(text) = item.as_str() { + return Some(text.to_string()); + } + let item_type = item.get("type").and_then(|v| v.as_str()); + if item_type == Some("text") { + return item + .get("text") + .and_then(|v| v.as_str()) + .map(|text| text.to_string()); + } + tracing::warn!( + chunk_type = item_type.unwrap_or("unknown"), + "DeepSeek V4 formatter dropped non-text content chunk while normalizing message content", + ); + None + }) + .collect::(), + _ => to_json(content), + } +} + +/// Normalize message `content` fields for text-only DeepSeek V4 rendering. +fn normalize_message_contents(messages: &mut [JsonValue]) { + for msg in messages { + let Some(content) = msg.get("content") else { + continue; + }; + // Leave non-string/non-array content untouched (null, etc.) + if !content.is_string() && !content.is_array() { + continue; + } + let normalized = extract_visible_text(content); + if let Some(obj) = msg.as_object_mut() { + obj.insert("content".to_string(), JsonValue::String(normalized)); + } + } +} + +/// Encode tool call arguments into DSML parameter format. +fn encode_arguments_to_dsml(tool_call: &JsonValue) -> Result { + let arguments_str = tool_call + .get("arguments") + .and_then(|a| a.as_str()) + .context("Missing or invalid 'arguments' field")?; + + // Python falls back to `{"arguments": raw_string}` on parse failure. + let arguments: JsonValue = match serde_json::from_str(arguments_str) { + Ok(v) => v, + Err(_) => serde_json::json!({ "arguments": arguments_str }), + }; + + let arguments_obj = arguments + .as_object() + .context("Arguments must be a JSON object")?; + + let mut params = Vec::new(); + for (key, value) in arguments_obj { + let is_string = value.is_string(); + let value_str = if is_string { + value.as_str().unwrap().to_string() + } else { + to_json(value) + }; + params.push(format!( + "<{}parameter name=\"{}\" string=\"{}\">{}", + tokens::DSML_TOKEN, + key, + if is_string { "true" } else { "false" }, + value_str, + tokens::DSML_TOKEN + )); + } + + Ok(params.join("\n")) +} + +/// Lookup the task token for a quick-instruction task. +fn task_token(task: &str) -> Option<&'static str> { + match task { + "action" => Some(tokens::TASK_ACTION), + "query" => Some(tokens::TASK_QUERY), + "authority" => Some(tokens::TASK_AUTHORITY), + "domain" => Some(tokens::TASK_DOMAIN), + "title" => Some(tokens::TASK_TITLE), + "read_url" => Some(tokens::TASK_READ_URL), + _ => None, + } +} + +/// Render a single message at the given index. +fn render_message( + index: usize, + messages: &[JsonValue], + thinking_mode: ThinkingMode, + drop_thinking: bool, + reasoning_effort: Option, +) -> Result { + let msg = &messages[index]; + let last_user_idx = find_last_user_index(messages); + + let role = msg + .get("role") + .and_then(|r| r.as_str()) + .context("Missing 'role' field")?; + + let mut prompt = String::new(); + + // Reasoning effort prefix (only at index 0 in thinking mode with max effort). + if index == 0 + && thinking_mode == ThinkingMode::Thinking + && reasoning_effort == Some(ReasoningEffort::Max) + { + prompt.push_str(REASONING_EFFORT_MAX); + } + + match role { + "system" => { + let content = msg.get("content").and_then(|c| c.as_str()).unwrap_or(""); + prompt.push_str(content); + if let Some(tools) = msg.get("tools").and_then(|t| t.as_array()) { + prompt.push_str("\n\n"); + prompt.push_str(&render_tools(tools)); + } + if let Some(response_format) = msg.get("response_format") { + prompt.push_str("\n\n"); + prompt.push_str( + &RESPONSE_FORMAT_TEMPLATE.replace("{schema}", &to_json(response_format)), + ); + } + } + + "developer" => { + let content = msg + .get("content") + .and_then(|c| c.as_str()) + .filter(|s| !s.is_empty()) + .context("Developer role requires content")?; + + let mut content_developer = String::from(tokens::USER_START); + content_developer.push_str(content); + + if let Some(tools) = msg.get("tools").and_then(|t| t.as_array()) { + content_developer.push_str("\n\n"); + content_developer.push_str(&render_tools(tools)); + } + if let Some(response_format) = msg.get("response_format") { + content_developer.push_str("\n\n"); + content_developer.push_str( + &RESPONSE_FORMAT_TEMPLATE.replace("{schema}", &to_json(response_format)), + ); + } + prompt.push_str(&content_developer); + } + + "user" => { + prompt.push_str(tokens::USER_START); + if let Some(blocks) = msg.get("content_blocks").and_then(|b| b.as_array()) { + let mut parts: Vec = Vec::with_capacity(blocks.len()); + for block in blocks { + let block_type = block.get("type").and_then(|v| v.as_str()).unwrap_or(""); + match block_type { + "text" => { + let text = block.get("text").and_then(|v| v.as_str()).unwrap_or(""); + parts.push(text.to_string()); + } + "tool_result" => { + let rendered = render_tool_result_content( + block.get("content").unwrap_or(&JsonValue::Null), + ); + parts.push(format!("{}", rendered)); + } + other => { + parts.push(format!("[Unsupported {}]", other)); + } + } + } + prompt.push_str(&parts.join("\n\n")); + } else { + let content = msg.get("content").and_then(|c| c.as_str()).unwrap_or(""); + prompt.push_str(content); + } + } + + "latest_reminder" => { + let content = msg.get("content").and_then(|c| c.as_str()).unwrap_or(""); + prompt.push_str(tokens::LATEST_REMINDER); + prompt.push_str(content); + } + + "tool" => { + anyhow::bail!( + "deepseek_v4 merges tool messages into user; preprocess with merge_tool_messages()" + ); + } + + "assistant" => { + let content = msg.get("content").and_then(|c| c.as_str()).unwrap_or(""); + let reasoning = msg + .get("reasoning_content") + .and_then(|c| c.as_str()) + .unwrap_or(""); + let wo_eos = msg + .get("wo_eos") + .and_then(|v| v.as_bool()) + .unwrap_or(false); + + let prev_has_task = index > 0 + && messages[index - 1] + .get("task") + .map(|v| !v.is_null()) + .unwrap_or(false); + + let mut thinking_part = String::new(); + if thinking_mode == ThinkingMode::Thinking && !prev_has_task { + let render_thinking = !drop_thinking + || last_user_idx.is_some_and(|u| index > u); + if render_thinking { + thinking_part.push_str(reasoning); + thinking_part.push_str(tokens::THINKING_END); + } + } + + prompt.push_str(&thinking_part); + prompt.push_str(content); + + if let Some(tool_calls) = msg.get("tool_calls").and_then(|t| t.as_array()) + && !tool_calls.is_empty() + { + prompt.push_str("\n\n"); + prompt.push_str(&format!( + "<{}{}>\n", + tokens::DSML_TOKEN, + TOOL_CALLS_BLOCK_NAME + )); + + let mut invocations = Vec::with_capacity(tool_calls.len()); + for tc in tool_calls { + // Accept both OpenAI-format (nested `function`) and internal + // `{name, arguments}` shape, matching Python's `tool_calls_from_openai_format`. + let fn_obj = tc.get("function").unwrap_or(tc); + let name = fn_obj + .get("name") + .and_then(|n| n.as_str()) + .context("Missing tool call name")?; + let arguments = encode_arguments_to_dsml(fn_obj)?; + invocations.push(format!( + "<{}invoke name=\"{}\">\n{}\n", + tokens::DSML_TOKEN, + name, + arguments, + tokens::DSML_TOKEN + )); + } + prompt.push_str(&invocations.join("\n")); + prompt.push_str(&format!( + "\n", + tokens::DSML_TOKEN, + TOOL_CALLS_BLOCK_NAME + )); + } + + if !wo_eos { + prompt.push_str(tokens::EOS); + } + } + + other => anyhow::bail!("Unknown role: {}", other), + } + + // Early return if the next message is not assistant/latest_reminder — no transition appended. + if index + 1 < messages.len() { + let next_role = messages[index + 1].get("role").and_then(|r| r.as_str()); + if !matches!(next_role, Some("assistant") | Some("latest_reminder")) { + return Ok(prompt); + } + } + + // Transition tokens based on task field and role. + let task = msg.get("task").and_then(|v| v.as_str()); + if let Some(task) = task { + let sp = task_token(task) + .with_context(|| format!("Invalid task: '{}'", task))?; + if task != "action" { + prompt.push_str(sp); + } else { + prompt.push_str(tokens::ASSISTANT_START); + prompt.push_str(if thinking_mode != ThinkingMode::Thinking { + tokens::THINKING_END + } else { + tokens::THINKING_START + }); + prompt.push_str(sp); + } + } else if matches!(role, "user" | "developer") { + prompt.push_str(tokens::ASSISTANT_START); + let seed_thinking = thinking_mode == ThinkingMode::Thinking + && (!drop_thinking || last_user_idx.is_some_and(|u| index >= u)); + prompt.push_str(if seed_thinking { + tokens::THINKING_START + } else { + tokens::THINKING_END + }); + } + + Ok(prompt) +} + +/// Render a tool_result `content` payload (string or content-block list). +fn render_tool_result_content(content: &JsonValue) -> String { + match content { + JsonValue::String(s) => s.clone(), + JsonValue::Array(items) => { + let mut parts: Vec = Vec::with_capacity(items.len()); + for item in items { + let item_type = item.get("type").and_then(|v| v.as_str()).unwrap_or(""); + if item_type == "text" { + parts.push( + item.get("text") + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(), + ); + } else { + parts.push(format!("[Unsupported {}]", item_type)); + } + } + parts.join("\n\n") + } + JsonValue::Null => String::new(), + _ => to_json(content), + } +} + +/// Merge `tool` role messages into preceding user `content_blocks` and collapse +/// consecutive user turns, matching Python's `merge_tool_messages`. +pub fn merge_tool_messages(messages: &[JsonValue]) -> Vec { + let mut merged: Vec = Vec::with_capacity(messages.len()); + + for msg in messages { + let msg = msg.clone(); + let role = msg.get("role").and_then(|r| r.as_str()).unwrap_or(""); + + if role == "tool" { + let tool_block = serde_json::json!({ + "type": "tool_result", + "tool_use_id": msg.get("tool_call_id").cloned().unwrap_or(JsonValue::String(String::new())), + "content": msg.get("content").cloned().unwrap_or(JsonValue::String(String::new())), + }); + + let can_merge = merged + .last() + .map(|m| { + m.get("role").and_then(|r| r.as_str()) == Some("user") + && m.get("content_blocks").is_some() + }) + .unwrap_or(false); + + if can_merge { + let last = merged.last_mut().unwrap(); + if let Some(blocks) = last + .as_object_mut() + .and_then(|o| o.get_mut("content_blocks")) + .and_then(|v| v.as_array_mut()) + { + blocks.push(tool_block); + } + } else { + merged.push(serde_json::json!({ + "role": "user", + "content_blocks": [tool_block], + })); + } + } else if role == "user" { + let text = msg + .get("content") + .and_then(|c| c.as_str()) + .unwrap_or("") + .to_string(); + let text_block = serde_json::json!({ "type": "text", "text": text }); + + let can_merge = merged + .last() + .map(|m| { + m.get("role").and_then(|r| r.as_str()) == Some("user") + && m.get("content_blocks").is_some() + && m.get("task").map(|v| v.is_null()).unwrap_or(true) + }) + .unwrap_or(false); + + if can_merge { + let last = merged.last_mut().unwrap(); + if let Some(blocks) = last + .as_object_mut() + .and_then(|o| o.get_mut("content_blocks")) + .and_then(|v| v.as_array_mut()) + { + blocks.push(text_block); + } + } else { + let mut new_msg = serde_json::json!({ + "role": "user", + "content": text, + "content_blocks": [text_block], + }); + // Preserve extra fields. + if let Some(obj) = new_msg.as_object_mut() { + for key in ["task", "wo_eos", "mask"] { + if let Some(v) = msg.get(key) { + obj.insert(key.to_string(), v.clone()); + } + } + } + merged.push(new_msg); + } + } else { + merged.push(msg); + } + } + + merged +} + +/// Sort `tool_result` blocks within user messages by the `tool_calls[].id` order +/// of the preceding assistant message. +pub fn sort_tool_results_by_call_order(mut messages: Vec) -> Vec { + use std::collections::HashMap; + let mut last_order: HashMap = HashMap::new(); + + for msg in &mut messages { + let role = msg.get("role").and_then(|r| r.as_str()).unwrap_or(""); + if role == "assistant" { + if let Some(tcs) = msg.get("tool_calls").and_then(|t| t.as_array()) { + last_order.clear(); + for (idx, tc) in tcs.iter().enumerate() { + let id = tc + .get("id") + .and_then(|v| v.as_str()) + .or_else(|| { + tc.get("function") + .and_then(|f| f.get("id")) + .and_then(|v| v.as_str()) + }) + .unwrap_or(""); + if !id.is_empty() { + last_order.insert(id.to_string(), idx); + } + } + } + } else if role == "user" && !last_order.is_empty() { + let Some(blocks) = msg + .as_object_mut() + .and_then(|o| o.get_mut("content_blocks")) + .and_then(|v| v.as_array_mut()) + else { + continue; + }; + + // Collect tool_result blocks with their positions. + let tool_positions: Vec = blocks + .iter() + .enumerate() + .filter(|(_, b)| b.get("type").and_then(|v| v.as_str()) == Some("tool_result")) + .map(|(i, _)| i) + .collect(); + + if tool_positions.len() > 1 { + let mut tool_blocks: Vec = tool_positions + .iter() + .map(|&i| blocks[i].clone()) + .collect(); + tool_blocks.sort_by_key(|b| { + let id = b.get("tool_use_id").and_then(|v| v.as_str()).unwrap_or(""); + *last_order.get(id).unwrap_or(&0) + }); + for (sorted_idx, &pos) in tool_positions.iter().enumerate() { + blocks[pos] = tool_blocks[sorted_idx].clone(); + } + } + } + } + + messages +} + +/// Drop reasoning and non-essential messages before the last user message. +fn drop_thinking_messages(messages: Vec) -> Vec { + let last_user_idx = find_last_user_index(&messages).unwrap_or(usize::MAX); + let mut out = Vec::with_capacity(messages.len()); + const KEEP: &[&str] = &[ + "user", + "system", + "tool", + "latest_reminder", + "direct_search_results", + ]; + + for (idx, mut msg) in messages.into_iter().enumerate() { + let role = msg.get("role").and_then(|r| r.as_str()).unwrap_or(""); + if KEEP.contains(&role) || idx >= last_user_idx { + out.push(msg); + } else if role == "assistant" { + if let Some(obj) = msg.as_object_mut() { + obj.remove("reasoning_content"); + } + out.push(msg); + } + // developer and other roles before last_user_idx are dropped. + } + out +} + +/// Encode messages to prompt string with default options. +/// +/// Equivalent to `encode_messages_with_options(.., drop_thinking=true, reasoning_effort=None)`. +pub fn encode_messages( + messages: &[JsonValue], + thinking_mode: ThinkingMode, + add_bos_token: bool, +) -> Result { + encode_messages_with_options(messages, thinking_mode, add_bos_token, true, None) +} + +/// Encode messages to prompt string. +/// +/// # Arguments +/// * `messages` - Array of messages in OpenAI format +/// * `thinking_mode` - Chat or Thinking +/// * `add_bos_token` - Whether to prepend BOS token +/// * `drop_thinking` - Drop reasoning_content from earlier turns (auto-disabled if tools present) +/// * `reasoning_effort` - Optional reasoning effort level (Max prepends a verbatim block) +pub fn encode_messages_with_options( + messages: &[JsonValue], + thinking_mode: ThinkingMode, + add_bos_token: bool, + drop_thinking: bool, + reasoning_effort: Option, +) -> Result { + let merged = merge_tool_messages(messages); + let mut full = sort_tool_results_by_call_order(merged); + + let mut prompt = String::new(); + if add_bos_token { + prompt.push_str(tokens::BOS); + } + + // Auto-disable drop_thinking when any message carries a `tools` field. + let has_tools = full.iter().any(|m| { + m.get("tools") + .map(|v| match v { + JsonValue::Array(a) => !a.is_empty(), + JsonValue::Null => false, + _ => true, + }) + .unwrap_or(false) + }); + let effective_drop_thinking = drop_thinking && !has_tools; + + if thinking_mode == ThinkingMode::Thinking && effective_drop_thinking { + full = drop_thinking_messages(full); + } + + for idx in 0..full.len() { + let part = render_message( + idx, + &full, + thinking_mode, + effective_drop_thinking, + reasoning_effort, + )?; + prompt.push_str(&part); + } + + Ok(prompt) +} + +/// DeepSeek V4 Prompt Formatter +#[derive(Debug)] +pub struct DeepSeekV4Formatter { + thinking_mode: ThinkingMode, +} + +impl DeepSeekV4Formatter { + pub fn new(thinking_mode: ThinkingMode) -> Self { + Self { thinking_mode } + } + + /// Create formatter with thinking mode enabled (default for DSV4) + pub fn new_thinking() -> Self { + Self::new(ThinkingMode::Thinking) + } + + /// Create formatter with chat mode + pub fn new_chat() -> Self { + Self::new(ThinkingMode::Chat) + } + + fn resolve_thinking_mode( + &self, + args: Option<&std::collections::HashMap>, + ) -> ThinkingMode { + if let Some(args) = args { + if let Some(thinking) = args.get("thinking").and_then(|v| v.as_bool()) { + return if thinking { + ThinkingMode::Thinking + } else { + ThinkingMode::Chat + }; + } + if let Some(mode) = args.get("thinking_mode").and_then(|v| v.as_str()) { + match mode { + "chat" => return ThinkingMode::Chat, + "thinking" => return ThinkingMode::Thinking, + _ => {} + } + } + } + self.thinking_mode + } +} + +impl super::OAIPromptFormatter for DeepSeekV4Formatter { + fn supports_add_generation_prompt(&self) -> bool { + true + } + + fn render(&self, req: &dyn super::OAIChatLikeRequest) -> Result { + let thinking_mode = self.resolve_thinking_mode(req.chat_template_args()); + + let messages_value = req.messages(); + let messages_json = + serde_json::to_value(&messages_value).context("Failed to convert messages to JSON")?; + + let mut messages_array = messages_json + .as_array() + .context("Messages is not an array")? + .clone(); + + normalize_message_contents(&mut messages_array); + + let tools_json = req + .tools() + .map(|t| serde_json::to_value(&t)) + .transpose() + .context("Failed to convert tools to JSON")?; + + let response_format_json = req + .response_format() + .map(|rf| serde_json::to_value(&rf)) + .transpose() + .context("Failed to convert response_format to JSON")?; + + if tools_json.is_some() || response_format_json.is_some() { + let system_idx = messages_array + .iter() + .position(|msg| msg.get("role").and_then(|r| r.as_str()) == Some("system")); + + if let Some(idx) = system_idx { + if let Some(msg) = messages_array.get_mut(idx) + && let Some(obj) = msg.as_object_mut() + { + if let Some(tools) = tools_json { + obj.insert("tools".to_string(), tools); + } + if let Some(rf) = response_format_json { + obj.insert("response_format".to_string(), rf); + } + } + } else { + let mut system_msg = serde_json::json!({ + "role": "system", + "content": "" + }); + if let Some(obj) = system_msg.as_object_mut() { + if let Some(tools) = tools_json { + obj.insert("tools".to_string(), tools); + } + if let Some(rf) = response_format_json { + obj.insert("response_format".to_string(), rf); + } + } + messages_array.insert(0, system_msg); + } + } + + encode_messages(&messages_array, thinking_mode, true) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn test_simple_conversation() { + let messages = json!([ + {"role": "system", "content": "You are a helpful assistant."}, + {"role": "user", "content": "Hello"}, + {"role": "assistant", "reasoning_content": "greet", "content": "Hi!"}, + {"role": "user", "content": "What is 2+2?"} + ]); + let out = encode_messages(messages.as_array().unwrap(), ThinkingMode::Thinking, true) + .unwrap(); + assert!(out.starts_with(tokens::BOS)); + assert!(out.ends_with(&format!("{}{}", tokens::ASSISTANT_START, tokens::THINKING_START))); + // drop_thinking default true → earlier reasoning stripped + assert!(!out.contains("greet")); + } + + #[test] + fn test_thinking_with_tools() { + let raw = std::fs::read_to_string( + "../../DeepSeek-V4-Pro/encoding/tests/test_input_1.json", + ) + .or_else(|_| { + std::fs::read_to_string( + "/home/ayush-lab/Work/dynamo/DeepSeek-V4-Pro/encoding/tests/test_input_1.json", + ) + }); + let Ok(data) = raw else { return }; + let parsed: JsonValue = serde_json::from_str(&data).unwrap(); + let mut messages = parsed.get("messages").unwrap().as_array().unwrap().clone(); + // Inject tools onto the system message (as the request layer would). + let tools = parsed.get("tools").unwrap().clone(); + messages[0] + .as_object_mut() + .unwrap() + .insert("tools".to_string(), tools); + let out = + encode_messages(&messages, ThinkingMode::Thinking, true).unwrap(); + let expected = std::fs::read_to_string( + "/home/ayush-lab/Work/dynamo/DeepSeek-V4-Pro/encoding/tests/test_output_1.txt", + ) + .unwrap(); + assert_eq!(out, expected); + } + + #[test] + fn test_latest_reminder_multi_turn() { + let data = std::fs::read_to_string( + "/home/ayush-lab/Work/dynamo/DeepSeek-V4-Pro/encoding/tests/test_input_3.json", + ); + let Ok(data) = data else { return }; + let messages: Vec = serde_json::from_str(&data).unwrap(); + let out = encode_messages(&messages, ThinkingMode::Thinking, true).unwrap(); + let expected = std::fs::read_to_string( + "/home/ayush-lab/Work/dynamo/DeepSeek-V4-Pro/encoding/tests/test_output_3.txt", + ) + .unwrap(); + assert_eq!(out, expected); + } + + #[test] + fn test_chat_mode_with_action_task() { + let data = std::fs::read_to_string( + "/home/ayush-lab/Work/dynamo/DeepSeek-V4-Pro/encoding/tests/test_input_4.json", + ); + let Ok(data) = data else { return }; + let messages: Vec = serde_json::from_str(&data).unwrap(); + let out = encode_messages(&messages, ThinkingMode::Chat, true).unwrap(); + let expected = std::fs::read_to_string( + "/home/ayush-lab/Work/dynamo/DeepSeek-V4-Pro/encoding/tests/test_output_4.txt", + ) + .unwrap(); + assert_eq!(out, expected); + } + + #[test] + fn test_reasoning_effort_max_prefix() { + let messages = json!([ + {"role": "system", "content": "hi"}, + {"role": "user", "content": "hello"} + ]); + let out = encode_messages_with_options( + messages.as_array().unwrap(), + ThinkingMode::Thinking, + true, + true, + Some(ReasoningEffort::Max), + ) + .unwrap(); + assert!(out.contains("Reasoning Effort: Absolute maximum")); + // Prefix comes between BOS and system content. + let after_bos = &out[tokens::BOS.len()..]; + assert!(after_bos.starts_with("Reasoning Effort:")); + + // High and None do not emit the prefix. + let out2 = encode_messages_with_options( + messages.as_array().unwrap(), + ThinkingMode::Thinking, + true, + true, + Some(ReasoningEffort::High), + ) + .unwrap(); + assert!(!out2.contains("Reasoning Effort: Absolute maximum")); + } + + #[test] + fn test_content_blocks_with_tool_result() { + let messages = json!([ + {"role": "user", "content_blocks": [ + {"type": "text", "text": "prefix"}, + {"type": "tool_result", "tool_use_id": "c1", "content": "RESULT"}, + {"type": "text", "text": "suffix"} + ]} + ]); + let out = encode_messages(messages.as_array().unwrap(), ThinkingMode::Chat, false) + .unwrap(); + assert!(out.contains("prefix\n\nRESULT\n\nsuffix")); + } + + #[test] + fn test_drop_thinking_auto_disable_when_tools_present() { + let messages = json!([ + {"role": "system", "content": "s", "tools": [{ + "type": "function", + "function": {"name": "f", "description": "", "parameters": {"type": "object", "properties": {}}} + }]}, + {"role": "user", "content": "hi"}, + {"role": "assistant", "reasoning_content": "PRIOR_REASONING", "content": "reply"}, + {"role": "user", "content": "again"} + ]); + let out = encode_messages(messages.as_array().unwrap(), ThinkingMode::Thinking, true) + .unwrap(); + // Tools present → drop_thinking auto-disabled → earlier reasoning preserved. + assert!(out.contains("PRIOR_REASONING")); + } +} diff --git a/lib/llm/tests/data/deepseek-v4/test_input_1.json b/lib/llm/tests/data/deepseek-v4/test_input_1.json new file mode 100644 index 000000000000..de0f6b8b2966 --- /dev/null +++ b/lib/llm/tests/data/deepseek-v4/test_input_1.json @@ -0,0 +1 @@ +{"tools":[{"type":"function","function":{"name":"get_weather","description":"Get the weather for a specific location","parameters":{"type":"object","properties":{"location":{"type":"string","description":"The city name"},"unit":{"type":"string","enum":["celsius","fahrenheit"],"description":"Temperature unit"}},"required":["location"]}}},{"type":"function","function":{"name":"search","description":"Search the web for information","parameters":{"type":"object","properties":{"query":{"type":"string","description":"Search query"},"num_results":{"type":"integer","description":"Number of results to return"}},"required":["query"]}}}],"messages":[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"What's the weather in Beijing?"},{"role":"assistant","reasoning_content":"The user wants to know the weather in Beijing. I should use the get_weather tool.","tool_calls":[{"id":"call_001","type":"function","function":{"name":"get_weather","arguments":"{\"location\": \"Beijing\", \"unit\": \"celsius\"}"}}]},{"role":"tool","tool_call_id":"call_001","content":"{\"temperature\": 22, \"condition\": \"sunny\", \"humidity\": 45}"},{"role":"assistant","reasoning_content":"Got the weather data. Let me format a nice response.","content":"The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity."}]} diff --git a/lib/llm/tests/data/deepseek-v4/test_input_2.json b/lib/llm/tests/data/deepseek-v4/test_input_2.json new file mode 100644 index 000000000000..81a0a589a717 --- /dev/null +++ b/lib/llm/tests/data/deepseek-v4/test_input_2.json @@ -0,0 +1 @@ +[{"role":"system","content":"You are a helpful assistant."},{"role":"user","content":"Hello"},{"role":"assistant","reasoning_content":"The user said hello, I should greet back.","content":"Hi there! How can I help you?"},{"role":"user","content":"What is the capital of France?"},{"role":"assistant","reasoning_content":"The user asks about the capital of France. It is Paris.","content":"The capital of France is Paris."}] diff --git a/lib/llm/tests/data/deepseek-v4/test_input_3.json b/lib/llm/tests/data/deepseek-v4/test_input_3.json new file mode 100644 index 000000000000..3468700632b4 --- /dev/null +++ b/lib/llm/tests/data/deepseek-v4/test_input_3.json @@ -0,0 +1 @@ +[{"role":"system","content":"该助手为DeepSeek，由深度求索公司创造。"},{"role":"latest_reminder","content":"2026-02-21,星期六,广州,App,中文"},{"role":"developer","content":"小柴胡冲剂和布洛芬能一起吃吗？\n\nCITATION FORMAT: 【{cursor_id}†L{start_line_id}(-L{end_line_id})?】","tools":[{"type":"function","function":{"name":"search","description":"Web search. Split multiple queries with '||'.","parameters":{"type":"object","properties":{"queries":{"type":"string","description":"query1||query2"}},"required":["queries"],"additionalProperties":false,"$schema":"http://json-schema.org/draft-07/schema#"}}},{"type":"function","function":{"name":"open","description":"Batch open IDs (format 【{id}†...】) or URLs.","parameters":{"type":"object","properties":{"open_list":{"type":"array","items":{"type":"object","properties":{"id":{"description":"ID or URL","anyOf":[{"type":"integer"},{"type":"string"}],"default":-1},"cursor":{"type":"integer","description":"","default":-1},"loc":{"type":"integer","description":"Start line","default":-1},"num_lines":{"type":"integer","description":"","default":-1},"view_source":{"type":"boolean","description":"","default":false}},"additionalProperties":false},"description":""}},"required":["open_list"],"additionalProperties":false,"$schema":"http://json-schema.org/draft-07/schema#"}}},{"type":"function","function":{"name":"find","description":"Find exact text pattern in pages.","parameters":{"type":"object","properties":{"find_list":{"type":"array","items":{"type":"object","properties":{"pattern":{"type":"string","description":""},"cursor":{"type":"integer","description":"","default":-1}},"required":["pattern"],"additionalProperties":false},"description":""}},"required":["find_list"],"additionalProperties":false,"$schema":"http://json-schema.org/draft-07/schema#"}}}]},{"role":"assistant","content":"","reasoning_content":"用户想知道小柴胡冲剂和布洛芬能否一起服用。","tool_calls":[{"type":"function","function":{"name":"search","arguments":"{\"queries\": \"小柴胡冲剂 布洛芬 相互作用 一起吃\"}"}}]},{"role":"tool","content":"[0]"},{"role":"assistant","content":"请及时就医。","reasoning_content":"现在开始组织回答。","tool_calls":[]}] diff --git a/lib/llm/tests/data/deepseek-v4/test_input_4.json b/lib/llm/tests/data/deepseek-v4/test_input_4.json new file mode 100644 index 000000000000..34feb3f2d896 --- /dev/null +++ b/lib/llm/tests/data/deepseek-v4/test_input_4.json @@ -0,0 +1 @@ +[{"role":"system","content":"该助手为DeepSeek-V3，由深度求索公司创造。\n今天是2025年10月17日，星期五。"},{"role":"latest_reminder","content":"2024-11-15,上海市,App,中文"},{"role":"user","content":"热海大滚锅是世界著名温泉吗"},{"role":"assistant","content":"关于热海大滚锅是否是世界著名温泉，可以这样总结：它在**中国乃至全球的地热奇观中占有重要地位**，但“世界著名”的称号更侧重于它作为**独特的地质现象和旅游景点**，而非普遍的温泉疗养体验。\n\n为了让你快速了解，我整理了一个简要的表格：\n\n| 维度 | 热海大滚锅的地位与特点 |\n| :--- | :--- |\n| **地质奇观** | **世界罕见**的高温地热系统，泉眼多、水温高、形态多样。 |\n| **国内声誉** | **中国三大地热区之一**，**国家5A级旅游景区**，自古闻名（徐霞客曾游历并记载）。 |\n| **国际知名度** | 在一些旅游平台被提及为“世界六大温泉”之一，但此说法流传不广，其国际声誉更多建立在地质独特性上。 |\n| **核心体验** | **观赏地热奇观**（如97℃沸腾的“大滚锅”）、**体验温泉煮鸡蛋**。 |\n\n### 💡 游玩攻略与温馨提示\n\n如果你计划前往热海大滚锅，这里有一些实用信息供你参考：\n\n- **门票与开放时间**：\n - **门票**：景区门票约为**50元/人**。如果选择包含温泉沐浴的套餐，价格会更高，例如约**288元**。\n - **开放时间**：景区一般**08:00-18:00**开放，但具体时间可能变动，建议提前核实。\n\n- **特色体验**：\n - **温泉煮鸡蛋**：这几乎是必试项目。可以在景区门口购买用草绳串起的生鸡蛋（约5-8元/串），然后到“大滚锅”旁的指定区域蒸煮，几分钟便可熟食，趣味十足。\n - **金汤足浴**：可以直接用从“大滚锅”流出的温泉水泡脚，缓解旅途疲劳。\n\n- **注意事项**：\n - **安全第一**：“大滚锅”水温极高，务必遵守游览规则，在指定区域内观赏，切勿随意触碰泉水。\n - **规划行程**：建议为热海景区预留**3-4小时**的游览时间。景区内步道不走回头路，出入口有观光车接送。\n\n希望这些信息能帮助你更好地了解热海大滚锅。如果你对腾冲的其他景点或者行程规划有更多疑问，我很乐意提供进一步的信息。","mask":1},{"role":"user","content":"世界著名温泉有哪些","task":"action"},{"role":"assistant","content":"Search"}] diff --git a/lib/llm/tests/data/deepseek-v4/test_output_1.txt b/lib/llm/tests/data/deepseek-v4/test_output_1.txt new file mode 100644 index 000000000000..7e3c9bd5a394 --- /dev/null +++ b/lib/llm/tests/data/deepseek-v4/test_output_1.txt @@ -0,0 +1,36 @@ +<｜begin▁of▁sentence｜>You are a helpful assistant. + +## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<｜DSML｜tool_calls>" block like the following: + +<｜DSML｜tool_calls> +<｜DSML｜invoke name="$TOOL_NAME"> +<｜DSML｜parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<｜DSML｜invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response. + +Otherwise, output directly after with tool calls or final response. + +### Available Tool Schemas + +{"name": "get_weather", "description": "Get the weather for a specific location", "parameters": {"type": "object", "properties": {"location": {"type": "string", "description": "The city name"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"], "description": "Temperature unit"}}, "required": ["location"]}} +{"name": "search", "description": "Search the web for information", "parameters": {"type": "object", "properties": {"query": {"type": "string", "description": "Search query"}, "num_results": {"type": "integer", "description": "Number of results to return"}}, "required": ["query"]}} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +<｜User｜>What's the weather in Beijing?<｜Assistant｜>The user wants to know the weather in Beijing. I should use the get_weather tool. + +<｜DSML｜tool_calls> +<｜DSML｜invoke name="get_weather"> +<｜DSML｜parameter name="location" string="true">Beijing +<｜DSML｜parameter name="unit" string="true">celsius + +<｜end▁of▁sentence｜><｜User｜>{"temperature": 22, "condition": "sunny", "humidity": 45}<｜Assistant｜>Got the weather data. Let me format a nice response.The weather in Beijing is currently sunny with a temperature of 22°C and 45% humidity.<｜end▁of▁sentence｜> \ No newline at end of file diff --git a/lib/llm/tests/data/deepseek-v4/test_output_2.txt b/lib/llm/tests/data/deepseek-v4/test_output_2.txt new file mode 100644 index 000000000000..fc397ef54972 --- /dev/null +++ b/lib/llm/tests/data/deepseek-v4/test_output_2.txt @@ -0,0 +1 @@ +<｜begin▁of▁sentence｜>You are a helpful assistant.<｜User｜>Hello<｜Assistant｜>Hi there! How can I help you?<｜end▁of▁sentence｜><｜User｜>What is the capital of France?<｜Assistant｜>The user asks about the capital of France. It is Paris.The capital of France is Paris.<｜end▁of▁sentence｜> \ No newline at end of file diff --git a/lib/llm/tests/data/deepseek-v4/test_output_3.txt b/lib/llm/tests/data/deepseek-v4/test_output_3.txt new file mode 100644 index 000000000000..edee563300d4 --- /dev/null +++ b/lib/llm/tests/data/deepseek-v4/test_output_3.txt @@ -0,0 +1,38 @@ +<｜begin▁of▁sentence｜>该助手为DeepSeek，由深度求索公司创造。<｜latest_reminder｜>2026-02-21,星期六,广州,App,中文<｜User｜>小柴胡冲剂和布洛芬能一起吃吗？ + +CITATION FORMAT: 【{cursor_id}†L{start_line_id}(-L{end_line_id})?】 + +## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<｜DSML｜tool_calls>" block like the following: + +<｜DSML｜tool_calls> +<｜DSML｜invoke name="$TOOL_NAME"> +<｜DSML｜parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<｜DSML｜invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by ), you MUST output your complete reasoning inside ... BEFORE any tool calls or final response. + +Otherwise, output directly after with tool calls or final response. + +### Available Tool Schemas + +{"name": "search", "description": "Web search. Split multiple queries with '||'.", "parameters": {"type": "object", "properties": {"queries": {"type": "string", "description": "query1||query2"}}, "required": ["queries"], "additionalProperties": false, "$schema": "http://json-schema.org/draft-07/schema#"}} +{"name": "open", "description": "Batch open IDs (format 【{id}†...】) or URLs.", "parameters": {"type": "object", "properties": {"open_list": {"type": "array", "items": {"type": "object", "properties": {"id": {"description": "ID or URL", "anyOf": [{"type": "integer"}, {"type": "string"}], "default": -1}, "cursor": {"type": "integer", "description": "", "default": -1}, "loc": {"type": "integer", "description": "Start line", "default": -1}, "num_lines": {"type": "integer", "description": "", "default": -1}, "view_source": {"type": "boolean", "description": "", "default": false}}, "additionalProperties": false}, "description": ""}}, "required": ["open_list"], "additionalProperties": false, "$schema": "http://json-schema.org/draft-07/schema#"}} +{"name": "find", "description": "Find exact text pattern in pages.", "parameters": {"type": "object", "properties": {"find_list": {"type": "array", "items": {"type": "object", "properties": {"pattern": {"type": "string", "description": ""}, "cursor": {"type": "integer", "description": "", "default": -1}}, "required": ["pattern"], "additionalProperties": false}, "description": ""}}, "required": ["find_list"], "additionalProperties": false, "$schema": "http://json-schema.org/draft-07/schema#"}} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +<｜Assistant｜>用户想知道小柴胡冲剂和布洛芬能否一起服用。 + +<｜DSML｜tool_calls> +<｜DSML｜invoke name="search"> +<｜DSML｜parameter name="queries" string="true">小柴胡冲剂 布洛芬 相互作用 一起吃 + +<｜end▁of▁sentence｜><｜User｜>[0]<｜Assistant｜>现在开始组织回答。请及时就医。<｜end▁of▁sentence｜> \ No newline at end of file diff --git a/lib/llm/tests/data/deepseek-v4/test_output_4.txt b/lib/llm/tests/data/deepseek-v4/test_output_4.txt new file mode 100644 index 000000000000..d30bd5d06cf3 --- /dev/null +++ b/lib/llm/tests/data/deepseek-v4/test_output_4.txt @@ -0,0 +1,29 @@ +<｜begin▁of▁sentence｜>该助手为DeepSeek-V3，由深度求索公司创造。 +今天是2025年10月17日，星期五。<｜latest_reminder｜>2024-11-15,上海市,App,中文<｜User｜>热海大滚锅是世界著名温泉吗<｜Assistant｜>关于热海大滚锅是否是世界著名温泉，可以这样总结：它在**中国乃至全球的地热奇观中占有重要地位**，但“世界著名”的称号更侧重于它作为**独特的地质现象和旅游景点**，而非普遍的温泉疗养体验。 + +为了让你快速了解，我整理了一个简要的表格： + +| 维度 | 热海大滚锅的地位与特点 | +| :--- | :--- | +| **地质奇观** | **世界罕见**的高温地热系统，泉眼多、水温高、形态多样。 | +| **国内声誉** | **中国三大地热区之一**，**国家5A级旅游景区**，自古闻名（徐霞客曾游历并记载）。 | +| **国际知名度** | 在一些旅游平台被提及为“世界六大温泉”之一，但此说法流传不广，其国际声誉更多建立在地质独特性上。 | +| **核心体验** | **观赏地热奇观**（如97℃沸腾的“大滚锅”）、**体验温泉煮鸡蛋**。 | + +### 💡 游玩攻略与温馨提示 + +如果你计划前往热海大滚锅，这里有一些实用信息供你参考： + +- **门票与开放时间**： + - **门票**：景区门票约为**50元/人**。如果选择包含温泉沐浴的套餐，价格会更高，例如约**288元**。 + - **开放时间**：景区一般**08:00-18:00**开放，但具体时间可能变动，建议提前核实。 + +- **特色体验**： + - **温泉煮鸡蛋**：这几乎是必试项目。可以在景区门口购买用草绳串起的生鸡蛋（约5-8元/串），然后到“大滚锅”旁的指定区域蒸煮，几分钟便可熟食，趣味十足。 + - **金汤足浴**：可以直接用从“大滚锅”流出的温泉水泡脚，缓解旅途疲劳。 + +- **注意事项**： + - **安全第一**：“大滚锅”水温极高，务必遵守游览规则，在指定区域内观赏，切勿随意触碰泉水。 + - **规划行程**：建议为热海景区预留**3-4小时**的游览时间。景区内步道不走回头路，出入口有观光车接送。 + +希望这些信息能帮助你更好地了解热海大滚锅。如果你对腾冲的其他景点或者行程规划有更多疑问，我很乐意提供进一步的信息。<｜end▁of▁sentence｜><｜User｜>世界著名温泉有哪些<｜Assistant｜><｜action｜>Search<｜end▁of▁sentence｜> \ No newline at end of file diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_content_before_tool.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_content_before_tool.json index 5fddf247c4fe..087e58bb4a20 100644 --- a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_content_before_tool.json +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_content_before_tool.json @@ -1,182 +1,15 @@ { "request_id": "deepseek-v4-content-before-tool-test", - "expected_output": { - "normal_content": "Let me check the forecast for Tokyo right now.", - "reasoning_content": "The user wants today's weather in Tokyo. I'll acknowledge the request, then call get_weather.", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_weather", - "arguments": "{\"location\": \"Tokyo\", \"unit\": \"celsius\"}" - } - } - ] - }, + "expected_output": {"normal_content": "Let me check the forecast for Tokyo right now.", "reasoning_content": "The user wants today's weather in Tokyo. I'll acknowledge the request, then call get_weather.", "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "get_weather", "arguments": "{\"location\": \"Tokyo\", \"unit\": \"celsius\"}"}}]}, "input_stream": [ - { - "data": { - "id": "chatcmpl-deepseek-v4-content-before-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "The user wants today's weather in Tokyo. I'll acknowledge the request, then call get_weather.", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "The user wants today's weather in Tokyo. I'll acknowledge the request, then call get_weather." - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-content-before-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "Let me check the forecast for Tokyo right now.", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-content-before-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜tool_calls>\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-content-before-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜invoke name=\"get_weather\">\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-content-before-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜parameter name=\"location\" string=\"true\">Tokyo\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-content-before-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜parameter name=\"unit\" string=\"true\">celsius\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-content-before-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-content-before-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-content-before-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": null, - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - }, - "finish_reason": "tool_calls" - } - ] - } - } + {"data":{"id":"chatcmpl-deepseek-v4-content-before-tool","choices":[{"index":0,"delta":{"content":"The user wants today's weather in Tokyo. I'll acknowledge the request, then call get_weather.","role":"assistant","reasoning_content":"The user wants today's weather in Tokyo. I'll acknowledge the request, then call get_weather."}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-content-before-tool","choices":[{"index":0,"delta":{"content":"Let me check the forecast for Tokyo right now.","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-content-before-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜tool_calls>\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-content-before-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜invoke name=\"get_weather\">\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-content-before-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"location\" string=\"true\">Tokyo\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-content-before-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"unit\" string=\"true\">celsius\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-content-before-tool","choices":[{"index":0,"delta":{"content":"\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-content-before-tool","choices":[{"index":0,"delta":{"content":"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-content-before-tool","choices":[{"index":0,"delta":{"content":null,"role":"assistant"},"finish_reason":"tool_calls"}]}} ] } diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_fragmented_tokens.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_fragmented_tokens.json index 8f9ccff8445a..322ec9f2af27 100644 --- a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_fragmented_tokens.json +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_fragmented_tokens.json @@ -1,3422 +1,195 @@ { "request_id": "deepseek-v4-fragmented-tokens-test", - "expected_output": { - "normal_content": "", - "reasoning_content": "Break tokens apart aggressively.", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "ping", - "arguments": "{\"host\": \"example.com\"}" - } - } - ] - }, + "expected_output": {"normal_content": "", "reasoning_content": "Break tokens apart aggressively.", "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "ping", "arguments": "{\"host\": \"example.com\"}"}}]}, "input_stream": [ - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "B", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "B" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "r", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "r" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "e", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "e" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "a", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "a" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "k", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "k" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": " ", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": " " - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "t", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "t" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "o", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "o" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "k", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "k" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "e", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "e" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "n" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "s", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "s" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": " ", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": " " - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "a", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "a" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "p", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "p" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "a", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "a" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "r", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "r" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "t", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "t" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": " ", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": " " - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "a", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "a" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "g", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "g" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "g", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "g" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "r", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "r" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "e", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "e" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "s", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "s" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "s", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "s" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "i", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "i" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "v", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "v" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "e", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "e" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "l", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "l" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "y", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "y" - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": ".", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "." - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "<", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "｜", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "D", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "S", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "M", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "L", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "｜", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "t", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "o", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "o", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "l", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "_", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "c", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "a", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "l", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "l", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "s", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": ">", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "<", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "｜", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "D", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "S", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "M", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "L", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "｜", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "i", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "v", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "o", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "k", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "e", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": " ", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "a", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "m", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "e", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "=", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "\"", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "p", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "i", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "g", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "\"", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": ">", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "<", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "｜", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "D", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "S", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "M", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "L", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "｜", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "p", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "a", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "r", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "a", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "m", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "e", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "t", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "e", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "r", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": " ", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "a", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "m", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "e", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "=", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "\"", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "h", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "o", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "s", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "t", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "\"", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": " ", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "s", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "t", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "r", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "i", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "g", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "=", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "\"", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "t", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "r", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "u", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "e", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "\"", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": ">", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "e", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "x", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "a", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "m", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "p", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "l", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "e", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": ".", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "c", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "o", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "m", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "<", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "/", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "｜", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "D", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "S", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "M", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "L", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "｜", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "p", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "a", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "r", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "a", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "m", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "e", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "t", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "e", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "r", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": ">", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "<", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "/", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "｜", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "D", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "S", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "M", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "L", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "｜", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "i", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "v", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "o", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "k", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "e", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": ">", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "<", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "/", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "｜", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "D", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "S", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "M", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "L", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "｜", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "t", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "o", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "o", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "l", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "_", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "c", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "a", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "l", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "l", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": "s", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": ">", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-fragmented-tokens", - "choices": [ - { - "index": 0, - "delta": { - "content": null, - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - }, - "finish_reason": "tool_calls" - } - ] - } - } + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"B","role":"assistant","reasoning_content":"B"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"r","role":"assistant","reasoning_content":"r"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant","reasoning_content":"e"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant","reasoning_content":"a"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"k","role":"assistant","reasoning_content":"k"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":" ","role":"assistant","reasoning_content":" "}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"t","role":"assistant","reasoning_content":"t"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"o","role":"assistant","reasoning_content":"o"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"k","role":"assistant","reasoning_content":"k"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant","reasoning_content":"e"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"n","role":"assistant","reasoning_content":"n"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"s","role":"assistant","reasoning_content":"s"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":" ","role":"assistant","reasoning_content":" "}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant","reasoning_content":"a"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"p","role":"assistant","reasoning_content":"p"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant","reasoning_content":"a"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"r","role":"assistant","reasoning_content":"r"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"t","role":"assistant","reasoning_content":"t"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":" ","role":"assistant","reasoning_content":" "}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant","reasoning_content":"a"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"g","role":"assistant","reasoning_content":"g"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"g","role":"assistant","reasoning_content":"g"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"r","role":"assistant","reasoning_content":"r"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant","reasoning_content":"e"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"s","role":"assistant","reasoning_content":"s"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"s","role":"assistant","reasoning_content":"s"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"i","role":"assistant","reasoning_content":"i"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"v","role":"assistant","reasoning_content":"v"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant","reasoning_content":"e"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"l","role":"assistant","reasoning_content":"l"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"y","role":"assistant","reasoning_content":"y"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":".","role":"assistant","reasoning_content":"."}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"<","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"D","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"S","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"M","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"L","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"t","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"o","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"o","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"l","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"_","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"c","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"l","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"l","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"s","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":">","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"<","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"D","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"S","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"M","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"L","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"i","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"v","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"o","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"k","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":" ","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"m","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"=","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"\"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"p","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"i","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"g","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"\"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":">","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"<","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"D","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"S","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"M","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"L","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"p","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"r","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"m","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"t","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"r","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":" ","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"m","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"=","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"\"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"h","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"o","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"s","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"t","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"\"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":" ","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"s","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"t","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"r","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"i","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"g","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"=","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"\"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"t","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"r","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"u","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"\"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":">","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"x","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"m","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"p","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"l","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":".","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"c","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"o","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"m","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"<","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"/","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"D","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"S","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"M","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"L","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"p","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"r","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"m","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"t","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"r","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":">","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"<","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"/","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"D","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"S","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"M","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"L","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"i","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"v","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"o","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"k","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"e","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":">","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"<","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"/","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"D","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"S","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"M","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"L","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"｜","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"t","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"o","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"o","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"l","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"_","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"c","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"a","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"l","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"l","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":"s","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":">","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-fragmented-tokens","choices":[{"index":0,"delta":{"content":null,"role":"assistant"},"finish_reason":"tool_calls"}]}} ] } diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_mixed_param_types.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_mixed_param_types.json index 711950300227..4210e1c28490 100644 --- a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_mixed_param_types.json +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_mixed_param_types.json @@ -1,218 +1,17 @@ { "request_id": "deepseek-v4-mixed-param-types-test", - "expected_output": { - "normal_content": "", - "reasoning_content": "The user asked me to send a high-priority overdue-billing notification. I'll call send_notification with the appropriate parameters.", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "send_notification", - "arguments": "{\"recipient\": \"user@example.com\", \"priority\": 3, \"urgent\": true, \"tags\": [\"billing\", \"overdue\"], \"metadata\": {\"ticket\": \"T-42\", \"retries\": 2}}" - } - } - ] - }, + "expected_output": {"normal_content": "", "reasoning_content": "The user asked me to send a high-priority overdue-billing notification. I'll call send_notification with the appropriate parameters.", "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "send_notification", "arguments": "{\"recipient\": \"user@example.com\", \"priority\": 3, \"urgent\": true, \"tags\": [\"billing\", \"overdue\"], \"metadata\": {\"ticket\": \"T-42\", \"retries\": 2}}"}}]}, "input_stream": [ - { - "data": { - "id": "chatcmpl-deepseek-v4-mixed", - "choices": [ - { - "index": 0, - "delta": { - "content": "The user asked me to send a high-priority overdue-billing notification. I'll call send_notification with the appropriate parameters.", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "The user asked me to send a high-priority overdue-billing notification. I'll call send_notification with the appropriate parameters." - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-mixed", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜tool_calls>\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-mixed", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜invoke name=\"send_notification\">\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-mixed", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜parameter name=\"recipient\" string=\"true\">user@example.com\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-mixed", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜parameter name=\"priority\" string=\"false\">3\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-mixed", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜parameter name=\"urgent\" string=\"false\">true\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-mixed", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜parameter name=\"tags\" string=\"false\">[\"billing\", \"overdue\"]\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-mixed", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜parameter name=\"metadata\" string=\"false\">{\"ticket\": \"T-42\", \"retries\": 2}\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-mixed", - "choices": [ - { - "index": 0, - "delta": { - "content": "\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-mixed", - "choices": [ - { - "index": 0, - "delta": { - "content": "", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-mixed", - "choices": [ - { - "index": 0, - "delta": { - "content": null, - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - }, - "finish_reason": "tool_calls" - } - ] - } - } + {"data":{"id":"chatcmpl-deepseek-v4-mixed","choices":[{"index":0,"delta":{"content":"The user asked me to send a high-priority overdue-billing notification. I'll call send_notification with the appropriate parameters.","role":"assistant","reasoning_content":"The user asked me to send a high-priority overdue-billing notification. I'll call send_notification with the appropriate parameters."}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-mixed","choices":[{"index":0,"delta":{"content":"<｜DSML｜tool_calls>\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-mixed","choices":[{"index":0,"delta":{"content":"<｜DSML｜invoke name=\"send_notification\">\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-mixed","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"recipient\" string=\"true\">user@example.com\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-mixed","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"priority\" string=\"false\">3\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-mixed","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"urgent\" string=\"false\">true\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-mixed","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"tags\" string=\"false\">[\"billing\", \"overdue\"]\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-mixed","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"metadata\" string=\"false\">{\"ticket\": \"T-42\", \"retries\": 2}\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-mixed","choices":[{"index":0,"delta":{"content":"\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-mixed","choices":[{"index":0,"delta":{"content":"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-mixed","choices":[{"index":0,"delta":{"content":null,"role":"assistant"},"finish_reason":"tool_calls"}]}} ] } diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_multi_tool.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_multi_tool.json index fca42aa16b75..0129265540d2 100644 --- a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_multi_tool.json +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_multi_tool.json @@ -1,244 +1,18 @@ { "request_id": "deepseek-v4-multi-tool-test", - "expected_output": { - "normal_content": "", - "reasoning_content": "The user wants to check the weather in Beijing and Shanghai, I need to call the get_current_weather tool to get this information.", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_current_weather", - "arguments": "{\"location\": \"Beijing\", \"format\": \"celsius\"}" - } - }, - { - "id": "call_2", - "type": "function", - "function": { - "name": "get_current_weather", - "arguments": "{\"location\": \"Shanghai\", \"format\": \"celsius\"}" - } - } - ] - }, + "expected_output": {"normal_content": "", "reasoning_content": "The user wants to check the weather in Beijing and Shanghai, I need to call the get_current_weather tool to get this information.", "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "get_current_weather", "arguments": "{\"location\": \"Beijing\", \"format\": \"celsius\"}"}}, {"id": "call_2", "type": "function", "function": {"name": "get_current_weather", "arguments": "{\"location\": \"Shanghai\", \"format\": \"celsius\"}"}}]}, "input_stream": [ - { - "data": { - "id": "chatcmpl-deepseek-v4-multi-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "The user wants to check the weather in Beijing and Shanghai, I need to call the get_current_weather tool to get this information.", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "The user wants to check the weather in Beijing and Shanghai, I need to call the get_current_weather tool to get this information." - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-multi-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜tool_calls>\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-multi-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜invoke name=\"get_current_weather\">\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-multi-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜parameter name=\"location\" string=\"true\">Beijing\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-multi-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜parameter name=\"format\" string=\"true\">celsius\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-multi-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-multi-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜invoke name=\"get_current_weather\">\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-multi-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜parameter name=\"location\" string=\"true\">Shanghai\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-multi-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜parameter name=\"format\" string=\"true\">celsius\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-multi-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-multi-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-multi-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": null, - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - }, - "finish_reason": "tool_calls" - } - ] - } - } + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":"The user wants to check the weather in Beijing and Shanghai, I need to call the get_current_weather tool to get this information.","role":"assistant","reasoning_content":"The user wants to check the weather in Beijing and Shanghai, I need to call the get_current_weather tool to get this information."}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜tool_calls>\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜invoke name=\"get_current_weather\">\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"location\" string=\"true\">Beijing\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"format\" string=\"true\">celsius\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":"\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜invoke name=\"get_current_weather\">\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"location\" string=\"true\">Shanghai\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"format\" string=\"true\">celsius\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":"\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-multi-tool","choices":[{"index":0,"delta":{"content":null,"role":"assistant"},"finish_reason":"tool_calls"}]}} ] } diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_no_params.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_no_params.json deleted file mode 100644 index 68668ac932dd..000000000000 --- a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_no_params.json +++ /dev/null @@ -1,146 +0,0 @@ -{ - "request_id": "deepseek-v4-no-params-test", - "expected_output": { - "normal_content": "", - "reasoning_content": "The user asked for the current time. I'll call get_current_time which takes no parameters.", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_current_time", - "arguments": "{}" - } - } - ] - }, - "input_stream": [ - { - "data": { - "id": "chatcmpl-deepseek-v4-no-params", - "choices": [ - { - "index": 0, - "delta": { - "content": "The user asked for the current time. I'll call get_current_time which takes no parameters.", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "The user asked for the current time. I'll call get_current_time which takes no parameters." - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-no-params", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜tool_calls>\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-no-params", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜invoke name=\"get_current_time\">\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-no-params", - "choices": [ - { - "index": 0, - "delta": { - "content": "\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-no-params", - "choices": [ - { - "index": 0, - "delta": { - "content": "\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-no-params", - "choices": [ - { - "index": 0, - "delta": { - "content": "", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-no-params", - "choices": [ - { - "index": 0, - "delta": { - "content": null, - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - }, - "finish_reason": "tool_calls" - } - ] - } - } - ] -} diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_no_tool.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_no_tool.json index a19ee9084343..df0781e1370e 100644 --- a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_no_tool.json +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_no_tool.json @@ -1,14 +1,10 @@ { "request_id": "deepseek-v4-no-tool-test", - "expected_output": { - "normal_content": "Hi! I'm here to help — what would you like to work on today?", - "reasoning_content": "User greeted me politely. A short friendly reply is appropriate; no tools needed.", - "tool_calls": [] - }, + "expected_output": {"normal_content": "Hi! I'm here to help — what would you like to work on today?", "reasoning_content": "User greeted me politely. A short friendly reply is appropriate; no tools needed.", "tool_calls": []}, "input_stream": [ - {"data":{"id":"chatcmpl-deepseek-v4-no-tool","choices":[{"index":0,"delta":{"content":"User greeted me politely. A short friendly reply is appropriate; no tools needed.","function_call":null,"tool_calls":null,"role":"assistant","refusal":null,"reasoning_content":"User greeted me politely. A short friendly reply is appropriate; no tools needed."}}]}}, - {"data":{"id":"chatcmpl-deepseek-v4-no-tool","choices":[{"index":0,"delta":{"content":"Hi! I'm here to help — ","function_call":null,"tool_calls":null,"role":"assistant","refusal":null,"reasoning_content":null}}]}}, - {"data":{"id":"chatcmpl-deepseek-v4-no-tool","choices":[{"index":0,"delta":{"content":"what would you like to work on today?","function_call":null,"tool_calls":null,"role":"assistant","refusal":null,"reasoning_content":null}}]}}, - {"data":{"id":"chatcmpl-deepseek-v4-no-tool","choices":[{"index":0,"delta":{"content":null,"function_call":null,"tool_calls":null,"role":"assistant","refusal":null,"reasoning_content":null},"finish_reason":"stop"}]}} + {"data":{"id":"chatcmpl-deepseek-v4-no-tool","choices":[{"index":0,"delta":{"content":"User greeted me politely. A short friendly reply is appropriate; no tools needed.","role":"assistant","reasoning_content":"User greeted me politely. A short friendly reply is appropriate; no tools needed."}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-no-tool","choices":[{"index":0,"delta":{"content":"Hi! I'm here to help — ","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-no-tool","choices":[{"index":0,"delta":{"content":"what would you like to work on today?","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-no-tool","choices":[{"index":0,"delta":{"content":null,"role":"assistant"},"finish_reason":"stop"}]}} ] } diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_special_chars.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_special_chars.json index 082ee79820e3..38ab2a150cd8 100644 --- a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_special_chars.json +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_special_chars.json @@ -1,201 +1,16 @@ { "request_id": "deepseek-v4-special-chars-test", - "expected_output": { - "normal_content": "", - "reasoning_content": "The user wants me to save a multiline note with special characters, quotes, unicode, and emoji. I'll call save_note.", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "save_note", - "arguments": "{\"note\": \"He said \\\"hello\\\".\\n\\t'world' `backtick` — 中文测试 — 🚀✨ & .\"}" - } - } - ], - "finish_reason": "tool_calls" - }, + "expected_output": {"normal_content": "", "reasoning_content": "The user wants me to save a multiline note with special characters, quotes, unicode, and emoji. I'll call save_note.", "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "save_note", "arguments": "{\"note\": \"He said \\\"hello\\\".\\n\\t'world' `backtick` — 中文测试 — 🚀✨ & .\"}"}}], "finish_reason": "tool_calls"}, "input_stream": [ - { - "data": { - "id": "chatcmpl-deepseek-v4-special", - "choices": [ - { - "index": 0, - "delta": { - "content": "The user wants me to save a multiline note with special characters, quotes, unicode, and emoji. I'll call save_note.", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "The user wants me to save a multiline note with special characters, quotes, unicode, and emoji. I'll call save_note." - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-special", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜tool_calls>\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-special", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜invoke name=\"save_note\">\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-special", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜parameter name=\"note\" string=\"true\">", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-special", - "choices": [ - { - "index": 0, - "delta": { - "content": "He said \"hello\".\n\t'world' `backtick` ", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-special", - "choices": [ - { - "index": 0, - "delta": { - "content": "— 中文测试 — 🚀✨ ", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-special", - "choices": [ - { - "index": 0, - "delta": { - "content": " & .\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-special", - "choices": [ - { - "index": 0, - "delta": { - "content": "\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-special", - "choices": [ - { - "index": 0, - "delta": { - "content": "", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-special", - "choices": [ - { - "index": 0, - "delta": { - "content": null, - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - }, - "finish_reason": "tool_calls" - } - ] - } - } + {"data":{"id":"chatcmpl-deepseek-v4-special","choices":[{"index":0,"delta":{"content":"The user wants me to save a multiline note with special characters, quotes, unicode, and emoji. I'll call save_note.","role":"assistant","reasoning_content":"The user wants me to save a multiline note with special characters, quotes, unicode, and emoji. I'll call save_note."}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-special","choices":[{"index":0,"delta":{"content":"<｜DSML｜tool_calls>\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-special","choices":[{"index":0,"delta":{"content":"<｜DSML｜invoke name=\"save_note\">\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-special","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"note\" string=\"true\">","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-special","choices":[{"index":0,"delta":{"content":"He said \"hello\".\n\t'world' `backtick` ","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-special","choices":[{"index":0,"delta":{"content":"— 中文测试 — 🚀✨ ","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-special","choices":[{"index":0,"delta":{"content":" & .\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-special","choices":[{"index":0,"delta":{"content":"\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-special","choices":[{"index":0,"delta":{"content":"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-special","choices":[{"index":0,"delta":{"content":null,"role":"assistant"},"finish_reason":"tool_calls"}]}} ] } diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_tool.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_tool.json index 85fddc391375..47e94f923073 100644 --- a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_tool.json +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_tool.json @@ -1,164 +1,14 @@ { "request_id": "deepseek-v4-tool-call-test", - "expected_output": { - "normal_content": "", - "reasoning_content": "User wants the current weather in Beijing. I'll call get_current_weather with celsius units.", - "tool_calls": [ - { - "id": "call_1", - "type": "function", - "function": { - "name": "get_current_weather", - "arguments": "{\"location\": \"Beijing\", \"format\": \"celsius\"}" - } - } - ] - }, + "expected_output": {"normal_content": "", "reasoning_content": "User wants the current weather in Beijing. I'll call get_current_weather with celsius units.", "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "get_current_weather", "arguments": "{\"location\": \"Beijing\", \"format\": \"celsius\"}"}}]}, "input_stream": [ - { - "data": { - "id": "chatcmpl-deepseek-v4-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "User wants the current weather in Beijing. I'll call get_current_weather with celsius units.", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": "User wants the current weather in Beijing. I'll call get_current_weather with celsius units." - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜tool_calls>\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜invoke name=\"get_current_weather\">\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜parameter name=\"location\" string=\"true\">Beijing\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "<｜DSML｜parameter name=\"format\" string=\"true\">celsius\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "\n", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": "", - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - } - } - ] - } - }, - { - "data": { - "id": "chatcmpl-deepseek-v4-tool", - "choices": [ - { - "index": 0, - "delta": { - "content": null, - "function_call": null, - "tool_calls": null, - "role": "assistant", - "refusal": null, - "reasoning_content": null - }, - "finish_reason": "tool_calls" - } - ] - } - } + {"data":{"id":"chatcmpl-deepseek-v4-tool","choices":[{"index":0,"delta":{"content":"User wants the current weather in Beijing. I'll call get_current_weather with celsius units.","role":"assistant","reasoning_content":"User wants the current weather in Beijing. I'll call get_current_weather with celsius units."}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜tool_calls>\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜invoke name=\"get_current_weather\">\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"location\" string=\"true\">Beijing\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-tool","choices":[{"index":0,"delta":{"content":"<｜DSML｜parameter name=\"format\" string=\"true\">celsius\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-tool","choices":[{"index":0,"delta":{"content":"\n","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-tool","choices":[{"index":0,"delta":{"content":"","role":"assistant"}}]}}, + {"data":{"id":"chatcmpl-deepseek-v4-tool","choices":[{"index":0,"delta":{"content":null,"role":"assistant"},"finish_reason":"tool_calls"}]}} ] } diff --git a/lib/llm/tests/deepseek_v4_encoding.rs b/lib/llm/tests/deepseek_v4_encoding.rs new file mode 100644 index 000000000000..8eda47107c66 --- /dev/null +++ b/lib/llm/tests/deepseek_v4_encoding.rs @@ -0,0 +1,100 @@ +// SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +//! Tests for DeepSeek V4 encoding against official test data +//! +//! These tests use the official test files from: +//! https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro/tree/main/encoding + +use dynamo_llm::preprocessor::prompt::deepseek_v4::{ThinkingMode, encode_messages}; +use serde_json::Value as JsonValue; +use std::fs; +use std::path::PathBuf; + +fn get_test_data_path() -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("tests/data/deepseek-v4") +} + +/// Load an input fixture. V4 fixtures come in two shapes: +/// 1. `{"tools": [...], "messages": [...]}` — tools injected on first (system) message +/// 2. bare `[...]` — just the messages array +fn load_messages(path: &PathBuf) -> Vec { + let raw: JsonValue = serde_json::from_str( + &fs::read_to_string(path).unwrap_or_else(|_| panic!("Failed to read {:?}", path)), + ) + .unwrap_or_else(|_| panic!("Failed to parse {:?}", path)); + + if let Some(messages) = raw.get("messages").and_then(|m| m.as_array()) { + let mut messages = messages.clone(); + if let Some(tools) = raw.get("tools") + && let Some(first) = messages.get_mut(0) + && let Some(obj) = first.as_object_mut() + { + obj.insert("tools".to_string(), tools.clone()); + } + messages + } else if let Some(arr) = raw.as_array() { + arr.clone() + } else { + panic!("Unexpected input shape in {:?}", path); + } +} + +fn run_official_test(input_file: &str, output_file: &str, thinking_mode: ThinkingMode) { + let test_dir = get_test_data_path(); + let messages = load_messages(&test_dir.join(input_file)); + let expected = fs::read_to_string(test_dir.join(output_file)) + .unwrap_or_else(|_| panic!("Failed to read {}", output_file)); + + let actual = encode_messages(&messages, thinking_mode, true) + .unwrap_or_else(|e| panic!("encode_messages failed for {}: {:?}", input_file, e)); + + let exp = expected.trim_end(); + let act = actual.trim_end(); + + if exp != act { + println!("=== Test: {} ===", input_file); + let exp_lines: Vec<&str> = exp.lines().collect(); + let act_lines: Vec<&str> = act.lines().collect(); + for (i, (el, al)) in exp_lines.iter().zip(act_lines.iter()).enumerate() { + if el != al { + println!("Line {} differs:", i + 1); + println!(" Expected: {:?}", el); + println!(" Actual: {:?}", al); + break; + } + } + if exp_lines.len() != act_lines.len() { + println!( + "\nLine count mismatch: expected {} lines, got {} lines", + exp_lines.len(), + act_lines.len() + ); + } + panic!("Output does not match expected for {}", input_file); + } +} + +/// Case 1 — thinking mode, single tool, tool result round-trip. +#[test] +fn test_official_thinking_with_tools() { + run_official_test("test_input_1.json", "test_output_1.txt", ThinkingMode::Thinking); +} + +/// Case 2 — thinking mode, no tools, multi-turn (drop_thinking strips earlier reasoning). +#[test] +fn test_official_thinking_no_tools_multiturn() { + run_official_test("test_input_2.json", "test_output_2.txt", ThinkingMode::Thinking); +} + +/// Case 3 — thinking mode, developer role with tools + latest_reminder + tool result. +#[test] +fn test_official_developer_with_tools_and_reminder() { + run_official_test("test_input_3.json", "test_output_3.txt", ThinkingMode::Thinking); +} + +/// Case 4 — chat mode, latest_reminder + task="action" + mask preservation. +#[test] +fn test_official_chat_mode_action_task() { + run_official_test("test_input_4.json", "test_output_4.txt", ThinkingMode::Chat); +} diff --git a/lib/llm/tests/test_streaming_tool_parsers.rs b/lib/llm/tests/test_streaming_tool_parsers.rs index 28beb5e3dce7..3035f77e7ec0 100644 --- a/lib/llm/tests/test_streaming_tool_parsers.rs +++ b/lib/llm/tests/test_streaming_tool_parsers.rs @@ -1226,16 +1226,6 @@ mod tests { run_deepseek_v4_tool_call_fixture(&file_path).await; } - /// Tool call with zero parameters — invoke body is empty; arguments = "{}". - #[tokio::test] - async fn test_deepseek_v4_e2e_no_params_vllm() { - let file_path = format!( - "{}/vllm/deepseek-v4/chat_completion_stream_no_params.json", - DATA_ROOT_PATH - ); - run_deepseek_v4_tool_call_fixture(&file_path).await; - } - /// Body text emitted before the DSML block — parser must populate both /// normal_content and tool_calls. #[tokio::test] From 6a159fedd8e4a1563aa647c31f622aedbf254b5b Mon Sep 17 00:00:00 2001 From: ayushag Date: Thu, 23 Apr 2026 22:27:21 -0700 Subject: [PATCH 04/21] chore: add formatter to template -- critical path Signed-off-by: ayushag --- lib/llm/src/preprocessor/prompt/template.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/llm/src/preprocessor/prompt/template.rs b/lib/llm/src/preprocessor/prompt/template.rs index fbdf2da1af4a..10e772df5d9b 100644 --- a/lib/llm/src/preprocessor/prompt/template.rs +++ b/lib/llm/src/preprocessor/prompt/template.rs @@ -19,8 +19,14 @@ use tokcfg::ChatTemplateValue; impl PromptFormatter { pub fn from_mdc(mdc: &ModelDeploymentCard) -> Result { - // Special handling for DeepSeek-V3.2(-Speciale) which doesn't provide Jinja chat_template + // Special handling for DeepSeek models whose HF repos don't ship a Jinja chat_template. let name_lower = mdc.display_name.to_lowercase(); + if name_lower.contains("deepseek") && name_lower.contains("v4") { + tracing::info!("Detected DeepSeek V4 model, using native Rust formatter"); + return Ok(Self::OAI(Arc::new( + super::deepseek_v4::DeepSeekV4Formatter::new_thinking(), + ))); + } if name_lower.contains("deepseek") && name_lower.contains("v3.2") && !name_lower.contains("exp") From e1bbc4656ce9ec389f78da925dbb96d8d55a7e3a Mon Sep 17 00:00:00 2001 From: ayushag Date: Thu, 23 Apr 2026 23:03:43 -0700 Subject: [PATCH 05/21] fix: reasoning Signed-off-by: ayushag --- lib/llm/src/preprocessor.rs | 52 +++++- .../src/preprocessor/prompt/deepseek_v4.rs | 160 ++++++++++++------ lib/llm/tests/deepseek_v4_encoding.rs | 18 +- 3 files changed, 175 insertions(+), 55 deletions(-) diff --git a/lib/llm/src/preprocessor.rs b/lib/llm/src/preprocessor.rs index eb312cf91e90..851c143f9e24 100644 --- a/lib/llm/src/preprocessor.rs +++ b/lib/llm/src/preprocessor.rs @@ -1227,8 +1227,9 @@ impl OpenAIPreprocessor { /// For kimi_k25: disabled when chat_template_args contains "thinking": false. /// For nemotron_nano: disabled when chat_template_args contains "enable_thinking": false /// or "force_nonempty_content": true. - /// For deepseek_r1: disabled when chat_template_args contains "thinking": false - /// or "thinking_mode": "chat". + /// For deepseek_r1 / deepseek_v4: disabled when chat_template_args contains + /// "thinking": false or "thinking_mode": "chat" — matches the V4 formatter's + /// `resolve_thinking_mode` convention, so the parser and the prompt stay in sync. fn is_reasoning_disabled_by_request( reasoning_parser: Option<&str>, chat_template_args: Option<&std::collections::HashMap>, @@ -1257,7 +1258,8 @@ impl OpenAIPreprocessor { } false } - Some("deepseek_r1") => { + Some("deepseek_r1") | Some("deepseek_v4") | Some("deepseek-v4") + | Some("deepseekv4") => { if let Some(args) = chat_template_args { if let Some(thinking) = args.get("thinking") { return thinking == &serde_json::Value::Bool(false); @@ -1829,6 +1831,50 @@ mod tests { false, "nemotron_nano + empty args → enabled", ), + // deepseek_v4 — same convention as deepseek_r1; verify all three aliases + // (deepseek_v4 / deepseek-v4 / deepseekv4) plus both signal keys. + ( + Some("deepseek_v4"), + Some(&thinking_false), + true, + "deepseek_v4 + thinking=false → disabled", + ), + ( + Some("deepseek_v4"), + Some(&thinking_true), + false, + "deepseek_v4 + thinking=true → enabled", + ), + ( + Some("deepseek_v4"), + Some(&thinking_mode_chat), + true, + "deepseek_v4 + thinking_mode=chat → disabled", + ), + ( + Some("deepseek_v4"), + Some(&thinking_mode_thinking), + false, + "deepseek_v4 + thinking_mode=thinking → enabled", + ), + ( + Some("deepseek_v4"), + None, + false, + "deepseek_v4 + no args → enabled", + ), + ( + Some("deepseek-v4"), + Some(&thinking_false), + true, + "deepseek-v4 (hyphen alias) + thinking=false → disabled", + ), + ( + Some("deepseekv4"), + Some(&thinking_mode_chat), + true, + "deepseekv4 (joined alias) + thinking_mode=chat → disabled", + ), ]; for (parser, args, expected, desc) in cases { diff --git a/lib/llm/src/preprocessor/prompt/deepseek_v4.rs b/lib/llm/src/preprocessor/prompt/deepseek_v4.rs index 5cdb7694a819..9ad9520b5566 100644 --- a/lib/llm/src/preprocessor/prompt/deepseek_v4.rs +++ b/lib/llm/src/preprocessor/prompt/deepseek_v4.rs @@ -88,29 +88,51 @@ pub enum ReasoningEffort { } /// Serialize a JSON value to match Python's `json.dumps(ensure_ascii=False)` spacing. -/// Python inserts a space after every `:` and `,` outside of strings. +/// Python's default separators are `(', ', ': ')`; we use a custom `Formatter` +/// so escape sequences inside strings can't confuse state tracking. fn to_json(value: &JsonValue) -> String { - let compact = serde_json::to_string(value).unwrap_or_else(|_| "{}".to_string()); - - let mut result = String::with_capacity(compact.len() + compact.len() / 4); - let mut in_string = false; - let mut prev_char = '\0'; - - for ch in compact.chars() { - if ch == '"' && prev_char != '\\' { - in_string = !in_string; + use serde::Serialize; + use serde_json::ser::Formatter; + use std::io; + + struct PythonFormatter; + + impl Formatter for PythonFormatter { + fn begin_array_value( + &mut self, + writer: &mut W, + first: bool, + ) -> io::Result<()> { + if first { + Ok(()) + } else { + writer.write_all(b", ") + } } - result.push(ch); - - if !in_string && (ch == ':' || ch == ',') { - result.push(' '); + fn begin_object_key( + &mut self, + writer: &mut W, + first: bool, + ) -> io::Result<()> { + if first { + Ok(()) + } else { + writer.write_all(b", ") + } } - prev_char = ch; + fn begin_object_value(&mut self, writer: &mut W) -> io::Result<()> { + writer.write_all(b": ") + } } - result + let mut buf = Vec::with_capacity(64); + let mut ser = serde_json::Serializer::with_formatter(&mut buf, PythonFormatter); + if value.serialize(&mut ser).is_err() { + return "{}".to_string(); + } + String::from_utf8(buf).unwrap_or_else(|_| "{}".to_string()) } /// Extract function definitions from OpenAI-format tool list. @@ -136,6 +158,10 @@ fn render_tools(tools: &[JsonValue]) -> String { } /// Find the index of the last user/developer message. +/// +/// Returns `None` when no such message exists. Callers should treat `None` +/// as Python's `-1` sentinel: `idx >= -1` is always true in Python, so use +/// `Option::is_none_or(|u| idx >= u)` (or `>`) to match the reference encoder. fn find_last_user_index(messages: &[JsonValue]) -> Option { messages .iter() @@ -253,9 +279,9 @@ fn render_message( thinking_mode: ThinkingMode, drop_thinking: bool, reasoning_effort: Option, + last_user_idx: Option, ) -> Result { let msg = &messages[index]; - let last_user_idx = find_last_user_index(messages); let role = msg .get("role") @@ -358,10 +384,7 @@ fn render_message( .get("reasoning_content") .and_then(|c| c.as_str()) .unwrap_or(""); - let wo_eos = msg - .get("wo_eos") - .and_then(|v| v.as_bool()) - .unwrap_or(false); + let wo_eos = msg.get("wo_eos").and_then(|v| v.as_bool()).unwrap_or(false); let prev_has_task = index > 0 && messages[index - 1] @@ -371,8 +394,7 @@ fn render_message( let mut thinking_part = String::new(); if thinking_mode == ThinkingMode::Thinking && !prev_has_task { - let render_thinking = !drop_thinking - || last_user_idx.is_some_and(|u| index > u); + let render_thinking = !drop_thinking || last_user_idx.is_none_or(|u| index > u); if render_thinking { thinking_part.push_str(reasoning); thinking_part.push_str(tokens::THINKING_END); @@ -437,8 +459,7 @@ fn render_message( // Transition tokens based on task field and role. let task = msg.get("task").and_then(|v| v.as_str()); if let Some(task) = task { - let sp = task_token(task) - .with_context(|| format!("Invalid task: '{}'", task))?; + let sp = task_token(task).with_context(|| format!("Invalid task: '{}'", task))?; if task != "action" { prompt.push_str(sp); } else { @@ -453,7 +474,7 @@ fn render_message( } else if matches!(role, "user" | "developer") { prompt.push_str(tokens::ASSISTANT_START); let seed_thinking = thinking_mode == ThinkingMode::Thinking - && (!drop_thinking || last_user_idx.is_some_and(|u| index >= u)); + && (!drop_thinking || last_user_idx.is_none_or(|u| index >= u)); prompt.push_str(if seed_thinking { tokens::THINKING_START } else { @@ -623,10 +644,8 @@ pub fn sort_tool_results_by_call_order(mut messages: Vec) -> Vec 1 { - let mut tool_blocks: Vec = tool_positions - .iter() - .map(|&i| blocks[i].clone()) - .collect(); + let mut tool_blocks: Vec = + tool_positions.iter().map(|&i| blocks[i].clone()).collect(); tool_blocks.sort_by_key(|b| { let id = b.get("tool_use_id").and_then(|v| v.as_str()).unwrap_or(""); *last_order.get(id).unwrap_or(&0) @@ -643,7 +662,7 @@ pub fn sort_tool_results_by_call_order(mut messages: Vec) -> Vec) -> Vec { - let last_user_idx = find_last_user_index(&messages).unwrap_or(usize::MAX); + let last_user_idx = find_last_user_index(&messages); let mut out = Vec::with_capacity(messages.len()); const KEEP: &[&str] = &[ "user", @@ -655,7 +674,7 @@ fn drop_thinking_messages(messages: Vec) -> Vec { for (idx, mut msg) in messages.into_iter().enumerate() { let role = msg.get("role").and_then(|r| r.as_str()).unwrap_or(""); - if KEEP.contains(&role) || idx >= last_user_idx { + if KEEP.contains(&role) || last_user_idx.is_none_or(|u| idx >= u) { out.push(msg); } else if role == "assistant" { if let Some(obj) = msg.as_object_mut() { @@ -718,6 +737,7 @@ pub fn encode_messages_with_options( full = drop_thinking_messages(full); } + let last_user_idx = find_last_user_index(&full); for idx in 0..full.len() { let part = render_message( idx, @@ -725,6 +745,7 @@ pub fn encode_messages_with_options( thinking_mode, effective_drop_thinking, reasoning_effort, + last_user_idx, )?; prompt.push_str(&part); } @@ -858,24 +879,26 @@ mod tests { {"role": "assistant", "reasoning_content": "greet", "content": "Hi!"}, {"role": "user", "content": "What is 2+2?"} ]); - let out = encode_messages(messages.as_array().unwrap(), ThinkingMode::Thinking, true) - .unwrap(); + let out = + encode_messages(messages.as_array().unwrap(), ThinkingMode::Thinking, true).unwrap(); assert!(out.starts_with(tokens::BOS)); - assert!(out.ends_with(&format!("{}{}", tokens::ASSISTANT_START, tokens::THINKING_START))); + assert!(out.ends_with(&format!( + "{}{}", + tokens::ASSISTANT_START, + tokens::THINKING_START + ))); // drop_thinking default true → earlier reasoning stripped assert!(!out.contains("greet")); } #[test] fn test_thinking_with_tools() { - let raw = std::fs::read_to_string( - "../../DeepSeek-V4-Pro/encoding/tests/test_input_1.json", - ) - .or_else(|_| { - std::fs::read_to_string( - "/home/ayush-lab/Work/dynamo/DeepSeek-V4-Pro/encoding/tests/test_input_1.json", - ) - }); + let raw = std::fs::read_to_string("../../DeepSeek-V4-Pro/encoding/tests/test_input_1.json") + .or_else(|_| { + std::fs::read_to_string( + "/home/ayush-lab/Work/dynamo/DeepSeek-V4-Pro/encoding/tests/test_input_1.json", + ) + }); let Ok(data) = raw else { return }; let parsed: JsonValue = serde_json::from_str(&data).unwrap(); let mut messages = parsed.get("messages").unwrap().as_array().unwrap().clone(); @@ -885,8 +908,7 @@ mod tests { .as_object_mut() .unwrap() .insert("tools".to_string(), tools); - let out = - encode_messages(&messages, ThinkingMode::Thinking, true).unwrap(); + let out = encode_messages(&messages, ThinkingMode::Thinking, true).unwrap(); let expected = std::fs::read_to_string( "/home/ayush-lab/Work/dynamo/DeepSeek-V4-Pro/encoding/tests/test_output_1.txt", ) @@ -964,8 +986,7 @@ mod tests { {"type": "text", "text": "suffix"} ]} ]); - let out = encode_messages(messages.as_array().unwrap(), ThinkingMode::Chat, false) - .unwrap(); + let out = encode_messages(messages.as_array().unwrap(), ThinkingMode::Chat, false).unwrap(); assert!(out.contains("prefix\n\nRESULT\n\nsuffix")); } @@ -980,9 +1001,50 @@ mod tests { {"role": "assistant", "reasoning_content": "PRIOR_REASONING", "content": "reply"}, {"role": "user", "content": "again"} ]); - let out = encode_messages(messages.as_array().unwrap(), ThinkingMode::Thinking, true) - .unwrap(); + let out = + encode_messages(messages.as_array().unwrap(), ThinkingMode::Thinking, true).unwrap(); // Tools present → drop_thinking auto-disabled → earlier reasoning preserved. assert!(out.contains("PRIOR_REASONING")); } + + // ---- Regression tests for known divergences from the Python reference ---- + + /// Bug: `last_user_idx = None` (no user/developer in history) should behave + /// like Python's `-1` sentinel — `index >= -1` / `idx >= -1` always true, so + /// earlier reasoning is preserved and the assistant's reasoning block is + /// rendered. Rust defaulting `None` to `usize::MAX` / `is_some_and` silently + /// stripped reasoning instead. + /// + /// Byte-equivalent to Python reference with the same input: + /// `sysREASONING_BLOCKhello` + #[test] + fn test_assistant_reasoning_preserved_when_no_user_in_history() { + let messages = json!([ + {"role": "system", "content": "sys"}, + {"role": "assistant", "content": "hello", "reasoning_content": "REASONING_BLOCK"} + ]); + let out = + encode_messages(messages.as_array().unwrap(), ThinkingMode::Thinking, true).unwrap(); + assert_eq!( + out, "<｜begin▁of▁sentence｜>sysREASONING_BLOCKhello<｜end▁of▁sentence｜>", + "Output must match Python reference byte-for-byte when no user/developer in history" + ); + } + + /// Bug: `to_json` tracks in-string state via `prev_char != '\\'` which + /// mis-handles consecutive backslashes. A value containing `\\` (one literal + /// backslash in JSON) makes the helper think the closing `"` is escaped, + /// so it stops inserting Python-compatible spaces after subsequent `:`/`,`. + /// + /// Python `json.dumps({"path": "\\", "count": 5}, ensure_ascii=False)` + /// emits `{"path": "\\", "count": 5}` — space after every `:` and `,`. + #[test] + fn test_to_json_preserves_spacing_past_escaped_backslash() { + let v = json!({"path": "\\", "count": 5}); + let got = to_json(&v); + assert_eq!( + got, r#"{"path": "\\", "count": 5}"#, + "to_json must match Python's json.dumps formatting past an escaped backslash" + ); + } } diff --git a/lib/llm/tests/deepseek_v4_encoding.rs b/lib/llm/tests/deepseek_v4_encoding.rs index 8eda47107c66..6a968e4c4b59 100644 --- a/lib/llm/tests/deepseek_v4_encoding.rs +++ b/lib/llm/tests/deepseek_v4_encoding.rs @@ -78,19 +78,31 @@ fn run_official_test(input_file: &str, output_file: &str, thinking_mode: Thinkin /// Case 1 — thinking mode, single tool, tool result round-trip. #[test] fn test_official_thinking_with_tools() { - run_official_test("test_input_1.json", "test_output_1.txt", ThinkingMode::Thinking); + run_official_test( + "test_input_1.json", + "test_output_1.txt", + ThinkingMode::Thinking, + ); } /// Case 2 — thinking mode, no tools, multi-turn (drop_thinking strips earlier reasoning). #[test] fn test_official_thinking_no_tools_multiturn() { - run_official_test("test_input_2.json", "test_output_2.txt", ThinkingMode::Thinking); + run_official_test( + "test_input_2.json", + "test_output_2.txt", + ThinkingMode::Thinking, + ); } /// Case 3 — thinking mode, developer role with tools + latest_reminder + tool result. #[test] fn test_official_developer_with_tools_and_reminder() { - run_official_test("test_input_3.json", "test_output_3.txt", ThinkingMode::Thinking); + run_official_test( + "test_input_3.json", + "test_output_3.txt", + ThinkingMode::Thinking, + ); } /// Case 4 — chat mode, latest_reminder + task="action" + mask preservation. From d187c88f0d9b5041bd389ce05900e4a6cbe090f0 Mon Sep 17 00:00:00 2001 From: ayushag Date: Thu, 23 Apr 2026 23:07:38 -0700 Subject: [PATCH 06/21] fix: copyright issues Signed-off-by: ayushag --- .github/workflows/copyright-check.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/copyright-check.ps1 b/.github/workflows/copyright-check.ps1 index 5613d51ff63f..9411e21183f8 100644 --- a/.github/workflows/copyright-check.ps1 +++ b/.github/workflows/copyright-check.ps1 @@ -84,7 +84,7 @@ $global:copyright_results = @{ $ignored_files = @('.clang-format', '.gitattributes', '.gitignore', '.gitkeep', '.patch', 'Cargo.lock', 'LICENSE', 'uv.lock', 'rust-toolchain.toml', 'codespell.txt', 'exclusions.txt') write-debug " ignored_files = ['$($ignored_files -join "','")']." -$ignored_paths = @('.github', '.mypy_cache', '.pytest_cache', 'lib/llm/tests/data/sample-models', 'lib/llm/tests/data/deepseek-v3.2') +$ignored_paths = @('.github', '.mypy_cache', '.pytest_cache', 'lib/llm/tests/data/sample-models', 'lib/llm/tests/data/deepseek-v3.2', 'lib/llm/tests/data/deepseek-v4') write-debug " ignored_paths = ['$($ignored_paths -join "','")']." $ignored_types = @('.bat', '.gif', '.ico', '.ipynb', '.jpg', '.jpeg', '.patch', '.png', '.pyc', '.pyi', '.rst', '.zip', '.md', '.json') write-debug " ignored_types = ['$($ignored_types -join "', '")']." From 95bda3734c84101e0c537bfc224954754d2b1379 Mon Sep 17 00:00:00 2001 From: ayushag Date: Fri, 24 Apr 2026 00:15:23 -0700 Subject: [PATCH 07/21] fix: harchoded path Signed-off-by: ayushag --- .../src/preprocessor/prompt/deepseek_v4.rs | 80 +++++-------------- 1 file changed, 18 insertions(+), 62 deletions(-) diff --git a/lib/llm/src/preprocessor/prompt/deepseek_v4.rs b/lib/llm/src/preprocessor/prompt/deepseek_v4.rs index 9ad9520b5566..528e917ef44c 100644 --- a/lib/llm/src/preprocessor/prompt/deepseek_v4.rs +++ b/lib/llm/src/preprocessor/prompt/deepseek_v4.rs @@ -891,61 +891,6 @@ mod tests { assert!(!out.contains("greet")); } - #[test] - fn test_thinking_with_tools() { - let raw = std::fs::read_to_string("../../DeepSeek-V4-Pro/encoding/tests/test_input_1.json") - .or_else(|_| { - std::fs::read_to_string( - "/home/ayush-lab/Work/dynamo/DeepSeek-V4-Pro/encoding/tests/test_input_1.json", - ) - }); - let Ok(data) = raw else { return }; - let parsed: JsonValue = serde_json::from_str(&data).unwrap(); - let mut messages = parsed.get("messages").unwrap().as_array().unwrap().clone(); - // Inject tools onto the system message (as the request layer would). - let tools = parsed.get("tools").unwrap().clone(); - messages[0] - .as_object_mut() - .unwrap() - .insert("tools".to_string(), tools); - let out = encode_messages(&messages, ThinkingMode::Thinking, true).unwrap(); - let expected = std::fs::read_to_string( - "/home/ayush-lab/Work/dynamo/DeepSeek-V4-Pro/encoding/tests/test_output_1.txt", - ) - .unwrap(); - assert_eq!(out, expected); - } - - #[test] - fn test_latest_reminder_multi_turn() { - let data = std::fs::read_to_string( - "/home/ayush-lab/Work/dynamo/DeepSeek-V4-Pro/encoding/tests/test_input_3.json", - ); - let Ok(data) = data else { return }; - let messages: Vec = serde_json::from_str(&data).unwrap(); - let out = encode_messages(&messages, ThinkingMode::Thinking, true).unwrap(); - let expected = std::fs::read_to_string( - "/home/ayush-lab/Work/dynamo/DeepSeek-V4-Pro/encoding/tests/test_output_3.txt", - ) - .unwrap(); - assert_eq!(out, expected); - } - - #[test] - fn test_chat_mode_with_action_task() { - let data = std::fs::read_to_string( - "/home/ayush-lab/Work/dynamo/DeepSeek-V4-Pro/encoding/tests/test_input_4.json", - ); - let Ok(data) = data else { return }; - let messages: Vec = serde_json::from_str(&data).unwrap(); - let out = encode_messages(&messages, ThinkingMode::Chat, true).unwrap(); - let expected = std::fs::read_to_string( - "/home/ayush-lab/Work/dynamo/DeepSeek-V4-Pro/encoding/tests/test_output_4.txt", - ) - .unwrap(); - assert_eq!(out, expected); - } - #[test] fn test_reasoning_effort_max_prefix() { let messages = json!([ @@ -979,15 +924,26 @@ mod tests { #[test] fn test_content_blocks_with_tool_result() { + // `merge_tool_messages` turns a `tool` role followed by a plain user text + // into a single user turn whose `content_blocks` interleave the tool result + // with the text, joined by "\n\n" at render time. Users don't construct + // `content_blocks` directly — both the Python reference and this port + // overwrite any user-supplied `content_blocks` with a single text block. let messages = json!([ - {"role": "user", "content_blocks": [ - {"type": "text", "text": "prefix"}, - {"type": "tool_result", "tool_use_id": "c1", "content": "RESULT"}, - {"type": "text", "text": "suffix"} - ]} + {"role": "user", "content": "call tool"}, + {"role": "assistant", "content": "", "tool_calls": [{ + "id": "c1", "type": "function", + "function": {"name": "f", "arguments": "{}"} + }]}, + {"role": "tool", "tool_call_id": "c1", "content": "RESULT"}, + {"role": "user", "content": "thanks"} ]); - let out = encode_messages(messages.as_array().unwrap(), ThinkingMode::Chat, false).unwrap(); - assert!(out.contains("prefix\n\nRESULT\n\nsuffix")); + let out = encode_messages(messages.as_array().unwrap(), ThinkingMode::Chat, true).unwrap(); + assert!( + out.contains("RESULT\n\nthanks"), + "expected tool_result block followed by 'thanks' in the merged user turn, got:\n{}", + out + ); } #[test] From 5cef9b62de1aa3be96a8606644eb64d31acd4969 Mon Sep 17 00:00:00 2001 From: Biswa Panda Date: Fri, 24 Apr 2026 00:42:16 -0700 Subject: [PATCH 08/21] fix(preprocessor): key DeepSeek-V4/V3.2 detection off config.json model_type MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Previously `PromptFormatter::from_mdc` activated the V4 / V3.2 native formatters based on a substring match against `mdc.display_name`: name_lower.contains("deepseek") && name_lower.contains("v4") Two problems: 1. **Rename-fragile** — `display_name` is overwritten by `--served-model-name`, so a valid DeepSeek-V4 checkpoint served as `dsflash` (or any other operator-chosen alias) silently fell through to the HF Jinja path and the native Rust formatter never fired. 2. **Loose** — `contains("v4")` mis-matched composite names like `deepseek-v3.2-v4-foo`, short-circuiting to V4 before the V3.2 branch. Rework detection to key off `config.json` `model_type`, which DeepSeek-V4-Pro / V4-Flash both ship as `"deepseek_v4"` (verified on the HF repo). The config value is set by the model author and survives any rename. `display_name` is kept only as a fallback for MDCs that don't have a loadable config.json (tokenizer-only deployments), and a concrete-but-different config.json value now authoritatively suppresses the fallback — a `model_type: "llama"` model served as `deepseek-v4-flash` no longer gets the V4 formatter. Also tightens the display-name fallback to an anchored match equivalent to `^deepseek(?:[-_.])?v4(?:[-_.]|$)` (no `regex` crate dep), closing the composite-name hole flagged as N5 in the V4 review. Unit tests cover: canonical V4 variants (hyphen / underscore / dot / concat separator), negative cases (composite / non-V4 / mis-prefixed), config-primary precedence, and the V3.2 symmetric case. --- lib/llm/src/preprocessor/prompt/template.rs | 174 +++++++++++++++++++- 1 file changed, 167 insertions(+), 7 deletions(-) diff --git a/lib/llm/src/preprocessor/prompt/template.rs b/lib/llm/src/preprocessor/prompt/template.rs index 10e772df5d9b..a81dbc27a281 100644 --- a/lib/llm/src/preprocessor/prompt/template.rs +++ b/lib/llm/src/preprocessor/prompt/template.rs @@ -20,17 +20,29 @@ use tokcfg::ChatTemplateValue; impl PromptFormatter { pub fn from_mdc(mdc: &ModelDeploymentCard) -> Result { // Special handling for DeepSeek models whose HF repos don't ship a Jinja chat_template. - let name_lower = mdc.display_name.to_lowercase(); - if name_lower.contains("deepseek") && name_lower.contains("v4") { - tracing::info!("Detected DeepSeek V4 model, using native Rust formatter"); + // + // Prefer the authoritative `model_type` from config.json — it's set by + // the model author and survives any `--served-model-name` rename. Fall + // back to a tight substring match on `display_name` only when config.json + // is absent (e.g., tokenizer-only MDCs) or unreadable. + let model_type_lower = mdc + .model_info + .as_ref() + .and_then(|info| info.get_model_info().ok()) + .map(|info| info.model_type().to_lowercase()); + let display_name_lower = mdc.display_name.to_lowercase(); + + if is_deepseek_v4(&model_type_lower, &display_name_lower) { + tracing::info!( + model_type = ?model_type_lower, + display_name = %mdc.display_name, + "Detected DeepSeek V4 model, using native Rust formatter", + ); return Ok(Self::OAI(Arc::new( super::deepseek_v4::DeepSeekV4Formatter::new_thinking(), ))); } - if name_lower.contains("deepseek") - && name_lower.contains("v3.2") - && !name_lower.contains("exp") - { + if is_deepseek_v3_2_non_exp(&model_type_lower, &display_name_lower) { tracing::info!("Detected DeepSeek V3.2 model (non-Exp), using native Rust formatter"); return Ok(Self::OAI(Arc::new( super::deepseek_v32::DeepSeekV32Formatter::new_thinking(), @@ -193,3 +205,151 @@ struct HfTokenizerConfigJsonFormatter { pub struct ContextMixins { context_mixins: HashSet, } + +/// Decides whether to activate the DeepSeek-V4 native formatter. +/// +/// Primary signal: config.json `model_type`. DeepSeek-V4-Pro and V4-Flash both +/// ship `"model_type": "deepseek_v4"`, set by the model author — this survives +/// any `--served-model-name` rename. +/// +/// Fallback: `display_name`, tight-matched against +/// `^deepseek(?:[-_.])?v4(?:[-_.]|$)`. Only consulted when config.json is +/// absent (tokenizer-only MDCs) or unreadable; a concrete config.json value +/// that is *not* `deepseek_v4` is authoritative and suppresses the fallback. +fn is_deepseek_v4(model_type_lower: &Option, display_name_lower: &str) -> bool { + match model_type_lower.as_deref() { + Some("deepseek_v4") => true, + Some(_) => false, // config.json says something else — trust it + None => is_deepseek_v4_name(display_name_lower), + } +} + +/// Decides whether to activate the DeepSeek-V3.2 (non-Exp) native formatter. +/// Same config-primary / name-fallback rule as V4. +fn is_deepseek_v3_2_non_exp(model_type_lower: &Option, display_name_lower: &str) -> bool { + let name_match = display_name_lower.contains("deepseek") + && display_name_lower.contains("v3.2") + && !display_name_lower.contains("exp"); + match model_type_lower.as_deref() { + Some("deepseek_v3_2") => !display_name_lower.contains("exp"), + Some(_) => false, + None => name_match, + } +} + +/// Tight, anchored match for DeepSeek-V4 display names. Equivalent to the +/// regex `^deepseek(?:[-_.])?v4(?:[-_.]|$)` over an already-lowercased string. +/// Written with string ops to avoid pulling in the `regex` crate. +/// +/// Rejects composite names that previously short-circuited the V4 branch: +/// - `deepseek-v3.2-v4-foo` (the `v3.2` variant is the real one) +/// - `deepseek-v40` / `deepseek-v4pro` (no separator after `v4`) +/// - `my-deepseek-v4` (prefix must be at the start) +fn is_deepseek_v4_name(name_lower: &str) -> bool { + let Some(rest) = name_lower.strip_prefix("deepseek") else { + return false; + }; + // Optional single separator between "deepseek" and "v4". + let rest = rest + .strip_prefix(|c: char| matches!(c, '-' | '_' | '.')) + .unwrap_or(rest); + let Some(after_v4) = rest.strip_prefix("v4") else { + return false; + }; + // `v4` must end the name or be followed by a separator — anything else + // (e.g. `v40`, `v4pro`) is a different model family. + after_v4.is_empty() || after_v4.starts_with(|c: char| matches!(c, '-' | '_' | '.')) +} + +#[cfg(test)] +mod detection_tests { + use super::{is_deepseek_v3_2_non_exp, is_deepseek_v4, is_deepseek_v4_name}; + + #[test] + fn v4_name_matches_canonical_variants() { + for name in [ + "deepseek-v4", + "deepseek_v4", + "deepseek.v4", + "deepseekv4", + "deepseek-v4-pro", + "deepseek-v4-flash", + "deepseek-v4-flash-2507", + "deepseek-v4.1", + "deepseek_v4_thinking", + ] { + assert!(is_deepseek_v4_name(name), "expected {name} to match V4"); + } + } + + #[test] + fn v4_name_rejects_non_v4() { + // Composite names that previously short-circuited to V4 before the + // V3.2 branch — now correctly rejected. + for name in [ + "deepseek-v3.2-v4-foo", + "my-deepseek-v4", + "deepseek-v40", + "deepseek-v4pro", + "deepseekv40", + "deepseek-v3", + "deepseek-v3.2", + "deepseek-r1", + "qwen3-v4", // only deepseek-prefixed names qualify + "dsflash", + "", + ] { + assert!( + !is_deepseek_v4_name(name), + "expected {name} to NOT match V4", + ); + } + } + + #[test] + fn v4_detection_prefers_config_model_type() { + // config.json `model_type = "deepseek_v4"` wins regardless of what + // the operator calls the model via --served-model-name. + let v4 = Some("deepseek_v4".to_string()); + for display in ["dsflash", "my-pet-model", "llama-3-8b", ""] { + assert!( + is_deepseek_v4(&v4, display), + "config says deepseek_v4, display {display:?} — expected V4", + ); + } + + // A concrete non-V4 config.json suppresses the display-name fallback. + // Even if the operator names the served model "deepseek-v4", a model + // with `model_type = "llama"` is NOT DeepSeek-V4. + let llama = Some("llama".to_string()); + for display in ["deepseek-v4", "deepseek-v4-flash", "anything"] { + assert!( + !is_deepseek_v4(&llama, display), + "config says llama, display {display:?} — expected NOT V4", + ); + } + + // No config.json — fall back to display-name match. + assert!(is_deepseek_v4(&None, "deepseek-v4-flash")); + assert!(!is_deepseek_v4(&None, "dsflash")); + } + + #[test] + fn v3_2_detection_prefers_config_model_type() { + // config says deepseek_v3_2, any non-"exp" display name triggers. + let v3_2 = Some("deepseek_v3_2".to_string()); + assert!(is_deepseek_v3_2_non_exp(&v3_2, "whatever")); + assert!(is_deepseek_v3_2_non_exp(&v3_2, "deepseek-v3.2")); + // V3.2-Exp is a separate model family; suppress even via config. + assert!(!is_deepseek_v3_2_non_exp(&v3_2, "deepseek-v3.2-exp")); + + // Other config types lose regardless of display name. + let other = Some("deepseek_v4".to_string()); + assert!(!is_deepseek_v3_2_non_exp(&other, "deepseek-v3.2")); + + // No config — fall back to the original display-name heuristic. + assert!(is_deepseek_v3_2_non_exp(&None, "deepseek-v3.2-pro")); + assert!(!is_deepseek_v3_2_non_exp(&None, "deepseek-v3.2-exp")); + assert!(!is_deepseek_v3_2_non_exp(&None, "deepseek-v4")); + } +} From 38a2b9575f9cc5d14570df65455204e323b6ba0c Mon Sep 17 00:00:00 2001 From: Biswa Panda Date: Fri, 24 Apr 2026 00:45:58 -0700 Subject: [PATCH 09/21] fix(preprocessor): treat empty config.json model_type as no-signal MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `HFConfig::model_type` is a required field at parse time, so a missing entry fails deserialization and `get_model_info().ok()` drops the whole `ModelInfo` — we correctly fall back to the display-name heuristic. But `"model_type": ""` (rare, legal JSON) parses cleanly into `Some(String::new())`, hits the `Some(_) => false` arm in `is_deepseek_v4`, and *suppresses* the display-name fallback. A DeepSeek-V4 model served as `deepseek-v4-flash` with an empty `model_type` would silently miss the native formatter. Normalize empty strings to `None` via `.filter(|s| !s.is_empty())` so "no signal from config.json" behaves uniformly regardless of whether the field was absent or blank. --- lib/llm/src/preprocessor/prompt/template.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/llm/src/preprocessor/prompt/template.rs b/lib/llm/src/preprocessor/prompt/template.rs index a81dbc27a281..375aeef5ce25 100644 --- a/lib/llm/src/preprocessor/prompt/template.rs +++ b/lib/llm/src/preprocessor/prompt/template.rs @@ -25,11 +25,16 @@ impl PromptFormatter { // the model author and survives any `--served-model-name` rename. Fall // back to a tight substring match on `display_name` only when config.json // is absent (e.g., tokenizer-only MDCs) or unreadable. + // + // An empty `model_type` string (rare but legal in the JSON) carries + // no signal — normalize it to `None` so the display-name fallback + // still runs instead of being silently suppressed. let model_type_lower = mdc .model_info .as_ref() .and_then(|info| info.get_model_info().ok()) - .map(|info| info.model_type().to_lowercase()); + .map(|info| info.model_type().to_lowercase()) + .filter(|s| !s.is_empty()); let display_name_lower = mdc.display_name.to_lowercase(); if is_deepseek_v4(&model_type_lower, &display_name_lower) { @@ -332,6 +337,13 @@ mod detection_tests { // No config.json — fall back to display-name match. assert!(is_deepseek_v4(&None, "deepseek-v4-flash")); assert!(!is_deepseek_v4(&None, "dsflash")); + + // A config.json with `"model_type": ""` is treated as "no signal" at + // the call site (normalized to None before is_deepseek_v4 is called), + // so the display-name fallback still runs — pin that contract. + let empty: Option = None; + assert!(is_deepseek_v4(&empty, "deepseek-v4-flash")); + assert!(!is_deepseek_v4(&empty, "dsflash")); } #[test] From 934e1176cb52e2460c5b562e5a675275110ff237 Mon Sep 17 00:00:00 2001 From: Biswa Panda Date: Fri, 24 Apr 2026 01:10:04 -0700 Subject: [PATCH 10/21] perf(deepseek_v4): size to_json buffer from compact length instead of 64 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `to_json` serializes tool schemas, response-format bodies, and arbitrary message-content fallbacks through a custom `PythonFormatter`. The buffer was `Vec::with_capacity(64)` — too small for anything beyond a trivial value, and worse than `Vec::new()` for tiny values since it forces an up-front heap alloc that's immediately outgrown. For a ~5 KB tool schema (200-field array), 64 bytes triggers 6 sequential doublings (64→128→256→512→1024→2048→4096→8192), each a memcpy of the current contents. Callers hit this on every request that includes tools or a response_format, so the overhead is per-request on the hot path. Pre-size the buffer from one compact `serde_json::to_string` pass. The `PythonFormatter` adds exactly 1 byte per structural separator (`,`→`, ` and `:`→`: `), which bounds the final length by ~12.5% over compact — `compact_len + compact_len/8` is a tight upper bound that eliminates all reallocations in the formatted pass. A 256-byte floor keeps tiny payloads from under-allocating after empty-map round-trips. Adds a ~5 KB round-trip test (`test_to_json_handles_large_payload`) that pins: - Output parses back to the input (no truncation). - Python-style spacing survives deep nesting (no bare `",", / `":"`). All 7 deepseek_v4 unit tests pass including the new one. --- .../src/preprocessor/prompt/deepseek_v4.rs | 50 ++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/lib/llm/src/preprocessor/prompt/deepseek_v4.rs b/lib/llm/src/preprocessor/prompt/deepseek_v4.rs index 528e917ef44c..039eeaf59ced 100644 --- a/lib/llm/src/preprocessor/prompt/deepseek_v4.rs +++ b/lib/llm/src/preprocessor/prompt/deepseek_v4.rs @@ -127,7 +127,14 @@ fn to_json(value: &JsonValue) -> String { } } - let mut buf = Vec::with_capacity(64); + // Size the buffer from a compact pre-serialization. Python-style spacing + // adds exactly one byte per structural separator (`,` → `, `, `:` → `: `), + // which bounds the final length by ~1.125× the compact length. This keeps + // small payloads cheap and avoids the 5–9 reallocations the prior fixed + // 64-byte hint forced on KB-sized tool schemas and response formats. + let compact_len = serde_json::to_string(value).map(|s| s.len()).unwrap_or(256); + let capacity = compact_len.saturating_add(compact_len / 8).max(256); + let mut buf = Vec::with_capacity(capacity); let mut ser = serde_json::Serializer::with_formatter(&mut buf, PythonFormatter); if value.serialize(&mut ser).is_err() { return "{}".to_string(); @@ -1003,4 +1010,45 @@ mod tests { "to_json must match Python's json.dumps formatting past an escaped backslash" ); } + + /// Larger-than-64-byte inputs (typical tool schemas / response formats) + /// must round-trip unchanged — pins that the capacity hint doesn't + /// truncate and that Python spacing holds across deep nesting. + #[test] + fn test_to_json_handles_large_payload() { + // Build a ~5 KB tool-like schema by nesting an array of items. + let items: Vec = (0..200) + .map(|i| json!({"name": format!("field_{i}"), "type": "string", "i": i})) + .collect(); + let v = json!({ + "type": "object", + "properties": {"items": {"type": "array", "items": items}}, + "required": ["items"], + }); + + let got = to_json(&v); + // Baseline: default serde_json::to_string round-trips. + let parsed: serde_json::Value = serde_json::from_str(&got).expect("round-trip parse"); + assert_eq!( + parsed, v, + "to_json output must round-trip back to the input" + ); + // Python spacing assertions: no bare `",` or `":` sequences outside of + // string literals. The payload contains no commas or colons inside + // string values, so a byte scan is sufficient. + assert!( + !got.contains("\",\""), + "expected ', ' between keys — raw '\",\"' should not appear", + ); + assert!( + !got.contains("\":\""), + "expected ': ' between key and value — raw '\":\"' should not appear", + ); + // Sanity: large payload exercised (> 5KB). + assert!( + got.len() > 5_000, + "test payload is too small: {}", + got.len() + ); + } } From 408daec961d05dd0a1c76f2edf9bbbdcc44ee3d9 Mon Sep 17 00:00:00 2001 From: Biswa Panda Date: Fri, 24 Apr 2026 01:34:56 -0700 Subject: [PATCH 11/21] perf(dsml): cache compiled regexes per config instead of recompiling per call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `extract_tool_calls`, `extract_invokes`, `parse_parameters` each built a regex pattern from `format!("...{}...", regex::escape(config.field))` and `Regex::new`'d it on every invocation. On streaming hot paths this recompiled three regexes per chunk — independently of what the chunk actually contained — despite the config strings being effectively fixed per backend (V3.2 and V4 are the only variants in practice). Introduce a `OnceLock>>>` module cache keyed on the six config strings, and look up (with a shared read lock on the fast path) before falling through to compile-and-insert. The cache has at most two entries for the process lifetime. `extract_invokes` and `parse_parameters` now take `&DsmlRegexes` instead of `&DsmlParserConfig`, so the inner call tree carries compiled regexes rather than repeatedly hashing and re-looking-up the same entry. All 19 DSML parser unit tests still pass unchanged — the cache is a pure behavioral no-op. --- lib/parsers/src/tool_calling/dsml/parser.rs | 128 +++++++++++++------- 1 file changed, 85 insertions(+), 43 deletions(-) diff --git a/lib/parsers/src/tool_calling/dsml/parser.rs b/lib/parsers/src/tool_calling/dsml/parser.rs index 5d9d72a628b0..badb4bf703d7 100644 --- a/lib/parsers/src/tool_calling/dsml/parser.rs +++ b/lib/parsers/src/tool_calling/dsml/parser.rs @@ -5,11 +5,84 @@ // https://huggingface.co/deepseek-ai/DeepSeek-V3.2/tree/main/encoding/encoding_dsv32.py use regex::Regex; +use std::collections::HashMap; +use std::sync::{Arc, OnceLock, RwLock}; use uuid::Uuid; use super::super::config::DsmlParserConfig; use super::super::response::{CalledFunction, ToolCallResponse, ToolCallType}; +/// Compiled regex trio for a given `DsmlParserConfig`. Compiled once and +/// reused across every subsequent parse/stream chunk. +struct DsmlRegexes { + block: Regex, + invoke: Regex, + parameter: Regex, +} + +/// Cache key = the six config strings that drive the three regex patterns. +/// V3.2 and V4 are the only variants in use, so the cache has at most two +/// entries for the lifetime of the process. +type DsmlRegexKey = (String, String, String, String, String, String); + +fn regex_cache() -> &'static RwLock>> { + static CACHE: OnceLock>>> = OnceLock::new(); + CACHE.get_or_init(|| RwLock::new(HashMap::new())) +} + +/// Return the compiled regex trio for `config`, compiling on first use. +/// +/// Each parse call previously recompiled three regexes from `format!`'d +/// patterns that embed the config strings — expensive on streaming hot paths. +/// The cache is keyed on the raw config strings (not the escaped patterns) +/// so distinct configs that happen to escape identically still get distinct +/// entries. +fn get_dsml_regexes(config: &DsmlParserConfig) -> anyhow::Result> { + let key: DsmlRegexKey = ( + config.function_calls_start.clone(), + config.function_calls_end.clone(), + config.invoke_start_prefix.clone(), + config.invoke_end.clone(), + config.parameter_prefix.clone(), + config.parameter_end.clone(), + ); + // Fast path: shared read lock, common after the first parse of each config. + if let Some(regexes) = regex_cache() + .read() + .expect("DSML regex cache read lock poisoned") + .get(&key) + { + return Ok(Arc::clone(regexes)); + } + // Slow path: compile and install. Use `entry` so a concurrent compiler of + // the same key only inserts once (we still compile speculatively, then + // drop the duplicate — cheap on a map of <= 2 keys). + let block = Regex::new(&format!( + r"(?s){}\s*(.*?)\s*{}", + regex::escape(&config.function_calls_start), + regex::escape(&config.function_calls_end), + ))?; + let invoke = Regex::new(&format!( + r#"(?s){}\"([^"]+)\"\s*>(.*?){}"#, + regex::escape(&config.invoke_start_prefix), + regex::escape(&config.invoke_end), + ))?; + let parameter = Regex::new(&format!( + r#"(?s){}\"([^"]+)\"\s+string=\"(true|false)\"\s*>(.*?){}"#, + regex::escape(&config.parameter_prefix), + regex::escape(&config.parameter_end), + ))?; + let regexes = Arc::new(DsmlRegexes { + block, + invoke, + parameter, + }); + let mut cache = regex_cache() + .write() + .expect("DSML regex cache write lock poisoned"); + Ok(Arc::clone(cache.entry(key).or_insert(regexes))) +} + /// DeepSeek V3.2 uses DSML (DeepSeek Markup Language) format for tool calls: /// /// <｜DSML｜function_calls> @@ -97,24 +170,16 @@ fn extract_tool_calls( config: &DsmlParserConfig, ) -> anyhow::Result> { let mut tool_calls = Vec::new(); + let regexes = get_dsml_regexes(config)?; - // Find all function_calls blocks - // Matches: <｜DSML｜function_calls> ... - // Pattern: (?s) = dot matches newlines - // \s*(.*?)\s* = capture content between start/end tags (non-greedy) - let block_pattern = format!( - r"(?s){}\s*(.*?)\s*{}", - regex::escape(&config.function_calls_start), - regex::escape(&config.function_calls_end) - ); - let block_regex = Regex::new(&block_pattern)?; - - for block_match in block_regex.captures_iter(text) { + // Find all function_calls blocks — the block regex captures the content + // between start/end tags (non-greedy, dot-matches-newline). + for block_match in regexes.block.captures_iter(text) { if let Some(block_content) = block_match.get(1) { let block = block_content.as_str(); // Extract individual invokes from this block - let invokes = extract_invokes(block, config)?; + let invokes = extract_invokes(block, ®exes)?; tool_calls.extend(invokes); } } @@ -123,29 +188,18 @@ fn extract_tool_calls( } /// Extract individual invoke blocks from function_calls content -fn extract_invokes( - block: &str, - config: &DsmlParserConfig, -) -> anyhow::Result> { +fn extract_invokes(block: &str, regexes: &DsmlRegexes) -> anyhow::Result> { let mut invokes = Vec::new(); - // Regex to match: <｜DSML｜invoke name="function_name">..content.. - // Note: invoke_start_prefix is "<｜DSML｜invoke name=" (no quotes, we add them in pattern) - let invoke_pattern = format!( - r#"(?s){}\"([^"]+)\"\s*>(.*?){}"#, - regex::escape(&config.invoke_start_prefix), - regex::escape(&config.invoke_end) - ); - let invoke_regex = Regex::new(&invoke_pattern)?; - - for invoke_match in invoke_regex.captures_iter(block) { + // Matches: <｜DSML｜invoke name="function_name">..content.. + for invoke_match in regexes.invoke.captures_iter(block) { if let (Some(name_match), Some(content_match)) = (invoke_match.get(1), invoke_match.get(2)) { let function_name = name_match.as_str().trim().to_string(); let invoke_content = content_match.as_str(); // Parse parameters from invoke content - let parameters = parse_parameters(invoke_content, config)?; + let parameters = parse_parameters(invoke_content, regexes)?; // Create tool call response let arguments_json = serde_json::to_string(¶meters)?; @@ -167,24 +221,12 @@ fn extract_invokes( /// Parse parameters from invoke content fn parse_parameters( content: &str, - config: &DsmlParserConfig, + regexes: &DsmlRegexes, ) -> anyhow::Result> { let mut parameters = serde_json::Map::new(); - // Build pattern with proper escaping - // Match: <｜DSML｜parameter name="param_name" string="true|false">value - // Note: parameter_prefix is "<｜DSML｜parameter name=" (no quotes, we add them in pattern) - let prefix_escaped = regex::escape(&config.parameter_prefix); - let end_escaped = regex::escape(&config.parameter_end); - - let param_pattern = format!( - r#"(?s){}\"([^"]+)\"\s+string=\"(true|false)\"\s*>(.*?){}"#, - prefix_escaped, end_escaped - ); - - let param_regex = Regex::new(¶m_pattern)?; - - for param_match in param_regex.captures_iter(content) { + // Matches: <｜DSML｜parameter name="param_name" string="true|false">value + for param_match in regexes.parameter.captures_iter(content) { if let (Some(name_match), Some(string_match), Some(value_match)) = (param_match.get(1), param_match.get(2), param_match.get(3)) { From b04a92fabfd2f6b1c69ae2a636cd82c02caad07a Mon Sep 17 00:00:00 2001 From: Biswa Panda Date: Fri, 24 Apr 2026 01:38:58 -0700 Subject: [PATCH 12/21] perf(deepseek_v4): collapse render_tools 4-replace chain into one format! MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `render_tools` built the tool-section of the system prompt by running four sequential `String::replace` passes over `TOOLS_TEMPLATE`: TOOLS_TEMPLATE .replace("{tool_schemas}", ...) // allocates a fresh String .replace("{dsml_token}", ...) // allocates again .replace("{thinking_start_token}", ...) .replace("{thinking_end_token}", ...) Each `replace` allocates a new `String` sized for the whole template — and after the first call the template already contains the inlined schemas, so each subsequent pass copies the full (potentially kB-scale) payload. Four tool-schema-inlined copies per render. Collapse into one `format!` with named arguments: a single allocation sized by the macro from the final length. Four placeholders, one pass. The standalone `TOOLS_TEMPLATE` const is no longer referenced and is removed — the template body now lives inside the `format!` macro where it's used. Byte-identical output: all four fixture-driven tests in `tests/deepseek_v4_encoding.rs` pass unchanged, including the tools-present fixtures that exercise every placeholder. --- .../src/preprocessor/prompt/deepseek_v4.rs | 71 ++++++++++--------- 1 file changed, 39 insertions(+), 32 deletions(-) diff --git a/lib/llm/src/preprocessor/prompt/deepseek_v4.rs b/lib/llm/src/preprocessor/prompt/deepseek_v4.rs index 039eeaf59ced..495199c2b21b 100644 --- a/lib/llm/src/preprocessor/prompt/deepseek_v4.rs +++ b/lib/llm/src/preprocessor/prompt/deepseek_v4.rs @@ -35,33 +35,6 @@ const TOOL_CALLS_BLOCK_NAME: &str = "tool_calls"; const RESPONSE_FORMAT_TEMPLATE: &str = "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{schema}"; -const TOOLS_TEMPLATE: &str = r#"## Tools - -You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml_token}tool_calls>" block like the following: - -<{dsml_token}tool_calls> -<{dsml_token}invoke name="$TOOL_NAME"> -<{dsml_token}parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE -... - -<{dsml_token}invoke name="$TOOL_NAME2"> -... - - - -String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. - -If thinking_mode is enabled (triggered by {thinking_start_token}), you MUST output your complete reasoning inside {thinking_start_token}...{thinking_end_token} BEFORE any tool calls or final response. - -Otherwise, output directly after {thinking_end_token} with tool calls or final response. - -### Available Tool Schemas - -{tool_schemas} - -You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. -"#; - const REASONING_EFFORT_MAX: &str = "Reasoning Effort: Absolute maximum with no shortcuts permitted.\nYou MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\nExplicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n"; /// Thinking mode for the model @@ -151,17 +124,51 @@ fn tools_from_openai_format(tools: &[JsonValue]) -> Vec { } /// Render tool schemas into the system prompt format. +/// +/// Previously built via four sequential `String::replace` passes over a +/// `{placeholder}`-based template, which allocated a fresh copy of the whole +/// (schema-inlined, potentially kB-scale) string on each substitution. A +/// single `format!` with named arguments collapses that to one allocation +/// sized from the final length. fn render_tools(tools: &[JsonValue]) -> String { let tools_json: Vec = tools_from_openai_format(tools) .iter() .map(to_json) .collect(); + let schemas = tools_json.join("\n"); + + format!( + r#"## Tools + +You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml}tool_calls>" block like the following: + +<{dsml}tool_calls> +<{dsml}invoke name="$TOOL_NAME"> +<{dsml}parameter name="$PARAMETER_NAME" string="true|false">$PARAMETER_VALUE +... + +<{dsml}invoke name="$TOOL_NAME2"> +... + + + +String parameters should be specified as is and set `string="true"`. For all other types (numbers, booleans, arrays, objects), pass the value in JSON format and set `string="false"`. + +If thinking_mode is enabled (triggered by {think_open}), you MUST output your complete reasoning inside {think_open}...{think_close} BEFORE any tool calls or final response. - TOOLS_TEMPLATE - .replace("{tool_schemas}", &tools_json.join("\n")) - .replace("{dsml_token}", tokens::DSML_TOKEN) - .replace("{thinking_start_token}", tokens::THINKING_START) - .replace("{thinking_end_token}", tokens::THINKING_END) +Otherwise, output directly after {think_close} with tool calls or final response. + +### Available Tool Schemas + +{schemas} + +You MUST strictly follow the above defined tool name and parameter schemas to invoke tool calls. +"#, + dsml = tokens::DSML_TOKEN, + think_open = tokens::THINKING_START, + think_close = tokens::THINKING_END, + schemas = schemas, + ) } /// Find the index of the last user/developer message. From e90c6ff0b224397e5052a978739088833422d1b0 Mon Sep 17 00:00:00 2001 From: Biswa Panda Date: Fri, 24 Apr 2026 01:43:05 -0700 Subject: [PATCH 13/21] perf(deepseek_v4): drop per-message clone in merge_tool_messages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `merge_tool_messages` cloned every input message at loop top (`let msg = msg.clone();`) regardless of which branch handled it: - `tool` role: extracts two fields into a fresh `tool_result` JSON — never pushes `msg`. The full clone was discarded. - `user` role: extracts the text content and preserves three named fields via per-field `v.clone()` into a fresh message — never pushes the original. The full clone was discarded. - other roles: passes `msg` through into `merged`. Only this branch actually needs to own the value. On long chat histories this cloned the entire conversation's JSON tree once per turn — tool calls with large payloads paid the worst cost. Iterate by reference (`for msg in messages` already binds `&JsonValue` from a slice), drop the top-level clone, and move the clone into the pass-through `else` branch where ownership is actually needed. The two "consecutive-user merge" branches are also tightened with let-chains to avoid the `merged.last_mut().unwrap()` unwrap that was guarded by the `can_merge` flag a few lines earlier — original malformed-data behavior (silent drop rather than fall-through) is preserved with an explicit comment. All 7 deepseek_v4 unit tests and all 4 fixture-driven integration tests pass unchanged — the output JSON is byte-identical for every shape the existing suite covers. --- .../src/preprocessor/prompt/deepseek_v4.rs | 35 ++++++++++++------- 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/lib/llm/src/preprocessor/prompt/deepseek_v4.rs b/lib/llm/src/preprocessor/prompt/deepseek_v4.rs index 495199c2b21b..15ca47afdfb6 100644 --- a/lib/llm/src/preprocessor/prompt/deepseek_v4.rs +++ b/lib/llm/src/preprocessor/prompt/deepseek_v4.rs @@ -527,11 +527,17 @@ fn render_tool_result_content(content: &JsonValue) -> String { /// Merge `tool` role messages into preceding user `content_blocks` and collapse /// consecutive user turns, matching Python's `merge_tool_messages`. +/// +/// Iterates the input by reference. Each message is cloned at most once — and +/// only when the control flow actually moves it into `merged` (the "other +/// role" pass-through branch). The `tool` and `user` branches extract the few +/// fields they need and build fresh JSON objects, so cloning the whole input +/// message up front (as the original implementation did) was pure overhead +/// on long chat histories. pub fn merge_tool_messages(messages: &[JsonValue]) -> Vec { let mut merged: Vec = Vec::with_capacity(messages.len()); for msg in messages { - let msg = msg.clone(); let role = msg.get("role").and_then(|r| r.as_str()).unwrap_or(""); if role == "tool" { @@ -550,11 +556,15 @@ pub fn merge_tool_messages(messages: &[JsonValue]) -> Vec { .unwrap_or(false); if can_merge { - let last = merged.last_mut().unwrap(); - if let Some(blocks) = last - .as_object_mut() - .and_then(|o| o.get_mut("content_blocks")) - .and_then(|v| v.as_array_mut()) + // `can_merge` already checked `content_blocks.is_some()`; if + // the subsequent `as_array_mut()` ever fails (data invariant + // violation) we match the original behavior and silently + // drop rather than falling through to push a new user msg. + if let Some(last) = merged.last_mut() + && let Some(blocks) = last + .as_object_mut() + .and_then(|o| o.get_mut("content_blocks")) + .and_then(|v| v.as_array_mut()) { blocks.push(tool_block); } @@ -582,11 +592,11 @@ pub fn merge_tool_messages(messages: &[JsonValue]) -> Vec { .unwrap_or(false); if can_merge { - let last = merged.last_mut().unwrap(); - if let Some(blocks) = last - .as_object_mut() - .and_then(|o| o.get_mut("content_blocks")) - .and_then(|v| v.as_array_mut()) + if let Some(last) = merged.last_mut() + && let Some(blocks) = last + .as_object_mut() + .and_then(|o| o.get_mut("content_blocks")) + .and_then(|v| v.as_array_mut()) { blocks.push(text_block); } @@ -607,7 +617,8 @@ pub fn merge_tool_messages(messages: &[JsonValue]) -> Vec { merged.push(new_msg); } } else { - merged.push(msg); + // Pass-through: clone only when we're actually moving the message. + merged.push(msg.clone()); } } From fffbe306b568118cec0a8d561a7cdcba8072a3a3 Mon Sep 17 00:00:00 2001 From: Biswa Panda Date: Fri, 24 Apr 2026 01:52:21 -0700 Subject: [PATCH 14/21] perf(deepseek_v4): dedupe find_last_user_index scans in encode path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `encode_messages_with_options` used to scan the message list twice: 1. `drop_thinking_messages` computed `find_last_user_index(&input)` at the top of its body to decide which pre-last-user messages to drop. 2. After drop (or instead of it), the encode loop computed `find_last_user_index(&full)` again to pass into `render_message`. Each `find_last_user_index` is a full O(n) reverse scan of a freshly- allocated JSON message vec. On long chat histories this is two redundant scans per request — one when drop runs, one always. Dedup: hoist `find_last_user_index(&full)` to the single site in `encode_messages_with_options` and thread both its input (used by drop to decide) and its *output* (computed inline while iterating) through `drop_thinking_messages`: - Caller computes `let mut last_user_idx = find_last_user_index(&full)` once on the post-merge/sort list. - `drop_thinking_messages(full, last_user_idx)` now returns `(Vec, Option)` — the dropped list plus the post-drop position of the same "last user/developer" message, tracked as `out.len()` at the moment it's pushed. - The encode loop reads the returned index directly — no rescan. The post-drop index is mathematically equivalent to a full `find_last_user_index(&dropped)` rescan: `drop_thinking_messages` never drops or reorders the message at `last_user_idx`, and every surviving message before it counts toward the new position by 1. Three new unit tests pin this equivalence for the three cases that matter: - `test_drop_thinking_messages_returns_post_drop_last_user_idx` — shape where a pre-last-user `developer` is dropped, forcing the trailing developer to shift from idx 3 to idx 2. Asserts the returned index equals a full rescan. - `test_drop_thinking_messages_preserves_idx_when_nothing_dropped` — no-shift sanity: returned idx equals the input idx. - `test_drop_thinking_messages_no_user_in_history` — edge case where both pre- and post-drop indices are `None`. All 10 deepseek_v4 unit tests and all 4 fixture-driven integration tests pass unchanged. --- .../src/preprocessor/prompt/deepseek_v4.rs | 124 +++++++++++++++++- 1 file changed, 119 insertions(+), 5 deletions(-) diff --git a/lib/llm/src/preprocessor/prompt/deepseek_v4.rs b/lib/llm/src/preprocessor/prompt/deepseek_v4.rs index 15ca47afdfb6..016d51c59bde 100644 --- a/lib/llm/src/preprocessor/prompt/deepseek_v4.rs +++ b/lib/llm/src/preprocessor/prompt/deepseek_v4.rs @@ -686,8 +686,15 @@ pub fn sort_tool_results_by_call_order(mut messages: Vec) -> Vec) -> Vec { - let last_user_idx = find_last_user_index(&messages); +/// +/// Takes `last_user_idx` (the pre-drop position of the last user/developer +/// message, as returned by [`find_last_user_index`]) from the caller so we +/// don't rescan the vector; returns the post-drop position of the same +/// message so the caller doesn't have to rescan either. +fn drop_thinking_messages( + messages: Vec, + last_user_idx: Option, +) -> (Vec, Option) { let mut out = Vec::with_capacity(messages.len()); const KEEP: &[&str] = &[ "user", @@ -697,9 +704,13 @@ fn drop_thinking_messages(messages: Vec) -> Vec { "direct_search_results", ]; + let mut new_last_user_idx: Option = None; for (idx, mut msg) in messages.into_iter().enumerate() { let role = msg.get("role").and_then(|r| r.as_str()).unwrap_or(""); if KEEP.contains(&role) || last_user_idx.is_none_or(|u| idx >= u) { + if last_user_idx == Some(idx) { + new_last_user_idx = Some(out.len()); + } out.push(msg); } else if role == "assistant" { if let Some(obj) = msg.as_object_mut() { @@ -709,7 +720,7 @@ fn drop_thinking_messages(messages: Vec) -> Vec { } // developer and other roles before last_user_idx are dropped. } - out + (out, new_last_user_idx) } /// Encode messages to prompt string with default options. @@ -758,11 +769,17 @@ pub fn encode_messages_with_options( }); let effective_drop_thinking = drop_thinking && !has_tools; + // Locate the last user/developer message once. `drop_thinking_messages` + // both consumes this (to know what to keep) and returns the new index + // (to avoid a second full scan over its output list). + let mut last_user_idx = find_last_user_index(&full); + if thinking_mode == ThinkingMode::Thinking && effective_drop_thinking { - full = drop_thinking_messages(full); + let (dropped, new_last_user) = drop_thinking_messages(full, last_user_idx); + full = dropped; + last_user_idx = new_last_user; } - let last_user_idx = find_last_user_index(&full); for idx in 0..full.len() { let part = render_message( idx, @@ -1069,4 +1086,101 @@ mod tests { got.len() ); } + + /// `drop_thinking_messages` now takes the pre-drop last-user index and + /// returns the post-drop index instead of forcing the caller to rescan + /// the output list. This test pins that the returned index is + /// equivalent to a full `find_last_user_index` rescan, for the shape + /// that actually shifts indices (non-KEEP role dropped before the + /// final user/developer). + #[test] + fn test_drop_thinking_messages_returns_post_drop_last_user_idx() { + // A `developer` message before the last user triggers an index + // shift: it's not in KEEP and is at idx < last_user_idx, so it + // gets dropped and every surviving message's index decreases by 1. + let messages = vec![ + json!({"role": "developer", "content": "dev1"}), + json!({"role": "assistant", "reasoning_content": "r", "content": "a1"}), + json!({"role": "user", "content": "u1"}), + json!({"role": "developer", "content": "dev2"}), + ]; + let pre_drop_idx = find_last_user_index(&messages); + // The last user-like message is the trailing developer at idx 3. + assert_eq!(pre_drop_idx, Some(3)); + + let (dropped, new_idx) = drop_thinking_messages(messages, pre_drop_idx); + + // developer at idx 0: dropped (not KEEP, before last_user_idx). + // assistant at idx 1: kept with reasoning stripped. + // user at idx 2: kept. + // developer at idx 3: kept (is last_user_idx). + assert_eq!(dropped.len(), 3, "expected 3 messages after drop"); + assert!( + dropped[0].get("reasoning_content").is_none(), + "assistant reasoning_content must be stripped", + ); + assert_eq!( + dropped[0].get("role").and_then(|v| v.as_str()), + Some("assistant"), + ); + assert_eq!( + dropped[1].get("role").and_then(|v| v.as_str()), + Some("user") + ); + assert_eq!( + dropped[2].get("role").and_then(|v| v.as_str()), + Some("developer"), + ); + + // The core invariant: the returned post-drop index is identical to + // what a full `find_last_user_index` rescan would produce. This + // is what allows `encode_messages_with_options` to skip the + // previously-redundant second scan. + let rescan_idx = find_last_user_index(&dropped); + assert_eq!( + new_idx, rescan_idx, + "drop_thinking_messages must return the same index find_last_user_index would compute on its output", + ); + assert_eq!(new_idx, Some(2), "developer shifted from idx 3 to idx 2"); + } + + /// Sanity check for the no-shift case: when nothing before the last + /// user/developer is droppable, the index returned by + /// `drop_thinking_messages` must equal the input index. + #[test] + fn test_drop_thinking_messages_preserves_idx_when_nothing_dropped() { + let messages = vec![ + json!({"role": "system", "content": "s"}), + json!({"role": "user", "content": "u1"}), + json!({"role": "assistant", "reasoning_content": "r", "content": "a"}), + json!({"role": "user", "content": "u2"}), + ]; + let pre_drop_idx = find_last_user_index(&messages); + assert_eq!(pre_drop_idx, Some(3)); + + let (dropped, new_idx) = drop_thinking_messages(messages, pre_drop_idx); + assert_eq!(dropped.len(), 4, "nothing should be dropped here"); + // No shift: last user still at the same index. + assert_eq!(new_idx, Some(3)); + assert_eq!(new_idx, find_last_user_index(&dropped)); + } + + /// Edge case: no user/developer anywhere in the history. Both the + /// pre-drop and post-drop indices must be `None` so the render loop + /// treats every message as "after last user" (Python `-1` sentinel). + #[test] + fn test_drop_thinking_messages_no_user_in_history() { + let messages = vec![ + json!({"role": "system", "content": "s"}), + json!({"role": "assistant", "reasoning_content": "r", "content": "a"}), + ]; + let pre_drop_idx = find_last_user_index(&messages); + assert_eq!(pre_drop_idx, None); + + let (dropped, new_idx) = drop_thinking_messages(messages, pre_drop_idx); + // With `last_user_idx == None`, the `idx >= u` guard is vacuously + // true for every index, so the assistant is kept (with reasoning). + assert_eq!(dropped.len(), 2); + assert_eq!(new_idx, None); + } } From b46e21a9a7f939c3992f733f9028ecbca41cf7be Mon Sep 17 00:00:00 2001 From: Biswa Panda Date: Fri, 24 Apr 2026 02:20:10 -0700 Subject: [PATCH 15/21] chore(deepseek_v4): code clean up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Focused pass over the DeepSeek-V4 formatter + reasoning registration. No behavior changes are intended for any fixture-covered input; every existing unit and integration test in `deepseek_v4` and `reasoning` passes unchanged. All items come from the review of 88478b4ba3..HEAD. Addressed: - Smell 4 / idiomaticity 4 — `to_json` now returns `Result` instead of silently collapsing serialization / UTF-8 errors to `"{}"`. Cascaded through `render_tools`, `extract_visible_text`, `normalize_message_contents`, `render_tool_result_content`, the two `render_message` tool/response-format sites, `encode_arguments_to_dsml`, and the `impl OAIPromptFormatter::render` entry point. Errors now surface as `anyhow::Error` with context instead of corrupting the prompt. - Smell 7 — `value.as_str().unwrap().to_string()` (guarded a few lines earlier by `.is_string()`) replaced with a `match value { JsonValue:: String(s) => ... }` pattern so the invariant holds at the type level rather than by convention. - Smell 8 — magic `&["user","system","tool","latest_reminder", "direct_search_results"]` list inside `drop_thinking_messages` hoisted to module-scope `const KEEP_ROLES: &[&str]` with a doc comment explaining what the list means. - Smell 9 — `"[Unsupported {}]"` literal injected into user-visible prompts now goes through a shared `UNSUPPORTED_PLACEHOLDER_FMT` constant and emits a `tracing::warn!` at both call sites (user-content blocks and tool_result items), matching the existing `extract_visible_text` drop-path warn. - Smell 10 — `ReasoningParserType::DeepSeekV4` dedicated variant added, replacing the silent `Qwen` alias. Verified against deepseek-ai/DeepSeek-V4-Pro/encoding/encoding_dsv4.py: V4 uses the same `` / `` delimiters as Qwen today, so the new variant's match arm points at the same `BasicReasoningParser` config — but future V4-specific divergence (max-thinking mode, different tokens) now has a place to land without rippling through Qwen. - Smell 11 — the three-alias entries (`deepseek_v4` / `deepseek-v4` / `deepseekv4`) in the reasoning parser map now carry an inline comment explaining why (callers wire the name through from heterogeneous sources: `--dyn-reasoning-parser` flag, vLLM recipes, chat-template authors; accepting all three avoids a canonical-form requirement). - Smell 12 — `RESPONSE_FORMAT_TEMPLATE` const + `replace("{schema}", ..)` collapsed into a direct `format!("## Response Format:...{}", schema)` at the two system/developer call sites. One fewer indirection, one fewer allocation per render. - Smell 13 — `TOOL_CALLS_BLOCK_NAME` const now has a docstring explaining its role (one DSML tag is extracted; the others are inline because they appear exactly once). The summary's alternative — also constifying `INVOKE_BLOCK_NAME` / `PARAMETER_BLOCK_NAME` — would introduce consts used in exactly one place each, net-negative for readability. Deferred (tracked for follow-up commits): - Smell 2 (V3.2/V4 helper duplication). The V4 versions of `to_json`, `extract_visible_text`, `normalize_message_contents`, etc. are not byte-identical to V3.2 — V4's `to_json` uses `PythonFormatter` while V3.2 uses a character-scan that has a known escape-handling bug. Extracting a shared module would change V3.2 behavior; needs its own commit with V3.2 regression coverage first. - Smell 3 (`render_message` split into `render_assistant_body` / `render_user_body` / `append_transition_token`). Worth doing, but mechanically sizable; keeping this commit focused. - Smell 5 (test move: keep invariants in-module, move scenarios to integration). Opinionated; deferred. Tests: 10 unit + 4 integration + 91 parsers reasoning tests green. --- .../src/preprocessor/prompt/deepseek_v4.rs | 128 +++++++++++------- lib/parsers/src/reasoning/mod.rs | 33 ++++- 2 files changed, 108 insertions(+), 53 deletions(-) diff --git a/lib/llm/src/preprocessor/prompt/deepseek_v4.rs b/lib/llm/src/preprocessor/prompt/deepseek_v4.rs index 016d51c59bde..baa8ff93d182 100644 --- a/lib/llm/src/preprocessor/prompt/deepseek_v4.rs +++ b/lib/llm/src/preprocessor/prompt/deepseek_v4.rs @@ -30,10 +30,25 @@ pub mod tokens { pub const TASK_READ_URL: &str = "<｜read_url｜>"; } +/// DSML outer block name for tool-call groups: `<｜DSML｜tool_calls>...`. const TOOL_CALLS_BLOCK_NAME: &str = "tool_calls"; -const RESPONSE_FORMAT_TEMPLATE: &str = - "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{schema}"; +/// Roles whose messages are always kept by `drop_thinking_messages`. All other +/// roles before the last user/developer message are dropped (assistant turns +/// keep their non-`reasoning_content` fields; everything else is removed). +const KEEP_ROLES: &[&str] = &[ + "user", + "system", + "tool", + "latest_reminder", + "direct_search_results", +]; + +/// Placeholder the Python reference inserts when it can't render an +/// unsupported content-block or tool-result item type. Rendered into the +/// assistant-visible prompt so a human can tell something was skipped; +/// dynamo additionally emits a `tracing::warn!` so ops see it too. +const UNSUPPORTED_PLACEHOLDER_FMT: &str = "[Unsupported {}]"; const REASONING_EFFORT_MAX: &str = "Reasoning Effort: Absolute maximum with no shortcuts permitted.\nYou MUST be very thorough in your thinking and comprehensively decompose the problem to resolve the root cause, rigorously stress-testing your logic against all potential paths, edge cases, and adversarial scenarios.\nExplicitly write out your entire deliberation process, documenting every intermediate step, considered alternative, and rejected hypothesis to ensure absolutely no assumption is left unchecked.\n\n"; @@ -63,7 +78,11 @@ pub enum ReasoningEffort { /// Serialize a JSON value to match Python's `json.dumps(ensure_ascii=False)` spacing. /// Python's default separators are `(', ', ': ')`; we use a custom `Formatter` /// so escape sequences inside strings can't confuse state tracking. -fn to_json(value: &JsonValue) -> String { +/// +/// Returns `Result` so serialization / UTF-8 errors propagate to the caller +/// rather than silently collapsing to `"{}"` (the old error-swallow behavior +/// dropped the entire request payload with no signal up the stack). +fn to_json(value: &JsonValue) -> Result { use serde::Serialize; use serde_json::ser::Formatter; use std::io; @@ -105,14 +124,16 @@ fn to_json(value: &JsonValue) -> String { // which bounds the final length by ~1.125× the compact length. This keeps // small payloads cheap and avoids the 5–9 reallocations the prior fixed // 64-byte hint forced on KB-sized tool schemas and response formats. - let compact_len = serde_json::to_string(value).map(|s| s.len()).unwrap_or(256); + let compact_len = serde_json::to_string(value) + .context("to_json: compact pre-serialization failed")? + .len(); let capacity = compact_len.saturating_add(compact_len / 8).max(256); let mut buf = Vec::with_capacity(capacity); let mut ser = serde_json::Serializer::with_formatter(&mut buf, PythonFormatter); - if value.serialize(&mut ser).is_err() { - return "{}".to_string(); - } - String::from_utf8(buf).unwrap_or_else(|_| "{}".to_string()) + value + .serialize(&mut ser) + .context("to_json: Python-formatter serialization failed")?; + String::from_utf8(buf).context("to_json: serialized output is not valid UTF-8") } /// Extract function definitions from OpenAI-format tool list. @@ -130,14 +151,14 @@ fn tools_from_openai_format(tools: &[JsonValue]) -> Vec { /// (schema-inlined, potentially kB-scale) string on each substitution. A /// single `format!` with named arguments collapses that to one allocation /// sized from the final length. -fn render_tools(tools: &[JsonValue]) -> String { +fn render_tools(tools: &[JsonValue]) -> Result { let tools_json: Vec = tools_from_openai_format(tools) .iter() .map(to_json) - .collect(); + .collect::>()?; let schemas = tools_json.join("\n"); - format!( + Ok(format!( r#"## Tools You have access to a set of tools to help answer the user's question. You can invoke tools by writing a "<{dsml}tool_calls>" block like the following: @@ -168,7 +189,7 @@ You MUST strictly follow the above defined tool name and parameter schemas to in think_open = tokens::THINKING_START, think_close = tokens::THINKING_END, schemas = schemas, - ) + )) } /// Find the index of the last user/developer message. @@ -191,8 +212,11 @@ fn find_last_user_index(messages: &[JsonValue]) -> Option { } /// Extract visible text from OpenAI-style message content. -fn extract_visible_text(content: &JsonValue) -> String { - match content { +/// +/// Returns `Result` because the `_ => to_json(content)` fallback can now fail +/// (to_json itself returns `Result`). Callers already sit in `Result` context. +fn extract_visible_text(content: &JsonValue) -> Result { + Ok(match content { JsonValue::String(text) => text.clone(), JsonValue::Array(items) => items .iter() @@ -214,12 +238,12 @@ fn extract_visible_text(content: &JsonValue) -> String { None }) .collect::(), - _ => to_json(content), - } + _ => to_json(content)?, + }) } /// Normalize message `content` fields for text-only DeepSeek V4 rendering. -fn normalize_message_contents(messages: &mut [JsonValue]) { +fn normalize_message_contents(messages: &mut [JsonValue]) -> Result<()> { for msg in messages { let Some(content) = msg.get("content") else { continue; @@ -228,11 +252,12 @@ fn normalize_message_contents(messages: &mut [JsonValue]) { if !content.is_string() && !content.is_array() { continue; } - let normalized = extract_visible_text(content); + let normalized = extract_visible_text(content)?; if let Some(obj) = msg.as_object_mut() { obj.insert("content".to_string(), JsonValue::String(normalized)); } } + Ok(()) } /// Encode tool call arguments into DSML parameter format. @@ -254,11 +279,12 @@ fn encode_arguments_to_dsml(tool_call: &JsonValue) -> Result { let mut params = Vec::new(); for (key, value) in arguments_obj { - let is_string = value.is_string(); - let value_str = if is_string { - value.as_str().unwrap().to_string() - } else { - to_json(value) + // Dispatch on the concrete variant so we don't do the `is_string` / `as_str` + // / `unwrap` dance (unwrap is technically safe after is_string but fragile + // against future refactors). + let (is_string, value_str) = match value { + JsonValue::String(s) => (true, s.clone()), + _ => (false, to_json(value)?), }; params.push(format!( "<{}parameter name=\"{}\" string=\"{}\">{}", @@ -318,13 +344,14 @@ fn render_message( prompt.push_str(content); if let Some(tools) = msg.get("tools").and_then(|t| t.as_array()) { prompt.push_str("\n\n"); - prompt.push_str(&render_tools(tools)); + prompt.push_str(&render_tools(tools)?); } if let Some(response_format) = msg.get("response_format") { prompt.push_str("\n\n"); - prompt.push_str( - &RESPONSE_FORMAT_TEMPLATE.replace("{schema}", &to_json(response_format)), - ); + prompt.push_str(&format!( + "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{}", + to_json(response_format)?, + )); } } @@ -340,13 +367,14 @@ fn render_message( if let Some(tools) = msg.get("tools").and_then(|t| t.as_array()) { content_developer.push_str("\n\n"); - content_developer.push_str(&render_tools(tools)); + content_developer.push_str(&render_tools(tools)?); } if let Some(response_format) = msg.get("response_format") { content_developer.push_str("\n\n"); - content_developer.push_str( - &RESPONSE_FORMAT_TEMPLATE.replace("{schema}", &to_json(response_format)), - ); + content_developer.push_str(&format!( + "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{}", + to_json(response_format)?, + )); } prompt.push_str(&content_developer); } @@ -365,11 +393,15 @@ fn render_message( "tool_result" => { let rendered = render_tool_result_content( block.get("content").unwrap_or(&JsonValue::Null), - ); + )?; parts.push(format!("{}", rendered)); } other => { - parts.push(format!("[Unsupported {}]", other)); + tracing::warn!( + block_type = other, + "DeepSeek V4 formatter emitted placeholder for unsupported user content block type", + ); + parts.push(UNSUPPORTED_PLACEHOLDER_FMT.replace("{}", other)); } } } @@ -500,8 +532,8 @@ fn render_message( } /// Render a tool_result `content` payload (string or content-block list). -fn render_tool_result_content(content: &JsonValue) -> String { - match content { +fn render_tool_result_content(content: &JsonValue) -> Result { + Ok(match content { JsonValue::String(s) => s.clone(), JsonValue::Array(items) => { let mut parts: Vec = Vec::with_capacity(items.len()); @@ -515,14 +547,18 @@ fn render_tool_result_content(content: &JsonValue) -> String { .to_string(), ); } else { - parts.push(format!("[Unsupported {}]", item_type)); + tracing::warn!( + item_type, + "DeepSeek V4 formatter emitted placeholder for unsupported tool_result content item type", + ); + parts.push(UNSUPPORTED_PLACEHOLDER_FMT.replace("{}", item_type)); } } parts.join("\n\n") } JsonValue::Null => String::new(), - _ => to_json(content), - } + _ => to_json(content)?, + }) } /// Merge `tool` role messages into preceding user `content_blocks` and collapse @@ -696,18 +732,10 @@ fn drop_thinking_messages( last_user_idx: Option, ) -> (Vec, Option) { let mut out = Vec::with_capacity(messages.len()); - const KEEP: &[&str] = &[ - "user", - "system", - "tool", - "latest_reminder", - "direct_search_results", - ]; - let mut new_last_user_idx: Option = None; for (idx, mut msg) in messages.into_iter().enumerate() { let role = msg.get("role").and_then(|r| r.as_str()).unwrap_or(""); - if KEEP.contains(&role) || last_user_idx.is_none_or(|u| idx >= u) { + if KEEP_ROLES.contains(&role) || last_user_idx.is_none_or(|u| idx >= u) { if last_user_idx == Some(idx) { new_last_user_idx = Some(out.len()); } @@ -857,7 +885,7 @@ impl super::OAIPromptFormatter for DeepSeekV4Formatter { .context("Messages is not an array")? .clone(); - normalize_message_contents(&mut messages_array); + normalize_message_contents(&mut messages_array)?; let tools_json = req .tools() @@ -1039,7 +1067,7 @@ mod tests { #[test] fn test_to_json_preserves_spacing_past_escaped_backslash() { let v = json!({"path": "\\", "count": 5}); - let got = to_json(&v); + let got = to_json(&v).expect("to_json must succeed on well-formed input"); assert_eq!( got, r#"{"path": "\\", "count": 5}"#, "to_json must match Python's json.dumps formatting past an escaped backslash" @@ -1061,7 +1089,7 @@ mod tests { "required": ["items"], }); - let got = to_json(&v); + let got = to_json(&v).expect("to_json must succeed on well-formed input"); // Baseline: default serde_json::to_string round-trips. let parsed: serde_json::Value = serde_json::from_str(&got).expect("round-trip parse"); assert_eq!( diff --git a/lib/parsers/src/reasoning/mod.rs b/lib/parsers/src/reasoning/mod.rs index bbfb67e4d651..91dd0d3399c5 100644 --- a/lib/parsers/src/reasoning/mod.rs +++ b/lib/parsers/src/reasoning/mod.rs @@ -29,9 +29,22 @@ fn get_reasoning_parser_map() -> &'static HashMap<&'static str, ReasoningParserT map.insert("basic", ReasoningParserType::Basic); map.insert("gpt_oss", ReasoningParserType::GptOss); map.insert("qwen3", ReasoningParserType::Qwen); - map.insert("deepseek_v4", ReasoningParserType::Qwen); - map.insert("deepseek-v4", ReasoningParserType::Qwen); - map.insert("deepseekv4", ReasoningParserType::Qwen); + // DeepSeek-V4 uses the same `` / `` delimiters as Qwen + // (confirmed against deepseek-ai/DeepSeek-V4-Pro's encoding_dsv4.py) + // so it delegates to the same `BasicReasoningParser` config today. We + // still route through a dedicated `DeepSeekV4` variant rather than + // hard-aliasing to `Qwen` so future divergence (different special + // tokens, max-thinking mode, etc.) has a place to land without rippling + // through Qwen's own config. + // + // The three name aliases exist because callers set this via + // `--dyn-reasoning-parser` / `--reasoning-parser` with whatever string + // the HF model / vLLM recipe / chat-template author picked. We accept + // all three separator conventions (snake / kebab / concat) rather than + // force a single canonical form on users. + map.insert("deepseek_v4", ReasoningParserType::DeepSeekV4); + map.insert("deepseek-v4", ReasoningParserType::DeepSeekV4); + map.insert("deepseekv4", ReasoningParserType::DeepSeekV4); map.insert("nemotron_deci", ReasoningParserType::NemotronDeci); map.insert("kimi", ReasoningParserType::Kimi); map.insert("kimi_k25", ReasoningParserType::KimiK25); @@ -113,6 +126,14 @@ pub enum ReasoningParserType { Basic, GptOss, Qwen, + /// DeepSeek-V4-Pro / V4-Flash. Currently uses the same `` / + /// `` `BasicReasoningParser` config as Qwen (V4 never appends + /// `` in the completion — the chat template always pre-injects it, + /// so the parser starts via `set_in_reasoning(true)` rather than + /// `force_reasoning`). A dedicated variant keeps future V4-specific + /// divergence (different delimiters, thinking-effort modes) from leaking + /// into Qwen's behavior. + DeepSeekV4, NemotronDeci, Kimi, KimiK25, @@ -164,6 +185,12 @@ impl ReasoningParserType { ReasoningParserType::Qwen => ReasoningParserWrapper { parser: Box::new(basic_parser), }, + // Same `` / `` config as Qwen today; kept as a + // distinct variant so V4-specific divergence has somewhere to land. + // See `ReasoningParserType::DeepSeekV4` docstring for rationale. + ReasoningParserType::DeepSeekV4 => ReasoningParserWrapper { + parser: Box::new(basic_parser), + }, ReasoningParserType::NemotronDeci => ReasoningParserWrapper { parser: Box::new(basic_parser), }, From fb4fbeb2cbddcb7a931827bb95f1e68e04e5ad5b Mon Sep 17 00:00:00 2001 From: Biswa Panda Date: Fri, 24 Apr 2026 02:26:26 -0700 Subject: [PATCH 16/21] =?UTF-8?q?chore(deepseek=5Fv4):=20code=20clean=20up?= =?UTF-8?q?=20part-2=20=E2=80=94=20Rust=20idiomaticity?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pure-style pass over the DeepSeek-V4 formatter. No behavior changes; all 10 unit + 4 integration tests still green. - Idiom 1: `find_last_user_index` role matcher `.map(|r| r == "user" || r == "developer").unwrap_or(false)` → `.is_some_and(|r| matches!(r, "user" | "developer"))` (closer to the `Python` `is_some_and` sentinel semantics the file uses elsewhere for last_user_idx comparisons.) - Idiom 2: `resolve_thinking_mode` nested-`if let` over the same `args` collapsed with 2024-edition let-chains; the outer scope `args` is matched twice but it's a cheap reference comparison and the flatter shape matches the rest of the file (e.g. the `can_merge` branches in `merge_tool_messages` already use this form). - Idiom 3: `.and_then(|v| v.as_str())` / `.as_array()` / `.as_bool()` / `.as_array_mut()` closures across ~30 sites replaced with `JsonValue::as_str` / `as_array` / `as_bool` / `as_array_mut` fn pointers. No closure allocation, one less turbofish-free indirection. - Idiom 5: `anyhow::bail!("Unknown role: {}", other)` → `anyhow::bail!("Unknown role: {other}")` (2024-edition captured identifier; matches `"{text_so_far}"` / `"{idx}"` usage elsewhere in the file). - Idiom 6: `#[inline]` on `ThinkingMode::as_str` (two-branch match returning `&'static str`) and `task_token` (seven-branch match returning `&'static str`). Both are on the per-message render hot path and smaller than the call overhead; inlining lets LLVM fold them into caller string-equality checks. Deferred from the idiomaticity list: - Idiom 4 (`to_json` → `Result`) — already landed in the previous cleanup commit alongside Smell 4. - Idiom 7 (`extract_visible_text` → `Cow<'_, str>`). The text-only path already avoids re-allocating (`text.clone()` is a `String::clone`, which is still O(n) but unavoidable without a `Cow`). Wrapping in `Cow<'_, str>` would change the return type through `normalize_message_contents` / `render` ripple and intermittently save one clone per message. Worth its own focused commit with a benchmark. Tests: 10 unit + 4 integration, all green. --- .../src/preprocessor/prompt/deepseek_v4.rs | 131 ++++++++++-------- 1 file changed, 73 insertions(+), 58 deletions(-) diff --git a/lib/llm/src/preprocessor/prompt/deepseek_v4.rs b/lib/llm/src/preprocessor/prompt/deepseek_v4.rs index baa8ff93d182..66159b549256 100644 --- a/lib/llm/src/preprocessor/prompt/deepseek_v4.rs +++ b/lib/llm/src/preprocessor/prompt/deepseek_v4.rs @@ -60,6 +60,9 @@ pub enum ThinkingMode { } impl ThinkingMode { + /// Tiny branch-free mapping to a static string. `#[inline]` because this + /// is called from inside the per-message render hot path. + #[inline] pub fn as_str(&self) -> &'static str { match self { ThinkingMode::Chat => "chat", @@ -204,9 +207,8 @@ fn find_last_user_index(messages: &[JsonValue]) -> Option { .rev() .find(|(_, msg)| { msg.get("role") - .and_then(|r| r.as_str()) - .map(|r| r == "user" || r == "developer") - .unwrap_or(false) + .and_then(JsonValue::as_str) + .is_some_and(|r| matches!(r, "user" | "developer")) }) .map(|(idx, _)| idx) } @@ -224,11 +226,11 @@ fn extract_visible_text(content: &JsonValue) -> Result { if let Some(text) = item.as_str() { return Some(text.to_string()); } - let item_type = item.get("type").and_then(|v| v.as_str()); + let item_type = item.get("type").and_then(JsonValue::as_str); if item_type == Some("text") { return item .get("text") - .and_then(|v| v.as_str()) + .and_then(JsonValue::as_str) .map(|text| text.to_string()); } tracing::warn!( @@ -264,7 +266,7 @@ fn normalize_message_contents(messages: &mut [JsonValue]) -> Result<()> { fn encode_arguments_to_dsml(tool_call: &JsonValue) -> Result { let arguments_str = tool_call .get("arguments") - .and_then(|a| a.as_str()) + .and_then(JsonValue::as_str) .context("Missing or invalid 'arguments' field")?; // Python falls back to `{"arguments": raw_string}` on parse failure. @@ -300,6 +302,11 @@ fn encode_arguments_to_dsml(tool_call: &JsonValue) -> Result { } /// Lookup the task token for a quick-instruction task. +/// +/// Called once per assistant-turn in `render_message` and once per transition +/// token lookup; `#[inline]` because the static-str match is smaller than the +/// call overhead. +#[inline] fn task_token(task: &str) -> Option<&'static str> { match task { "action" => Some(tokens::TASK_ACTION), @@ -325,7 +332,7 @@ fn render_message( let role = msg .get("role") - .and_then(|r| r.as_str()) + .and_then(JsonValue::as_str) .context("Missing 'role' field")?; let mut prompt = String::new(); @@ -340,9 +347,9 @@ fn render_message( match role { "system" => { - let content = msg.get("content").and_then(|c| c.as_str()).unwrap_or(""); + let content = msg.get("content").and_then(JsonValue::as_str).unwrap_or(""); prompt.push_str(content); - if let Some(tools) = msg.get("tools").and_then(|t| t.as_array()) { + if let Some(tools) = msg.get("tools").and_then(JsonValue::as_array) { prompt.push_str("\n\n"); prompt.push_str(&render_tools(tools)?); } @@ -358,14 +365,14 @@ fn render_message( "developer" => { let content = msg .get("content") - .and_then(|c| c.as_str()) + .and_then(JsonValue::as_str) .filter(|s| !s.is_empty()) .context("Developer role requires content")?; let mut content_developer = String::from(tokens::USER_START); content_developer.push_str(content); - if let Some(tools) = msg.get("tools").and_then(|t| t.as_array()) { + if let Some(tools) = msg.get("tools").and_then(JsonValue::as_array) { content_developer.push_str("\n\n"); content_developer.push_str(&render_tools(tools)?); } @@ -381,13 +388,13 @@ fn render_message( "user" => { prompt.push_str(tokens::USER_START); - if let Some(blocks) = msg.get("content_blocks").and_then(|b| b.as_array()) { + if let Some(blocks) = msg.get("content_blocks").and_then(JsonValue::as_array) { let mut parts: Vec = Vec::with_capacity(blocks.len()); for block in blocks { - let block_type = block.get("type").and_then(|v| v.as_str()).unwrap_or(""); + let block_type = block.get("type").and_then(JsonValue::as_str).unwrap_or(""); match block_type { "text" => { - let text = block.get("text").and_then(|v| v.as_str()).unwrap_or(""); + let text = block.get("text").and_then(JsonValue::as_str).unwrap_or(""); parts.push(text.to_string()); } "tool_result" => { @@ -407,13 +414,13 @@ fn render_message( } prompt.push_str(&parts.join("\n\n")); } else { - let content = msg.get("content").and_then(|c| c.as_str()).unwrap_or(""); + let content = msg.get("content").and_then(JsonValue::as_str).unwrap_or(""); prompt.push_str(content); } } "latest_reminder" => { - let content = msg.get("content").and_then(|c| c.as_str()).unwrap_or(""); + let content = msg.get("content").and_then(JsonValue::as_str).unwrap_or(""); prompt.push_str(tokens::LATEST_REMINDER); prompt.push_str(content); } @@ -425,12 +432,15 @@ fn render_message( } "assistant" => { - let content = msg.get("content").and_then(|c| c.as_str()).unwrap_or(""); + let content = msg.get("content").and_then(JsonValue::as_str).unwrap_or(""); let reasoning = msg .get("reasoning_content") - .and_then(|c| c.as_str()) + .and_then(JsonValue::as_str) .unwrap_or(""); - let wo_eos = msg.get("wo_eos").and_then(|v| v.as_bool()).unwrap_or(false); + let wo_eos = msg + .get("wo_eos") + .and_then(JsonValue::as_bool) + .unwrap_or(false); let prev_has_task = index > 0 && messages[index - 1] @@ -450,7 +460,7 @@ fn render_message( prompt.push_str(&thinking_part); prompt.push_str(content); - if let Some(tool_calls) = msg.get("tool_calls").and_then(|t| t.as_array()) + if let Some(tool_calls) = msg.get("tool_calls").and_then(JsonValue::as_array) && !tool_calls.is_empty() { prompt.push_str("\n\n"); @@ -467,7 +477,7 @@ fn render_message( let fn_obj = tc.get("function").unwrap_or(tc); let name = fn_obj .get("name") - .and_then(|n| n.as_str()) + .and_then(JsonValue::as_str) .context("Missing tool call name")?; let arguments = encode_arguments_to_dsml(fn_obj)?; invocations.push(format!( @@ -491,19 +501,19 @@ fn render_message( } } - other => anyhow::bail!("Unknown role: {}", other), + other => anyhow::bail!("Unknown role: {other}"), } // Early return if the next message is not assistant/latest_reminder — no transition appended. if index + 1 < messages.len() { - let next_role = messages[index + 1].get("role").and_then(|r| r.as_str()); + let next_role = messages[index + 1].get("role").and_then(JsonValue::as_str); if !matches!(next_role, Some("assistant") | Some("latest_reminder")) { return Ok(prompt); } } // Transition tokens based on task field and role. - let task = msg.get("task").and_then(|v| v.as_str()); + let task = msg.get("task").and_then(JsonValue::as_str); if let Some(task) = task { let sp = task_token(task).with_context(|| format!("Invalid task: '{}'", task))?; if task != "action" { @@ -538,11 +548,11 @@ fn render_tool_result_content(content: &JsonValue) -> Result { JsonValue::Array(items) => { let mut parts: Vec = Vec::with_capacity(items.len()); for item in items { - let item_type = item.get("type").and_then(|v| v.as_str()).unwrap_or(""); + let item_type = item.get("type").and_then(JsonValue::as_str).unwrap_or(""); if item_type == "text" { parts.push( item.get("text") - .and_then(|v| v.as_str()) + .and_then(JsonValue::as_str) .unwrap_or("") .to_string(), ); @@ -574,7 +584,7 @@ pub fn merge_tool_messages(messages: &[JsonValue]) -> Vec { let mut merged: Vec = Vec::with_capacity(messages.len()); for msg in messages { - let role = msg.get("role").and_then(|r| r.as_str()).unwrap_or(""); + let role = msg.get("role").and_then(JsonValue::as_str).unwrap_or(""); if role == "tool" { let tool_block = serde_json::json!({ @@ -586,7 +596,7 @@ pub fn merge_tool_messages(messages: &[JsonValue]) -> Vec { let can_merge = merged .last() .map(|m| { - m.get("role").and_then(|r| r.as_str()) == Some("user") + m.get("role").and_then(JsonValue::as_str) == Some("user") && m.get("content_blocks").is_some() }) .unwrap_or(false); @@ -600,7 +610,7 @@ pub fn merge_tool_messages(messages: &[JsonValue]) -> Vec { && let Some(blocks) = last .as_object_mut() .and_then(|o| o.get_mut("content_blocks")) - .and_then(|v| v.as_array_mut()) + .and_then(JsonValue::as_array_mut) { blocks.push(tool_block); } @@ -613,7 +623,7 @@ pub fn merge_tool_messages(messages: &[JsonValue]) -> Vec { } else if role == "user" { let text = msg .get("content") - .and_then(|c| c.as_str()) + .and_then(JsonValue::as_str) .unwrap_or("") .to_string(); let text_block = serde_json::json!({ "type": "text", "text": text }); @@ -621,7 +631,7 @@ pub fn merge_tool_messages(messages: &[JsonValue]) -> Vec { let can_merge = merged .last() .map(|m| { - m.get("role").and_then(|r| r.as_str()) == Some("user") + m.get("role").and_then(JsonValue::as_str) == Some("user") && m.get("content_blocks").is_some() && m.get("task").map(|v| v.is_null()).unwrap_or(true) }) @@ -632,7 +642,7 @@ pub fn merge_tool_messages(messages: &[JsonValue]) -> Vec { && let Some(blocks) = last .as_object_mut() .and_then(|o| o.get_mut("content_blocks")) - .and_then(|v| v.as_array_mut()) + .and_then(JsonValue::as_array_mut) { blocks.push(text_block); } @@ -668,18 +678,18 @@ pub fn sort_tool_results_by_call_order(mut messages: Vec) -> Vec = HashMap::new(); for msg in &mut messages { - let role = msg.get("role").and_then(|r| r.as_str()).unwrap_or(""); + let role = msg.get("role").and_then(JsonValue::as_str).unwrap_or(""); if role == "assistant" { - if let Some(tcs) = msg.get("tool_calls").and_then(|t| t.as_array()) { + if let Some(tcs) = msg.get("tool_calls").and_then(JsonValue::as_array) { last_order.clear(); for (idx, tc) in tcs.iter().enumerate() { let id = tc .get("id") - .and_then(|v| v.as_str()) + .and_then(JsonValue::as_str) .or_else(|| { tc.get("function") .and_then(|f| f.get("id")) - .and_then(|v| v.as_str()) + .and_then(JsonValue::as_str) }) .unwrap_or(""); if !id.is_empty() { @@ -691,7 +701,7 @@ pub fn sort_tool_results_by_call_order(mut messages: Vec) -> Vec) -> Vec = blocks .iter() .enumerate() - .filter(|(_, b)| b.get("type").and_then(|v| v.as_str()) == Some("tool_result")) + .filter(|(_, b)| b.get("type").and_then(JsonValue::as_str) == Some("tool_result")) .map(|(i, _)| i) .collect(); @@ -708,7 +718,10 @@ pub fn sort_tool_results_by_call_order(mut messages: Vec) -> Vec = tool_positions.iter().map(|&i| blocks[i].clone()).collect(); tool_blocks.sort_by_key(|b| { - let id = b.get("tool_use_id").and_then(|v| v.as_str()).unwrap_or(""); + let id = b + .get("tool_use_id") + .and_then(JsonValue::as_str) + .unwrap_or(""); *last_order.get(id).unwrap_or(&0) }); for (sorted_idx, &pos) in tool_positions.iter().enumerate() { @@ -734,7 +747,7 @@ fn drop_thinking_messages( let mut out = Vec::with_capacity(messages.len()); let mut new_last_user_idx: Option = None; for (idx, mut msg) in messages.into_iter().enumerate() { - let role = msg.get("role").and_then(|r| r.as_str()).unwrap_or(""); + let role = msg.get("role").and_then(JsonValue::as_str).unwrap_or(""); if KEEP_ROLES.contains(&role) || last_user_idx.is_none_or(|u| idx >= u) { if last_user_idx == Some(idx) { new_last_user_idx = Some(out.len()); @@ -848,20 +861,22 @@ impl DeepSeekV4Formatter { &self, args: Option<&std::collections::HashMap>, ) -> ThinkingMode { - if let Some(args) = args { - if let Some(thinking) = args.get("thinking").and_then(|v| v.as_bool()) { - return if thinking { - ThinkingMode::Thinking - } else { - ThinkingMode::Chat - }; - } - if let Some(mode) = args.get("thinking_mode").and_then(|v| v.as_str()) { - match mode { - "chat" => return ThinkingMode::Chat, - "thinking" => return ThinkingMode::Thinking, - _ => {} - } + if let Some(args) = args + && let Some(thinking) = args.get("thinking").and_then(JsonValue::as_bool) + { + return if thinking { + ThinkingMode::Thinking + } else { + ThinkingMode::Chat + }; + } + if let Some(args) = args + && let Some(mode) = args.get("thinking_mode").and_then(JsonValue::as_str) + { + match mode { + "chat" => return ThinkingMode::Chat, + "thinking" => return ThinkingMode::Thinking, + _ => {} } } self.thinking_mode @@ -902,7 +917,7 @@ impl super::OAIPromptFormatter for DeepSeekV4Formatter { if tools_json.is_some() || response_format_json.is_some() { let system_idx = messages_array .iter() - .position(|msg| msg.get("role").and_then(|r| r.as_str()) == Some("system")); + .position(|msg| msg.get("role").and_then(JsonValue::as_str) == Some("system")); if let Some(idx) = system_idx { if let Some(msg) = messages_array.get_mut(idx) @@ -1148,15 +1163,15 @@ mod tests { "assistant reasoning_content must be stripped", ); assert_eq!( - dropped[0].get("role").and_then(|v| v.as_str()), + dropped[0].get("role").and_then(JsonValue::as_str), Some("assistant"), ); assert_eq!( - dropped[1].get("role").and_then(|v| v.as_str()), + dropped[1].get("role").and_then(JsonValue::as_str), Some("user") ); assert_eq!( - dropped[2].get("role").and_then(|v| v.as_str()), + dropped[2].get("role").and_then(JsonValue::as_str), Some("developer"), ); From 77602e441a0c438fc9be66b79043fd6a5b90d4b2 Mon Sep 17 00:00:00 2001 From: ayushag Date: Fri, 24 Apr 2026 09:29:19 -0700 Subject: [PATCH 17/21] fix(llm,v4): use char slice for clippy::manual_pattern_char_comparison Signed-off-by: ayushag --- lib/llm/src/preprocessor/prompt/template.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/llm/src/preprocessor/prompt/template.rs b/lib/llm/src/preprocessor/prompt/template.rs index 375aeef5ce25..b9b86c599f2c 100644 --- a/lib/llm/src/preprocessor/prompt/template.rs +++ b/lib/llm/src/preprocessor/prompt/template.rs @@ -263,7 +263,7 @@ fn is_deepseek_v4_name(name_lower: &str) -> bool { }; // `v4` must end the name or be followed by a separator — anything else // (e.g. `v40`, `v4pro`) is a different model family. - after_v4.is_empty() || after_v4.starts_with(|c: char| matches!(c, '-' | '_' | '.')) + after_v4.is_empty() || after_v4.starts_with(['-', '_', '.']) } #[cfg(test)] From 3fcb0452ecd908cc7a8a48e9d3b48105f1bbbe3a Mon Sep 17 00:00:00 2001 From: Keiven Chang Date: Fri, 24 Apr 2026 17:41:51 +0000 Subject: [PATCH 18/21] test(parsers): add CASE.* taxonomy docs and V4 corner-case pinning tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces a `CASE.` / `CASE..` taxonomy for parser test categories: - new `lib/parsers/README.md` — crate landing page (parser families, request-flow diagram, how-to-add-a-parser) - new `lib/parsers/TESTING.md` — 16 generic + 2 XML-family + 1 Harmony categories, with per-category definitions, applicability, and example anchors Adds 4 DSML pinning tests that document current behavior on truncated V4 output: - `CASE.5` missing outer `` fence — whole block silently dropped (same structural class as Kimi K2 pre-recovery) - `CASE.4` missing inner `` fence — call silently dropped - `CASE.4` malformed JSON in `string="false"` param — falls back to raw string (intentional, pins the `unwrap_or_else` behavior) Annotates every existing V4 test with a `/// CASE.` doc comment naming the category it pins, across 4 files (dsml parser tests, parsers registry, reasoning registry, streaming e2e). Adds a coverage manifest block above the V4 test section listing what's still NOT covered so future contributors see the gaps without reading the full test file: - `CASE.5` mid-stream truncation recovery (TODO) - `CASE.4` parameter-close and middle-invoke-bleed variants - `CASE.11` tool_choice auto/required/named/none - `CASE.12` `FinishReason::Length` - `CASE.14` empty-content / null at e2e - `CASE.15` duplicate calls (universal gap) - `CASE.16` regression (V4 is brand new, no bugs filed yet) No production code changes. All 389 dynamo-parsers tests + 4 V4 encoding golden tests + 24 V4 streaming tests pass. Signed-off-by: Keiven Chang --- lib/llm/tests/test_streaming_tool_parsers.rs | 7 + lib/parsers/README.md | 129 ++++++++ lib/parsers/TESTING.md | 317 +++++++++++++++++++ lib/parsers/src/reasoning/mod.rs | 3 + lib/parsers/src/tool_calling/dsml/parser.rs | 167 ++++++++++ lib/parsers/src/tool_calling/parsers.rs | 2 + 6 files changed, 625 insertions(+) create mode 100644 lib/parsers/README.md create mode 100644 lib/parsers/TESTING.md diff --git a/lib/llm/tests/test_streaming_tool_parsers.rs b/lib/llm/tests/test_streaming_tool_parsers.rs index 3035f77e7ec0..983b59dcf42d 100644 --- a/lib/llm/tests/test_streaming_tool_parsers.rs +++ b/lib/llm/tests/test_streaming_tool_parsers.rs @@ -1157,6 +1157,7 @@ mod tests { } /// Single tool call, thinking mode (direct V4 analog of the V3 tool fixture). + /// `CASE.1` + `CASE.8` + `CASE.9` — single tool call, streaming assembly, paired with reasoning. Also validates `CASE.12` finish_reason=tool_calls. #[tokio::test] async fn test_deepseek_v4_e2e_with_tools_vllm() { let file_path = format!( @@ -1167,6 +1168,7 @@ mod tests { } /// No tool call — thinking + plain body; finish_reason=stop. + /// `CASE.3` + `CASE.10` — no tool call + reasoning only. Also validates `CASE.12` finish_reason=stop. #[tokio::test] async fn test_deepseek_v4_e2e_with_no_tools_vllm() { let file_path = format!( @@ -1206,6 +1208,7 @@ mod tests { } /// Two parallel tool calls inside one DSML block. + /// `CASE.2` — parallel tool calls in one DSML block. #[tokio::test] async fn test_deepseek_v4_e2e_multi_tool_vllm() { let file_path = format!( @@ -1217,6 +1220,7 @@ mod tests { /// string="true" vs string="false" — numbers, booleans, arrays, objects must /// round-trip as their proper JSON types inside arguments. + /// `CASE.7` — complex args (mixed string="true|false" → strings / numbers / bools / arrays / objects round-trip). #[tokio::test] async fn test_deepseek_v4_e2e_mixed_param_types_vllm() { let file_path = format!( @@ -1228,6 +1232,7 @@ mod tests { /// Body text emitted before the DSML block — parser must populate both /// normal_content and tool_calls. + /// `CASE.13` — normal text interleaved before the DSML block. #[tokio::test] async fn test_deepseek_v4_e2e_content_before_tool_vllm() { let file_path = format!( @@ -1240,6 +1245,7 @@ mod tests { /// Parameter value containing unicode, emoji, embedded quotes/newlines/tabs, /// and fragments that look like sentinels but aren't — must not confuse the /// parser, which anchors only on the exact token. + /// `CASE.7` — Unicode / special characters inside argument values. (`CASE.xml.entities` is N/A for DSML — no entity decoding.) #[tokio::test] async fn test_deepseek_v4_e2e_special_chars_vllm() { let file_path = format!( @@ -1251,6 +1257,7 @@ mod tests { /// Adversarial streaming: every DSML character is its own delta (~200 chunks). /// Exercises buffer accumulation across chunk boundaries. + /// `CASE.8` — streaming chunk-boundary splits (tokens straddle chunks). #[tokio::test] async fn test_deepseek_v4_e2e_fragmented_tokens_vllm() { let file_path = format!( diff --git a/lib/parsers/README.md b/lib/parsers/README.md new file mode 100644 index 000000000000..3a6bcac2e62f --- /dev/null +++ b/lib/parsers/README.md @@ -0,0 +1,129 @@ +# dynamo-parsers + +Rust crate for parsing **tool calls** and **reasoning content** out of raw LLM +output. Wire-format-aware, streaming-first, model-family-aware. + +This is the post-model side of Dynamo's chat-completions pipeline: given a +token stream from vLLM or SGLang, extract structured `Vec` + +`reasoning_content` for the client. The pre-model side (prompt formatting) +lives in `lib/llm/src/preprocessor/prompt/`. + +## What's in the crate + +Two top-level modules, each with its own parser registry: + +``` +lib/parsers/ +├── src/ +│ ├── tool_calling/ ← tool-call extraction (17 registered parsers) +│ │ ├── parsers.rs — registry + dispatch (detect_and_parse_tool_call) +│ │ ├── config.rs — per-parser ToolCallConfig +│ │ ├── response.rs — ToolCallResponse shape (wire type) +│ │ ├── dsml/ — DeepSeek V3.2 / V4 DSML grammar +│ │ ├── xml/ — hermes, glm47, kimi_k2, minimax_m2, qwen3_coder +│ │ ├── json/ — deepseek_v3, deepseek_v3_1, nemotron_deci/nano, jamba, mistral, phi4, llama3_json +│ │ ├── harmony/ — OpenAI gpt-oss (Harmony token stream, uses openai_harmony crate) +│ │ └── pythonic/ — Python function-call syntax (some Llama variants) +│ └── reasoning/ ← reasoning-content extraction (14 registered parsers) +│ ├── mod.rs — registry + dispatch +│ ├── base_parser.rs — BasicReasoningParser ( ... ) +│ ├── gpt_oss_parser.rs — Harmony channel parsing +│ ├── granite_parser.rs — Granite-style +│ └── minimax_append_think_parser.rs — MiniMax inline-reasoning +``` + +## How a request flows through the crate + +``` + token stream from engine + │ + ▼ + ┌─────────────────────────────────┐ + │ reasoning parser │ — registered by name via + │ (basic / gpt_oss / ...) │ reasoning::mod.rs get_reasoning_parser_map() + │ │ returns: (reasoning_content, non_reasoning_tail) + └─────────────────────────────────┘ + │ + ▼ (non-reasoning tail) + ┌─────────────────────────────────┐ + │ tool-call parser │ — registered by name via + │ dispatched on parser name │ tool_calling::parsers::get_tool_parser_map() + │ which picks a ParserConfig: │ + │ - Dsml(DsmlParserConfig) │ → try_tool_call_parse_dsml + │ - Json(JsonParserConfig) │ → try_tool_call_parse_json + │ - Xml(XmlParserConfig) │ → try_tool_call_parse_xml + │ - KimiK2(KimiK2ParserConfig)│ → try_tool_call_parse_kimi_k2 + │ - Pythonic / Harmony │ + └─────────────────────────────────┘ + │ + ▼ + Vec + normal_text +``` + +Main public entry points in `tool_calling/parsers.rs`: + +- `detect_and_parse_tool_call(input, parser_name, schema) -> (calls, normal_text)` +- `try_tool_call_parse(input, config) -> (calls, normal_text)` (lower-level, bypasses the registry) +- `detect_tool_call_start(chunk, parser_name)` — streaming: "is this chunk starting a tool-call block?" +- `find_tool_call_end_position(chunk, parser_name)` — streaming: "where does the block end in this chunk?" + +## Parser-family cheat sheet + +When adding a new model, the right parser family is usually one of: + +| Family | Grammar | Shared engine | Examples | +| -- | -- | -- | -- | +| **DSML** | `<｜DSML｜tool_calls>...` with typed `string="true|false"` parameters | `dsml/parser.rs` | DeepSeek V3.2, V4 | +| **XML** | `...` with nested `` or `` | `xml/parser.rs` (generic) or own file for variants | hermes, qwen3_coder, minimax_m2, glm47 (own), kimi_k2 (own, special-token XML) | +| **JSON** | Start sentinel + bare JSON array of `{name, arguments}` | `json/base_json_parser.rs` | deepseek_v3, deepseek_v3_1, nemotron_deci/nano | +| **Harmony** | OpenAI Harmony token stream with `<\|channel\|>`, `<\|message\|>`, `<\|call\|>` | `harmony/harmony_parser.rs` (wraps external `openai_harmony` crate) | gpt-oss-20B / 120B | +| **Pythonic** | `[func_name(arg=value, ...)]` Python function-call syntax | `pythonic/pythonic_parser.rs` | some Llama variants | + +Reasoning parsers: + +| Family | Grammar | Shared engine | Examples | +| -- | -- | -- | -- | +| **Basic (think-tag)** | `...` | `reasoning/base_parser.rs` (BasicReasoningParser) | Qwen3, Nemotron, Kimi K2.5, DeepSeek R1 / V4, GLM-4.5+ | +| **Append-think** | `...` left inline as text, with `` prefix on first chunk | `reasoning/minimax_append_think_parser.rs` | MiniMax M2 | +| **Harmony channel** | Hidden `analysis` channel | `reasoning/gpt_oss_parser.rs` (wraps external `openai_harmony`) | gpt-oss-20B / 120B | +| **Granite** | Custom start/end tokens | `reasoning/granite_parser.rs` | IBM Granite | + +## Adding a new parser + +1. **Pick the family** from the cheat sheet above. If an existing config-driven + family fits, add a `ToolCallConfig::()` constructor in + `tool_calling/config.rs`, register it in `tool_calling/parsers.rs`. Done — + you inherit all the shared parser and tests. + +2. **If the grammar is genuinely new**, add a module under `tool_calling/` and + add a `ParserConfig` variant in `config.rs`. Follow the existing parser + modules for layout. + +3. **For reasoning**, prefer aliasing to `BasicReasoningParser` unless the + grammar truly diverges (append-think, Harmony channels). Most new models + use plain `...` and can share. + +4. **Write tests.** Minimum viable set is in [`TESTING.md`](./TESTING.md) (T1–T20 + taxonomy). At minimum: T1/T2/T3 for correctness, T5 for truncation + behavior, T8/T9 for streaming, T14 for interleaved text. `N/A` categories + should be explicitly called out in a comment rather than silently skipped. + +## Related docs + +- [`TESTING.md`](./TESTING.md) — corner-case taxonomy (T1–T20). What every + parser should be tested against, what's N/A per family, what's a universal + gap today. +- `lib/llm/tests/data/` — captured streaming fixtures per (engine × model) + that feed `test_streaming_tool_parsers.rs`. The replay side of the testing + story. + +## Integration with the rest of Dynamo + +- `lib/llm/src/preprocessor/prompt/` — pre-model side. Writes the prompts + that (eventually) come back and get parsed here. +- `lib/llm/src/preprocessor.rs` — top-level request/response pipeline. + Decides whether to run the reasoning parser based on + `is_reasoning_disabled_by_request`, then hands the reasoning-stripped + tail to the tool-call parser. +- `components/src/dynamo/frontend/` — Python frontend that surfaces parsed + output as OpenAI-compatible SSE chunks to the client. diff --git a/lib/parsers/TESTING.md b/lib/parsers/TESTING.md new file mode 100644 index 000000000000..a977196da8ac --- /dev/null +++ b/lib/parsers/TESTING.md @@ -0,0 +1,317 @@ +# Tool-Call / Reasoning Parser Corner Cases + +Reference taxonomy for unit testing tool-call and reasoning parsers. Each parser +added under `src/tool_calling/` or `src/reasoning/` should cover the generic +`CASE.` categories; family-specific parsers also cover their respective +`CASE.xml` / `CASE.harmony` categories. `N/A` should be called out +explicitly in the test file rather than silently omitted. + +Category layout: +- **`CASE.1`–`CASE.16`** — **Generic**. Apply to every parser regardless of grammar. +- **`CASE.xml1`–`CASE.xml2`** — XML-family only (hermes, glm47, qwen3_coder, minimax_m2, kimi_k2). +- **`CASE.harmony1`** — Harmony only (gpt-oss). + +Per-model gap tracking lives elsewhere (not in this repo). + +## Quick reference + +### Generic (all parsers) + +- **`CASE.1`** Single tool call — happy path (one complete, well-formed call). Ex: `xml::test_parse_simple_tool_call`. +- **`CASE.2`** Multiple tool calls — sequential or parallel (2+ in one response). Ex: `tool_choice::test_streaming_required_tool_parallel`. +- **`CASE.3`** No tool call (response is text only). Ex: `xml::test_parse_no_tool_calls`. +- **`CASE.4`** Malformed / partial JSON args (truncated, missing close brace, invalid syntax). Ex: `deepseek_v3::test_parse_..._with_invalid_json`, `xml::test_parse_missing_..._closing_tag`. +- **`CASE.5`** Missing end-token recovery (recover calls when `section_end` is absent due to max_tokens / EOS). Ex: `kimi_k2::test_parse_malformed_no_section_end`. +- **`CASE.6`** Empty args (`arguments={}` / no-arg call). Ex: `kimi_k2::test_parse_no_arg_call`. +- **`CASE.7`** Complex arg types (nested objects, arrays, bool, number, Unicode / newlines in values). Ex: `kimi_k2::test_parse_complex_json_arguments`. +- **`CASE.8`** Streaming — token-by-token assembly + chunk-boundary splits. Ex: `basic::test_buffer_state_persistence_across_calls`, `basic::test_partial_token_matching_closing_tag`, `basic::test_kimi_k2_one_shot_split`. +- **`CASE.9`** Paired reasoning + tool in same response. Ex: `test_reasoning_parser::test_nemotron_with_reasoning_and_tool_calls`. +- **`CASE.10`** Reasoning only (think tags, no tool call). Ex: `basic::test_detect_and_parse_reasoning_reasoning`. +- **`CASE.11`** `tool_choice` = auto / required / named / none. Ex: `tool_choice::test_named_tool_choice_parses_json` (**hermes only today**). +- **`CASE.12`** `finish_reason` semantics (`stop` / `tool_calls` / `length` mapping). Ex: `tool_choice_finish_reasons::*` (hermes only); `test_streaming_tool_parsers::test_qwen_finish_reason_length_vllm`. +- **`CASE.13`** Normal text interleaved with tool calls. Ex: `xml::test_parse_with_normal_text`. +- **`CASE.14`** Empty content / empty `tool_calls` array / null response. Ex: `parallel_tool_call_integration::test_empty_tool_calls`. +- **`CASE.15`** Duplicate tool calls (same name twice). No test anywhere in the repo; universal gap. +- **`CASE.16`** Regression for a specific customer bug (ticket ID referenced in test name or body). Ex: `kimi_k2::test_parse_malformed_no_section_end`. + +### XML-family (`CASE.xml*`) + +- **`CASE.xml1`** XML entity / HTML unescape handling (`<`, `&`, `"` in parameter values). Ex: `xml::test_html_unescape`, `glm47::test_xml_entity_decoding`. +- **`CASE.xml2`** Schema-aware type coercion (string → number/bool/array based on declared parameter schema). Ex: `xml::test_schema_aware_type_conversion`, `glm47::test_type_coercion_*`. + +### Harmony (`CASE.harmony*`) + +- **`CASE.harmony1`** Channel / recipient parsing (analysis / commentary / final channels). Ex: `harmony_parser::test_parse_tool_calls_harmony_*`. + +### Universal gaps (no test anywhere, not promoted to numbered categories) + +- Unicode in function names (non-ASCII tool names, emoji). +- Numeric overflow in args (very large int / float outside JSON spec range). +- Empty function name (`"name": ""`). +- Concurrent parallel requests (process-level contention during parse). +- Guided-decoding ↔ tool-call interaction (constrained generation emits malformed args). +- Extremely long output (≥10 KB tool-call JSON in a single call). +- Mid-stream error injection / interruption (worker kill, network drop mid-parse). +- Schema arg-count mismatch (model emits extra or missing args vs declared schema). + +--- + +## `CASE.1` — Single tool call, happy path + +One complete, well-formed call in the response. + +- Applies to every tool-call parser. +- Baseline correctness check. If `CASE.1` fails, nothing else below matters. +- Example: `dsml/parser.rs::test_parse_single_tool_call_string_param`. + +## `CASE.2` — Multiple tool calls (sequential or parallel) + +Two or more calls in one response, in the same block or back-to-back. + +- Applies to every tool-call parser. +- Some grammars emit parallel calls in one block (DSML, XML); others emit + sequential top-level sentinels (JSON dialects). Either way, extract all. +- Example: `dsml/parser.rs::test_parse_multiple_tool_calls`, + `tool_choice::test_streaming_required_tool_parallel`. + +## `CASE.3` — No tool call + +Response is plain text, no tool-call grammar present. + +- Applies to every tool-call parser. +- Must return empty `Vec` and the input as `normal_text`. Zero false + positives. +- Example: `dsml/parser.rs::test_parse_no_tool_calls`. + +## `CASE.4` — Malformed / partial JSON args + +Truncated JSON, missing close brace, invalid syntax inside the arguments +payload. + +- Applies to every tool-call parser. For parsers whose grammar never embeds + JSON (none today — all top-N families embed JSON somewhere), mark explicit + `N/A`. +- Behavior must be documented: either graceful fallback to string (DSML's + current behavior via `serde_json::from_str(...).unwrap_or_else(|_| String(...))`) + or explicit error. Silent drop is the failure mode. +- Example: `dsml/parser.rs::test_parse_deepseek_v4_malformed_json_value_falls_back_to_string`, + `deepseek_v3_parser.rs::test_parse_tool_calls_deepseek_v3_with_invalid_json`. + +## `CASE.5` — Missing end-token recovery + +The model's response is truncated before the closing fence arrives +(`<|tool_calls_section_end|>` for Kimi, `` for DeepSeek +DSML, etc.) — typically because the engine hit `max_tokens` or the model +emitted EOS mid-generation. + +- Applies to every tool-call parser with paired start/end fences. +- Customer-facing bug class: silent drop of the in-flight call looks like a + successful HTTP 200 with no tool_calls and no error. +- Two acceptable resolutions: (a) recover completed invokes even without the + outer close fence (Kimi K2 does this post-fix), or (b) return an explicit + error. Either way, pin the behavior with a test so a future change is + intentional. +- Example (post-recovery): `kimi_k2_parser.rs::test_parse_malformed_no_section_end`. +- Example (behavior-pinning, pre-recovery): `dsml/parser.rs::test_parse_deepseek_v4_missing_end_token`. + +## `CASE.6` — Empty args + +Tool call with `arguments={}`, or a no-parameter invoke. + +- Applies to every tool-call parser. +- Must still return the call — empty args is a valid call, not a missing one. +- Example: `kimi_k2_parser.rs::test_parse_no_arg_call`, + `dsml/parser.rs::test_parse_deepseek_v4_no_parameters`. + +## `CASE.7` — Complex argument types + +Nested objects, arrays, booleans, numbers, mixed types, Unicode values, and +newlines inside argument values. + +- Applies to every tool-call parser. +- For grammars that carry type hints (DSML's `string="true|false"`), verify + JSON round-tripping. For XML grammars without hints, the type-coercion + half of the test is covered under `CASE.xml2` instead — here just verify + that complex values make it through without truncation or escape bugs. +- Example: `dsml/parser.rs::test_parse_mixed_types_realistic`, + `kimi_k2_parser.rs::test_parse_complex_json_arguments`. + +## `CASE.8` — Streaming + +Chunked input arriving over SSE. Covers two concerns that tend to fail +together: + +1. **Token-by-token assembly** — the parser incrementally reconstructs the + tool-call structure across many small chunks. +2. **Chunk-boundary splits** — start fence, end fence, or parameter name / + value straddles a chunk boundary. Partial-token matching must return + `true` (keep buffering, don't flush as plain text) and complete the + match on the next chunk. + +- Applies to every tool-call parser. Dominant production path. +- Example: `basic::test_buffer_state_persistence_across_calls`, + `basic::test_partial_token_matching_closing_tag`, + `basic::test_kimi_k2_one_shot_split`, + `test_streaming_tool_parsers::test_deepseek_v4_e2e_fragmented_tokens_vllm`. + +## `CASE.9` — Paired reasoning + tool in same response + +Model emits `...` (or analog) followed by a tool call. Both +must be extracted: `reasoning_content` populated AND `tool_calls` populated. + +- Applies to every (tool, reasoning) parser pair. +- Watch for the "unclosed think-tag swallows tool call" bug — if the reasoning + parser is greedy it may eat the tool-call content that follows. +- Example: `test_reasoning_parser::test_nemotron_with_reasoning_and_tool_calls`, + `test_reasoning_parser::test_kimi_k25_with_reasoning_and_tool_calls`. + +## `CASE.10` — Reasoning only + +`...` or analog present, no tool call. Parser must populate +`reasoning_content` and leave `tool_calls` empty. + +- Applies to every reasoning parser. +- Example: `reasoning/base_parser.rs::test_detect_and_parse_reasoning_reasoning`, + `reasoning/mod.rs::test_deepseek_v4_detect_and_parse`. + +## `CASE.11` — `tool_choice` = auto / required / named / none + +Each of the four OpenAI `tool_choice` modes exercised per parser. + +- Applies to every tool-call parser. +- Cross-parser suites at `lib/llm/tests/tool_choice.rs` / + `parallel_tool_call_integration.rs` / `tool_choice_finish_reasons.rs` + run `hermes` only today. Adding a new parser requires parametrizing those + suites or adding a per-parser equivalent. +- Universal gap across most parsers in the repo as of 2026-04. +- Example: `tool_choice::test_named_tool_choice_parses_json`, + `tool_choice::test_required_tool_choice_parses_json_array`. + +## `CASE.12` — `finish_reason` semantics + +`stop` vs `tool_calls` vs `length` mapping, in both streaming and +non-streaming paths. + +- Applies to every tool-call parser. +- When a tool call lands, `finish_reason` must become `tool_calls`. When + `max_tokens` truncates mid-stream, `length` must propagate — this is + often the signal that should trigger `CASE.5` recovery on the parser side. +- Example: `tool_choice_finish_reasons::test_named_tool_choice_normal_stop_becomes_tool_calls`, + `test_streaming_tool_parsers::test_qwen_finish_reason_length_vllm`. + +## `CASE.13` — Normal text interleaved with tool calls + +Model emits narration text before / after / between tool-call blocks. Parser +must split content correctly: text → `normal_content`, calls → `tool_calls`. + +- Applies to every tool-call parser. +- Example: `dsml/parser.rs::test_parse_with_normal_text`, + `test_streaming_tool_parsers.rs::test_deepseek_v4_e2e_content_before_tool_vllm`. + +## `CASE.14` — Empty content / empty `tool_calls` array / null response + +Engine emits a chunk with `delta.content = ""`, or a final response with +`tool_calls: []`, or `null` values inside arguments. + +- Applies to every tool-call parser. +- Null-value handling inside parameters is parser-level (`parse_parameters` + in DSML handles it via `serde_json::Value::Null`). Empty-choices / + empty-stream handling is typically at the e2e integration layer. +- Example: `dsml/parser.rs::test_parse_null_parameter`, + `parallel_tool_call_integration::test_empty_tool_calls`. + +## `CASE.15` — Duplicate tool calls (same name twice) + +Two calls to the same function name in one response, possibly with the same +arguments. + +- Applies to every tool-call parser. +- **Zero coverage across the entire repo as of 2026-04.** Universal gap. +- Expected behavior: both calls must appear in `tool_calls` with distinct + IDs. (The runtime / client is responsible for deciding whether duplicate + invocation is intended.) + +## `CASE.16` — Regression for a specific customer bug + +Test named after (or containing) a ticket reference, pinning the fix for +a customer-reported failure. + +- Applies per-incident. Not a category every parser needs to cover in + advance; populated as bugs are reported and fixed. +- Existing example: `kimi_k2_parser.rs::test_parse_malformed_no_section_end`. + +--- + +## `CASE.xml1` — XML entity / HTML unescape handling + +Parameter values contain XML-encoded entities (`<`, `&`, `"`, +`'`, numeric entities like `&`) that must be decoded before the +value is surfaced to the client. + +- Applies only to XML-family tool-call parsers: `hermes`, `glm47`, + `qwen3_coder`, `minimax_m2`, `kimi_k2` (despite its special-token outer + fence, the inner parameter payload is XML-ish). +- **N/A for DSML** — the `string="true|false"` attribute tells the parser + whether to JSON-decode or pass through verbatim; no entity decoding pass. +- **N/A for JSON-family and Harmony** — JSON has its own escape semantics + handled by `serde_json`. +- Example: `xml/parser.rs::test_html_unescape`, + `glm47_parser.rs::test_xml_entity_decoding`. + +## `CASE.xml2` — Schema-aware type coercion + +Parser uses the declared tool schema to coerce string args to +number / bool / array based on the declared parameter type. + +- Applies only to XML-family parsers without explicit type annotations in + the wire format. `xml/parser.rs`, `glm47_parser.rs` do this. +- **N/A for DSML** — the `string="true|false"` attribute carries the type + intent per parameter, so no schema lookup is needed. +- **N/A for JSON-family** — JSON has native types. +- **N/A for Harmony** — payload is JSON inside the channel envelope. +- Example: `xml/parser.rs::test_schema_aware_type_conversion`, + `glm47_parser.rs::test_type_coercion_array_comma_separated`. + +--- + +## `CASE.harmony1` — Channel / recipient parsing + +OpenAI Harmony's token stream carries channel metadata +(`<|channel|>analysis|commentary|final<|message|>`) and recipient targets +(`to=functions.foo`). Parser must route the `commentary` channel content +into tool-call extraction while surfacing `analysis` as reasoning and +`final` as the user-visible output. + +- **Harmony only.** N/A for every other family. +- Example: `harmony/harmony_parser.rs::test_parse_tool_calls_harmony_with_multi_args`, + `harmony/harmony_parser.rs::test_parse_tool_calls_harmony_with_normal_text`. + +--- + +## Applicability summary + +| Category block | Parsers | Notes | +| -- | -- | -- | +| `CASE.1`–`CASE.16` (generic) | All | Required contract for every parser | +| `CASE.xml1`–`CASE.xml2` | XML-family only | Entity decoding + schema-aware coercion | +| `CASE.harmony1` | Harmony only | Channel routing | + +## Adding a new parser: what you must include + +Minimum viable set for a new tool-call parser: + +1. `CASE.1`, `CASE.2`, `CASE.3` — baseline correctness. +2. `CASE.4` or explicit N/A justification — handle or refuse malformed input. +3. `CASE.5` — pin behavior when the outer fence is missing. Silent drop is a + regression waiting to happen. +4. `CASE.6`, `CASE.7` — empty and complex args. +5. `CASE.8` — streaming. Essentially non-negotiable for any parser that sits + behind a streaming frontend. +6. `CASE.13` — interleaved text. +7. `CASE.15` — document whether duplicate calls are supported. Flat gap + today; landing a test with the parser establishes the contract. +8. Family-specific categories where applicable: `CASE.xml1` / `CASE.xml2` + for XML grammars, `CASE.harmony1` for Harmony. + +For reasoning parsers, replace `CASE.4` / `CASE.5` / `CASE.8`-assembly with +`CASE.8`-partial-close-tag and `CASE.10` (reasoning-only). diff --git a/lib/parsers/src/reasoning/mod.rs b/lib/parsers/src/reasoning/mod.rs index 91dd0d3399c5..5f259bc80f74 100644 --- a/lib/parsers/src/reasoning/mod.rs +++ b/lib/parsers/src/reasoning/mod.rs @@ -294,6 +294,7 @@ mod tests { assert!(parsers.contains(&parser)); } } + /// `CASE.10` — reasoning-only (V4 ``/``). #[test] fn test_deepseek_v4_detect_and_parse() { @@ -304,6 +305,7 @@ mod tests { assert_eq!(result.normal_text, "answer"); } } + /// `CASE.3` / `CASE.10` — no reasoning tags ⇒ no `reasoning_content`. #[test] fn test_deepseek_v4_no_forced_reasoning_without_tags() { @@ -312,6 +314,7 @@ mod tests { assert_eq!(result.reasoning_text, ""); assert_eq!(result.normal_text, "answer only"); } + /// `CASE.8` — streaming reasoning parse (chunked). #[test] fn test_deepseek_v4_streaming() { diff --git a/lib/parsers/src/tool_calling/dsml/parser.rs b/lib/parsers/src/tool_calling/dsml/parser.rs index badb4bf703d7..cec8e75f67db 100644 --- a/lib/parsers/src/tool_calling/dsml/parser.rs +++ b/lib/parsers/src/tool_calling/dsml/parser.rs @@ -289,6 +289,58 @@ mod tests { assert!(!detect_tool_call_start_dsml("no tool call here", &config)); } + // ------------------------------------------------------------------- + // DeepSeek V4 coverage (see lib/parsers/TESTING.md for CASE.* taxonomy). + // + // Covered by the V4 tests below (or by a shared DSML generic test): + // - CASE.1 single-call (parsers.rs :: test_deepseek_v4_single_tool_call) + // - CASE.2 multi-calls (test_parse_deepseek_v4_multiple_tool_calls) + // - CASE.3 no-call (shared: test_parse_no_tool_calls) + // - CASE.4 malformed-args (test_parse_deepseek_v4_malformed_json_value_falls_back_to_string, + // test_parse_deepseek_v4_missing_invoke_close_drops_call) + // - CASE.5 missing-end-token (test_parse_deepseek_v4_missing_end_token{,_multiple_calls}) + // — PINNED AS BROKEN: parser drops the call. See TODO below. + // - CASE.6 empty-args (test_parse_deepseek_v4_no_parameters) + // - CASE.7 complex-args (shared: test_parse_mixed_types_realistic, test_parse_nested_object_parameter, + // lib/llm/tests/test_streaming_tool_parsers :: ..._mixed_param_types_vllm, + // ..._special_chars_vllm) + // - CASE.8 streaming (test_detect_tool_call_start_v4, test_find_tool_call_end_position_v4, + // lib/llm/tests/test_streaming_tool_parsers :: ..._fragmented_tokens_vllm) + // - CASE.9 reasoning-plus-tool (lib/llm/tests/test_streaming_tool_parsers :: ..._with_tools_vllm + // — fixtures include ... alongside DSML) + // - CASE.10 reasoning-only (reasoning/mod.rs :: test_deepseek_v4_detect_and_parse etc.) + // - CASE.12 finish-reason (lib/llm/tests/test_streaming_tool_parsers :: ..._with_tools_vllm → + // FinishReason::ToolCalls; ..._with_no_tools_vllm → FinishReason::Stop + // — Length variant NOT covered, see TODO) + // - CASE.13 interleaved-text (test_parse_deepseek_v4_multiple_tool_calls prefix text; + // lib/llm/tests/test_streaming_tool_parsers :: ..._content_before_tool_vllm) + // + // - CASE.xml.* N/A — DSML carries per-parameter string="true|false" type hints, + // so XML entity decoding (CASE.xml.entities) and schema-aware + // coercion (CASE.xml.schema-coercion) don't apply. + // - CASE.harmony.* N/A — Harmony-only. + // + // TODO — not yet covered for V4: + // - CASE.5 Fix mid-stream truncation: parser currently drops all calls when + // is absent (max_tokens / EOS before close). + // Same class as Kimi K2 pre-DIS-1765. Recovery pattern: scan for + // complete <｜DSML｜invoke>... pairs even without + // the outer close fence (see kimi_k2_parser.rs for precedent). + // Pinning tests below capture the current silent-drop behavior; + // flip them when recovery lands. + // - CASE.4 Variants not pinned: missing close tag, + // middle-invoke truncation corrupting subsequent invokes (non-greedy + // regex bleed-through). Same structural class as CASE.5. + // - CASE.11 tool_choice auto/required/named/none — cross-parser suites at + // lib/llm/tests/tool_choice.rs run hermes only; V4 not exercised. + // - CASE.12 FinishReason::Length — current E2E fixtures only cover Stop and + // ToolCalls finish reasons. No truncation-forcing fixture. + // - CASE.14 empty-content / null response at the e2e layer. + // - CASE.15 duplicate-calls (same name twice) — universal gap across all parsers. + // - CASE.16 regression — V4 is hours old (2026-04-24); no customer bugs filed yet. + // ------------------------------------------------------------------- + + /// `CASE.8` — streaming start-token detection (V4 variant). #[test] fn test_detect_tool_call_start_v4() { let config = get_v4_test_config(); @@ -313,6 +365,7 @@ mod tests { assert_eq!(&text[pos..], "more"); } + /// `CASE.8` — streaming end-position lookup (V4 variant). #[test] fn test_find_tool_call_end_position_v4() { let config = get_v4_test_config(); @@ -394,6 +447,7 @@ mod tests { assert_eq!(args2["location"], "Hangzhou"); } + /// `CASE.2` multi-calls + `CASE.13` interleaved-text (prefix text before the block). #[test] fn test_parse_deepseek_v4_multiple_tool_calls() { let input = r#"Let's check this. <｜DSML｜tool_calls> @@ -423,6 +477,7 @@ mod tests { assert_eq!(args2["source"], "web"); } + /// `CASE.6` — empty args (no-parameter invoke). #[test] fn test_parse_deepseek_v4_no_parameters() { let input = r#"<｜DSML｜tool_calls> @@ -610,4 +665,116 @@ mod tests { let (_, args) = extract_name_and_args(calls[0].clone()); assert!(args["value"].is_null()); } + + // Corner-case pinning tests. See the V4 coverage manifest above for the + // full mapping from CASE.* → test. Each test's doc-comment names the + // specific CASE it pins. + + /// `CASE.5` — missing end-token recovery. + /// **Pinned as broken** — parser drops the call; see the TODO block above. + /// + /// If a DeepSeek V4 stream is truncated before `` + /// arrives (max_tokens cut-off, EOS mid-generation, connection drop), + /// the block regex requires both fences and matches zero times. The + /// entire DSML-looking payload falls through as raw `normal_text`; no + /// tool calls are recovered. + /// + /// This is the same structural failure mode Kimi K2 had before its + /// parser gained end-token recovery; see + /// `kimi_k2_parser.rs::test_parse_malformed_no_section_end` for the + /// post-fix recovery pattern. + #[test] + fn test_parse_deepseek_v4_missing_end_token() { + // Start fence + complete invoke, but no . + let input = "<｜DSML｜tool_calls>\n\ +<｜DSML｜invoke name=\"get_weather\">\n\ +<｜DSML｜parameter name=\"city\" string=\"true\">NYC\n\ +"; + + let config = get_v4_test_config(); + let (calls, normal_text) = try_tool_call_parse_dsml(input, &config).unwrap(); + + assert!( + calls.is_empty(), + "V4 DSML parser currently drops tool calls when \ + is missing. \ + If recovery is added, flip this assertion." + ); + assert_eq!( + normal_text.as_deref(), + Some(input), + "Unrecovered payload should fall through to normal_text verbatim." + ); + } + + /// `CASE.5` — multiple complete invokes, missing end fence. + /// + /// Even with multiple fully-formed invokes inside the start fence, the + /// absence of the closing fence prevents the block regex from matching. + /// All calls are dropped. If the parser ever gains partial-block + /// recovery, this test will fail and force an intentional update. + #[test] + fn test_parse_deepseek_v4_missing_end_token_multiple_calls() { + let input = "<｜DSML｜tool_calls>\n\ +<｜DSML｜invoke name=\"a\">\n\ +<｜DSML｜parameter name=\"x\" string=\"true\">1\n\ +\n\ +<｜DSML｜invoke name=\"b\">\n\ +<｜DSML｜parameter name=\"y\" string=\"true\">2\n\ +"; + + let config = get_v4_test_config(); + let (calls, _) = try_tool_call_parse_dsml(input, &config).unwrap(); + + assert!( + calls.is_empty(), + "Even two fully-formed invokes are dropped when the outer \ + is missing." + ); + } + + /// `CASE.4` — malformed JSON in a `string="false"` parameter value falls back + /// to a string. `parse_parameters` explicitly swallows the serde error + /// (unwrap_or_else → Value::String). Pin the fallback so removing it + /// (which would cause the whole call to 500 on ragged-edge JSON) is a + /// deliberate change. + #[test] + fn test_parse_deepseek_v4_malformed_json_value_falls_back_to_string() { + let input = "<｜DSML｜tool_calls>\n\ +<｜DSML｜invoke name=\"test\">\n\ +<｜DSML｜parameter name=\"payload\" string=\"false\">{this is not valid json\n\ +\n\ +"; + + let config = get_v4_test_config(); + let (calls, _) = try_tool_call_parse_dsml(input, &config).unwrap(); + assert_eq!(calls.len(), 1); + + let (name, args) = extract_name_and_args(calls[0].clone()); + assert_eq!(name, "test"); + assert_eq!( + args["payload"], "{this is not valid json", + "Malformed JSON should fall back to the raw string, not drop \ + the parameter or the call." + ); + } + + /// `CASE.4` — malformed invoke (missing `` but block fences + /// intact). The invoke regex requires its own close tag, so the call is + /// silently dropped. Pin the behavior. + #[test] + fn test_parse_deepseek_v4_missing_invoke_close_drops_call() { + let input = "<｜DSML｜tool_calls>\n\ +<｜DSML｜invoke name=\"test\">\n\ +<｜DSML｜parameter name=\"x\" string=\"true\">value\n\ +"; + + let config = get_v4_test_config(); + let (calls, _) = try_tool_call_parse_dsml(input, &config).unwrap(); + assert!( + calls.is_empty(), + "Malformed invoke (missing ) is dropped today. \ + If recovery is added, flip this assertion." + ); + } } diff --git a/lib/parsers/src/tool_calling/parsers.rs b/lib/parsers/src/tool_calling/parsers.rs index 96846d7be7e9..c3b0c9f34ab1 100644 --- a/lib/parsers/src/tool_calling/parsers.rs +++ b/lib/parsers/src/tool_calling/parsers.rs @@ -1707,6 +1707,7 @@ Remember, San Francisco weather can be quite unpredictable, particularly with it assert_eq!(args["topn"], 10); // Should be number, not string assert_eq!(args["source"], "web"); } + /// `CASE.1` — single-call happy path (V4). #[tokio::test] async fn test_deepseek_v4_single_tool_call() { @@ -1729,6 +1730,7 @@ Remember, San Francisco weather can be quite unpredictable, particularly with it serde_json::from_str(&tool_calls[0].function.arguments).unwrap(); assert_eq!(args["timezone"], "Asia/Shanghai"); } + /// Alias registration: verifies `deepseek-v4` and `deepseekv4` route to the same parser as `deepseek_v4`. Not a CASE.*; covers registry plumbing. #[tokio::test] async fn test_deepseek_v4_compatibility_aliases() { From bfc6004f8b65543162790ee7bfe53ddbe639fa60 Mon Sep 17 00:00:00 2001 From: Keiven Chang Date: Fri, 24 Apr 2026 18:03:30 +0000 Subject: [PATCH 19/21] refactor(llm,v4): split render_message into per-role helpers + extract const strings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses biswapanda's review comments on #8665: - L389 "split into separate funcs" — render_message's 221-line match is split into six per-role helpers: render_system_role, render_developer_role, render_user_role, render_latest_reminder_role, render_assistant_role, plus append_response_format / append_tools_section to dedup the sys/developer response-format and tools blocks. render_message itself stays as a slim dispatcher that handles the reasoning-effort prefix and the transition-token tail. - L400 "const strings" — hard-coded `` / `` tags extracted to TOOL_RESULT_OPEN / TOOL_RESULT_CLOSE consts alongside the other wire-format constants. The response-format preamble (previously duplicated inline in the system and developer branches) becomes RESPONSE_FORMAT_PREAMBLE with a {} placeholder. Already addressed by prior commits on the branch (no-op here): - L32 "Should be ReasoningParserType::DeepSeekV4" — the three deepseek_v4 aliases now map to ReasoningParserType::DeepSeekV4. - L899 "hard coded path" — the `/home/ayush-lab/...` absolute path in the test has been removed. - L266 "clippy manual_pattern_char_comparison" — fixed in 77602e441a0 by using a char array. Pure refactor — behavior is identical. All 4 V4 encoding golden tests + 7 V4 streaming tests pass unchanged. Clippy clean on dynamo-llm. Signed-off-by: Keiven Chang --- .../src/preprocessor/prompt/deepseek_v4.rs | 352 ++++++++++-------- 1 file changed, 197 insertions(+), 155 deletions(-) diff --git a/lib/llm/src/preprocessor/prompt/deepseek_v4.rs b/lib/llm/src/preprocessor/prompt/deepseek_v4.rs index 66159b549256..37c534ed3ef6 100644 --- a/lib/llm/src/preprocessor/prompt/deepseek_v4.rs +++ b/lib/llm/src/preprocessor/prompt/deepseek_v4.rs @@ -33,6 +33,16 @@ pub mod tokens { /// DSML outer block name for tool-call groups: `<｜DSML｜tool_calls>...`. const TOOL_CALLS_BLOCK_NAME: &str = "tool_calls"; +/// Wire-format tags that wrap a tool result inside a user content block. +const TOOL_RESULT_OPEN: &str = ""; +const TOOL_RESULT_CLOSE: &str = ""; + +/// Preamble line that introduces a response-format schema in both the +/// system and developer roles. The `{}` placeholder is filled by the +/// caller with the JSON-serialized schema. +const RESPONSE_FORMAT_PREAMBLE: &str = + "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{}"; + /// Roles whose messages are always kept by `drop_thinking_messages`. All other /// roles before the last user/developer message are dropped (assistant turns /// keep their non-`reasoning_content` fields; everything else is removed). @@ -319,6 +329,177 @@ fn task_token(task: &str) -> Option<&'static str> { } } +/// Append the "Response Format" schema block to `prompt` if the message has one. +fn append_response_format(msg: &JsonValue, prompt: &mut String) -> Result<()> { + if let Some(response_format) = msg.get("response_format") { + prompt.push_str("\n\n"); + prompt.push_str(&RESPONSE_FORMAT_PREAMBLE.replace("{}", &to_json(response_format)?)); + } + Ok(()) +} + +/// Append the tools section to `prompt` if the message has a `tools` array. +fn append_tools_section(msg: &JsonValue, prompt: &mut String) -> Result<()> { + if let Some(tools) = msg.get("tools").and_then(JsonValue::as_array) { + prompt.push_str("\n\n"); + prompt.push_str(&render_tools(tools)?); + } + Ok(()) +} + +/// Render the `system` role: raw content, then optional tools + response-format. +fn render_system_role(msg: &JsonValue, prompt: &mut String) -> Result<()> { + let content = msg.get("content").and_then(JsonValue::as_str).unwrap_or(""); + prompt.push_str(content); + append_tools_section(msg, prompt)?; + append_response_format(msg, prompt)?; + Ok(()) +} + +/// Render the `developer` role: wraps content in USER_START, then optional +/// tools + response-format. Developer content is required (non-empty). +fn render_developer_role(msg: &JsonValue, prompt: &mut String) -> Result<()> { + let content = msg + .get("content") + .and_then(JsonValue::as_str) + .filter(|s| !s.is_empty()) + .context("Developer role requires content")?; + + prompt.push_str(tokens::USER_START); + prompt.push_str(content); + append_tools_section(msg, prompt)?; + append_response_format(msg, prompt)?; + Ok(()) +} + +/// Render the `user` role: either content-blocks (with tool_result wrapping) +/// or a plain string `content` field. +fn render_user_role(msg: &JsonValue, prompt: &mut String) -> Result<()> { + prompt.push_str(tokens::USER_START); + if let Some(blocks) = msg.get("content_blocks").and_then(JsonValue::as_array) { + let mut parts: Vec = Vec::with_capacity(blocks.len()); + for block in blocks { + let block_type = block.get("type").and_then(JsonValue::as_str).unwrap_or(""); + match block_type { + "text" => { + let text = block.get("text").and_then(JsonValue::as_str).unwrap_or(""); + parts.push(text.to_string()); + } + "tool_result" => { + let rendered = render_tool_result_content( + block.get("content").unwrap_or(&JsonValue::Null), + )?; + parts.push(format!( + "{}{}{}", + TOOL_RESULT_OPEN, rendered, TOOL_RESULT_CLOSE + )); + } + other => { + tracing::warn!( + block_type = other, + "DeepSeek V4 formatter emitted placeholder for unsupported user content block type", + ); + parts.push(UNSUPPORTED_PLACEHOLDER_FMT.replace("{}", other)); + } + } + } + prompt.push_str(&parts.join("\n\n")); + } else { + let content = msg.get("content").and_then(JsonValue::as_str).unwrap_or(""); + prompt.push_str(content); + } + Ok(()) +} + +/// Render the `latest_reminder` role: LATEST_REMINDER token + content. +fn render_latest_reminder_role(msg: &JsonValue, prompt: &mut String) { + let content = msg.get("content").and_then(JsonValue::as_str).unwrap_or(""); + prompt.push_str(tokens::LATEST_REMINDER); + prompt.push_str(content); +} + +/// Render the `assistant` role: optional thinking prefix, content, optional +/// `tool_calls` DSML block, optional EOS. +/// +/// Needs `index` and `messages` to peek at the previous turn's `task` field +/// (suppresses thinking prefix when the prior message was a task message). +fn render_assistant_role( + msg: &JsonValue, + index: usize, + messages: &[JsonValue], + thinking_mode: ThinkingMode, + drop_thinking: bool, + last_user_idx: Option, + prompt: &mut String, +) -> Result<()> { + let content = msg.get("content").and_then(JsonValue::as_str).unwrap_or(""); + let reasoning = msg + .get("reasoning_content") + .and_then(JsonValue::as_str) + .unwrap_or(""); + let wo_eos = msg + .get("wo_eos") + .and_then(JsonValue::as_bool) + .unwrap_or(false); + + let prev_has_task = index > 0 + && messages[index - 1] + .get("task") + .map(|v| !v.is_null()) + .unwrap_or(false); + + if thinking_mode == ThinkingMode::Thinking && !prev_has_task { + let render_thinking = !drop_thinking || last_user_idx.is_none_or(|u| index > u); + if render_thinking { + prompt.push_str(reasoning); + prompt.push_str(tokens::THINKING_END); + } + } + + prompt.push_str(content); + + if let Some(tool_calls) = msg.get("tool_calls").and_then(JsonValue::as_array) + && !tool_calls.is_empty() + { + prompt.push_str("\n\n"); + prompt.push_str(&format!( + "<{}{}>\n", + tokens::DSML_TOKEN, + TOOL_CALLS_BLOCK_NAME + )); + + let mut invocations = Vec::with_capacity(tool_calls.len()); + for tc in tool_calls { + // Accept both OpenAI-format (nested `function`) and internal + // `{name, arguments}` shape, matching Python's `tool_calls_from_openai_format`. + let fn_obj = tc.get("function").unwrap_or(tc); + let name = fn_obj + .get("name") + .and_then(JsonValue::as_str) + .context("Missing tool call name")?; + let arguments = encode_arguments_to_dsml(fn_obj)?; + invocations.push(format!( + "<{}invoke name=\"{}\">\n{}\n", + tokens::DSML_TOKEN, + name, + arguments, + tokens::DSML_TOKEN + )); + } + prompt.push_str(&invocations.join("\n")); + prompt.push_str(&format!( + "\n", + tokens::DSML_TOKEN, + TOOL_CALLS_BLOCK_NAME + )); + } + + if !wo_eos { + prompt.push_str(tokens::EOS); + } + Ok(()) +} + /// Render a single message at the given index. fn render_message( index: usize, @@ -346,161 +527,22 @@ fn render_message( } match role { - "system" => { - let content = msg.get("content").and_then(JsonValue::as_str).unwrap_or(""); - prompt.push_str(content); - if let Some(tools) = msg.get("tools").and_then(JsonValue::as_array) { - prompt.push_str("\n\n"); - prompt.push_str(&render_tools(tools)?); - } - if let Some(response_format) = msg.get("response_format") { - prompt.push_str("\n\n"); - prompt.push_str(&format!( - "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{}", - to_json(response_format)?, - )); - } - } - - "developer" => { - let content = msg - .get("content") - .and_then(JsonValue::as_str) - .filter(|s| !s.is_empty()) - .context("Developer role requires content")?; - - let mut content_developer = String::from(tokens::USER_START); - content_developer.push_str(content); - - if let Some(tools) = msg.get("tools").and_then(JsonValue::as_array) { - content_developer.push_str("\n\n"); - content_developer.push_str(&render_tools(tools)?); - } - if let Some(response_format) = msg.get("response_format") { - content_developer.push_str("\n\n"); - content_developer.push_str(&format!( - "## Response Format:\n\nYou MUST strictly adhere to the following schema to reply:\n{}", - to_json(response_format)?, - )); - } - prompt.push_str(&content_developer); - } - - "user" => { - prompt.push_str(tokens::USER_START); - if let Some(blocks) = msg.get("content_blocks").and_then(JsonValue::as_array) { - let mut parts: Vec = Vec::with_capacity(blocks.len()); - for block in blocks { - let block_type = block.get("type").and_then(JsonValue::as_str).unwrap_or(""); - match block_type { - "text" => { - let text = block.get("text").and_then(JsonValue::as_str).unwrap_or(""); - parts.push(text.to_string()); - } - "tool_result" => { - let rendered = render_tool_result_content( - block.get("content").unwrap_or(&JsonValue::Null), - )?; - parts.push(format!("{}", rendered)); - } - other => { - tracing::warn!( - block_type = other, - "DeepSeek V4 formatter emitted placeholder for unsupported user content block type", - ); - parts.push(UNSUPPORTED_PLACEHOLDER_FMT.replace("{}", other)); - } - } - } - prompt.push_str(&parts.join("\n\n")); - } else { - let content = msg.get("content").and_then(JsonValue::as_str).unwrap_or(""); - prompt.push_str(content); - } - } - - "latest_reminder" => { - let content = msg.get("content").and_then(JsonValue::as_str).unwrap_or(""); - prompt.push_str(tokens::LATEST_REMINDER); - prompt.push_str(content); - } - - "tool" => { - anyhow::bail!( - "deepseek_v4 merges tool messages into user; preprocess with merge_tool_messages()" - ); - } - - "assistant" => { - let content = msg.get("content").and_then(JsonValue::as_str).unwrap_or(""); - let reasoning = msg - .get("reasoning_content") - .and_then(JsonValue::as_str) - .unwrap_or(""); - let wo_eos = msg - .get("wo_eos") - .and_then(JsonValue::as_bool) - .unwrap_or(false); - - let prev_has_task = index > 0 - && messages[index - 1] - .get("task") - .map(|v| !v.is_null()) - .unwrap_or(false); - - let mut thinking_part = String::new(); - if thinking_mode == ThinkingMode::Thinking && !prev_has_task { - let render_thinking = !drop_thinking || last_user_idx.is_none_or(|u| index > u); - if render_thinking { - thinking_part.push_str(reasoning); - thinking_part.push_str(tokens::THINKING_END); - } - } - - prompt.push_str(&thinking_part); - prompt.push_str(content); - - if let Some(tool_calls) = msg.get("tool_calls").and_then(JsonValue::as_array) - && !tool_calls.is_empty() - { - prompt.push_str("\n\n"); - prompt.push_str(&format!( - "<{}{}>\n", - tokens::DSML_TOKEN, - TOOL_CALLS_BLOCK_NAME - )); - - let mut invocations = Vec::with_capacity(tool_calls.len()); - for tc in tool_calls { - // Accept both OpenAI-format (nested `function`) and internal - // `{name, arguments}` shape, matching Python's `tool_calls_from_openai_format`. - let fn_obj = tc.get("function").unwrap_or(tc); - let name = fn_obj - .get("name") - .and_then(JsonValue::as_str) - .context("Missing tool call name")?; - let arguments = encode_arguments_to_dsml(fn_obj)?; - invocations.push(format!( - "<{}invoke name=\"{}\">\n{}\n", - tokens::DSML_TOKEN, - name, - arguments, - tokens::DSML_TOKEN - )); - } - prompt.push_str(&invocations.join("\n")); - prompt.push_str(&format!( - "\n", - tokens::DSML_TOKEN, - TOOL_CALLS_BLOCK_NAME - )); - } - - if !wo_eos { - prompt.push_str(tokens::EOS); - } - } - + "system" => render_system_role(msg, &mut prompt)?, + "developer" => render_developer_role(msg, &mut prompt)?, + "user" => render_user_role(msg, &mut prompt)?, + "latest_reminder" => render_latest_reminder_role(msg, &mut prompt), + "tool" => anyhow::bail!( + "deepseek_v4 merges tool messages into user; preprocess with merge_tool_messages()" + ), + "assistant" => render_assistant_role( + msg, + index, + messages, + thinking_mode, + drop_thinking, + last_user_idx, + &mut prompt, + )?, other => anyhow::bail!("Unknown role: {other}"), } From 2aab621e26efe7070c172e4839ea3b59a578eac0 Mon Sep 17 00:00:00 2001 From: Keiven Chang Date: Fri, 24 Apr 2026 19:30:51 +0000 Subject: [PATCH 20/21] test(llm,v4): drop unused finish_reason from special_chars fixture MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The fixture's expected_output carried a finish_reason field that the streaming test harness never reads — finish_reason validation is hardcoded against the stream chunks themselves, not the expected_output struct. None of the other 6 deepseek-v4 fixtures have it. Drop it for consistency. Addresses CodeRabbit comment 2 on #8665. Signed-off-by: Keiven Chang Signed-off-by: Keiven Chang --- .../vllm/deepseek-v4/chat_completion_stream_special_chars.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_special_chars.json b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_special_chars.json index 38ab2a150cd8..2d5375f10014 100644 --- a/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_special_chars.json +++ b/lib/llm/tests/data/vllm/deepseek-v4/chat_completion_stream_special_chars.json @@ -1,6 +1,6 @@ { "request_id": "deepseek-v4-special-chars-test", - "expected_output": {"normal_content": "", "reasoning_content": "The user wants me to save a multiline note with special characters, quotes, unicode, and emoji. I'll call save_note.", "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "save_note", "arguments": "{\"note\": \"He said \\\"hello\\\".\\n\\t'world' `backtick` — 中文测试 — 🚀✨ & .\"}"}}], "finish_reason": "tool_calls"}, + "expected_output": {"normal_content": "", "reasoning_content": "The user wants me to save a multiline note with special characters, quotes, unicode, and emoji. I'll call save_note.", "tool_calls": [{"id": "call_1", "type": "function", "function": {"name": "save_note", "arguments": "{\"note\": \"He said \\\"hello\\\".\\n\\t'world' `backtick` — 中文测试 — 🚀✨ & .\"}"}}]}, "input_stream": [ {"data":{"id":"chatcmpl-deepseek-v4-special","choices":[{"index":0,"delta":{"content":"The user wants me to save a multiline note with special characters, quotes, unicode, and emoji. I'll call save_note.","role":"assistant","reasoning_content":"The user wants me to save a multiline note with special characters, quotes, unicode, and emoji. I'll call save_note."}}]}}, {"data":{"id":"chatcmpl-deepseek-v4-special","choices":[{"index":0,"delta":{"content":"<｜DSML｜tool_calls>\n","role":"assistant"}}]}}, From 44fb33a5ad53caead03780b5a2a4593b5388c88f Mon Sep 17 00:00:00 2001 From: ishandhanani Date: Fri, 24 Apr 2026 02:55:03 -0500 Subject: [PATCH 21/21] investigate tmrw --- components/src/dynamo/frontend/sglang_prepost.py | 2 +- .../src/dynamo/sglang/request_handlers/llm/decode_handler.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/components/src/dynamo/frontend/sglang_prepost.py b/components/src/dynamo/frontend/sglang_prepost.py index c66ee450760f..5918f93d7bae 100644 --- a/components/src/dynamo/frontend/sglang_prepost.py +++ b/components/src/dynamo/frontend/sglang_prepost.py @@ -174,7 +174,7 @@ def build_tool_call_guided_decoding( ) constraint = parser.get_structure_constraint( tool_choice, - parallel_tool_calls=parallel_tool_calls, + # parallel_tool_calls=parallel_tool_calls, ) if isinstance(constraint, tuple) and len(constraint) == 2: diff --git a/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py b/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py index 81579d3fba9d..66efcd6c68b3 100644 --- a/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py +++ b/components/src/dynamo/sglang/request_handlers/llm/decode_handler.py @@ -308,7 +308,7 @@ async def generate( **input_param, sampling_params=sampling_params, stream=True, - return_routed_experts=return_routed_experts, + # return_routed_experts=return_routed_experts, bootstrap_host=bootstrap_info["bootstrap_host"], bootstrap_port=bootstrap_info["bootstrap_port"], bootstrap_room=bootstrap_info["bootstrap_room"], @@ -346,7 +346,7 @@ async def generate( video_data=video_data, sampling_params=sampling_params, stream=True, - return_routed_experts=return_routed_experts, + # return_routed_experts=return_routed_experts, external_trace_header=trace_header, rid=trace_id, data_parallel_rank=dp_rank,