From 8c84cc581761c4bed512965c8773eec12f02b46a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 30 Mar 2026 13:39:14 +0000 Subject: [PATCH 01/14] studio: improve GGUF tool calling accuracy and reliability - Add URL fetching to web_search tool so models can read full page content instead of only getting search snippets. Uses html2text for clean markdown conversion with regex fallback. - Inject current date and behavioral guidance (URL fetch workflow, no repeated queries, use code for data processing) into the tool-use system prompt. - Append error recovery nudge to tool results that indicate failure, helping small models avoid looping on the same broken call. - Strip leaked XML from assistant messages in conversation history and from the outgoing SSE stream. - Raise default max tool iterations from 10 to 25 across backend, model schema, and frontend defaults. - Increase _MAX_PAGE_CHARS from 4k to 16k so fetched pages contain enough content for the model to extract useful information. - Add "IMPORTANT: These are only short snippets" hint to search results so models know to fetch full pages when needed. Tested with Qwen3.5-4B-GGUF (UD-Q4_K_XL), 10 runs before/after: - XML leaks in responses: 10/10 -> 0/10 - URL fetch usage: 0 -> 4/10 runs - Runs producing actual correct answers: 0/10 -> 2/10 - Average tool calls per query: 5.5 -> 3.8 (more efficient) - Average response time: 12.3s -> 9.8s --- studio/backend/core/inference/llama_cpp.py | 23 ++++- studio/backend/core/inference/tools.py | 95 +++++++++++++++++-- studio/backend/models/inference.py | 2 +- studio/backend/routes/inference.py | 68 ++++++++++++- .../chat/stores/chat-runtime-store.ts | 2 +- 5 files changed, 177 insertions(+), 13 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index ca39054ec0b..d03d20a29d6 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -11,6 +11,7 @@ import atexit import contextlib import json +import re import struct import structlog from loggers import get_logger @@ -2099,7 +2100,7 @@ def generate_chat_completion_with_tools( stop: Optional[list[str]] = None, cancel_event: Optional[threading.Event] = None, enable_thinking: Optional[bool] = None, - max_tool_iterations: int = 10, + max_tool_iterations: int = 25, auto_heal_tool_calls: bool = True, tool_call_timeout: int = 300, session_id: Optional[str] = None, @@ -2547,6 +2548,12 @@ def _strip_tool_markup(text: str, *, final: bool = False) -> str: # Merge accumulated metrics from prior tool # iterations so they are not silently dropped. yield {"type": "status", "text": ""} + if content_accum: + # Strip leaked tool-call XML before yielding + content_accum = re.sub( + r".*?", "", + content_accum, flags=re.DOTALL, + ).strip() if content_accum: yield {"type": "content", "text": content_accum} _fu = _iter_usage or {} @@ -2658,10 +2665,22 @@ def _strip_tool_markup(text: str, *, final: bool = False) -> str: "result": result, } + # Nudge model to try a different approach on errors + _error_prefixes = ( + "Error", "Search failed", "Execution error", + "Blocked", "No ", "Exit code", "Failed to fetch", + ) + _result_content = result + if isinstance(result, str) and result.lstrip().startswith(_error_prefixes): + _result_content = ( + result + "\n\nThe tool call encountered an issue. " + "Please try a different approach or rephrase your request." + ) + tool_msg = { "role": "tool", "name": tool_name, - "content": result, + "content": _result_content, } tool_call_id = tc.get("id") if tool_call_id: diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 55bfa095f9d..2a740ec34ec 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -57,16 +57,23 @@ def _get_workdir(session_id: str | None = None) -> str: "type": "function", "function": { "name": "web_search", - "description": "Search the web for current information, recent events, or facts you are uncertain about.", + "description": ( + "Search the web and fetch page content. Returns snippets for all results. " + "Use the url parameter to fetch full page text from a specific URL." + ), "parameters": { "type": "object", "properties": { "query": { "type": "string", "description": "The search query", - } + }, + "url": { + "type": "string", + "description": "A URL to fetch full page content from (instead of searching). Use this to read a page found in search results.", + }, }, - "required": ["query"], + "required": [], }, }, } @@ -131,7 +138,11 @@ def execute_tool( ) effective_timeout = _EXEC_TIMEOUT if timeout is _TIMEOUT_UNSET else timeout if name == "web_search": - return _web_search(arguments.get("query", ""), timeout = effective_timeout) + return _web_search( + arguments.get("query", ""), + url = arguments.get("url"), + timeout = effective_timeout, + ) if name == "python": return _python_exec( arguments.get("code", ""), cancel_event, effective_timeout, session_id @@ -143,9 +154,71 @@ def execute_tool( return f"Unknown tool: {name}" -def _web_search(query: str, max_results: int = 5, timeout: int = _EXEC_TIMEOUT) -> str: - """Search the web using DuckDuckGo and return formatted results.""" - if not query.strip(): +_MAX_PAGE_CHARS = 16000 # limit fetched page text + + +def _fetch_page_text(url: str, max_chars: int = _MAX_PAGE_CHARS, timeout: int = 30) -> str: + """Fetch a URL and return plain text content (HTML tags stripped). + + Only http:// and https:// schemes are allowed (SSRF protection). + """ + import re as _re + from urllib.parse import urlparse + + parsed = urlparse(url) + if parsed.scheme not in ("http", "https"): + return f"Blocked: only http/https URLs are allowed (got {parsed.scheme!r})." + + try: + import urllib.request + + req = urllib.request.Request( + url, + headers={"User-Agent": "UnslothStudio/1.0"}, + ) + with urllib.request.urlopen(req, timeout=timeout) as resp: + raw_html = resp.read().decode("utf-8", errors="replace") + except Exception as e: + return f"Failed to fetch URL: {e}" + + # Convert HTML to text -- prefer html2text for clean markdown output + try: + import html2text as _h2t + + converter = _h2t.HTML2Text() + converter.ignore_links = False + converter.ignore_images = True + converter.body_width = 0 # no wrapping + text = converter.handle(raw_html).strip() + except ImportError: + # Fallback: regex-based stripping + text = _re.sub(r"]*>.*?", "", raw_html, flags=_re.DOTALL | _re.IGNORECASE) + text = _re.sub(r"]*>.*?", "", text, flags=_re.DOTALL | _re.IGNORECASE) + text = _re.sub(r"<[^>]+>", " ", text) + text = _re.sub(r"\s+", " ", text).strip() + + if not text: + return "(page returned no readable text)" + if len(text) > max_chars: + text = text[:max_chars] + f"\n\n... (truncated, {len(text)} chars total)" + return text + + +def _web_search( + query: str, + max_results: int = 5, + timeout: int = _EXEC_TIMEOUT, + url: str | None = None, +) -> str: + """Search the web using DuckDuckGo and return formatted results. + + If ``url`` is provided, fetches that page directly instead of searching. + """ + # Direct URL fetch mode + if url and url.strip(): + return _fetch_page_text(url.strip(), timeout=min(timeout, 60)) + + if not query or not query.strip(): return "No query provided." try: from ddgs import DDGS @@ -160,7 +233,13 @@ def _web_search(query: str, max_results: int = 5, timeout: int = _EXEC_TIMEOUT) f"URL: {r.get('href', '')}\n" f"Snippet: {r.get('body', '')}" ) - return "\n\n---\n\n".join(parts) + text = "\n\n---\n\n".join(parts) + text += ( + "\n\n---\n\nIMPORTANT: These are only short snippets. " + "To get the full page content, call web_search with " + "the url parameter (e.g. {\"url\": \"\"})." + ) + return text except Exception as e: return f"Search failed: {e}" diff --git a/studio/backend/models/inference.py b/studio/backend/models/inference.py index a20c2052aa7..3b4dc923d08 100644 --- a/studio/backend/models/inference.py +++ b/studio/backend/models/inference.py @@ -340,7 +340,7 @@ class ChatCompletionRequest(BaseModel): description = "[x-unsloth] Auto-detect and fix malformed tool calls from model output.", ) max_tool_calls_per_message: Optional[int] = Field( - 10, + 25, ge = 0, description = "[x-unsloth] Maximum number of tool call iterations per message (0 = disabled, 9999 = unlimited).", ) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 7d48198d427..0f0b677e72c 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1065,6 +1065,70 @@ async def audio_input_stream(): else: tools_to_use = ALL_TOOLS + # ── Tool-use system prompt nudge ────────────────────── + _tool_names = {t["function"]["name"] for t in tools_to_use} + _has_web = "web_search" in _tool_names + _has_code = "python" in _tool_names or "terminal" in _tool_names + + from datetime import date as _date + _date_line = f"The current date is {_date.today().isoformat()}." + + _web_tips = ( + "When you search and find a relevant URL in the results, " + "fetch its full content by calling web_search with the url parameter. " + "Do not repeat the same search query. If a search returns " + "no useful results, try rephrasing or fetching a result URL directly." + ) + _code_tips = ( + "Use code execution for math, calculations, data processing, " + "or to parse and analyze information from tool results." + ) + + if _has_web and _has_code: + _nudge = ( + _date_line + " " + "You have access to tools. When appropriate, prefer using " + "tools rather than answering from memory. " + + _web_tips + " " + _code_tips + ) + elif _has_code: + _nudge = ( + _date_line + " " + "You have access to tools. When appropriate, prefer using " + "code execution rather than answering from memory. " + + _code_tips + ) + elif _has_web: + _nudge = ( + _date_line + " " + "You have access to tools. When appropriate, prefer using " + "web search for up-to-date or uncertain factual " + "information rather than answering from memory. " + + _web_tips + ) + else: + _nudge = "" + + if _nudge: + # Append nudge to system prompt (preserve user's prompt) + if system_prompt: + system_prompt = system_prompt.rstrip() + "\n\n" + _nudge + else: + system_prompt = _nudge + # Rebuild gguf_messages with updated system prompt + gguf_messages = [] + if system_prompt: + gguf_messages.append({"role": "system", "content": system_prompt}) + gguf_messages.extend(chat_messages) + + # ── Strip stale tool-call XML from conversation history ─ + _TOOL_XML_RE = _re.compile( + r".*?", _re.DOTALL + ) + for _msg in gguf_messages: + if _msg.get("role") == "assistant" and isinstance(_msg.get("content"), str): + _msg["content"] = _TOOL_XML_RE.sub("", _msg["content"]).strip() + def gguf_generate_with_tools(): return llama_backend.generate_chat_completion_with_tools( messages = gguf_messages, @@ -1083,7 +1147,7 @@ def gguf_generate_with_tools(): else True, max_tool_iterations = payload.max_tool_calls_per_message if payload.max_tool_calls_per_message is not None - else 10, + else 25, tool_call_timeout = payload.tool_call_timeout if payload.tool_call_timeout is not None else 300, @@ -1146,6 +1210,8 @@ async def gguf_tool_stream(): # "content" type -- cumulative text cumulative = event.get("text", "") + # Strip leaked tool-call XML from outgoing stream + cumulative = _TOOL_XML_RE.sub("", cumulative) new_text = cumulative[len(prev_text) :] prev_text = cumulative if not new_text: diff --git a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts index ca1044b3dc4..8cea234f21b 100644 --- a/studio/frontend/src/features/chat/stores/chat-runtime-store.ts +++ b/studio/frontend/src/features/chat/stores/chat-runtime-store.ts @@ -224,7 +224,7 @@ export const useChatRuntimeStore = create((set) => ({ toolStatus: null, generatingStatus: null, autoHealToolCalls: loadBool(AUTO_HEAL_TOOL_CALLS_KEY, true), - maxToolCallsPerMessage: loadInt(MAX_TOOL_CALLS_KEY, 10), + maxToolCallsPerMessage: loadInt(MAX_TOOL_CALLS_KEY, 25), toolCallTimeout: loadInt(TOOL_CALL_TIMEOUT_KEY, 5), kvCacheDtype: null, loadedKvCacheDtype: null, From 68546d7aaf7083963727848752c414c942be4f85 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 30 Mar 2026 13:39:58 +0000 Subject: [PATCH 02/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/core/inference/llama_cpp.py | 19 +++++++++++----- studio/backend/core/inference/tools.py | 25 +++++++++++++++------- studio/backend/routes/inference.py | 19 ++++++++-------- 3 files changed, 41 insertions(+), 22 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index d03d20a29d6..2885f535ef1 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -2551,8 +2551,10 @@ def _strip_tool_markup(text: str, *, final: bool = False) -> str: if content_accum: # Strip leaked tool-call XML before yielding content_accum = re.sub( - r".*?", "", - content_accum, flags=re.DOTALL, + r".*?", + "", + content_accum, + flags = re.DOTALL, ).strip() if content_accum: yield {"type": "content", "text": content_accum} @@ -2667,11 +2669,18 @@ def _strip_tool_markup(text: str, *, final: bool = False) -> str: # Nudge model to try a different approach on errors _error_prefixes = ( - "Error", "Search failed", "Execution error", - "Blocked", "No ", "Exit code", "Failed to fetch", + "Error", + "Search failed", + "Execution error", + "Blocked", + "No ", + "Exit code", + "Failed to fetch", ) _result_content = result - if isinstance(result, str) and result.lstrip().startswith(_error_prefixes): + if isinstance(result, str) and result.lstrip().startswith( + _error_prefixes + ): _result_content = ( result + "\n\nThe tool call encountered an issue. " "Please try a different approach or rephrase your request." diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 2a740ec34ec..ed327e17ab8 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -157,7 +157,9 @@ def execute_tool( _MAX_PAGE_CHARS = 16000 # limit fetched page text -def _fetch_page_text(url: str, max_chars: int = _MAX_PAGE_CHARS, timeout: int = 30) -> str: +def _fetch_page_text( + url: str, max_chars: int = _MAX_PAGE_CHARS, timeout: int = 30 +) -> str: """Fetch a URL and return plain text content (HTML tags stripped). Only http:// and https:// schemes are allowed (SSRF protection). @@ -174,10 +176,10 @@ def _fetch_page_text(url: str, max_chars: int = _MAX_PAGE_CHARS, timeout: int = req = urllib.request.Request( url, - headers={"User-Agent": "UnslothStudio/1.0"}, + headers = {"User-Agent": "UnslothStudio/1.0"}, ) - with urllib.request.urlopen(req, timeout=timeout) as resp: - raw_html = resp.read().decode("utf-8", errors="replace") + with urllib.request.urlopen(req, timeout = timeout) as resp: + raw_html = resp.read().decode("utf-8", errors = "replace") except Exception as e: return f"Failed to fetch URL: {e}" @@ -192,8 +194,15 @@ def _fetch_page_text(url: str, max_chars: int = _MAX_PAGE_CHARS, timeout: int = text = converter.handle(raw_html).strip() except ImportError: # Fallback: regex-based stripping - text = _re.sub(r"]*>.*?", "", raw_html, flags=_re.DOTALL | _re.IGNORECASE) - text = _re.sub(r"]*>.*?", "", text, flags=_re.DOTALL | _re.IGNORECASE) + text = _re.sub( + r"]*>.*?", + "", + raw_html, + flags = _re.DOTALL | _re.IGNORECASE, + ) + text = _re.sub( + r"]*>.*?", "", text, flags = _re.DOTALL | _re.IGNORECASE + ) text = _re.sub(r"<[^>]+>", " ", text) text = _re.sub(r"\s+", " ", text).strip() @@ -216,7 +225,7 @@ def _web_search( """ # Direct URL fetch mode if url and url.strip(): - return _fetch_page_text(url.strip(), timeout=min(timeout, 60)) + return _fetch_page_text(url.strip(), timeout = min(timeout, 60)) if not query or not query.strip(): return "No query provided." @@ -237,7 +246,7 @@ def _web_search( text += ( "\n\n---\n\nIMPORTANT: These are only short snippets. " "To get the full page content, call web_search with " - "the url parameter (e.g. {\"url\": \"\"})." + 'the url parameter (e.g. {"url": ""}).' ) return text except Exception as e: diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 0f0b677e72c..3be8ef1549a 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1071,6 +1071,7 @@ async def audio_input_stream(): _has_code = "python" in _tool_names or "terminal" in _tool_names from datetime import date as _date + _date_line = f"The current date is {_date.today().isoformat()}." _web_tips = ( @@ -1089,22 +1090,22 @@ async def audio_input_stream(): _date_line + " " "You have access to tools. When appropriate, prefer using " "tools rather than answering from memory. " - + _web_tips + " " + _code_tips + + _web_tips + + " " + + _code_tips ) elif _has_code: _nudge = ( _date_line + " " "You have access to tools. When appropriate, prefer using " - "code execution rather than answering from memory. " - + _code_tips + "code execution rather than answering from memory. " + _code_tips ) elif _has_web: _nudge = ( _date_line + " " "You have access to tools. When appropriate, prefer using " "web search for up-to-date or uncertain factual " - "information rather than answering from memory. " - + _web_tips + "information rather than answering from memory. " + _web_tips ) else: _nudge = "" @@ -1122,11 +1123,11 @@ async def audio_input_stream(): gguf_messages.extend(chat_messages) # ── Strip stale tool-call XML from conversation history ─ - _TOOL_XML_RE = _re.compile( - r".*?", _re.DOTALL - ) + _TOOL_XML_RE = _re.compile(r".*?", _re.DOTALL) for _msg in gguf_messages: - if _msg.get("role") == "assistant" and isinstance(_msg.get("content"), str): + if _msg.get("role") == "assistant" and isinstance( + _msg.get("content"), str + ): _msg["content"] = _TOOL_XML_RE.sub("", _msg["content"]).strip() def gguf_generate_with_tools(): From 8f60cea2f56d8aba86a3767031dcd544fd87171a Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Mon, 30 Mar 2026 21:38:56 +0000 Subject: [PATCH 03/14] Add tool calling benchmark results across model sizes and quants Tested 16 configurations (4 models x 2 quants x 2 KV cache types) with 10 runs each on NVIDIA B200. Best config: 27B UD-Q4_K_XL + bf16 KV -- 6/10 runs found all 4 correct songs, 0 XML leaks, 131s average response time. --- .../tests/tool_calling_benchmark_results.md | 62 +++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 studio/backend/tests/tool_calling_benchmark_results.md diff --git a/studio/backend/tests/tool_calling_benchmark_results.md b/studio/backend/tests/tool_calling_benchmark_results.md new file mode 100644 index 00000000000..c2b0687895e --- /dev/null +++ b/studio/backend/tests/tool_calling_benchmark_results.md @@ -0,0 +1,62 @@ +# GGUF Tool Calling Benchmark Results + +Prompt: "List and categorize all the songs that charted #3 on the Billboard Hot 100 in 2015." +10 runs per configuration, web search + code execution + thinking enabled. +GPU: NVIDIA B200, CUDA_VISIBLE_DEVICES=2. + +Ground truth: 4 songs peaked at #3 in 2015 -- "Love Me like You Do" (Ellie Goulding), "Earned It" (The Weeknd), "Watch Me" (Silento), "Drag Me Down" (One Direction). + +## Cartesian Grid: Model x Quant x KV Cache + +| Model | Quant | KV Cache | OK/10 | Avg Time | Avg Tools | XML Leaks | URL Fetch | Peak3 Avg | All 4/4 | Best Songs | +|-------|-------|----------|-------|----------|-----------|-----------|-----------|-----------|---------|------------| +| 4B | UD-Q4_K_XL | f16 | 10/10 | 9.8s | 3.5 | 0/10 | 4/10 | 0.8/4 | 2/10 | 9 | +| 4B | UD-Q4_K_XL | bf16 | 10/10 | 10.6s | 4.5 | 0/10 | 4/10 | 0.4/4 | 1/10 | 5 | +| 4B | Q8_0 | f16 | 10/10 | 4.9s | 2.4 | 0/10 | 8/10 | 0.4/4 | 1/10 | 5 | +| 4B | Q8_0 | bf16 | 10/10 | 8.0s | 3.0 | 0/10 | 5/10 | 0.0/4 | 0/10 | 0 | +| 9B | UD-Q4_K_XL | f16 | 10/10 | 6.7s | 2.0 | 0/10 | 5/10 | 0.0/4 | 0/10 | 3 | +| 9B | UD-Q4_K_XL | bf16 | 9/10 | 49.5s | 2.4 | 0/10 | 5/10 | 0.0/4 | 0/10 | 1 | +| 9B | Q8_0 | f16 | 10/10 | 7.4s | 2.5 | 0/10 | 5/10 | 0.0/4 | 0/10 | 2 | +| 9B | Q8_0 | bf16 | 10/10 | 10.4s | 2.7 | 0/10 | 6/10 | 1.0/4 | 2/10 | 15 | +| **27B** | **UD-Q4_K_XL** | **bf16** | **9/10** | **131.1s** | **13.8** | **0/10** | **7/10** | **2.7/4** | **6/10** | **27** | +| 27B | UD-Q4_K_XL | f16 | 7/10 | 201.6s | 14.1 | 0/10 | 8/10 | 2.0/4 | 5/10 | 26 | +| 27B | Q8_0 | f16 | 4/10 | 312.5s | 16.0 | 1/10 | 10/10 | 2.4/4 | 6/10 | 28 | +| 27B | Q8_0 | bf16 | 5/10 | 258.4s | 16.5 | 2/10 | 10/10 | 0.9/4 | 1/10 | 27 | +| 35B-A3B | UD-Q4_K_XL | f16 | 3/10 | 353.6s | 14.7 | 1/10 | 6/10 | 1.2/4 | 3/10 | 27 | +| 35B-A3B | UD-Q4_K_XL | bf16 | 3/10 | 356.2s | 17.2 | 1/10 | 8/10 | 1.6/4 | 4/10 | 27 | +| 35B-A3B | Q8_0 | f16 | 2/10 | 372.1s | 17.6 | 1/10 | 7/10 | 1.2/4 | 3/10 | 26 | +| 35B-A3B | Q8_0 | bf16 | 6/10 | 267.7s | 17.5 | 1/10 | 8/10 | 2.4/4 | 6/10 | 27 | + +**Column definitions:** +- **Peak3 Avg**: Average number of correct peak-#3 songs found per run (out of 4) +- **All 4/4**: Runs where all 4 correct songs were identified +- **Best Songs**: Maximum number of Billboard 2015 songs mentioned in any single run (out of 31 tracked) +- **URL Fetch**: Runs where the model used web_search with `url` parameter to fetch full page content + +## Key Findings + +1. **27B UD-Q4_K_XL + bf16 KV is the sweet spot.** 6/10 runs found all 4 correct songs, 0 XML leaks, 131s average. Best balance of accuracy, speed, and reliability. + +2. **Larger models use tools more effectively.** 27B and 35B-A3B models used 13-17 tool calls per query (vs 2-4 for 4B/9B), performing multiple searches and URL fetches to find the answer. + +3. **27B Q8_0 had the highest raw accuracy (6/10 all-4/4) but lower reliability** -- only 4/10 OK runs due to timeouts on long agentic chains. The UD-Q4_K_XL quant is more practical. + +4. **4B models were fastest (5-10s) but least accurate.** They occasionally found all 4 songs (2/10 best case) when they happened to fetch the right Wikipedia page. + +5. **9B was surprisingly weaker than 4B on this task.** It used fewer tool calls and rarely extracted song data from fetched pages. The 9B model may need higher temperature or different prompting for this specific task type. + +6. **35B-A3B had reliability issues.** Most runs timed out or errored due to slow per-token generation with many tool iterations. When it completed (2-6/10 OK), accuracy was comparable to 27B. + +7. **bf16 KV cache had mixed effects.** For 27B it improved both speed (131s vs 202s) and accuracy (6/10 vs 5/10 all-4/4). For smaller models it had no consistent benefit. + +8. **XML leaks are nearly eliminated.** 0/10 for all 4B and 9B configs, and only 1-2/10 for the largest models (which generate much more text in complex agentic loops). + +## Before vs After (4B UD-Q4_K_XL, f16 KV) + +| Metric | Before Changes | After Changes | +|--------|---------------|---------------| +| XML leaks | 10/10 | 0/10 | +| URL fetches | 0/10 | 4/10 | +| Peak3 accuracy | 0.0/4 | 0.8/4 | +| Runs with all 4 songs | 0/10 | 2/10 | +| Avg time | 12.3s | 9.8s | From a4c29d8d3c4979de05d3267a9ebf49441f365223 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 31 Mar 2026 07:53:29 +0000 Subject: [PATCH 04/14] Add duplicate tool-call detection and final-answer synthesis When the model repeats the exact same tool call (same name + arguments) twice in a row, skip execution and return a redirect message telling it to try a different approach. This prevents the 8x-repeated-query loops observed on 27B and 35B models. When the tool iteration cap (25) is reached, inject a "provide your final answer now" message before the final streaming pass. This lets the model synthesize a useful answer from everything it gathered instead of being silently cut off. Tested on Qwen3.5-27B UD-Q4_K_XL (10 runs): - Repeated query runs: 4/10 -> 2/10 - Cap hits: 1/10 -> 0/10 - All 4/4 accuracy: 5/10 -> 7/10 --- studio/backend/core/inference/llama_cpp.py | 66 ++++++++++++++++++---- 1 file changed, 56 insertions(+), 10 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 2885f535ef1..c5a36877074 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -2152,6 +2152,28 @@ def _strip_tool_markup(text: str, *, final: bool = False) -> str: ) _MAX_BUFFER_CHARS = 32 + # ── Duplicate tool-call detection ──────────────────────── + # Track recent (tool_name, arguments) hashes to detect loops + # where the model repeats the exact same call. + import hashlib as _hl + _tool_call_history: list[str] = [] + _DEDUP_WINDOW = 2 # flag if same call appears this many times in a row + + def _tool_call_key(name: str, args: dict) -> str: + raw = json.dumps({"t": name, "a": args}, sort_keys=True) + return _hl.md5(raw.encode()).hexdigest() + + def _is_duplicate_call(name: str, args: dict) -> bool: + key = _tool_call_key(name, args) + _tool_call_history.append(key) + if len(_tool_call_history) >= _DEDUP_WINDOW: + tail = _tool_call_history[-_DEDUP_WINDOW:] + if len(set(tail)) == 1: + return True + return False + + _hit_tool_cap = False + for iteration in range(max_tool_iterations): if cancel_event is not None and cancel_event.is_set(): return @@ -2649,16 +2671,27 @@ def _strip_tool_markup(text: str, *, final: bool = False) -> str: "arguments": arguments, } - _effective_timeout = ( - None if tool_call_timeout >= 9999 else tool_call_timeout - ) - result = execute_tool( - tool_name, - arguments, - cancel_event = cancel_event, - timeout = _effective_timeout, - session_id = session_id, - ) + # ── Duplicate call detection ────────────── + if _is_duplicate_call(tool_name, arguments): + result = ( + "You already made this exact call. " + "Do not repeat the same tool call. " + "Try a different approach: fetch a URL " + "from previous results, use Python to " + "process data you already have, or " + "provide your final answer now." + ) + else: + _effective_timeout = ( + None if tool_call_timeout >= 9999 else tool_call_timeout + ) + result = execute_tool( + tool_name, + arguments, + cancel_event = cancel_event, + timeout = _effective_timeout, + session_id = session_id, + ) yield { "type": "tool_end", @@ -2706,6 +2739,19 @@ def _strip_tool_markup(text: str, *, final: bool = False) -> str: return raise + # ── Tool iteration cap reached -- synthesize final answer ── + # The model used all iterations without producing a final text + # response. Inject a nudge so the final streaming pass produces + # a useful answer instead of continuing to request tools. + conversation.append({ + "role": "user", + "content": ( + "You have used all available tool calls. Based on " + "everything you have found so far, provide your final " + "answer now. Do not call any more tools." + ), + }) + # Clear status yield {"type": "status", "text": ""} From cf9deae02c69e1989aa8cb3f79274a55336703c4 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 31 Mar 2026 07:53:43 +0000 Subject: [PATCH 05/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/core/inference/llama_cpp.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index c5a36877074..7435a5836c2 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -2156,11 +2156,12 @@ def _strip_tool_markup(text: str, *, final: bool = False) -> str: # Track recent (tool_name, arguments) hashes to detect loops # where the model repeats the exact same call. import hashlib as _hl + _tool_call_history: list[str] = [] _DEDUP_WINDOW = 2 # flag if same call appears this many times in a row def _tool_call_key(name: str, args: dict) -> str: - raw = json.dumps({"t": name, "a": args}, sort_keys=True) + raw = json.dumps({"t": name, "a": args}, sort_keys = True) return _hl.md5(raw.encode()).hexdigest() def _is_duplicate_call(name: str, args: dict) -> bool: @@ -2743,14 +2744,16 @@ def _is_duplicate_call(name: str, args: dict) -> bool: # The model used all iterations without producing a final text # response. Inject a nudge so the final streaming pass produces # a useful answer instead of continuing to request tools. - conversation.append({ - "role": "user", - "content": ( - "You have used all available tool calls. Based on " - "everything you have found so far, provide your final " - "answer now. Do not call any more tools." - ), - }) + conversation.append( + { + "role": "user", + "content": ( + "You have used all available tool calls. Based on " + "everything you have found so far, provide your final " + "answer now. Do not call any more tools." + ), + } + ) # Clear status yield {"type": "status", "text": ""} From ba5226121d7f97327274a1a103dedc293e250a34 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 31 Mar 2026 08:03:07 +0000 Subject: [PATCH 06/14] Fix CodeQL alert: handle whitespace in script/style closing tags The regex fallback for HTML stripping did not match closing tags with whitespace before the angle bracket (e.g. ). Use \s* before > in both script and style patterns. --- studio/backend/core/inference/tools.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index ed327e17ab8..e86e31c0000 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -195,13 +195,13 @@ def _fetch_page_text( except ImportError: # Fallback: regex-based stripping text = _re.sub( - r"]*>.*?", + r"]*>.*?", "", raw_html, flags = _re.DOTALL | _re.IGNORECASE, ) text = _re.sub( - r"]*>.*?", "", text, flags = _re.DOTALL | _re.IGNORECASE + r"]*>.*?", "", text, flags = _re.DOTALL | _re.IGNORECASE ) text = _re.sub(r"<[^>]+>", " ", text) text = _re.sub(r"\s+", " ", text).strip() From cb2f98897d626378f53cf92f77bafe6a9fb51627 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 31 Mar 2026 08:20:32 +0000 Subject: [PATCH 07/14] Address reviewer findings: SSRF, timeout crash, XML regex, dedup - SSRF: resolve hostname via getaddrinfo and reject private, loopback, link-local, multicast, and reserved addresses before fetching - Timeout: handle timeout=None (unlimited mode) in URL fetch path by defaulting to 60s instead of crashing on min(None, 60) - Download cap: read at most max_chars*4+1 bytes instead of the full response body before truncating - XML regex: match both and markup in the history/stream cleanup (inference.py) - CodeQL: use [^>]* in closing script/style tags to handle any whitespace or attributes before > - Dedup: track whether each tool call failed so retries after transient errors are allowed; only block consecutive identical calls that both succeeded - Final-answer synthesis: guard on max_tool_iterations > 0 so callers who disable tools do not get a false "used all calls" turn --- studio/backend/core/inference/llama_cpp.py | 45 +++++++++++------- studio/backend/core/inference/tools.py | 55 ++++++++++++++++++---- studio/backend/routes/inference.py | 5 +- 3 files changed, 78 insertions(+), 27 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 7435a5836c2..a32b9719b43 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -2154,25 +2154,30 @@ def _strip_tool_markup(text: str, *, final: bool = False) -> str: # ── Duplicate tool-call detection ──────────────────────── # Track recent (tool_name, arguments) hashes to detect loops - # where the model repeats the exact same call. + # where the model repeats the exact same call. Retries after + # a transient failure are allowed (only block when the previous + # identical call succeeded). import hashlib as _hl - _tool_call_history: list[str] = [] + _tool_call_history: list[tuple[str, bool]] = [] # (key, failed) _DEDUP_WINDOW = 2 # flag if same call appears this many times in a row def _tool_call_key(name: str, args: dict) -> str: - raw = json.dumps({"t": name, "a": args}, sort_keys = True) + raw = json.dumps({"t": name, "a": args}, sort_keys=True) return _hl.md5(raw.encode()).hexdigest() def _is_duplicate_call(name: str, args: dict) -> bool: key = _tool_call_key(name, args) - _tool_call_history.append(key) if len(_tool_call_history) >= _DEDUP_WINDOW: tail = _tool_call_history[-_DEDUP_WINDOW:] - if len(set(tail)) == 1: + if all(k == key and not failed for k, failed in tail): return True return False + def _record_tool_call(name: str, args: dict, failed: bool) -> None: + key = _tool_call_key(name, args) + _tool_call_history.append((key, failed)) + _hit_tool_cap = False for iteration in range(max_tool_iterations): @@ -2682,6 +2687,7 @@ def _is_duplicate_call(name: str, args: dict) -> bool: "process data you already have, or " "provide your final answer now." ) + _record_tool_call(tool_name, arguments, failed=False) else: _effective_timeout = ( None if tool_call_timeout >= 9999 else tool_call_timeout @@ -2711,10 +2717,12 @@ def _is_duplicate_call(name: str, args: dict) -> bool: "Exit code", "Failed to fetch", ) - _result_content = result - if isinstance(result, str) and result.lstrip().startswith( + _is_error = isinstance(result, str) and result.lstrip().startswith( _error_prefixes - ): + ) + _record_tool_call(tool_name, arguments, failed=_is_error) + _result_content = result + if _is_error: _result_content = ( result + "\n\nThe tool call encountered an issue. " "Please try a different approach or rephrase your request." @@ -2744,16 +2752,17 @@ def _is_duplicate_call(name: str, args: dict) -> bool: # The model used all iterations without producing a final text # response. Inject a nudge so the final streaming pass produces # a useful answer instead of continuing to request tools. - conversation.append( - { - "role": "user", - "content": ( - "You have used all available tool calls. Based on " - "everything you have found so far, provide your final " - "answer now. Do not call any more tools." - ), - } - ) + if max_tool_iterations > 0: + conversation.append( + { + "role": "user", + "content": ( + "You have used all available tool calls. Based on " + "everything you have found so far, provide your final " + "answer now. Do not call any more tools." + ), + } + ) # Clear status yield {"type": "status", "text": ""} diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index e86e31c0000..5fa8b8f5e69 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -155,6 +155,31 @@ def execute_tool( _MAX_PAGE_CHARS = 16000 # limit fetched page text +_MAX_FETCH_BYTES = _MAX_PAGE_CHARS * 4 + 1 # cap raw download size + + +def _is_public_host(hostname: str, port: int) -> tuple[bool, str]: + """Resolve *hostname* and reject private/loopback/link-local addresses.""" + import ipaddress + import socket + + try: + infos = socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM) + except OSError as e: + return False, f"Failed to resolve host: {e}" + + for *_, sockaddr in infos: + ip = ipaddress.ip_address(sockaddr[0]) + if ( + ip.is_private + or ip.is_loopback + or ip.is_link_local + or ip.is_multicast + or ip.is_reserved + or ip.is_unspecified + ): + return False, f"Blocked: refusing to fetch non-public address {ip}." + return True, "" def _fetch_page_text( @@ -162,7 +187,8 @@ def _fetch_page_text( ) -> str: """Fetch a URL and return plain text content (HTML tags stripped). - Only http:// and https:// schemes are allowed (SSRF protection). + Blocks private/loopback/link-local targets (SSRF protection) and caps + the download size to avoid unbounded memory usage. """ import re as _re from urllib.parse import urlparse @@ -170,16 +196,28 @@ def _fetch_page_text( parsed = urlparse(url) if parsed.scheme not in ("http", "https"): return f"Blocked: only http/https URLs are allowed (got {parsed.scheme!r})." + if not parsed.hostname: + return "Blocked: URL is missing a hostname." + + ok, reason = _is_public_host( + parsed.hostname, + parsed.port or (443 if parsed.scheme == "https" else 80), + ) + if not ok: + return reason try: import urllib.request req = urllib.request.Request( url, - headers = {"User-Agent": "UnslothStudio/1.0"}, + headers={"User-Agent": "UnslothStudio/1.0"}, ) - with urllib.request.urlopen(req, timeout = timeout) as resp: - raw_html = resp.read().decode("utf-8", errors = "replace") + max_bytes = max_chars * 4 + 1 + with urllib.request.urlopen(req, timeout=timeout) as resp: + # Cap download size to avoid unbounded memory usage + raw_bytes = resp.read(max_bytes) + raw_html = raw_bytes.decode("utf-8", errors="replace") except Exception as e: return f"Failed to fetch URL: {e}" @@ -195,13 +233,13 @@ def _fetch_page_text( except ImportError: # Fallback: regex-based stripping text = _re.sub( - r"]*>.*?", + r"]*>.*?]*>", "", raw_html, - flags = _re.DOTALL | _re.IGNORECASE, + flags=_re.DOTALL | _re.IGNORECASE, ) text = _re.sub( - r"]*>.*?", "", text, flags = _re.DOTALL | _re.IGNORECASE + r"]*>.*?]*>", "", text, flags=_re.DOTALL | _re.IGNORECASE ) text = _re.sub(r"<[^>]+>", " ", text) text = _re.sub(r"\s+", " ", text).strip() @@ -225,7 +263,8 @@ def _web_search( """ # Direct URL fetch mode if url and url.strip(): - return _fetch_page_text(url.strip(), timeout = min(timeout, 60)) + fetch_timeout = 60 if timeout is None else min(timeout, 60) + return _fetch_page_text(url.strip(), timeout=fetch_timeout) if not query or not query.strip(): return "No query provided." diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 3be8ef1549a..b0e7839a975 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1123,7 +1123,10 @@ async def audio_input_stream(): gguf_messages.extend(chat_messages) # ── Strip stale tool-call XML from conversation history ─ - _TOOL_XML_RE = _re.compile(r".*?", _re.DOTALL) + _TOOL_XML_RE = _re.compile( + r".*?|.*?", + _re.DOTALL, + ) for _msg in gguf_messages: if _msg.get("role") == "assistant" and isinstance( _msg.get("content"), str From 6abf067d9e0eecf7f230e8655d24e6cc5aeefa6d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 31 Mar 2026 08:20:49 +0000 Subject: [PATCH 08/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/core/inference/llama_cpp.py | 6 +++--- studio/backend/core/inference/tools.py | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index a32b9719b43..4ec70e57206 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -2163,7 +2163,7 @@ def _strip_tool_markup(text: str, *, final: bool = False) -> str: _DEDUP_WINDOW = 2 # flag if same call appears this many times in a row def _tool_call_key(name: str, args: dict) -> str: - raw = json.dumps({"t": name, "a": args}, sort_keys=True) + raw = json.dumps({"t": name, "a": args}, sort_keys = True) return _hl.md5(raw.encode()).hexdigest() def _is_duplicate_call(name: str, args: dict) -> bool: @@ -2687,7 +2687,7 @@ def _record_tool_call(name: str, args: dict, failed: bool) -> None: "process data you already have, or " "provide your final answer now." ) - _record_tool_call(tool_name, arguments, failed=False) + _record_tool_call(tool_name, arguments, failed = False) else: _effective_timeout = ( None if tool_call_timeout >= 9999 else tool_call_timeout @@ -2720,7 +2720,7 @@ def _record_tool_call(name: str, args: dict, failed: bool) -> None: _is_error = isinstance(result, str) and result.lstrip().startswith( _error_prefixes ) - _record_tool_call(tool_name, arguments, failed=_is_error) + _record_tool_call(tool_name, arguments, failed = _is_error) _result_content = result if _is_error: _result_content = ( diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 5fa8b8f5e69..1a6caa6e3e0 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -164,7 +164,7 @@ def _is_public_host(hostname: str, port: int) -> tuple[bool, str]: import socket try: - infos = socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM) + infos = socket.getaddrinfo(hostname, port, type = socket.SOCK_STREAM) except OSError as e: return False, f"Failed to resolve host: {e}" @@ -211,13 +211,13 @@ def _fetch_page_text( req = urllib.request.Request( url, - headers={"User-Agent": "UnslothStudio/1.0"}, + headers = {"User-Agent": "UnslothStudio/1.0"}, ) max_bytes = max_chars * 4 + 1 - with urllib.request.urlopen(req, timeout=timeout) as resp: + with urllib.request.urlopen(req, timeout = timeout) as resp: # Cap download size to avoid unbounded memory usage raw_bytes = resp.read(max_bytes) - raw_html = raw_bytes.decode("utf-8", errors="replace") + raw_html = raw_bytes.decode("utf-8", errors = "replace") except Exception as e: return f"Failed to fetch URL: {e}" @@ -236,10 +236,10 @@ def _fetch_page_text( r"]*>.*?]*>", "", raw_html, - flags=_re.DOTALL | _re.IGNORECASE, + flags = _re.DOTALL | _re.IGNORECASE, ) text = _re.sub( - r"]*>.*?]*>", "", text, flags=_re.DOTALL | _re.IGNORECASE + r"]*>.*?]*>", "", text, flags = _re.DOTALL | _re.IGNORECASE ) text = _re.sub(r"<[^>]+>", " ", text) text = _re.sub(r"\s+", " ", text).strip() @@ -264,7 +264,7 @@ def _web_search( # Direct URL fetch mode if url and url.strip(): fetch_timeout = 60 if timeout is None else min(timeout, 60) - return _fetch_page_text(url.strip(), timeout=fetch_timeout) + return _fetch_page_text(url.strip(), timeout = fetch_timeout) if not query or not query.strip(): return "No query provided." From fec5f2cb00ae9d5802a9c0ea6b9d2b3bc0059be1 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 31 Mar 2026 08:36:57 +0000 Subject: [PATCH 09/14] Fix redirect SSRF, SSE streaming regression, dedup off-by-one - SSRF redirect bypass: disable auto-redirect in urllib, manually follow up to 5 hops with host validation at each step. Prevents public URLs from redirecting to loopback/private targets. - SSE streaming: track prev_text on the raw cumulative and strip XML from the delta only, so completed tool_call tags do not cause the cumulative to shrink and drop trailing real text. - Dedup off-by-one: check the immediately previous call (window=1) instead of requiring 2 matching history entries, so the second identical successful call is blocked rather than the third. --- studio/backend/core/inference/llama_cpp.py | 11 +++--- studio/backend/core/inference/tools.py | 41 ++++++++++++++++++---- studio/backend/routes/inference.py | 10 +++--- 3 files changed, 45 insertions(+), 17 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index d594b754a2e..9551443e38e 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -2181,19 +2181,18 @@ def _strip_tool_markup(text: str, *, final: bool = False) -> str: import hashlib as _hl _tool_call_history: list[tuple[str, bool]] = [] # (key, failed) - _DEDUP_WINDOW = 2 # flag if same call appears this many times in a row def _tool_call_key(name: str, args: dict) -> str: raw = json.dumps({"t": name, "a": args}, sort_keys = True) return _hl.md5(raw.encode()).hexdigest() def _is_duplicate_call(name: str, args: dict) -> bool: + """Block if the immediately previous call was identical and succeeded.""" + if not _tool_call_history: + return False key = _tool_call_key(name, args) - if len(_tool_call_history) >= _DEDUP_WINDOW: - tail = _tool_call_history[-_DEDUP_WINDOW:] - if all(k == key and not failed for k, failed in tail): - return True - return False + last_key, last_failed = _tool_call_history[-1] + return last_key == key and not last_failed def _record_tool_call(name: str, args: dict, failed: bool) -> None: key = _tool_call_key(name, args) diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 1a6caa6e3e0..b35a0176148 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -208,15 +208,44 @@ def _fetch_page_text( try: import urllib.request + from urllib.parse import urljoin - req = urllib.request.Request( - url, - headers = {"User-Agent": "UnslothStudio/1.0"}, - ) + # Disable auto-redirect so we can validate each hop for SSRF + class _NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + opener = urllib.request.build_opener(_NoRedirect) max_bytes = max_chars * 4 + 1 - with urllib.request.urlopen(req, timeout = timeout) as resp: - # Cap download size to avoid unbounded memory usage + current_url = url + + for _hop in range(5): + req = urllib.request.Request( + current_url, + headers = {"User-Agent": "UnslothStudio/1.0"}, + ) + resp = opener.open(req, timeout = timeout) + if resp.status in (301, 302, 303, 307, 308): + location = resp.headers.get("Location") + if not location: + return "Failed to fetch URL: redirect missing Location header." + current_url = urljoin(current_url, location) + rp = urlparse(current_url) + if rp.scheme not in ("http", "https") or not rp.hostname: + return "Blocked: redirect target is not a valid http/https URL." + ok2, reason2 = _is_public_host( + rp.hostname, + rp.port or (443 if rp.scheme == "https" else 80), + ) + if not ok2: + return reason2 + continue + # Success -- read capped body raw_bytes = resp.read(max_bytes) + break + else: + return "Failed to fetch URL: too many redirects." + raw_html = raw_bytes.decode("utf-8", errors = "replace") except Exception as e: return f"Failed to fetch URL: {e}" diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 5920155fabe..a4179f4e21f 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1226,11 +1226,11 @@ async def gguf_tool_stream(): continue # "content" type -- cumulative text - cumulative = event.get("text", "") - # Strip leaked tool-call XML from outgoing stream - cumulative = _TOOL_XML_RE.sub("", cumulative) - new_text = cumulative[len(prev_text) :] - prev_text = cumulative + raw_cumulative = event.get("text", "") + raw_delta = raw_cumulative[len(prev_text):] + prev_text = raw_cumulative + # Strip leaked tool-call XML from the delta only + new_text = _TOOL_XML_RE.sub("", raw_delta) if not new_text: continue chunk = ChatCompletionChunk( From 1d184bf815857acdeb8c3b4c5ee3f500976fe8e0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 31 Mar 2026 08:38:13 +0000 Subject: [PATCH 10/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/routes/inference.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index a4179f4e21f..40f41578a7b 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -1227,7 +1227,7 @@ async def gguf_tool_stream(): # "content" type -- cumulative text raw_cumulative = event.get("text", "") - raw_delta = raw_cumulative[len(prev_text):] + raw_delta = raw_cumulative[len(prev_text) :] prev_text = raw_cumulative # Strip leaked tool-call XML from the delta only new_text = _TOOL_XML_RE.sub("", raw_delta) From c80d30f4fc22383875fae2edc0b875c7f263fd0c Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 31 Mar 2026 08:53:29 +0000 Subject: [PATCH 11/14] Fix redirect HTTPError handling and tighten error prefixes - Redirect fix: urllib raises HTTPError (not a normal response) when the redirect handler returns None. Catch HTTPError for 3xx codes and extract the Location header from the exception object. - Error prefixes: remove overly broad "No " prefix that matched "No results found." (a valid empty-search outcome, not an error). Replace with specific prefixes like "Blocked:", "No query provided", "Failed to resolve". This ensures empty search results are correctly classified as non-errors for duplicate-call tracking. --- studio/backend/core/inference/llama_cpp.py | 5 +++-- studio/backend/core/inference/tools.py | 16 ++++++++++++---- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 9551443e38e..b24721cd4e3 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -2732,10 +2732,11 @@ def _record_tool_call(name: str, args: dict, failed: bool) -> None: "Error", "Search failed", "Execution error", - "Blocked", - "No ", + "Blocked:", "Exit code", "Failed to fetch", + "Failed to resolve", + "No query provided", ) _is_error = isinstance(result, str) and result.lstrip().startswith( _error_prefixes diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index b35a0176148..2b799ef43f0 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -208,9 +208,12 @@ def _fetch_page_text( try: import urllib.request + from urllib.error import HTTPError as _HTTPError from urllib.parse import urljoin - # Disable auto-redirect so we can validate each hop for SSRF + # Disable auto-redirect so we can validate each hop for SSRF. + # urllib raises HTTPError for 3xx when the handler returns None, + # so we catch that and extract the Location header manually. class _NoRedirect(urllib.request.HTTPRedirectHandler): def redirect_request(self, req, fp, code, msg, headers, newurl): return None @@ -224,9 +227,12 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): current_url, headers = {"User-Agent": "UnslothStudio/1.0"}, ) - resp = opener.open(req, timeout = timeout) - if resp.status in (301, 302, 303, 307, 308): - location = resp.headers.get("Location") + try: + resp = opener.open(req, timeout = timeout) + except _HTTPError as e: + if e.code not in (301, 302, 303, 307, 308): + return f"Failed to fetch URL: HTTP {e.code}" + location = e.headers.get("Location") if not location: return "Failed to fetch URL: redirect missing Location header." current_url = urljoin(current_url, location) @@ -247,6 +253,8 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): return "Failed to fetch URL: too many redirects." raw_html = raw_bytes.decode("utf-8", errors = "replace") + except _HTTPError: + raise except Exception as e: return f"Failed to fetch URL: {e}" From aac109ce8ed51405d463074a6a6b9489b7428312 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 31 Mar 2026 09:27:10 +0000 Subject: [PATCH 12/14] Fix SSE cross-chunk XML leaks, cleanup review findings - SSE streaming: sanitize the full cumulative text before diffing against the previous sanitized snapshot, so XML tags that span chunk boundaries are stripped correctly. The previous delta-based approach leaked split tags. - DRAINING fallback: use _strip_tool_markup() helper instead of a manual regex that only handled but not . - Move hashlib import, _TOOL_XML_RE compile, and datetime import to module level per style guide. - Remove unused _hit_tool_cap variable. --- studio/backend/core/inference/llama_cpp.py | 16 +++++---------- studio/backend/routes/inference.py | 23 ++++++++++++---------- 2 files changed, 18 insertions(+), 21 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index b24721cd4e3..03974f5a663 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -10,6 +10,7 @@ import atexit import contextlib +import hashlib import json import re import struct @@ -2178,13 +2179,11 @@ def _strip_tool_markup(text: str, *, final: bool = False) -> str: # where the model repeats the exact same call. Retries after # a transient failure are allowed (only block when the previous # identical call succeeded). - import hashlib as _hl - _tool_call_history: list[tuple[str, bool]] = [] # (key, failed) def _tool_call_key(name: str, args: dict) -> str: raw = json.dumps({"t": name, "a": args}, sort_keys = True) - return _hl.md5(raw.encode()).hexdigest() + return hashlib.md5(raw.encode()).hexdigest() def _is_duplicate_call(name: str, args: dict) -> bool: """Block if the immediately previous call was identical and succeeded.""" @@ -2198,8 +2197,6 @@ def _record_tool_call(name: str, args: dict, failed: bool) -> None: key = _tool_call_key(name, args) _tool_call_history.append((key, failed)) - _hit_tool_cap = False - for iteration in range(max_tool_iterations): if cancel_event is not None and cancel_event.is_set(): return @@ -2598,12 +2595,9 @@ def _record_tool_call(name: str, args: dict, failed: bool) -> None: yield {"type": "status", "text": ""} if content_accum: # Strip leaked tool-call XML before yielding - content_accum = re.sub( - r".*?", - "", - content_accum, - flags = re.DOTALL, - ).strip() + content_accum = _strip_tool_markup( + content_accum, final = True + ) if content_accum: yield {"type": "content", "text": content_accum} _fu = _iter_usage or {} diff --git a/studio/backend/routes/inference.py b/studio/backend/routes/inference.py index 40f41578a7b..9bce371775a 100644 --- a/studio/backend/routes/inference.py +++ b/studio/backend/routes/inference.py @@ -86,8 +86,15 @@ def _friendly_error(exc: Exception) -> str: import wave import base64 import numpy as np +from datetime import date as _date router = APIRouter() + +# Regex for stripping leaked tool-call XML from assistant messages/stream +_TOOL_XML_RE = _re.compile( + r".*?|.*?", + _re.DOTALL, +) logger = get_logger(__name__) @@ -1083,8 +1090,6 @@ async def audio_input_stream(): _has_web = "web_search" in _tool_names _has_code = "python" in _tool_names or "terminal" in _tool_names - from datetime import date as _date - _date_line = f"The current date is {_date.today().isoformat()}." _web_tips = ( @@ -1136,10 +1141,6 @@ async def audio_input_stream(): gguf_messages.extend(chat_messages) # ── Strip stale tool-call XML from conversation history ─ - _TOOL_XML_RE = _re.compile( - r".*?|.*?", - _re.DOTALL, - ) for _msg in gguf_messages: if _msg.get("role") == "assistant" and isinstance( _msg.get("content"), str @@ -1226,11 +1227,13 @@ async def gguf_tool_stream(): continue # "content" type -- cumulative text + # Sanitize the full cumulative then diff against + # the last sanitized snapshot so cross-chunk XML + # tags are handled correctly. raw_cumulative = event.get("text", "") - raw_delta = raw_cumulative[len(prev_text) :] - prev_text = raw_cumulative - # Strip leaked tool-call XML from the delta only - new_text = _TOOL_XML_RE.sub("", raw_delta) + clean_cumulative = _TOOL_XML_RE.sub("", raw_cumulative) + new_text = clean_cumulative[len(prev_text) :] + prev_text = clean_cumulative if not new_text: continue chunk = ChatCompletionChunk( From 0ad9d09950b411e828d46ac7796de927a384c440 Mon Sep 17 00:00:00 2001 From: Daniel Han Date: Tue, 31 Mar 2026 09:52:14 +0000 Subject: [PATCH 13/14] Fix DNS rebinding, charset detection, HTTPError handling, dedup double-record - DNS rebinding: resolve hostname once via getaddrinfo, pin the returned IP, rewrite the URL to connect to the pinned IP with a Host header. Each redirect hop re-resolves and re-validates. Closes the TOCTOU window between validation and connection. - Charset: use resp.headers.get_content_charset() instead of hardcoding utf-8, so pages with other encodings decode correctly. - HTTPError: return descriptive "HTTP {code} {reason}" instead of re-raising into a generic "Search failed" message. - Dedup: remove redundant _record_tool_call in the duplicate branch; the single call at the end of the loop handles all cases. --- studio/backend/core/inference/llama_cpp.py | 1 - studio/backend/core/inference/tools.py | 65 +++++++++++++++------- 2 files changed, 44 insertions(+), 22 deletions(-) diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py index 03974f5a663..c1f87ff9369 100644 --- a/studio/backend/core/inference/llama_cpp.py +++ b/studio/backend/core/inference/llama_cpp.py @@ -2701,7 +2701,6 @@ def _record_tool_call(name: str, args: dict, failed: bool) -> None: "process data you already have, or " "provide your final answer now." ) - _record_tool_call(tool_name, arguments, failed = False) else: _effective_timeout = ( None if tool_call_timeout >= 9999 else tool_call_timeout diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 2b799ef43f0..b292178c00c 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -158,15 +158,25 @@ def execute_tool( _MAX_FETCH_BYTES = _MAX_PAGE_CHARS * 4 + 1 # cap raw download size -def _is_public_host(hostname: str, port: int) -> tuple[bool, str]: - """Resolve *hostname* and reject private/loopback/link-local addresses.""" +def _validate_and_resolve_host( + hostname: str, port: int +) -> tuple[bool, str, str]: + """Resolve *hostname*, reject non-public IPs, return a pinned IP string. + + Returns ``(ok, reason_or_empty, resolved_ip)``. The caller should + connect to *resolved_ip* (with a ``Host`` header) to prevent DNS + rebinding between validation and the actual fetch. + """ import ipaddress import socket try: - infos = socket.getaddrinfo(hostname, port, type = socket.SOCK_STREAM) + infos = socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM) except OSError as e: - return False, f"Failed to resolve host: {e}" + return False, f"Failed to resolve host: {e}", "" + + if not infos: + return False, f"Failed to resolve host: no addresses for {hostname!r}", "" for *_, sockaddr in infos: ip = ipaddress.ip_address(sockaddr[0]) @@ -178,8 +188,11 @@ def _is_public_host(hostname: str, port: int) -> tuple[bool, str]: or ip.is_reserved or ip.is_unspecified ): - return False, f"Blocked: refusing to fetch non-public address {ip}." - return True, "" + return False, f"Blocked: refusing to fetch non-public address {ip}.", "" + + # Return the first resolved address for pinning + first_ip = infos[0][4][0] + return True, "", first_ip def _fetch_page_text( @@ -199,17 +212,15 @@ def _fetch_page_text( if not parsed.hostname: return "Blocked: URL is missing a hostname." - ok, reason = _is_public_host( - parsed.hostname, - parsed.port or (443 if parsed.scheme == "https" else 80), - ) + port = parsed.port or (443 if parsed.scheme == "https" else 80) + ok, reason, pinned_ip = _validate_and_resolve_host(parsed.hostname, port) if not ok: return reason try: import urllib.request from urllib.error import HTTPError as _HTTPError - from urllib.parse import urljoin + from urllib.parse import urljoin, urlunparse # Disable auto-redirect so we can validate each hop for SSRF. # urllib raises HTTPError for 3xx when the handler returns None, @@ -221,17 +232,27 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): opener = urllib.request.build_opener(_NoRedirect) max_bytes = max_chars * 4 + 1 current_url = url + current_host = parsed.hostname for _hop in range(5): + # Pin to the validated IP to prevent DNS rebinding. + # Rewrite the URL to use the IP and set the Host header. + cp = urlparse(current_url) + ip_netloc = f"{pinned_ip}:{cp.port}" if cp.port else pinned_ip + pinned_url = urlunparse(cp._replace(netloc=ip_netloc)) + req = urllib.request.Request( - current_url, - headers = {"User-Agent": "UnslothStudio/1.0"}, + pinned_url, + headers={ + "User-Agent": "UnslothStudio/1.0", + "Host": current_host, + }, ) try: - resp = opener.open(req, timeout = timeout) + resp = opener.open(req, timeout=timeout) except _HTTPError as e: if e.code not in (301, 302, 303, 307, 308): - return f"Failed to fetch URL: HTTP {e.code}" + return f"Failed to fetch URL: HTTP {e.code} {getattr(e, 'reason', '')}" location = e.headers.get("Location") if not location: return "Failed to fetch URL: redirect missing Location header." @@ -239,12 +260,13 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): rp = urlparse(current_url) if rp.scheme not in ("http", "https") or not rp.hostname: return "Blocked: redirect target is not a valid http/https URL." - ok2, reason2 = _is_public_host( - rp.hostname, - rp.port or (443 if rp.scheme == "https" else 80), + rp_port = rp.port or (443 if rp.scheme == "https" else 80) + ok2, reason2, pinned_ip = _validate_and_resolve_host( + rp.hostname, rp_port, ) if not ok2: return reason2 + current_host = rp.hostname continue # Success -- read capped body raw_bytes = resp.read(max_bytes) @@ -252,9 +274,10 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): else: return "Failed to fetch URL: too many redirects." - raw_html = raw_bytes.decode("utf-8", errors = "replace") - except _HTTPError: - raise + charset = resp.headers.get_content_charset() or "utf-8" + raw_html = raw_bytes.decode(charset, errors="replace") + except _HTTPError as e: + return f"Failed to fetch URL: HTTP {e.code} {getattr(e, 'reason', '')}" except Exception as e: return f"Failed to fetch URL: {e}" From d41b483688cc1d36e409a96be16defd290c2e527 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 31 Mar 2026 09:52:28 +0000 Subject: [PATCH 14/14] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/core/inference/tools.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index b292178c00c..65302fe2f34 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -158,9 +158,7 @@ def execute_tool( _MAX_FETCH_BYTES = _MAX_PAGE_CHARS * 4 + 1 # cap raw download size -def _validate_and_resolve_host( - hostname: str, port: int -) -> tuple[bool, str, str]: +def _validate_and_resolve_host(hostname: str, port: int) -> tuple[bool, str, str]: """Resolve *hostname*, reject non-public IPs, return a pinned IP string. Returns ``(ok, reason_or_empty, resolved_ip)``. The caller should @@ -171,7 +169,7 @@ def _validate_and_resolve_host( import socket try: - infos = socket.getaddrinfo(hostname, port, type=socket.SOCK_STREAM) + infos = socket.getaddrinfo(hostname, port, type = socket.SOCK_STREAM) except OSError as e: return False, f"Failed to resolve host: {e}", "" @@ -239,20 +237,22 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): # Rewrite the URL to use the IP and set the Host header. cp = urlparse(current_url) ip_netloc = f"{pinned_ip}:{cp.port}" if cp.port else pinned_ip - pinned_url = urlunparse(cp._replace(netloc=ip_netloc)) + pinned_url = urlunparse(cp._replace(netloc = ip_netloc)) req = urllib.request.Request( pinned_url, - headers={ + headers = { "User-Agent": "UnslothStudio/1.0", "Host": current_host, }, ) try: - resp = opener.open(req, timeout=timeout) + resp = opener.open(req, timeout = timeout) except _HTTPError as e: if e.code not in (301, 302, 303, 307, 308): - return f"Failed to fetch URL: HTTP {e.code} {getattr(e, 'reason', '')}" + return ( + f"Failed to fetch URL: HTTP {e.code} {getattr(e, 'reason', '')}" + ) location = e.headers.get("Location") if not location: return "Failed to fetch URL: redirect missing Location header." @@ -262,7 +262,8 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): return "Blocked: redirect target is not a valid http/https URL." rp_port = rp.port or (443 if rp.scheme == "https" else 80) ok2, reason2, pinned_ip = _validate_and_resolve_host( - rp.hostname, rp_port, + rp.hostname, + rp_port, ) if not ok2: return reason2 @@ -275,7 +276,7 @@ def redirect_request(self, req, fp, code, msg, headers, newurl): return "Failed to fetch URL: too many redirects." charset = resp.headers.get_content_charset() or "utf-8" - raw_html = raw_bytes.decode(charset, errors="replace") + raw_html = raw_bytes.decode(charset, errors = "replace") except _HTTPError as e: return f"Failed to fetch URL: HTTP {e.code} {getattr(e, 'reason', '')}" except Exception as e: