Real text content here.
" + out = _fetch_with(monkeypatch, html, "text/html; charset=utf-8") + assert "Hello" in out + assert "non-text content" not in out and "binary content" not in out + + +def test_content_type_sanitized_in_message(monkeypatch): + # An obs-folded Content-Type can smuggle control chars into get_content_type(); + # the returned message must be trimmed to a clean MIME token. + out = _fetch_with(monkeypatch, b"\x00\x01\x02" * 500, "application/octet-stream\r\n data: injected") + assert "\n" not in out and "\r" not in out + assert "injected" not in out + assert "application/octet-stream" in out + + +@pytest.mark.parametrize( + "n_bad,n_total,expect_binary", + [ + # Straddle the 12.5% ratio (well above the 16-char floor): just under vs + # just over len//8. Locks the divisor so it can't silently drift. + (120, 1000, False), # 120 <= 1000//8 (125) -> kept + (130, 1000, True), # 130 > 1000//8 (125) -> binary + ], +) +def test_replacement_ratio_boundary(monkeypatch, n_bad, n_total, expect_binary): + # Body of n_total chars: n_bad undecodable bytes + ASCII filler. Labeled + # text/plain so only the ratio fallback (not the type check) can fire. + body = b"\xff" * n_bad + b"a" * (n_total - n_bad) + out = _fetch_with(monkeypatch, body, "text/plain") + assert ("binary content" in out) is expect_binary + + +def test_text_with_a_few_stray_replacement_chars_kept(monkeypatch): + # A mostly-clean page with a handful of bad bytes stays (below the floor), + # so we don't drop legitimate pages over minor encoding glitches. + body = ("Real article text. " * 200).encode() + b"\xff\xfe\xff" + out = _fetch_with(monkeypatch, body, "text/html") + assert "Real article text." in out + assert "binary content" not in out From e39234332908239dcc07c5aca0dfffc6fbd25ae0 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 14 Jul 2026 21:07:35 +0000 Subject: [PATCH 02/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/core/inference/tools.py | 3 +-- .../tests/test_web_fetch_binary_guard.py | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 01ed9affe80c..11e7328b6f6e 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -1598,8 +1598,7 @@ def _fetch_page_text( m = re.match(r"[\w.+-]+/[\w.+-]+", content_type or "") safe_type = m.group(0) if m else "unknown type" return ( - f"(non-text content: {safe_type}, {len(raw_bytes)} bytes; " - "not readable as text)" + f"(non-text content: {safe_type}, {len(raw_bytes)} bytes; not readable as text)" ) charset = resp.headers.get_content_charset() or "utf-8" diff --git a/studio/backend/tests/test_web_fetch_binary_guard.py b/studio/backend/tests/test_web_fetch_binary_guard.py index 2f77d5da4593..cf9e0ce493c4 100644 --- a/studio/backend/tests/test_web_fetch_binary_guard.py +++ b/studio/backend/tests/test_web_fetch_binary_guard.py @@ -38,7 +38,11 @@ class _FakeOpener: def __init__(self, resp): self._resp = resp - def open(self, req, timeout=None): + def open( + self, + req, + timeout = None, + ): return self._resp @@ -48,9 +52,11 @@ def _fetch_with(monkeypatch, body: bytes, content_type: str | None) -> str: tools, "_validate_and_resolve_host", lambda host, port: (True, "", "93.184.216.34") ) monkeypatch.setattr( - tools.urllib.request, "build_opener", lambda *a, **k: _FakeOpener(_FakeResp(body, content_type)) + tools.urllib.request, + "build_opener", + lambda *a, **k: _FakeOpener(_FakeResp(body, content_type)), ) - return tools._fetch_page_text("https://example.com/thing", timeout=5) + return tools._fetch_page_text("https://example.com/thing", timeout = 5) # ── content-type classifier ── @@ -118,7 +124,9 @@ def test_html_page_unaffected(monkeypatch): def test_content_type_sanitized_in_message(monkeypatch): # An obs-folded Content-Type can smuggle control chars into get_content_type(); # the returned message must be trimmed to a clean MIME token. - out = _fetch_with(monkeypatch, b"\x00\x01\x02" * 500, "application/octet-stream\r\n data: injected") + out = _fetch_with( + monkeypatch, b"\x00\x01\x02" * 500, "application/octet-stream\r\n data: injected" + ) assert "\n" not in out and "\r" not in out assert "injected" not in out assert "application/octet-stream" in out @@ -130,7 +138,7 @@ def test_content_type_sanitized_in_message(monkeypatch): # Straddle the 12.5% ratio (well above the 16-char floor): just under vs # just over len//8. Locks the divisor so it can't silently drift. (120, 1000, False), # 120 <= 1000//8 (125) -> kept - (130, 1000, True), # 130 > 1000//8 (125) -> binary + (130, 1000, True), # 130 > 1000//8 (125) -> binary ], ) def test_replacement_ratio_boundary(monkeypatch, n_bad, n_total, expect_binary): From 75dfba2eaed0f5d6cbdd3282a67b13e4d9d7fe63 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 14 Jul 2026 19:25:15 -0300 Subject: [PATCH 03/15] Match web-fetch MIME subtypes exactly and detect control-char binary --- studio/backend/core/inference/tools.py | 50 ++++++++++++------- .../tests/test_web_fetch_binary_guard.py | 26 +++++++++- 2 files changed, 55 insertions(+), 21 deletions(-) diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 11e7328b6f6e..16dce26b55c6 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -1381,12 +1381,14 @@ def build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> di # sections stripped during conversion; 512 KB reaches article content even # where alone is ~200 KB. _MAX_FETCH_BYTES = 512 * 1024 -# A decoded page is treated as binary when more than 1/_BINARY_REPLACEMENT_DIVISOR -# (12.5%) of its chars are U+FFFD, but always tolerating up to -# _MIN_REPLACEMENT_CHARS stray bad bytes so minor encoding glitches don't drop a -# real page. -_MIN_REPLACEMENT_CHARS = 16 -_BINARY_REPLACEMENT_DIVISOR = 8 +# Chars that don't occur in real text: undecodable bytes (U+FFFD) plus control +# chars (C0 minus tab/newline/CR and ESC, DEL, C1). ESC is excluded so a text +# page of ANSI-colored terminal output isn't mistaken for binary. A decoded page +# is treated as binary when more than 1/_BINARY_CHAR_DIVISOR (12.5%) of its chars +# are these, tolerating up to _MIN_BINARY_CHARS so minor glitches don't drop it. +_BINARY_CHAR_RE = re.compile("[\\x00-\\x08\\x0b\\x0c\\x0e-\\x1a\\x1c-\\x1f\\x7f-\\x9f\\ufffd]") +_MIN_BINARY_CHARS = 16 +_BINARY_CHAR_DIVISOR = 8 _USER_AGENTS = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", @@ -1488,23 +1490,33 @@ def _validate_and_resolve_host(hostname: str, port: int) -> tuple[bool, str, str return True, "", first_ip -def _is_texty_content_type(content_type: str) -> bool: +# Bare application/* subtypes that are text (the leading x- and the +json/+xml +# structured-syntax suffixes are handled in the check). +_TEXT_APPLICATION_SUBTYPES = frozenset( + {"json", "xml", "javascript", "ecmascript", "csv", "yaml", "ndjson", "jsonl"} +) + + +def _is_texty_content_type(content_type: str | None) -> bool: """True for MIME types safe to decode as text (HTML pages, plain text, - JSON/XML feeds). Binary types (PDF, images, archives, octet-stream) return - False so the fetcher never decodes them into a flood of U+FFFD replacement - chars that poison the model context (unslothai/unsloth#7084). + JSON/XML/YAML feeds). Binary types (PDF, images, archives, Office/ZIP, + octet-stream) return False so the fetcher never decodes them into a flood of + replacement chars that poison the model context (unslothai/unsloth#7084). A missing Content-Type coerces to ``text/plain`` upstream, so unlabeled - bodies pass here and are caught instead by the replacement-char fallback. + bodies pass here and are caught instead by the binary-char fallback. """ ct = (content_type or "").lower() if not ct: - return True # unlabeled: let the replacement-char fallback decide + return True # unlabeled: let the binary-char fallback decide if ct.startswith("text/"): return True if ct.startswith("application/"): - # "xml" also covers xhtml+xml / *+xml; "json" covers *+json. - return any(marker in ct for marker in ("json", "xml", "javascript", "csv")) + # Match the exact subtype, not a loose substring, so a .docx labeled + # application/vnd.openxmlformats-... isn't read as xml. RFC 6839 + # +json/+xml suffixes (ld+json, xhtml+xml, ...) pass too. + subtype = ct[len("application/") :].removeprefix("x-") + return subtype in _TEXT_APPLICATION_SUBTYPES or subtype.endswith(("+json", "+xml")) return False @@ -1604,11 +1616,11 @@ def _fetch_page_text( charset = resp.headers.get_content_charset() or "utf-8" raw_html = raw_bytes.decode(charset, errors = "replace") - # Fallback for binary mislabeled as text/* or sent with no Content-Type: - # a real text page has only a few replacement chars, if any. - if raw_html.count("\ufffd") > max( - _MIN_REPLACEMENT_CHARS, len(raw_html) // _BINARY_REPLACEMENT_DIVISOR - ): + # Fallback for binary mislabeled as text/* or sent with no Content-Type, + # including valid-UTF-8 binary (NUL/control-heavy payloads) that decodes + # without replacement chars: a real text page has few binary chars. + binary_chars = len(_BINARY_CHAR_RE.findall(raw_html)) + if binary_chars > max(_MIN_BINARY_CHARS, len(raw_html) // _BINARY_CHAR_DIVISOR): return f"(binary content, {len(raw_bytes)} bytes; not readable as text)" except _HTTPError as e: return f"Failed to fetch URL: HTTP {e.code} {getattr(e, 'reason', '')}" diff --git a/studio/backend/tests/test_web_fetch_binary_guard.py b/studio/backend/tests/test_web_fetch_binary_guard.py index cf9e0ce493c4..e5a1870de0a1 100644 --- a/studio/backend/tests/test_web_fetch_binary_guard.py +++ b/studio/backend/tests/test_web_fetch_binary_guard.py @@ -71,12 +71,18 @@ def _fetch_with(monkeypatch, body: bytes, content_type: str | None) -> str: ("application/xml", True), ("application/xhtml+xml", True), ("application/ld+json", True), + ("application/yaml", True), # raw configs / API specs + ("application/x-yaml", True), + ("application/x-ndjson", True), # newline-delimited JSON is text + ("application/ndjson", True), ("application/pdf", False), ("image/png", False), ("image/svg+xml", False), # SVG source isn't extracted downstream; reject ("application/octet-stream", False), ("application/zip", False), - ("", True), # unlabeled: defer to the replacement-char fallback + # A .docx ZIP must not pass just because "xml" is inside "openxmlformats". + ("application/vnd.openxmlformats-officedocument.wordprocessingml.document", False), + ("", True), # unlabeled: defer to the binary-char fallback (None, True), ], ) @@ -101,12 +107,28 @@ def test_image_rejected_by_content_type(monkeypatch): def test_binary_mislabeled_as_text_caught_by_fallback(monkeypatch): # A server sends binary but labels it text/plain -> the type check passes, - # so the replacement-char fallback must catch it. + # so the binary-char fallback must catch it. out = _fetch_with(monkeypatch, bytes(range(256)) * 20, "text/plain") assert "�" not in out assert "binary content" in out +def test_valid_utf8_binary_caught_by_control_chars(monkeypatch): + # NUL/control-heavy binary is valid UTF-8, so it decodes with no U+FFFD; + # the fallback must still catch it via control-char density (codex #2). + body = bytes([0, 1, 2, 3, 4, 5, 6, 7]) * 400 # all valid UTF-8, 0 replacement chars + out = _fetch_with(monkeypatch, body, "text/plain") + assert "binary content" in out + + +def test_ansi_colored_text_log_kept(monkeypatch): + # A text log with per-token ANSI color codes is text; ESC is excluded from + # the binary-char set so it isn't dropped as binary. + line = "".join(f"\x1b[32m+{i}\x1b[0m\n" for i in range(300)).encode() + out = _fetch_with(monkeypatch, line, "text/plain") + assert "binary content" not in out + + def test_binary_unlabeled_caught_by_fallback(monkeypatch): # No Content-Type coerces to text/plain upstream; the fallback still catches it. out = _fetch_with(monkeypatch, bytes(range(256)) * 20, None) From 9fcbc1a943626bd66d52ec3e79288cb9dcc3c7b3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:33:01 +0000 Subject: [PATCH 04/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/core/inference/tools.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 16dce26b55c6..2356c1a32307 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -1609,9 +1609,7 @@ def _fetch_page_text( # the model. m = re.match(r"[\w.+-]+/[\w.+-]+", content_type or "") safe_type = m.group(0) if m else "unknown type" - return ( - f"(non-text content: {safe_type}, {len(raw_bytes)} bytes; not readable as text)" - ) + return f"(non-text content: {safe_type}, {len(raw_bytes)} bytes; not readable as text)" charset = resp.headers.get_content_charset() or "utf-8" raw_html = raw_bytes.decode(charset, errors = "replace") From 71fbfd26c49762cf6b50851ad7ba1b872da3067e Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 14 Jul 2026 19:57:24 -0300 Subject: [PATCH 05/15] Sniff binary magic bytes and retry undeclared non-UTF-8 pages as text --- studio/backend/core/inference/tools.py | 44 +++++++++++++++-- .../tests/test_web_fetch_binary_guard.py | 47 +++++++++++++++++-- 2 files changed, 83 insertions(+), 8 deletions(-) diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 2356c1a32307..63ade9ed0649 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -1389,6 +1389,29 @@ def build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> di _BINARY_CHAR_RE = re.compile("[\\x00-\\x08\\x0b\\x0c\\x0e-\\x1a\\x1c-\\x1f\\x7f-\\x9f\\ufffd]") _MIN_BINARY_CHARS = 16 _BINARY_CHAR_DIVISOR = 8 +# Leading bytes of common binary formats that servers sometimes mislabel as +# text/*; matched by signature because their replacement-char density alone can +# be low (e.g. a PDF whose first chunk is mostly ASCII object/xref syntax). +_BINARY_MAGIC = ( + b"%PDF-", # PDF + b"PK\x03\x04", # zip / docx / xlsx / pptx / epub / jar + b"\x89PNG\r\n\x1a\n", # PNG + b"\xff\xd8\xff", # JPEG + b"GIF87a", + b"GIF89a", + b"\x1f\x8b", # gzip + b"BZh", # bzip2 + b"\xfd7zXZ\x00", # xz + b"\x28\xb5\x2f\xfd", # zstd +) + + +def _looks_binary(text: str) -> bool: + """True when more than 1/_BINARY_CHAR_DIVISOR of ``text`` is binary chars + (control/undecodable, per _BINARY_CHAR_RE), past the _MIN_BINARY_CHARS floor.""" + return len(_BINARY_CHAR_RE.findall(text)) > max( + _MIN_BINARY_CHARS, len(text) // _BINARY_CHAR_DIVISOR + ) _USER_AGENTS = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", @@ -1611,15 +1634,26 @@ def _fetch_page_text( safe_type = m.group(0) if m else "unknown type" return f"(non-text content: {safe_type}, {len(raw_bytes)} bytes; not readable as text)" - charset = resp.headers.get_content_charset() or "utf-8" - raw_html = raw_bytes.decode(charset, errors = "replace") + # A PDF/image/archive mislabeled as text/*: catch by signature, since a + # text-heavy first chunk can slip past the replacement-char density check. + if raw_bytes.startswith(_BINARY_MAGIC): + return f"(binary content, {len(raw_bytes)} bytes; not readable as text)" + + declared = resp.headers.get_content_charset() + raw_html = raw_bytes.decode(declared or "utf-8", errors = "replace") # Fallback for binary mislabeled as text/* or sent with no Content-Type, # including valid-UTF-8 binary (NUL/control-heavy payloads) that decodes # without replacement chars: a real text page has few binary chars. - binary_chars = len(_BINARY_CHAR_RE.findall(raw_html)) - if binary_chars > max(_MIN_BINARY_CHARS, len(raw_html) // _BINARY_CHAR_DIVISOR): - return f"(binary content, {len(raw_bytes)} bytes; not readable as text)" + if _looks_binary(raw_html): + # An undeclared non-UTF-8 page (latin-1/cp1252) also decodes to many + # U+FFFD. Retry as cp1252 (maps almost every byte to a printable + # char): if that reads as text it was a charset mismatch, not binary. + alt = raw_bytes.decode("cp1252", "replace") if declared is None else None + if alt is not None and not _looks_binary(alt): + raw_html = alt + else: + return f"(binary content, {len(raw_bytes)} bytes; not readable as text)" except _HTTPError as e: return f"Failed to fetch URL: HTTP {e.code} {getattr(e, 'reason', '')}" except Exception as e: diff --git a/studio/backend/tests/test_web_fetch_binary_guard.py b/studio/backend/tests/test_web_fetch_binary_guard.py index e5a1870de0a1..62231eeff9f5 100644 --- a/studio/backend/tests/test_web_fetch_binary_guard.py +++ b/studio/backend/tests/test_web_fetch_binary_guard.py @@ -121,6 +121,47 @@ def test_valid_utf8_binary_caught_by_control_chars(monkeypatch): assert "binary content" in out +def test_pdf_mislabeled_as_text_caught_by_magic(monkeypatch): + # A PDF served as text/plain: the content-type gate passes, so the magic-byte + # sniff must catch it (its ASCII-heavy head can stay under the ratio). + pdf = b"%PDF-1.7\n" + b"1 0 obj<>endobj\n" * 300 + out = _fetch_with(monkeypatch, pdf, "text/plain") + assert "binary content" in out + + +def test_zip_mislabeled_as_text_caught_by_magic(monkeypatch): + out = _fetch_with(monkeypatch, b"PK\x03\x04" + b"filename.txt content " * 200, "text/plain") + assert "binary content" in out + + +@pytest.mark.parametrize( + "magic", + [b"\x1f\x8b", b"BZh", b"\xfd7zXZ\x00", b"\x28\xb5\x2f\xfd"], # gzip, bzip2, xz, zstd +) +def test_compression_mislabeled_as_text_caught_by_magic(monkeypatch, magic): + # A small compressed body mislabeled as text can slip past the density check; + # the signature must reject it. + out = _fetch_with(monkeypatch, magic + b"some short compressed-looking body", "text/plain") + assert "binary content" in out + + +def test_latin1_text_without_charset_kept(monkeypatch): + # Accent-dense cp1252 text with no declared charset decodes to many U+FFFD as + # UTF-8; the cp1252 retry must keep it instead of dropping it as binary. + body = ("Muller lauft uber die Strasse: schoene, groesse. MARKERWORD ".replace("ue", "ü") + + "äöüß éèà ") * 30 + out = _fetch_with(monkeypatch, body.encode("cp1252"), "text/plain") + assert "binary content" not in out + assert "MARKERWORD" in out + + +def test_control_heavy_binary_survives_cp1252_retry(monkeypatch): + # Genuine control-heavy binary (no magic) must stay rejected even after the + # cp1252 retry, which cannot rescue real binary. + out = _fetch_with(monkeypatch, bytes([0, 1, 2, 3, 4, 5, 6, 7]) * 400, "text/plain") + assert "binary content" in out + + def test_ansi_colored_text_log_kept(monkeypatch): # A text log with per-token ANSI color codes is text; ESC is excluded from # the binary-char set so it isn't dropped as binary. @@ -164,9 +205,9 @@ def test_content_type_sanitized_in_message(monkeypatch): ], ) def test_replacement_ratio_boundary(monkeypatch, n_bad, n_total, expect_binary): - # Body of n_total chars: n_bad undecodable bytes + ASCII filler. Labeled - # text/plain so only the ratio fallback (not the type check) can fire. - body = b"\xff" * n_bad + b"a" * (n_total - n_bad) + # Body of n_total chars: n_bad NUL control bytes + ASCII filler. NUL stays + # binary through the cp1252 retry, so only the ratio threshold decides here. + body = b"\x00" * n_bad + b"a" * (n_total - n_bad) out = _fetch_with(monkeypatch, body, "text/plain") assert ("binary content" in out) is expect_binary From ef43072848ca560f1e311b16c6f0d7e7a64a9cb3 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:59:03 +0000 Subject: [PATCH 06/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/core/inference/tools.py | 15 ++++++++------- .../backend/tests/test_web_fetch_binary_guard.py | 6 ++++-- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 63ade9ed0649..93826bf2af6b 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -1393,16 +1393,16 @@ def build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> di # text/*; matched by signature because their replacement-char density alone can # be low (e.g. a PDF whose first chunk is mostly ASCII object/xref syntax). _BINARY_MAGIC = ( - b"%PDF-", # PDF - b"PK\x03\x04", # zip / docx / xlsx / pptx / epub / jar + b"%PDF-", # PDF + b"PK\x03\x04", # zip / docx / xlsx / pptx / epub / jar b"\x89PNG\r\n\x1a\n", # PNG - b"\xff\xd8\xff", # JPEG + b"\xff\xd8\xff", # JPEG b"GIF87a", b"GIF89a", - b"\x1f\x8b", # gzip - b"BZh", # bzip2 - b"\xfd7zXZ\x00", # xz - b"\x28\xb5\x2f\xfd", # zstd + b"\x1f\x8b", # gzip + b"BZh", # bzip2 + b"\xfd7zXZ\x00", # xz + b"\x28\xb5\x2f\xfd", # zstd ) @@ -1413,6 +1413,7 @@ def _looks_binary(text: str) -> bool: _MIN_BINARY_CHARS, len(text) // _BINARY_CHAR_DIVISOR ) + _USER_AGENTS = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", diff --git a/studio/backend/tests/test_web_fetch_binary_guard.py b/studio/backend/tests/test_web_fetch_binary_guard.py index 62231eeff9f5..92dd9411bef3 100644 --- a/studio/backend/tests/test_web_fetch_binary_guard.py +++ b/studio/backend/tests/test_web_fetch_binary_guard.py @@ -148,8 +148,10 @@ def test_compression_mislabeled_as_text_caught_by_magic(monkeypatch, magic): def test_latin1_text_without_charset_kept(monkeypatch): # Accent-dense cp1252 text with no declared charset decodes to many U+FFFD as # UTF-8; the cp1252 retry must keep it instead of dropping it as binary. - body = ("Muller lauft uber die Strasse: schoene, groesse. MARKERWORD ".replace("ue", "ü") - + "äöüß éèà ") * 30 + body = ( + "Muller lauft uber die Strasse: schoene, groesse. MARKERWORD ".replace("ue", "ü") + + "äöüß éèà " + ) * 30 out = _fetch_with(monkeypatch, body.encode("cp1252"), "text/plain") assert "binary content" not in out assert "MARKERWORD" in out From c4404dd2a1778ef76f29ee2ad592d1835632ac12 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:07:50 -0300 Subject: [PATCH 07/15] Harden web fetch binary sniffing --- studio/backend/core/inference/tools.py | 44 +++++++++++++++---- .../tests/test_web_fetch_binary_guard.py | 39 +++++++++++----- 2 files changed, 64 insertions(+), 19 deletions(-) diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 93826bf2af6b..937219fbf15c 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -1405,6 +1405,13 @@ def build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> di b"\x28\xb5\x2f\xfd", # zstd ) +# Undeclared legacy Western text should have substantial ASCII structure even +# when its non-ASCII bytes are cp1252. Requiring at least 75% ASCII text bytes +# prevents arbitrary high-byte binary from becoming printable-looking gibberish +# merely because cp1252 defines a character for nearly every byte. +_MIN_SINGLE_BYTE_ASCII_RATIO = 3 / 4 +_ASCII_TEXT_BYTES = frozenset((*range(0x20, 0x7F), 0x09, 0x0A, 0x0D, 0x1B)) + def _looks_binary(text: str) -> bool: """True when more than 1/_BINARY_CHAR_DIVISOR of ``text`` is binary chars @@ -1413,6 +1420,13 @@ def _looks_binary(text: str) -> bool: _MIN_BINARY_CHARS, len(text) // _BINARY_CHAR_DIVISOR ) +def _has_single_byte_text_evidence(data: bytes) -> bool: + """True when *data* has enough ASCII structure for a cp1252 text retry.""" + if not data: + return True + ascii_text_bytes = sum(byte in _ASCII_TEXT_BYTES for byte in data) + return ascii_text_bytes / len(data) >= _MIN_SINGLE_BYTE_ASCII_RATIO + _USER_AGENTS = ( "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36", @@ -1522,15 +1536,18 @@ def _validate_and_resolve_host(hostname: str, port: int) -> tuple[bool, str, str def _is_texty_content_type(content_type: str | None) -> bool: - """True for MIME types safe to decode as text (HTML pages, plain text, - JSON/XML/YAML feeds). Binary types (PDF, images, archives, Office/ZIP, - octet-stream) return False so the fetcher never decodes them into a flood of - replacement chars that poison the model context (unslothai/unsloth#7084). + """True for MIME types safe to decode or inspect as possible text. + + This includes HTML, plain text, JSON/XML/YAML feeds, and generic + ``application/octet-stream`` downloads whose bytes must be sniffed. Known + binary types (PDF, images, archives, Office/ZIP) return False so the fetcher + never decodes them into replacement chars that poison the model context + (unslothai/unsloth#7084). A missing Content-Type coerces to ``text/plain`` upstream, so unlabeled bodies pass here and are caught instead by the binary-char fallback. """ - ct = (content_type or "").lower() + ct = (content_type or "").partition(";")[0].strip().lower() if not ct: return True # unlabeled: let the binary-char fallback decide if ct.startswith("text/"): @@ -1540,7 +1557,11 @@ def _is_texty_content_type(content_type: str | None) -> bool: # application/vnd.openxmlformats-... isn't read as xml. RFC 6839 # +json/+xml suffixes (ld+json, xhtml+xml, ...) pass too. subtype = ct[len("application/") :].removeprefix("x-") - return subtype in _TEXT_APPLICATION_SUBTYPES or subtype.endswith(("+json", "+xml")) + return ( + subtype == "octet-stream" + or subtype in _TEXT_APPLICATION_SUBTYPES + or subtype.endswith(("+json", "+xml")) + ) return False @@ -1648,9 +1669,14 @@ def _fetch_page_text( # without replacement chars: a real text page has few binary chars. if _looks_binary(raw_html): # An undeclared non-UTF-8 page (latin-1/cp1252) also decodes to many - # U+FFFD. Retry as cp1252 (maps almost every byte to a printable - # char): if that reads as text it was a charset mismatch, not binary. - alt = raw_bytes.decode("cp1252", "replace") if declared is None else None + # U+FFFD. Retry as cp1252 only when the bytes have substantial ASCII + # text structure; otherwise high-byte binary would become printable + # looking gibberish because cp1252 maps nearly every byte. + alt = ( + raw_bytes.decode("cp1252", "replace") + if declared is None and _has_single_byte_text_evidence(raw_bytes) + else None + ) if alt is not None and not _looks_binary(alt): raw_html = alt else: diff --git a/studio/backend/tests/test_web_fetch_binary_guard.py b/studio/backend/tests/test_web_fetch_binary_guard.py index 92dd9411bef3..d69526e898dd 100644 --- a/studio/backend/tests/test_web_fetch_binary_guard.py +++ b/studio/backend/tests/test_web_fetch_binary_guard.py @@ -2,10 +2,9 @@ # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 """Regression for unslothai/unsloth#7084: the web_search fetcher must not decode -a binary body (PDF, image, octet-stream) into a flood of U+FFFD replacement -chars that poison the model context. It rejects non-text Content-Types up front -and, for binary mislabeled as text/* or sent unlabeled, falls back to a -replacement-char ratio check. Real HTML pages are unaffected. +a binary body into control characters or a flood of U+FFFD replacement chars +that poison the model context. It rejects known binary Content-Types up front +and sniffs generic, mislabeled, or unlabeled bodies. Real text is unaffected. """ from __future__ import annotations @@ -68,6 +67,7 @@ def _fetch_with(monkeypatch, body: bytes, content_type: str | None) -> str: ("text/html", True), ("text/plain; charset=utf-8", True), ("application/json", True), + ("application/json; charset=utf-8", True), ("application/xml", True), ("application/xhtml+xml", True), ("application/ld+json", True), @@ -78,7 +78,8 @@ def _fetch_with(monkeypatch, body: bytes, content_type: str | None) -> str: ("application/pdf", False), ("image/png", False), ("image/svg+xml", False), # SVG source isn't extracted downstream; reject - ("application/octet-stream", False), + # Generic downloads need byte sniffing because they can contain text. + ("application/octet-stream", True), ("application/zip", False), # A .docx ZIP must not pass just because "xml" is inside "openxmlformats". ("application/vnd.openxmlformats-officedocument.wordprocessingml.document", False), @@ -105,6 +106,18 @@ def test_image_rejected_by_content_type(monkeypatch): assert "non-text content" in out and "image/png" in out +def test_text_octet_stream_kept_after_sniffing(monkeypatch): + body = b"level=info\nmessage=plain text artifact\n" * 100 + out = _fetch_with(monkeypatch, body, "application/octet-stream") + assert "plain text artifact" in out + assert "non-text content" not in out and "binary content" not in out + + +def test_binary_octet_stream_rejected_after_sniffing(monkeypatch): + out = _fetch_with(monkeypatch, bytes(range(256)) * 20, "application/octet-stream") + assert "binary content" in out + + def test_binary_mislabeled_as_text_caught_by_fallback(monkeypatch): # A server sends binary but labels it text/plain -> the type check passes, # so the binary-char fallback must catch it. @@ -164,6 +177,14 @@ def test_control_heavy_binary_survives_cp1252_retry(monkeypatch): assert "binary content" in out +def test_high_byte_binary_not_rescued_as_cp1252(monkeypatch): + # cp1252 maps almost every high byte to a printable character. Without the + # ASCII-structure requirement, this binary payload would be returned as text. + body = bytes(range(0xA0, 0x100)) * 40 + out = _fetch_with(monkeypatch, body, "text/plain") + assert "binary content" in out + + def test_ansi_colored_text_log_kept(monkeypatch): # A text log with per-token ANSI color codes is text; ESC is excluded from # the binary-char set so it isn't dropped as binary. @@ -189,12 +210,10 @@ def test_html_page_unaffected(monkeypatch): def test_content_type_sanitized_in_message(monkeypatch): # An obs-folded Content-Type can smuggle control chars into get_content_type(); # the returned message must be trimmed to a clean MIME token. - out = _fetch_with( - monkeypatch, b"\x00\x01\x02" * 500, "application/octet-stream\r\n data: injected" - ) + out = _fetch_with(monkeypatch, b"\x00\x01\x02" * 500, "application/pdf\r\n data: injected") assert "\n" not in out and "\r" not in out assert "injected" not in out - assert "application/octet-stream" in out + assert "application/pdf" in out @pytest.mark.parametrize( @@ -206,7 +225,7 @@ def test_content_type_sanitized_in_message(monkeypatch): (130, 1000, True), # 130 > 1000//8 (125) -> binary ], ) -def test_replacement_ratio_boundary(monkeypatch, n_bad, n_total, expect_binary): +def test_binary_char_ratio_boundary(monkeypatch, n_bad, n_total, expect_binary): # Body of n_total chars: n_bad NUL control bytes + ASCII filler. NUL stays # binary through the cp1252 retry, so only the ratio threshold decides here. body = b"\x00" * n_bad + b"a" * (n_total - n_bad) From c5902a6266e7475c77fab380539609f5d6743e48 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 01:13:22 +0000 Subject: [PATCH 08/15] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- studio/backend/core/inference/tools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 937219fbf15c..19c1c8aefa44 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -1420,6 +1420,7 @@ def _looks_binary(text: str) -> bool: _MIN_BINARY_CHARS, len(text) // _BINARY_CHAR_DIVISOR ) + def _has_single_byte_text_evidence(data: bytes) -> bool: """True when *data* has enough ASCII structure for a cp1252 text retry.""" if not data: From 10592853fcb6377abfeac59f9c031c74d59c3fdc Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:15:48 -0300 Subject: [PATCH 09/15] Simplify web fetch binary guard --- studio/backend/core/inference/tools.py | 61 +++------- .../tests/test_web_fetch_binary_guard.py | 107 ++++-------------- 2 files changed, 38 insertions(+), 130 deletions(-) diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 19c1c8aefa44..d655c8939798 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -1381,17 +1381,12 @@ def build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> di # sections stripped during conversion; 512 KB reaches article content even # where alone is ~200 KB. _MAX_FETCH_BYTES = 512 * 1024 -# Chars that don't occur in real text: undecodable bytes (U+FFFD) plus control -# chars (C0 minus tab/newline/CR and ESC, DEL, C1). ESC is excluded so a text -# page of ANSI-colored terminal output isn't mistaken for binary. A decoded page -# is treated as binary when more than 1/_BINARY_CHAR_DIVISOR (12.5%) of its chars -# are these, tolerating up to _MIN_BINARY_CHARS so minor glitches don't drop it. +# Undecodable bytes and controls, excluding text whitespace and ESC for ANSI logs. +# More than 12.5% is binary after allowing 16 minor encoding glitches. _BINARY_CHAR_RE = re.compile("[\\x00-\\x08\\x0b\\x0c\\x0e-\\x1a\\x1c-\\x1f\\x7f-\\x9f\\ufffd]") _MIN_BINARY_CHARS = 16 _BINARY_CHAR_DIVISOR = 8 -# Leading bytes of common binary formats that servers sometimes mislabel as -# text/*; matched by signature because their replacement-char density alone can -# be low (e.g. a PDF whose first chunk is mostly ASCII object/xref syntax). +# Common binary signatures that can otherwise look text-heavy when mislabeled. _BINARY_MAGIC = ( b"%PDF-", # PDF b"PK\x03\x04", # zip / docx / xlsx / pptx / epub / jar @@ -1405,17 +1400,13 @@ def build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> di b"\x28\xb5\x2f\xfd", # zstd ) -# Undeclared legacy Western text should have substantial ASCII structure even -# when its non-ASCII bytes are cp1252. Requiring at least 75% ASCII text bytes -# prevents arbitrary high-byte binary from becoming printable-looking gibberish -# merely because cp1252 defines a character for nearly every byte. +# A cp1252 retry needs 75% ASCII structure so it cannot rescue high-byte binary. _MIN_SINGLE_BYTE_ASCII_RATIO = 3 / 4 _ASCII_TEXT_BYTES = frozenset((*range(0x20, 0x7F), 0x09, 0x0A, 0x0D, 0x1B)) def _looks_binary(text: str) -> bool: - """True when more than 1/_BINARY_CHAR_DIVISOR of ``text`` is binary chars - (control/undecodable, per _BINARY_CHAR_RE), past the _MIN_BINARY_CHARS floor.""" + """Whether control or undecodable characters exceed the binary threshold.""" return len(_BINARY_CHAR_RE.findall(text)) > max( _MIN_BINARY_CHARS, len(text) // _BINARY_CHAR_DIVISOR ) @@ -1536,27 +1527,16 @@ def _validate_and_resolve_host(hostname: str, port: int) -> tuple[bool, str, str ) -def _is_texty_content_type(content_type: str | None) -> bool: - """True for MIME types safe to decode or inspect as possible text. - - This includes HTML, plain text, JSON/XML/YAML feeds, and generic - ``application/octet-stream`` downloads whose bytes must be sniffed. Known - binary types (PDF, images, archives, Office/ZIP) return False so the fetcher - never decodes them into replacement chars that poison the model context - (unslothai/unsloth#7084). - - A missing Content-Type coerces to ``text/plain`` upstream, so unlabeled - bodies pass here and are caught instead by the binary-char fallback. - """ +def _is_text_candidate_content_type(content_type: str | None) -> bool: + """Whether a MIME type is textual or ambiguous enough for byte sniffing.""" ct = (content_type or "").partition(";")[0].strip().lower() if not ct: - return True # unlabeled: let the binary-char fallback decide + return True if ct.startswith("text/"): return True if ct.startswith("application/"): - # Match the exact subtype, not a loose substring, so a .docx labeled - # application/vnd.openxmlformats-... isn't read as xml. RFC 6839 - # +json/+xml suffixes (ld+json, xhtml+xml, ...) pass too. + # Exact matches avoid treating "openxmlformats" as XML. Structured + # +json/+xml suffixes remain valid text candidates. subtype = ct[len("application/") :].removeprefix("x-") return ( subtype == "octet-stream" @@ -1646,33 +1626,24 @@ def _fetch_page_text( else: return "Failed to fetch URL: too many redirects." - # Reject binary bodies (PDF, image, archive): decoding them as text - # floods the model context with U+FFFD replacement chars (#7084). + # Reject MIME types known to be binary before decoding. content_type = resp.headers.get_content_type() - if not _is_texty_content_type(content_type): - # Trim to a clean MIME token: get_content_type() can echo control - # chars from an obs-folded header, and this string is returned to - # the model. + if not _is_text_candidate_content_type(content_type): + # Only echo a clean MIME token back to the model. m = re.match(r"[\w.+-]+/[\w.+-]+", content_type or "") safe_type = m.group(0) if m else "unknown type" return f"(non-text content: {safe_type}, {len(raw_bytes)} bytes; not readable as text)" - # A PDF/image/archive mislabeled as text/*: catch by signature, since a - # text-heavy first chunk can slip past the replacement-char density check. + # Catch text-labeled binary whose header and first chunk look textual. if raw_bytes.startswith(_BINARY_MAGIC): return f"(binary content, {len(raw_bytes)} bytes; not readable as text)" declared = resp.headers.get_content_charset() raw_html = raw_bytes.decode(declared or "utf-8", errors = "replace") - # Fallback for binary mislabeled as text/* or sent with no Content-Type, - # including valid-UTF-8 binary (NUL/control-heavy payloads) that decodes - # without replacement chars: a real text page has few binary chars. + # Catch mislabeled or unlabeled binary, including valid UTF-8 controls. if _looks_binary(raw_html): - # An undeclared non-UTF-8 page (latin-1/cp1252) also decodes to many - # U+FFFD. Retry as cp1252 only when the bytes have substantial ASCII - # text structure; otherwise high-byte binary would become printable - # looking gibberish because cp1252 maps nearly every byte. + # Rescue undeclared cp1252 only when the bytes have text structure. alt = ( raw_bytes.decode("cp1252", "replace") if declared is None and _has_single_byte_text_evidence(raw_bytes) diff --git a/studio/backend/tests/test_web_fetch_binary_guard.py b/studio/backend/tests/test_web_fetch_binary_guard.py index d69526e898dd..1feaf066b23a 100644 --- a/studio/backend/tests/test_web_fetch_binary_guard.py +++ b/studio/backend/tests/test_web_fetch_binary_guard.py @@ -1,11 +1,7 @@ # SPDX-License-Identifier: AGPL-3.0-only # Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0 -"""Regression for unslothai/unsloth#7084: the web_search fetcher must not decode -a binary body into control characters or a flood of U+FFFD replacement chars -that poison the model context. It rejects known binary Content-Types up front -and sniffs generic, mislabeled, or unlabeled bodies. Real text is unaffected. -""" +"""Regression tests for binary bodies poisoning web_search model context (#7084).""" from __future__ import annotations @@ -58,9 +54,6 @@ def _fetch_with(monkeypatch, body: bytes, content_type: str | None) -> str: return tools._fetch_page_text("https://example.com/thing", timeout = 5) -# ── content-type classifier ── - - @pytest.mark.parametrize( "content_type,expected", [ @@ -71,27 +64,22 @@ def _fetch_with(monkeypatch, body: bytes, content_type: str | None) -> str: ("application/xml", True), ("application/xhtml+xml", True), ("application/ld+json", True), - ("application/yaml", True), # raw configs / API specs + ("application/yaml", True), ("application/x-yaml", True), - ("application/x-ndjson", True), # newline-delimited JSON is text + ("application/x-ndjson", True), ("application/ndjson", True), ("application/pdf", False), ("image/png", False), - ("image/svg+xml", False), # SVG source isn't extracted downstream; reject - # Generic downloads need byte sniffing because they can contain text. + ("image/svg+xml", False), ("application/octet-stream", True), ("application/zip", False), - # A .docx ZIP must not pass just because "xml" is inside "openxmlformats". ("application/vnd.openxmlformats-officedocument.wordprocessingml.document", False), - ("", True), # unlabeled: defer to the binary-char fallback + ("", True), (None, True), ], ) -def test_is_texty_content_type(content_type, expected): - assert tools._is_texty_content_type(content_type) is expected - - -# ── fetcher end-to-end (mocked network) ── +def test_is_text_candidate_content_type(content_type, expected): + assert tools._is_text_candidate_content_type(content_type) is expected def test_pdf_rejected_by_content_type(monkeypatch): @@ -100,12 +88,6 @@ def test_pdf_rejected_by_content_type(monkeypatch): assert "non-text content" in out and "application/pdf" in out -def test_image_rejected_by_content_type(monkeypatch): - out = _fetch_with(monkeypatch, b"\x89PNG\r\n\x1a\n" + bytes(range(256)) * 4, "image/png") - assert "�" not in out - assert "non-text content" in out and "image/png" in out - - def test_text_octet_stream_kept_after_sniffing(monkeypatch): body = b"level=info\nmessage=plain text artifact\n" * 100 out = _fetch_with(monkeypatch, body, "application/octet-stream") @@ -113,54 +95,31 @@ def test_text_octet_stream_kept_after_sniffing(monkeypatch): assert "non-text content" not in out and "binary content" not in out -def test_binary_octet_stream_rejected_after_sniffing(monkeypatch): - out = _fetch_with(monkeypatch, bytes(range(256)) * 20, "application/octet-stream") - assert "binary content" in out - - -def test_binary_mislabeled_as_text_caught_by_fallback(monkeypatch): - # A server sends binary but labels it text/plain -> the type check passes, - # so the binary-char fallback must catch it. - out = _fetch_with(monkeypatch, bytes(range(256)) * 20, "text/plain") +@pytest.mark.parametrize("content_type", ["application/octet-stream", "text/plain", None]) +def test_binary_candidates_rejected_after_sniffing(monkeypatch, content_type): + out = _fetch_with(monkeypatch, bytes(range(256)) * 20, content_type) assert "�" not in out assert "binary content" in out def test_valid_utf8_binary_caught_by_control_chars(monkeypatch): - # NUL/control-heavy binary is valid UTF-8, so it decodes with no U+FFFD; - # the fallback must still catch it via control-char density (codex #2). - body = bytes([0, 1, 2, 3, 4, 5, 6, 7]) * 400 # all valid UTF-8, 0 replacement chars + # These controls are valid UTF-8 and therefore produce no replacement chars. + body = bytes([0, 1, 2, 3, 4, 5, 6, 7]) * 400 out = _fetch_with(monkeypatch, body, "text/plain") assert "binary content" in out -def test_pdf_mislabeled_as_text_caught_by_magic(monkeypatch): - # A PDF served as text/plain: the content-type gate passes, so the magic-byte - # sniff must catch it (its ASCII-heavy head can stay under the ratio). - pdf = b"%PDF-1.7\n" + b"1 0 obj<>endobj\n" * 300 - out = _fetch_with(monkeypatch, pdf, "text/plain") - assert "binary content" in out - - -def test_zip_mislabeled_as_text_caught_by_magic(monkeypatch): - out = _fetch_with(monkeypatch, b"PK\x03\x04" + b"filename.txt content " * 200, "text/plain") - assert "binary content" in out - - @pytest.mark.parametrize( "magic", - [b"\x1f\x8b", b"BZh", b"\xfd7zXZ\x00", b"\x28\xb5\x2f\xfd"], # gzip, bzip2, xz, zstd + [b"%PDF-", b"PK\x03\x04", b"\x1f\x8b", b"BZh", b"\xfd7zXZ\x00", b"\x28\xb5\x2f\xfd"], ) -def test_compression_mislabeled_as_text_caught_by_magic(monkeypatch, magic): - # A small compressed body mislabeled as text can slip past the density check; - # the signature must reject it. - out = _fetch_with(monkeypatch, magic + b"some short compressed-looking body", "text/plain") +def test_text_labeled_binary_caught_by_magic(monkeypatch, magic): + out = _fetch_with(monkeypatch, magic + b" printable text-heavy body" * 100, "text/plain") assert "binary content" in out def test_latin1_text_without_charset_kept(monkeypatch): - # Accent-dense cp1252 text with no declared charset decodes to many U+FFFD as - # UTF-8; the cp1252 retry must keep it instead of dropping it as binary. + # The cp1252 retry should rescue accent-heavy text with ASCII structure. body = ( "Muller lauft uber die Strasse: schoene, groesse. MARKERWORD ".replace("ue", "ü") + "äöüß éèà " @@ -170,36 +129,20 @@ def test_latin1_text_without_charset_kept(monkeypatch): assert "MARKERWORD" in out -def test_control_heavy_binary_survives_cp1252_retry(monkeypatch): - # Genuine control-heavy binary (no magic) must stay rejected even after the - # cp1252 retry, which cannot rescue real binary. - out = _fetch_with(monkeypatch, bytes([0, 1, 2, 3, 4, 5, 6, 7]) * 400, "text/plain") - assert "binary content" in out - - def test_high_byte_binary_not_rescued_as_cp1252(monkeypatch): - # cp1252 maps almost every high byte to a printable character. Without the - # ASCII-structure requirement, this binary payload would be returned as text. + # cp1252 maps these bytes to printable characters, but they lack ASCII structure. body = bytes(range(0xA0, 0x100)) * 40 out = _fetch_with(monkeypatch, body, "text/plain") assert "binary content" in out def test_ansi_colored_text_log_kept(monkeypatch): - # A text log with per-token ANSI color codes is text; ESC is excluded from - # the binary-char set so it isn't dropped as binary. + # ESC is excluded from the binary set so ANSI logs remain readable. line = "".join(f"\x1b[32m+{i}\x1b[0m\n" for i in range(300)).encode() out = _fetch_with(monkeypatch, line, "text/plain") assert "binary content" not in out -def test_binary_unlabeled_caught_by_fallback(monkeypatch): - # No Content-Type coerces to text/plain upstream; the fallback still catches it. - out = _fetch_with(monkeypatch, bytes(range(256)) * 20, None) - assert "�" not in out - assert "binary content" in out - - def test_html_page_unaffected(monkeypatch): html = b"Real text content here.
" out = _fetch_with(monkeypatch, html, "text/html; charset=utf-8") @@ -208,8 +151,7 @@ def test_html_page_unaffected(monkeypatch): def test_content_type_sanitized_in_message(monkeypatch): - # An obs-folded Content-Type can smuggle control chars into get_content_type(); - # the returned message must be trimmed to a clean MIME token. + # Do not echo obs-folded header content into the model response. out = _fetch_with(monkeypatch, b"\x00\x01\x02" * 500, "application/pdf\r\n data: injected") assert "\n" not in out and "\r" not in out assert "injected" not in out @@ -219,23 +161,18 @@ def test_content_type_sanitized_in_message(monkeypatch): @pytest.mark.parametrize( "n_bad,n_total,expect_binary", [ - # Straddle the 12.5% ratio (well above the 16-char floor): just under vs - # just over len//8. Locks the divisor so it can't silently drift. - (120, 1000, False), # 120 <= 1000//8 (125) -> kept - (130, 1000, True), # 130 > 1000//8 (125) -> binary + (120, 1000, False), + (130, 1000, True), ], ) def test_binary_char_ratio_boundary(monkeypatch, n_bad, n_total, expect_binary): - # Body of n_total chars: n_bad NUL control bytes + ASCII filler. NUL stays - # binary through the cp1252 retry, so only the ratio threshold decides here. body = b"\x00" * n_bad + b"a" * (n_total - n_bad) out = _fetch_with(monkeypatch, body, "text/plain") assert ("binary content" in out) is expect_binary def test_text_with_a_few_stray_replacement_chars_kept(monkeypatch): - # A mostly-clean page with a handful of bad bytes stays (below the floor), - # so we don't drop legitimate pages over minor encoding glitches. + # Minor encoding glitches below the floor should not drop a real page. body = ("Real article text. " * 200).encode() + b"\xff\xfe\xff" out = _fetch_with(monkeypatch, body, "text/html") assert "Real article text." in out From 1bfd58252a9d1f78580cd756affa48cdca295134 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:32:24 -0300 Subject: [PATCH 10/15] Sniff unknown MIME types and handle Latin-1 --- studio/backend/core/inference/tools.py | 50 +++++++++++++------ .../tests/test_web_fetch_binary_guard.py | 22 +++++++- 2 files changed, 57 insertions(+), 15 deletions(-) diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index d655c8939798..de772fd96592 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -5,6 +5,7 @@ (DuckDuckGo), Python code execution, and terminal commands.""" import ast +import codecs import http.client import os import signal @@ -1520,28 +1521,47 @@ def _validate_and_resolve_host(hostname: str, port: int) -> tuple[bool, str, str return True, "", first_ip -# Bare application/* subtypes that are text (the leading x- and the +json/+xml -# structured-syntax suffixes are handled in the check). -_TEXT_APPLICATION_SUBTYPES = frozenset( - {"json", "xml", "javascript", "ecmascript", "csv", "yaml", "ndjson", "jsonl"} +# Known binary application subtypes are rejected by MIME type. Unknown +# application types are sniffed so textual artifacts such as SQL remain usable. +_BINARY_APPLICATION_SUBTYPES = frozenset( + { + "epub+zip", + "gzip", + "java-archive", + "msword", + "pdf", + "vnd.apple.installer+xml", + "vnd.ms-excel", + "vnd.ms-powerpoint", + "wasm", + "x-7z-compressed", + "x-bzip2", + "x-gzip", + "x-rar-compressed", + "x-tar", + "x-xz", + "zip", + "zstd", + } +) +_BINARY_APPLICATION_PREFIXES = ( + "vnd.oasis.opendocument.", + "vnd.openxmlformats-officedocument.", ) def _is_text_candidate_content_type(content_type: str | None) -> bool: """Whether a MIME type is textual or ambiguous enough for byte sniffing.""" - ct = (content_type or "").partition(";")[0].strip().lower() - if not ct: + match = re.match(r"[\w.+-]+/[\w.+-]+", content_type or "") + if not match: return True + ct = match.group(0).lower() if ct.startswith("text/"): return True if ct.startswith("application/"): - # Exact matches avoid treating "openxmlformats" as XML. Structured - # +json/+xml suffixes remain valid text candidates. - subtype = ct[len("application/") :].removeprefix("x-") - return ( - subtype == "octet-stream" - or subtype in _TEXT_APPLICATION_SUBTYPES - or subtype.endswith(("+json", "+xml")) + subtype = ct[len("application/") :] + return subtype not in _BINARY_APPLICATION_SUBTYPES and not subtype.startswith( + _BINARY_APPLICATION_PREFIXES ) return False @@ -1639,6 +1659,7 @@ def _fetch_page_text( return f"(binary content, {len(raw_bytes)} bytes; not readable as text)" declared = resp.headers.get_content_charset() + declared_codec = codecs.lookup(declared).name if declared else None raw_html = raw_bytes.decode(declared or "utf-8", errors = "replace") # Catch mislabeled or unlabeled binary, including valid UTF-8 controls. @@ -1646,7 +1667,8 @@ def _fetch_page_text( # Rescue undeclared cp1252 only when the bytes have text structure. alt = ( raw_bytes.decode("cp1252", "replace") - if declared is None and _has_single_byte_text_evidence(raw_bytes) + if declared_codec in (None, "iso8859-1") + and _has_single_byte_text_evidence(raw_bytes) else None ) if alt is not None and not _looks_binary(alt): diff --git a/studio/backend/tests/test_web_fetch_binary_guard.py b/studio/backend/tests/test_web_fetch_binary_guard.py index 1feaf066b23a..9e767b64c132 100644 --- a/studio/backend/tests/test_web_fetch_binary_guard.py +++ b/studio/backend/tests/test_web_fetch_binary_guard.py @@ -68,6 +68,8 @@ def _fetch_with(monkeypatch, body: bytes, content_type: str | None) -> str: ("application/x-yaml", True), ("application/x-ndjson", True), ("application/ndjson", True), + ("application/sql", True), + ("application/x-www-form-urlencoded", True), ("application/pdf", False), ("image/png", False), ("image/svg+xml", False), @@ -95,13 +97,23 @@ def test_text_octet_stream_kept_after_sniffing(monkeypatch): assert "non-text content" not in out and "binary content" not in out -@pytest.mark.parametrize("content_type", ["application/octet-stream", "text/plain", None]) +@pytest.mark.parametrize( + "content_type", + ["application/octet-stream", "application/x-custom-binary", "text/plain", None], +) def test_binary_candidates_rejected_after_sniffing(monkeypatch, content_type): out = _fetch_with(monkeypatch, bytes(range(256)) * 20, content_type) assert "�" not in out assert "binary content" in out +@pytest.mark.parametrize("content_type", ["application/sql", "application/x-www-form-urlencoded"]) +def test_unknown_application_text_kept_after_sniffing(monkeypatch, content_type): + out = _fetch_with(monkeypatch, b"select readable_text from artifacts;\n" * 100, content_type) + assert "readable_text" in out + assert "non-text content" not in out and "binary content" not in out + + def test_valid_utf8_binary_caught_by_control_chars(monkeypatch): # These controls are valid UTF-8 and therefore produce no replacement chars. body = bytes([0, 1, 2, 3, 4, 5, 6, 7]) * 400 @@ -129,6 +141,14 @@ def test_latin1_text_without_charset_kept(monkeypatch): assert "MARKERWORD" in out +@pytest.mark.parametrize("charset", ["iso-8859-1", "latin-1", "latin1"]) +def test_declared_latin1_cp1252_punctuation_kept(monkeypatch, charset): + body = ("“quoted” " * 100).encode("cp1252") + out = _fetch_with(monkeypatch, body, f"text/plain; charset={charset}") + assert "quoted" in out + assert "binary content" not in out + + def test_high_byte_binary_not_rescued_as_cp1252(monkeypatch): # cp1252 maps these bytes to printable characters, but they lack ASCII structure. body = bytes(range(0xA0, 0x100)) * 40 From 14dca2482e417b3970b6038e36c458af64f159eb Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Tue, 14 Jul 2026 22:48:36 -0300 Subject: [PATCH 11/15] Sniff ambiguous Office MIME and prefixed magic --- studio/backend/core/inference/tools.py | 20 ++++----- .../tests/test_web_fetch_binary_guard.py | 42 ++++++++++++++++++- 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index de772fd96592..5f9f1bf53d4c 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -1391,6 +1391,7 @@ def build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> di _BINARY_MAGIC = ( b"%PDF-", # PDF b"PK\x03\x04", # zip / docx / xlsx / pptx / epub / jar + b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1", # OLE / legacy Office b"\x89PNG\r\n\x1a\n", # PNG b"\xff\xd8\xff", # JPEG b"GIF87a", @@ -1413,6 +1414,12 @@ def _looks_binary(text: str) -> bool: ) +def _has_binary_magic(data: bytes) -> bool: + """Whether a common binary signature follows optional BOM or whitespace.""" + head = data[:1024].lstrip().removeprefix(b"\xef\xbb\xbf").lstrip() + return head.startswith(_BINARY_MAGIC) + + def _has_single_byte_text_evidence(data: bytes) -> bool: """True when *data* has enough ASCII structure for a cp1252 text retry.""" if not data: @@ -1528,11 +1535,8 @@ def _validate_and_resolve_host(hostname: str, port: int) -> tuple[bool, str, str "epub+zip", "gzip", "java-archive", - "msword", "pdf", "vnd.apple.installer+xml", - "vnd.ms-excel", - "vnd.ms-powerpoint", "wasm", "x-7z-compressed", "x-bzip2", @@ -1544,10 +1548,6 @@ def _validate_and_resolve_host(hostname: str, port: int) -> tuple[bool, str, str "zstd", } ) -_BINARY_APPLICATION_PREFIXES = ( - "vnd.oasis.opendocument.", - "vnd.openxmlformats-officedocument.", -) def _is_text_candidate_content_type(content_type: str | None) -> bool: @@ -1560,9 +1560,7 @@ def _is_text_candidate_content_type(content_type: str | None) -> bool: return True if ct.startswith("application/"): subtype = ct[len("application/") :] - return subtype not in _BINARY_APPLICATION_SUBTYPES and not subtype.startswith( - _BINARY_APPLICATION_PREFIXES - ) + return subtype not in _BINARY_APPLICATION_SUBTYPES return False @@ -1655,7 +1653,7 @@ def _fetch_page_text( return f"(non-text content: {safe_type}, {len(raw_bytes)} bytes; not readable as text)" # Catch text-labeled binary whose header and first chunk look textual. - if raw_bytes.startswith(_BINARY_MAGIC): + if _has_binary_magic(raw_bytes): return f"(binary content, {len(raw_bytes)} bytes; not readable as text)" declared = resp.headers.get_content_charset() diff --git a/studio/backend/tests/test_web_fetch_binary_guard.py b/studio/backend/tests/test_web_fetch_binary_guard.py index 9e767b64c132..411db2276f66 100644 --- a/studio/backend/tests/test_web_fetch_binary_guard.py +++ b/studio/backend/tests/test_web_fetch_binary_guard.py @@ -75,7 +75,8 @@ def _fetch_with(monkeypatch, body: bytes, content_type: str | None) -> str: ("image/svg+xml", False), ("application/octet-stream", True), ("application/zip", False), - ("application/vnd.openxmlformats-officedocument.wordprocessingml.document", False), + ("application/vnd.ms-excel", True), + ("application/vnd.openxmlformats-officedocument.wordprocessingml.document", True), ("", True), (None, True), ], @@ -114,6 +115,13 @@ def test_unknown_application_text_kept_after_sniffing(monkeypatch, content_type) assert "non-text content" not in out and "binary content" not in out +def test_excel_labeled_csv_kept_after_sniffing(monkeypatch): + body = b"name,value\nreadable,42\n" * 100 + out = _fetch_with(monkeypatch, body, "application/vnd.ms-excel") + assert "readable" in out + assert "binary content" not in out + + def test_valid_utf8_binary_caught_by_control_chars(monkeypatch): # These controls are valid UTF-8 and therefore produce no replacement chars. body = bytes([0, 1, 2, 3, 4, 5, 6, 7]) * 400 @@ -123,13 +131,43 @@ def test_valid_utf8_binary_caught_by_control_chars(monkeypatch): @pytest.mark.parametrize( "magic", - [b"%PDF-", b"PK\x03\x04", b"\x1f\x8b", b"BZh", b"\xfd7zXZ\x00", b"\x28\xb5\x2f\xfd"], + [ + b"%PDF-", + b"PK\x03\x04", + b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1", + b"\x1f\x8b", + b"BZh", + b"\xfd7zXZ\x00", + b"\x28\xb5\x2f\xfd", + ], ) def test_text_labeled_binary_caught_by_magic(monkeypatch, magic): out = _fetch_with(monkeypatch, magic + b" printable text-heavy body" * 100, "text/plain") assert "binary content" in out +@pytest.mark.parametrize("prefix", [b"\xef\xbb\xbf", b" \r\n", b"\t\xef\xbb\xbf "]) +def test_pdf_magic_after_harmless_prefix(monkeypatch, prefix): + body = prefix + b"%PDF-1.7\n" + b"1 0 obj<>endobj\n" * 100 + out = _fetch_with(monkeypatch, body, "text/plain") + assert "binary content" in out + + +@pytest.mark.parametrize( + "content_type,magic", + [ + ("application/vnd.ms-excel", b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"), + ( + "application/vnd.openxmlformats-officedocument.wordprocessingml.document", + b"PK\x03\x04", + ), + ], +) +def test_office_labeled_binary_caught_by_magic(monkeypatch, content_type, magic): + out = _fetch_with(monkeypatch, magic + b" printable text-heavy body" * 100, content_type) + assert "binary content" in out + + def test_latin1_text_without_charset_kept(monkeypatch): # The cp1252 retry should rescue accent-heavy text with ASCII structure. body = ( From 4eb12832c4e9c0e6d8f27f2f672e2cb0f1401623 Mon Sep 17 00:00:00 2001 From: oobabooga <112222186+oobabooga@users.noreply.github.com> Date: Wed, 15 Jul 2026 11:33:49 -0300 Subject: [PATCH 12/15] Decode BOM-marked Unicode web content --- studio/backend/core/inference/tools.py | 21 +++++++++++-- .../tests/test_web_fetch_binary_guard.py | 31 ++++++++++++++++++- 2 files changed, 49 insertions(+), 3 deletions(-) diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py index 8f9241d51f1c..9f2e7a0f6293 100644 --- a/studio/backend/core/inference/tools.py +++ b/studio/backend/core/inference/tools.py @@ -3602,6 +3602,15 @@ def build_rag_autoinject(conversation: list[dict], rag_scope: dict | None) -> di b"\x28\xb5\x2f\xfd", # zstd ) +# Check UTF-32 first because its little-endian BOM starts with the UTF-16 BOM. +_UNICODE_BOM_CODECS = ( + (codecs.BOM_UTF32_LE, "utf-32"), + (codecs.BOM_UTF32_BE, "utf-32"), + (codecs.BOM_UTF16_LE, "utf-16"), + (codecs.BOM_UTF16_BE, "utf-16"), + (codecs.BOM_UTF8, "utf-8-sig"), +) + # A cp1252 retry needs 75% ASCII structure so it cannot rescue high-byte binary. _MIN_SINGLE_BYTE_ASCII_RATIO = 3 / 4 _ASCII_TEXT_BYTES = frozenset((*range(0x20, 0x7F), 0x09, 0x0A, 0x0D, 0x1B)) @@ -3616,7 +3625,11 @@ def _looks_binary(text: str) -> bool: def _has_binary_magic(data: bytes) -> bool: """Whether a common binary signature follows optional BOM or whitespace.""" - head = data[:1024].lstrip().removeprefix(b"\xef\xbb\xbf").lstrip() + head = data[:1024].lstrip() + for bom, _codec in _UNICODE_BOM_CODECS: + if head.startswith(bom): + head = head.removeprefix(bom).lstrip() + break return head.startswith(_BINARY_MAGIC) @@ -3858,7 +3871,11 @@ def _fetch_page_text( declared = resp.headers.get_content_charset() declared_codec = codecs.lookup(declared).name if declared else None - raw_html = raw_bytes.decode(declared or "utf-8", errors = "replace") + bom_codec = next( + (codec for bom, codec in _UNICODE_BOM_CODECS if raw_bytes.startswith(bom)), + None, + ) + raw_html = raw_bytes.decode(declared or bom_codec or "utf-8", errors = "replace") # Catch mislabeled or unlabeled binary, including valid UTF-8 controls. if _looks_binary(raw_html): diff --git a/studio/backend/tests/test_web_fetch_binary_guard.py b/studio/backend/tests/test_web_fetch_binary_guard.py index 411db2276f66..57159c024f4d 100644 --- a/studio/backend/tests/test_web_fetch_binary_guard.py +++ b/studio/backend/tests/test_web_fetch_binary_guard.py @@ -5,6 +5,7 @@ from __future__ import annotations +import codecs import sys from email.message import Message from pathlib import Path @@ -122,6 +123,23 @@ def test_excel_labeled_csv_kept_after_sniffing(monkeypatch): assert "binary content" not in out +@pytest.mark.parametrize( + "bom,encoding", + [ + (codecs.BOM_UTF16_LE, "utf-16-le"), + (codecs.BOM_UTF16_BE, "utf-16-be"), + (codecs.BOM_UTF32_LE, "utf-32-le"), + (codecs.BOM_UTF32_BE, "utf-32-be"), + ], +) +@pytest.mark.parametrize("content_type", ["text/plain", "application/vnd.ms-excel"]) +def test_bom_unicode_text_without_charset_kept(monkeypatch, bom, encoding, content_type): + body = bom + ("name,value\nreadable,42\n" * 100).encode(encoding) + out = _fetch_with(monkeypatch, body, content_type) + assert "readable" in out + assert "binary content" not in out + + def test_valid_utf8_binary_caught_by_control_chars(monkeypatch): # These controls are valid UTF-8 and therefore produce no replacement chars. body = bytes([0, 1, 2, 3, 4, 5, 6, 7]) * 400 @@ -146,7 +164,18 @@ def test_text_labeled_binary_caught_by_magic(monkeypatch, magic): assert "binary content" in out -@pytest.mark.parametrize("prefix", [b"\xef\xbb\xbf", b" \r\n", b"\t\xef\xbb\xbf "]) +@pytest.mark.parametrize( + "prefix", + [ + codecs.BOM_UTF8, + codecs.BOM_UTF16_LE, + codecs.BOM_UTF16_BE, + codecs.BOM_UTF32_LE, + codecs.BOM_UTF32_BE, + b" \r\n", + b"\t\xef\xbb\xbf ", + ], +) def test_pdf_magic_after_harmless_prefix(monkeypatch, prefix): body = prefix + b"%PDF-1.7\n" + b"1 0 obj<>endobj\n" * 100 out = _fetch_with(monkeypatch, body, "text/plain") From c7fbec216c8ccf712da3b1e4c94154b4dbcbe891 Mon Sep 17 00:00:00 2001 From: danielhanchen