diff --git a/agent/document_processing/__init__.py b/agent/document_processing/__init__.py
new file mode 100644
index 000000000000..705b3bd5c3e4
--- /dev/null
+++ b/agent/document_processing/__init__.py
@@ -0,0 +1,9 @@
+"""Document Intelligence Layer — unified document normalization pipeline.
+
+All document understanding goes through this module. Gateway platforms
+(Telegram, Discord, etc.) only handle *receiving* files; parsing and
+normalization happen here so the logic is reusable across every platform.
+"""
+
+from agent.document_processing.router import process_document # noqa: F401
+from agent.document_processing.types import DocumentResult # noqa: F401
diff --git a/agent/document_processing/html_parser.py b/agent/document_processing/html_parser.py
new file mode 100644
index 000000000000..6a1a37ce0fad
--- /dev/null
+++ b/agent/document_processing/html_parser.py
@@ -0,0 +1,137 @@
+"""HTML parser — extracts clean text, title, and links from HTML content.
+
+Uses BeautifulSoup with the stdlib html.parser backend so there is no
+hard dependency on lxml. If beautifulsoup4 is missing at runtime the
+parser falls back to a regex-based lightweight extractor.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import List, Optional, Tuple
+
+# ---------------------------------------------------------------------------
+# Try importing BeautifulSoup; provide a graceful fallback.
+# ---------------------------------------------------------------------------
+try:
+ from bs4 import BeautifulSoup, Comment # type: ignore[import-untyped]
+
+ _HAS_BS4 = True
+except ImportError:
+ _HAS_BS4 = False
+
+# Tags whose *content* should be removed entirely (not just the tag).
+_REMOVE_TAGS = {"script", "style", "noscript", "svg", "canvas", "template", "iframe"}
+
+# Maximum whitespace‐collapsed gap between blocks.
+_MAX_BLANK_LINES = 2
+
+
+def parse_html(raw_html: str) -> Tuple[str, str, List[str]]:
+ """Parse *raw_html* and return ``(title, text, links)``.
+
+ * ``title`` — content of ```` or first ``<h1>``, empty string if
+ neither exists.
+ * ``text`` — human-readable text with paragraph breaks preserved.
+ * ``links`` — deduplicated list of ``href`` values from ``<a>`` tags.
+ """
+ if _HAS_BS4:
+ return _parse_with_bs4(raw_html)
+ return _parse_fallback(raw_html)
+
+
+# ---------------------------------------------------------------------------
+# Primary: BeautifulSoup
+# ---------------------------------------------------------------------------
+
+
+def _parse_with_bs4(raw_html: str) -> Tuple[str, str, List[str]]:
+ soup = BeautifulSoup(raw_html, "html.parser")
+
+ # 1. Remove unwanted elements ----------------------------------------
+ for tag_name in _REMOVE_TAGS:
+ for tag in soup.find_all(tag_name):
+ tag.decompose()
+
+ # Remove HTML comments
+ for comment in soup.find_all(string=lambda t: isinstance(t, Comment)):
+ comment.extract()
+
+ # 2. Title -----------------------------------------------------------
+ title = ""
+ title_tag = soup.find("title")
+ if title_tag:
+ title = title_tag.get_text(strip=True)
+ if not title:
+ h1 = soup.find("h1")
+ if h1:
+ title = h1.get_text(strip=True)
+
+ # 3. Links -----------------------------------------------------------
+ seen_links: set[str] = set()
+ links: List[str] = []
+ for a_tag in soup.find_all("a", href=True):
+ href = a_tag["href"].strip()
+ if href and href not in seen_links and not href.startswith(("#", "javascript:")):
+ seen_links.add(href)
+ links.append(href)
+
+ # 4. Text ------------------------------------------------------------
+ # get_text with a separator that lets us collapse later.
+ raw_text = soup.get_text(separator="\n")
+ text = _normalise_whitespace(raw_text)
+
+ return title, text, links
+
+
+# ---------------------------------------------------------------------------
+# Fallback: regex (no external deps)
+# ---------------------------------------------------------------------------
+
+
+def _parse_fallback(raw_html: str) -> Tuple[str, str, List[str]]:
+ """Best-effort extraction when BeautifulSoup is unavailable."""
+ # Remove unwanted blocks
+ for tag_name in _REMOVE_TAGS:
+ raw_html = re.sub(
+ rf"<{tag_name}[^>]*>.*?</{tag_name}>",
+ "",
+ raw_html,
+ flags=re.IGNORECASE | re.DOTALL,
+ )
+ # Remove all remaining HTML tags
+ title_match = re.search(r"<title[^>]*>(.*?)", raw_html, re.IGNORECASE | re.DOTALL)
+ title = title_match.group(1).strip() if title_match else ""
+
+ # Links
+ links = list(dict.fromkeys(
+ href
+ for href in re.findall(r']+href=["\']([^"\']+)["\']', raw_html, re.IGNORECASE)
+ if not href.startswith(("#", "javascript:"))
+ ))
+
+ text = re.sub(r"<[^>]+>", " ", raw_html)
+ text = _normalise_whitespace(text)
+
+ return title, text, links
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _normalise_whitespace(text: str) -> str:
+ """Collapse runs of blank lines and trim each line."""
+ lines = [line.strip() for line in text.splitlines()]
+ result: List[str] = []
+ blank_count = 0
+ for line in lines:
+ if not line:
+ blank_count += 1
+ if blank_count <= _MAX_BLANK_LINES:
+ result.append("")
+ else:
+ blank_count = 0
+ result.append(line)
+ return "\n".join(result).strip()
diff --git a/agent/document_processing/normalizer.py b/agent/document_processing/normalizer.py
new file mode 100644
index 000000000000..c5bab275f778
--- /dev/null
+++ b/agent/document_processing/normalizer.py
@@ -0,0 +1,33 @@
+"""Normalizer — wraps parser output into a canonical :class:`DocumentResult`."""
+
+from __future__ import annotations
+
+from agent.document_processing.types import DocumentResult
+
+
+def normalise(
+ *,
+ source_type: str,
+ document_type: str,
+ title: str = "",
+ text: str = "",
+ links: list[str] | None = None,
+ filename: str = "",
+ url: str = "",
+ mime_type: str = "",
+ size: int = 0,
+) -> DocumentResult:
+ """Build a :class:`DocumentResult` with a validated *metadata* dict."""
+ return DocumentResult(
+ source_type=source_type,
+ document_type=document_type,
+ title=title,
+ text=text,
+ links=links or [],
+ metadata={
+ "filename": filename,
+ "url": url,
+ "mime_type": mime_type,
+ "size": size,
+ },
+ )
diff --git a/agent/document_processing/router.py b/agent/document_processing/router.py
new file mode 100644
index 000000000000..7b4e72c9e79c
--- /dev/null
+++ b/agent/document_processing/router.py
@@ -0,0 +1,143 @@
+"""Router — single entry-point for the Document Intelligence Layer.
+
+Gateway platforms call :func:`process_document` (for uploaded files) or
+:func:`process_url` (for URLs detected in messages). The router picks the
+right parser, normalises the output, and returns a :class:`DocumentResult`.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+from agent.document_processing.html_parser import parse_html
+from agent.document_processing.normalizer import normalise
+from agent.document_processing.types import DocumentResult
+from agent.document_processing.url_fetcher import FetchError, fetch_url
+
+logger = logging.getLogger(__name__)
+
+# Extensions that can be decoded as plain text and injected directly.
+_PLAINTEXT_EXTENSIONS = {".txt", ".md", ".log", ".ini", ".cfg", ".csv", ".json", ".xml", ".yaml", ".yml", ".toml"}
+
+# Extensions handled by the HTML parser.
+_HTML_EXTENSIONS = {".html", ".htm"}
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def process_document(
+ raw_bytes: bytes,
+ *,
+ filename: str = "",
+ ext: str = "",
+ mime_type: str = "",
+ source_type: str = "telegram_file",
+) -> DocumentResult:
+ """Parse an uploaded file and return a normalised :class:`DocumentResult`.
+
+ Parameters
+ ----------
+ raw_bytes:
+ The raw file content.
+ filename:
+ Original filename (e.g. ``"api-docs.html"``).
+ ext:
+ Lowercase extension including the dot (e.g. ``".html"``).
+ mime_type:
+ MIME type reported by the platform (informational).
+ source_type:
+ One of ``"telegram_file"``, ``"local_file"``, etc.
+ """
+ ext = ext.lower() if ext else ""
+
+ if ext in _HTML_EXTENSIONS:
+ try:
+ html_text = raw_bytes.decode("utf-8", errors="replace")
+ except Exception:
+ html_text = raw_bytes.decode("latin-1")
+ title, text, links = parse_html(html_text)
+ return normalise(
+ source_type=source_type,
+ document_type="html",
+ title=title,
+ text=text,
+ links=links,
+ filename=filename,
+ mime_type=mime_type,
+ size=len(raw_bytes),
+ )
+
+ if ext in _PLAINTEXT_EXTENSIONS:
+ try:
+ text = raw_bytes.decode("utf-8")
+ except UnicodeDecodeError:
+ text = raw_bytes.decode("latin-1")
+ doc_type = ext.lstrip(".")
+ # Normalise some extensions to canonical type names
+ type_map = {
+ "yml": "yaml",
+ "log": "txt",
+ "ini": "txt",
+ "cfg": "txt",
+ }
+ doc_type = type_map.get(doc_type, doc_type)
+ return normalise(
+ source_type=source_type,
+ document_type=doc_type,
+ text=text,
+ filename=filename,
+ mime_type=mime_type,
+ size=len(raw_bytes),
+ )
+
+ # Fallback — unsupported (PDF, DOCX, etc. can be added later)
+ return normalise(
+ source_type=source_type,
+ document_type=ext.lstrip(".") or "unknown",
+ text=f"[Document received: {filename or 'unnamed'} ({ext or 'unknown type'}). "
+ f"Automatic text extraction for this format is not yet supported. "
+ f"The file has been cached for manual inspection.]",
+ filename=filename,
+ mime_type=mime_type,
+ size=len(raw_bytes),
+ )
+
+
+def process_url(url: str, *, source_type: str = "url") -> DocumentResult:
+ """Fetch a URL and return a normalised :class:`DocumentResult`.
+
+ Raises nothing — errors are captured into the result text.
+ """
+ try:
+ html_text = fetch_url(url)
+ except FetchError as exc:
+ logger.warning("URL fetch failed for %s: %s", url, exc)
+ return normalise(
+ source_type=source_type,
+ document_type="html",
+ text=f"[Failed to fetch URL: {exc}]",
+ url=url,
+ )
+ except Exception as exc:
+ logger.warning("Unexpected error fetching %s: %s", url, exc, exc_info=True)
+ return normalise(
+ source_type=source_type,
+ document_type="html",
+ text=f"[Failed to fetch URL: {exc}]",
+ url=url,
+ )
+
+ title, text, links = parse_html(html_text)
+ return normalise(
+ source_type=source_type,
+ document_type="html",
+ title=title,
+ text=text,
+ links=links,
+ url=url,
+ size=len(html_text.encode("utf-8", errors="replace")),
+ )
diff --git a/agent/document_processing/types.py b/agent/document_processing/types.py
new file mode 100644
index 000000000000..1be7928a52d9
--- /dev/null
+++ b/agent/document_processing/types.py
@@ -0,0 +1,51 @@
+"""Canonical types for the Document Intelligence Layer."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional
+
+
+@dataclass
+class DocumentResult:
+ """Standardised output produced by every parser in the pipeline.
+
+ All fields follow the schema specified in the design doc (section C).
+ """
+
+ source_type: str # "telegram_file" | "url" | "local_file"
+ document_type: str # "html" | "pdf" | "docx" | "txt" | "md" | "json" | "csv"
+ title: str = ""
+ text: str = ""
+ links: List[str] = field(default_factory=list)
+ metadata: Dict[str, object] = field(default_factory=dict)
+
+ # Convenience helpers --------------------------------------------------
+
+ def to_dict(self) -> dict:
+ return {
+ "source_type": self.source_type,
+ "document_type": self.document_type,
+ "title": self.title,
+ "text": self.text,
+ "links": self.links,
+ "metadata": self.metadata,
+ }
+
+ def to_injection_text(self, max_chars: int = 100_000) -> str:
+ """Return a compact text representation for injecting into the LLM context."""
+ parts: List[str] = []
+ source_label = self.metadata.get("filename") or self.metadata.get("url") or self.source_type
+ parts.append(f"[Document: {source_label} ({self.document_type})]")
+ if self.title:
+ parts.append(f"Title: {self.title}")
+ if self.text:
+ text = self.text[:max_chars]
+ if len(self.text) > max_chars:
+ text += f"\n… (truncated, {len(self.text):,} chars total)"
+ parts.append(text)
+ if self.links:
+ parts.append(f"\nLinks ({len(self.links)}):")
+ for link in self.links[:50]: # cap at 50 links
+ parts.append(f" - {link}")
+ return "\n".join(parts)
diff --git a/agent/document_processing/url_fetcher.py b/agent/document_processing/url_fetcher.py
new file mode 100644
index 000000000000..397070aac927
--- /dev/null
+++ b/agent/document_processing/url_fetcher.py
@@ -0,0 +1,141 @@
+"""URL fetcher — download web pages with SSRF protection.
+
+Security constraints:
+* Timeout: 10 seconds
+* Max HTML size: 5 MB
+* Blocked: localhost, 127.0.0.0/8, 0.0.0.0, ::1, and all RFC-1918 private
+ ranges (10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16).
+* Only http:// and https:// schemes accepted.
+"""
+
+from __future__ import annotations
+
+import ipaddress
+import logging
+import socket
+from typing import Optional
+from urllib.parse import urlparse
+
+import requests
+
+logger = logging.getLogger(__name__)
+
+# ---------------------------------------------------------------------------
+# Constants
+# ---------------------------------------------------------------------------
+FETCH_TIMEOUT_SECONDS = 10
+MAX_CONTENT_BYTES = 5 * 1024 * 1024 # 5 MB
+
+_BLOCKED_NETWORKS = [
+ ipaddress.ip_network("127.0.0.0/8"),
+ ipaddress.ip_network("10.0.0.0/8"),
+ ipaddress.ip_network("172.16.0.0/12"),
+ ipaddress.ip_network("192.168.0.0/16"),
+ ipaddress.ip_network("0.0.0.0/8"),
+ ipaddress.ip_network("169.254.0.0/16"), # link-local
+ ipaddress.ip_network("::1/128"),
+ ipaddress.ip_network("fc00::/7"), # unique-local
+ ipaddress.ip_network("fe80::/10"), # link-local v6
+]
+
+_USER_AGENT = (
+ "Mozilla/5.0 (compatible; HermesAgent/1.0; +https://github.com/NousResearch/hermes-agent)"
+)
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+class FetchError(Exception):
+ """Raised when a URL cannot be fetched for a known reason."""
+
+
+def fetch_url(url: str) -> str:
+ """Fetch *url* and return the response body as a string.
+
+ Raises :class:`FetchError` with a human-readable message on failure.
+ """
+ # 1. Validate scheme --------------------------------------------------
+ parsed = urlparse(url)
+ if parsed.scheme not in ("http", "https"):
+ raise FetchError(f"Unsupported URL scheme '{parsed.scheme}'. Only http:// and https:// are allowed.")
+
+ hostname = parsed.hostname
+ if not hostname:
+ raise FetchError(f"Invalid URL: could not determine hostname from '{url}'.")
+
+ # 2. SSRF protection — resolve hostname and check against blocklist ---
+ _check_ssrf(hostname)
+
+ # 3. Fetch ------------------------------------------------------------
+ try:
+ resp = requests.get(
+ url,
+ timeout=FETCH_TIMEOUT_SECONDS,
+ headers={"User-Agent": _USER_AGENT},
+ allow_redirects=True,
+ stream=True,
+ )
+ resp.raise_for_status()
+
+ # Enforce size limit while streaming
+ chunks: list[bytes] = []
+ total = 0
+ for chunk in resp.iter_content(chunk_size=64 * 1024):
+ total += len(chunk)
+ if total > MAX_CONTENT_BYTES:
+ resp.close()
+ raise FetchError(
+ f"Response exceeded maximum size ({MAX_CONTENT_BYTES // (1024 * 1024)} MB). "
+ "The document is too large to process."
+ )
+ chunks.append(chunk)
+
+ raw_bytes = b"".join(chunks)
+
+ # Attempt charset detection from the Content-Type header
+ encoding = resp.encoding or "utf-8"
+ try:
+ return raw_bytes.decode(encoding)
+ except (UnicodeDecodeError, LookupError):
+ return raw_bytes.decode("utf-8", errors="replace")
+
+ except FetchError:
+ raise
+ except requests.exceptions.Timeout:
+ raise FetchError(f"Request to '{url}' timed out after {FETCH_TIMEOUT_SECONDS}s.")
+ except requests.exceptions.ConnectionError as exc:
+ raise FetchError(f"Could not connect to '{url}': {exc}")
+ except requests.exceptions.HTTPError as exc:
+ raise FetchError(f"HTTP error fetching '{url}': {exc}")
+ except requests.exceptions.RequestException as exc:
+ raise FetchError(f"Failed to fetch '{url}': {exc}")
+
+
+# ---------------------------------------------------------------------------
+# SSRF protection
+# ---------------------------------------------------------------------------
+
+
+def _check_ssrf(hostname: str) -> None:
+ """Resolve *hostname* and raise :class:`FetchError` if it points at a
+ private/loopback address."""
+ try:
+ infos = socket.getaddrinfo(hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM)
+ except socket.gaierror:
+ raise FetchError(f"Could not resolve hostname '{hostname}'.")
+
+ for family, _type, _proto, _canonname, sockaddr in infos:
+ ip_str = sockaddr[0]
+ try:
+ addr = ipaddress.ip_address(ip_str)
+ except ValueError:
+ continue
+ for network in _BLOCKED_NETWORKS:
+ if addr in network:
+ raise FetchError(
+ f"Access to '{hostname}' ({ip_str}) is blocked — "
+ "internal/private network addresses are not allowed."
+ )
diff --git a/gateway/platforms/base.py b/gateway/platforms/base.py
index 78e0dd7e25c3..ee1273e2528f 100644
--- a/gateway/platforms/base.py
+++ b/gateway/platforms/base.py
@@ -774,6 +774,8 @@ def cache_video_from_bytes(data: bytes, ext: str = ".mp4") -> str:
".log": "text/plain",
".json": "application/json",
".xml": "application/xml",
+ ".html": "text/html",
+ ".htm": "text/html",
".yaml": "application/yaml",
".yml": "application/yaml",
".toml": "application/toml",
diff --git a/gateway/platforms/telegram.py b/gateway/platforms/telegram.py
index 188038a1adbf..944062e93101 100644
--- a/gateway/platforms/telegram.py
+++ b/gateway/platforms/telegram.py
@@ -3229,23 +3229,49 @@ async def _handle_media_message(self, update: Update, context: ContextTypes.DEFA
event.media_types = [mime_type]
logger.info("[Telegram] Cached user document at %s", cached_path)
- # For text files, inject content into event.text (capped at 100 KB)
+ # --- Document Intelligence Layer ---
+ # Delegate parsing to agent/document_processing instead of
+ # handling it inline. The router returns a DocumentResult
+ # whose .to_injection_text() gives a clean LLM-ready string.
MAX_TEXT_INJECT_BYTES = 100 * 1024
- if ext in (".md", ".txt") and len(raw_bytes) <= MAX_TEXT_INJECT_BYTES:
+ if len(raw_bytes) <= MAX_TEXT_INJECT_BYTES:
try:
- text_content = raw_bytes.decode("utf-8")
- display_name = original_filename or f"document{ext}"
- display_name = re.sub(r'[^\w.\- ]', '_', display_name)
- injection = f"[Content of {display_name}]:\n{text_content}"
- if event.text:
- event.text = f"{injection}\n\n{event.text}"
- else:
- event.text = injection
- except UnicodeDecodeError:
+ from agent.document_processing.router import process_document as _process_doc
+
+ doc_result = _process_doc(
+ raw_bytes,
+ filename=original_filename,
+ ext=ext,
+ mime_type=mime_type,
+ source_type="telegram_file",
+ )
+ injection = doc_result.to_injection_text()
+ if injection:
+ if event.text:
+ event.text = f"{injection}\n\n{event.text}"
+ else:
+ event.text = injection
+ except Exception:
logger.warning(
- "[Telegram] Could not decode text file as UTF-8, skipping content injection",
+ "[Telegram] document_processing failed, falling back to raw text injection",
exc_info=True,
)
+ # Fallback: try raw UTF-8 decode for text-ish files
+ if ext in (".md", ".txt", ".html", ".htm", ".csv", ".json", ".xml", ".yaml", ".yml", ".toml", ".log", ".ini", ".cfg"):
+ try:
+ text_content = raw_bytes.decode("utf-8")
+ display_name = original_filename or f"document{ext}"
+ display_name = re.sub(r'[^\w.\- ]', '_', display_name)
+ fallback_injection = f"[Content of {display_name}]:\n{text_content}"
+ if event.text:
+ event.text = f"{fallback_injection}\n\n{event.text}"
+ else:
+ event.text = fallback_injection
+ except UnicodeDecodeError:
+ logger.warning(
+ "[Telegram] Could not decode document as UTF-8",
+ exc_info=True,
+ )
except Exception as e:
logger.warning("[Telegram] Failed to cache document: %s", e, exc_info=True)
diff --git a/pyproject.toml b/pyproject.toml
index a58e172795e6..ed09593e89ca 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -21,6 +21,7 @@ dependencies = [
"tenacity>=9.1.4,<10",
"pyyaml>=6.0.2,<7",
"requests>=2.33.0,<3", # CVE-2026-25645
+ "beautifulsoup4>=4.13.0,<5", # Document Intelligence Layer — HTML parsing
"jinja2>=3.1.5,<4",
"pydantic>=2.12.5,<3",
# Interactive CLI (prompt_toolkit is used directly by cli.py)
diff --git a/tests/agent/test_document_processing.py b/tests/agent/test_document_processing.py
new file mode 100644
index 000000000000..cb327a42301d
--- /dev/null
+++ b/tests/agent/test_document_processing.py
@@ -0,0 +1,241 @@
+"""Acceptance tests for the Document Intelligence Layer.
+
+Tests cover the 7 scenarios from the design doc (section J):
+1. Upload test.html → Hermes reads body text
+2. Upload test.htm → Hermes reads body text
+3. Process URL https://example.com → reads title & body
+4. Upload unsupported file → clear error message
+5. localhost URL → rejected by SSRF protection
+6. Oversized HTML → rejected
+7. script/style tags stripped from HTML
+"""
+
+from __future__ import annotations
+
+import json
+import pytest
+
+from agent.document_processing.html_parser import parse_html
+from agent.document_processing.normalizer import normalise
+from agent.document_processing.router import process_document, process_url
+from agent.document_processing.types import DocumentResult
+from agent.document_processing.url_fetcher import FetchError, _check_ssrf, MAX_CONTENT_BYTES
+
+
+# ---------------------------------------------------------------------------
+# Fixtures
+# ---------------------------------------------------------------------------
+
+SAMPLE_HTML = """\
+
+
+
+ Royal Pay API Documentation
+
+
+
+
+
+ Royal Pay — USDT Payment Gateway
+ Welcome to the Royal Pay API docs.
+ Base URL: https://api.royalpay.vip/v1
+ Authentication: Auth endpoint
+
+ Jump to section 2
+ Do nothing
+
+
+"""
+
+SAMPLE_HTML_BYTES = SAMPLE_HTML.encode("utf-8")
+
+
+# ---------------------------------------------------------------------------
+# Test 1: Upload .html — reads body text
+# ---------------------------------------------------------------------------
+
+class TestUploadHtml:
+ def test_html_body_extracted(self):
+ result = process_document(SAMPLE_HTML_BYTES, filename="api-docs.html", ext=".html")
+ assert isinstance(result, DocumentResult)
+ assert result.document_type == "html"
+ assert result.source_type == "telegram_file"
+ assert "Royal Pay" in result.title
+ assert "USDT Payment Gateway" in result.text
+ assert "Welcome to the Royal Pay API docs" in result.text
+ assert result.metadata["filename"] == "api-docs.html"
+
+ def test_html_links_extracted(self):
+ result = process_document(SAMPLE_HTML_BYTES, filename="api-docs.html", ext=".html")
+ assert "https://api.royalpay.vip/v1" in result.links
+ assert "https://api.royalpay.vip/v1/auth" in result.links
+ # Fragment and javascript links should be excluded
+ assert "#section2" not in result.links
+ assert "javascript:void(0)" not in result.links
+
+
+# ---------------------------------------------------------------------------
+# Test 2: Upload .htm — reads body text (same parser)
+# ---------------------------------------------------------------------------
+
+class TestUploadHtm:
+ def test_htm_body_extracted(self):
+ result = process_document(SAMPLE_HTML_BYTES, filename="docs.htm", ext=".htm")
+ assert result.document_type == "html"
+ assert "Royal Pay" in result.title
+ assert "USDT Payment Gateway" in result.text
+
+
+# ---------------------------------------------------------------------------
+# Test 3: Process URL — reads title & body
+# ---------------------------------------------------------------------------
+
+class TestProcessUrl:
+ def test_example_com(self):
+ """Fetch https://example.com and verify title + body text."""
+ result = process_url("https://example.com")
+ assert result.source_type == "url"
+ assert result.document_type == "html"
+ # example.com has a well-known title
+ assert "example" in result.title.lower() or "example" in result.text.lower()
+ assert result.metadata["url"] == "https://example.com"
+
+
+# ---------------------------------------------------------------------------
+# Test 4: Unsupported file → clear error
+# ---------------------------------------------------------------------------
+
+class TestUnsupportedFile:
+ def test_unsupported_extension(self):
+ result = process_document(b"\x00\x01\x02", filename="data.bin", ext=".bin")
+ assert "not yet supported" in result.text.lower() or "unsupported" in result.text.lower() or "received" in result.text.lower()
+ assert result.document_type == "bin"
+
+
+# ---------------------------------------------------------------------------
+# Test 5: localhost URL → SSRF rejection
+# ---------------------------------------------------------------------------
+
+class TestSsrfProtection:
+ def test_localhost_blocked(self):
+ with pytest.raises(FetchError, match="blocked"):
+ _check_ssrf("localhost")
+
+ def test_127_0_0_1_blocked(self):
+ with pytest.raises(FetchError, match="blocked"):
+ _check_ssrf("127.0.0.1")
+
+ def test_private_10_blocked(self):
+ with pytest.raises(FetchError, match="blocked"):
+ _check_ssrf("10.0.0.1")
+
+ def test_private_192_168_blocked(self):
+ with pytest.raises(FetchError, match="blocked"):
+ _check_ssrf("192.168.1.1")
+
+ def test_process_url_localhost_safe(self):
+ result = process_url("http://127.0.0.1/secret")
+ assert "blocked" in result.text.lower() or "failed" in result.text.lower()
+
+
+# ---------------------------------------------------------------------------
+# Test 6: Oversized HTML → rejected
+# ---------------------------------------------------------------------------
+
+class TestOversizedHtml:
+ def test_oversize_detection(self):
+ """A >5 MB HTML file should not be processed by the URL fetcher."""
+ # We test the constant directly — the streaming check happens in
+ # url_fetcher.fetch_url which we can't easily unit-test without a
+ # real server. But we verify the limit is correctly defined.
+ assert MAX_CONTENT_BYTES == 5 * 1024 * 1024
+
+ def test_large_html_document_still_parseable(self):
+ """process_document should still work for large files (it's the URL
+ fetcher that enforces the 5 MB limit, not the document processor)."""
+ big_html = b"Big" + b"x
" * 100_000 + b""
+ result = process_document(big_html, filename="big.html", ext=".html")
+ assert result.title == "Big"
+ assert "x" in result.text
+
+
+# ---------------------------------------------------------------------------
+# Test 7: script/style tags stripped
+# ---------------------------------------------------------------------------
+
+class TestScriptStyleRemoval:
+ def test_script_content_removed(self):
+ result = process_document(SAMPLE_HTML_BYTES, filename="test.html", ext=".html")
+ assert "console.log" not in result.text
+ assert "tracking" not in result.text
+ assert "analytics.track" not in result.text
+ assert "window.analytics" not in result.text
+
+ def test_style_content_removed(self):
+ result = process_document(SAMPLE_HTML_BYTES, filename="test.html", ext=".html")
+ assert "font-family" not in result.text
+ assert "sans-serif" not in result.text
+
+ def test_noscript_removed(self):
+ result = process_document(SAMPLE_HTML_BYTES, filename="test.html", ext=".html")
+ assert "Enable JavaScript" not in result.text
+
+
+# ---------------------------------------------------------------------------
+# Additional: DocumentResult serialization
+# ---------------------------------------------------------------------------
+
+class TestDocumentResult:
+ def test_to_dict(self):
+ r = normalise(
+ source_type="telegram_file",
+ document_type="html",
+ title="Test",
+ text="Hello",
+ filename="test.html",
+ )
+ d = r.to_dict()
+ assert d["source_type"] == "telegram_file"
+ assert d["document_type"] == "html"
+ assert d["title"] == "Test"
+ assert d["metadata"]["filename"] == "test.html"
+
+ def test_to_injection_text(self):
+ r = normalise(
+ source_type="telegram_file",
+ document_type="html",
+ title="Royal Pay API",
+ text="Some content here",
+ filename="api.html",
+ )
+ injection = r.to_injection_text()
+ assert "api.html" in injection
+ assert "Royal Pay API" in injection
+ assert "Some content here" in injection
+
+
+# ---------------------------------------------------------------------------
+# Additional: plaintext file processing
+# ---------------------------------------------------------------------------
+
+class TestPlaintextFiles:
+ def test_txt_file(self):
+ result = process_document(b"Hello World", filename="readme.txt", ext=".txt")
+ assert result.document_type == "txt"
+ assert result.text == "Hello World"
+
+ def test_md_file(self):
+ result = process_document(b"# Title\n\nContent", filename="readme.md", ext=".md")
+ assert result.document_type == "md"
+ assert "# Title" in result.text
+
+ def test_json_file(self):
+ content = json.dumps({"key": "value"}).encode()
+ result = process_document(content, filename="data.json", ext=".json")
+ assert result.document_type == "json"
+ assert "key" in result.text
+
+ def test_csv_file(self):
+ result = process_document(b"name,age\nAlice,30", filename="data.csv", ext=".csv")
+ assert result.document_type == "csv"
+ assert "Alice" in result.text