feat: Document Intelligence Layer — HTML/HTM file support + unified document pipeline

agent/document_processing/__init__.py

@@ -0,0 +1,9 @@
+"""Document Intelligence Layer — unified document normalization pipeline.
+
+All document understanding goes through this module. Gateway platforms
+(Telegram, Discord, etc.) only handle *receiving* files; parsing and
+normalization happen here so the logic is reusable across every platform.
+"""
+
+from agent.document_processing.router import process_document  # noqa: F401
+from agent.document_processing.types import DocumentResult  # noqa: F401

agent/document_processing/html_parser.py

@@ -0,0 +1,137 @@
+"""HTML parser — extracts clean text, title, and links from HTML content.
+
+Uses BeautifulSoup with the stdlib html.parser backend so there is no
+hard dependency on lxml.  If beautifulsoup4 is missing at runtime the
+parser falls back to a regex-based lightweight extractor.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import List, Optional, Tuple
+
+# ---------------------------------------------------------------------------
+# Try importing BeautifulSoup; provide a graceful fallback.
+# ---------------------------------------------------------------------------
+try:
+    from bs4 import BeautifulSoup, Comment  # type: ignore[import-untyped]
+
+    _HAS_BS4 = True
+except ImportError:
+    _HAS_BS4 = False
+
+# Tags whose *content* should be removed entirely (not just the tag).
+_REMOVE_TAGS = {"script", "style", "noscript", "svg", "canvas", "template", "iframe"}
+
+# Maximum whitespace‐collapsed gap between blocks.
+_MAX_BLANK_LINES = 2
+
+
+def parse_html(raw_html: str) -> Tuple[str, str, List[str]]:
+    """Parse *raw_html* and return ``(title, text, links)``.
+
+    * ``title`` — content of ``<title>`` or first ``<h1>``, empty string if
+      neither exists.
+    * ``text`` — human-readable text with paragraph breaks preserved.
+    * ``links`` — deduplicated list of ``href`` values from ``<a>`` tags.
+    """
+    if _HAS_BS4:
+        return _parse_with_bs4(raw_html)
+    return _parse_fallback(raw_html)
+
+
+# ---------------------------------------------------------------------------
+# Primary: BeautifulSoup
+# ---------------------------------------------------------------------------
+
+
+def _parse_with_bs4(raw_html: str) -> Tuple[str, str, List[str]]:
+    soup = BeautifulSoup(raw_html, "html.parser")
+
+    # 1. Remove unwanted elements ----------------------------------------
+    for tag_name in _REMOVE_TAGS:
+        for tag in soup.find_all(tag_name):
+            tag.decompose()
+
+    # Remove HTML comments
+    for comment in soup.find_all(string=lambda t: isinstance(t, Comment)):
+        comment.extract()
+
+    # 2. Title -----------------------------------------------------------
+    title = ""
+    title_tag = soup.find("title")
+    if title_tag:
+        title = title_tag.get_text(strip=True)
+    if not title:
+        h1 = soup.find("h1")
+        if h1:
+            title = h1.get_text(strip=True)
+
+    # 3. Links -----------------------------------------------------------
+    seen_links: set[str] = set()
+    links: List[str] = []
+    for a_tag in soup.find_all("a", href=True):
+        href = a_tag["href"].strip()
+        if href and href not in seen_links and not href.startswith(("#", "javascript:")):
+            seen_links.add(href)
+            links.append(href)
+
+    # 4. Text ------------------------------------------------------------
+    # get_text with a separator that lets us collapse later.
+    raw_text = soup.get_text(separator="\n")
+    text = _normalise_whitespace(raw_text)
+
+    return title, text, links
+
+
+# ---------------------------------------------------------------------------
+# Fallback: regex (no external deps)
+# ---------------------------------------------------------------------------
+
+
+def _parse_fallback(raw_html: str) -> Tuple[str, str, List[str]]:
+    """Best-effort extraction when BeautifulSoup is unavailable."""
+    # Remove unwanted blocks
+    for tag_name in _REMOVE_TAGS:
+        raw_html = re.sub(
+            rf"<{tag_name}[^>]*>.*?</{tag_name}>",
+            "",
+            raw_html,
+            flags=re.IGNORECASE | re.DOTALL,
+        )
+    # Remove all remaining HTML tags
+    title_match = re.search(r"<title[^>]*>(.*?)</title>", raw_html, re.IGNORECASE | re.DOTALL)
+    title = title_match.group(1).strip() if title_match else ""
+
+    # Links
+    links = list(dict.fromkeys(
+        href
+        for href in re.findall(r'<a[^>]+href=["\']([^"\']+)["\']', raw_html, re.IGNORECASE)
+        if not href.startswith(("#", "javascript:"))
+    ))
+
+    text = re.sub(r"<[^>]+>", " ", raw_html)
+    text = _normalise_whitespace(text)
+
+    return title, text, links
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+
+def _normalise_whitespace(text: str) -> str:
+    """Collapse runs of blank lines and trim each line."""
+    lines = [line.strip() for line in text.splitlines()]
+    result: List[str] = []
+    blank_count = 0
+    for line in lines:
+        if not line:
+            blank_count += 1
+            if blank_count <= _MAX_BLANK_LINES:
+                result.append("")
+        else:
+            blank_count = 0
+            result.append(line)
+    return "\n".join(result).strip()

agent/document_processing/normalizer.py

@@ -0,0 +1,33 @@
+"""Normalizer — wraps parser output into a canonical :class:`DocumentResult`."""
+
+from __future__ import annotations
+
+from agent.document_processing.types import DocumentResult
+
+
+def normalise(
+    *,
+    source_type: str,
+    document_type: str,
+    title: str = "",
+    text: str = "",
+    links: list[str] | None = None,
+    filename: str = "",
+    url: str = "",
+    mime_type: str = "",
+    size: int = 0,
+) -> DocumentResult:
+    """Build a :class:`DocumentResult` with a validated *metadata* dict."""
+    return DocumentResult(
+        source_type=source_type,
+        document_type=document_type,
+        title=title,
+        text=text,
+        links=links or [],
+        metadata={
+            "filename": filename,
+            "url": url,
+            "mime_type": mime_type,
+            "size": size,
+        },
+    )

agent/document_processing/router.py

@@ -0,0 +1,143 @@
+"""Router — single entry-point for the Document Intelligence Layer.
+
+Gateway platforms call :func:`process_document` (for uploaded files) or
+:func:`process_url` (for URLs detected in messages).  The router picks the
+right parser, normalises the output, and returns a :class:`DocumentResult`.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import Optional
+
+from agent.document_processing.html_parser import parse_html
+from agent.document_processing.normalizer import normalise
+from agent.document_processing.types import DocumentResult
+from agent.document_processing.url_fetcher import FetchError, fetch_url
+
+logger = logging.getLogger(__name__)
+
+# Extensions that can be decoded as plain text and injected directly.
+_PLAINTEXT_EXTENSIONS = {".txt", ".md", ".log", ".ini", ".cfg", ".csv", ".json", ".xml", ".yaml", ".yml", ".toml"}
+
+# Extensions handled by the HTML parser.
+_HTML_EXTENSIONS = {".html", ".htm"}
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def process_document(
+    raw_bytes: bytes,
+    *,
+    filename: str = "",
+    ext: str = "",
+    mime_type: str = "",
+    source_type: str = "telegram_file",
+) -> DocumentResult:
+    """Parse an uploaded file and return a normalised :class:`DocumentResult`.
+
+    Parameters
+    ----------
+    raw_bytes:
+        The raw file content.
+    filename:
+        Original filename (e.g. ``"api-docs.html"``).
+    ext:
+        Lowercase extension including the dot (e.g. ``".html"``).
+    mime_type:
+        MIME type reported by the platform (informational).
+    source_type:
+        One of ``"telegram_file"``, ``"local_file"``, etc.
+    """
+    ext = ext.lower() if ext else ""
+
+    if ext in _HTML_EXTENSIONS:
+        try:
+            html_text = raw_bytes.decode("utf-8", errors="replace")
+        except Exception:
+            html_text = raw_bytes.decode("latin-1")
+        title, text, links = parse_html(html_text)
+        return normalise(
+            source_type=source_type,
+            document_type="html",
+            title=title,
+            text=text,
+            links=links,
+            filename=filename,
+            mime_type=mime_type,
+            size=len(raw_bytes),
+        )
+
+    if ext in _PLAINTEXT_EXTENSIONS:
+        try:
+            text = raw_bytes.decode("utf-8")
+        except UnicodeDecodeError:
+            text = raw_bytes.decode("latin-1")
+        doc_type = ext.lstrip(".")
+        # Normalise some extensions to canonical type names
+        type_map = {
+            "yml": "yaml",
+            "log": "txt",
+            "ini": "txt",
+            "cfg": "txt",
+        }
+        doc_type = type_map.get(doc_type, doc_type)
+        return normalise(
+            source_type=source_type,
+            document_type=doc_type,
+            text=text,
+            filename=filename,
+            mime_type=mime_type,
+            size=len(raw_bytes),
+        )
+
+    # Fallback — unsupported (PDF, DOCX, etc. can be added later)
+    return normalise(
+        source_type=source_type,
+        document_type=ext.lstrip(".") or "unknown",
+        text=f"[Document received: {filename or 'unnamed'} ({ext or 'unknown type'}). "
+             f"Automatic text extraction for this format is not yet supported. "
+             f"The file has been cached for manual inspection.]",
+        filename=filename,
+        mime_type=mime_type,
+        size=len(raw_bytes),
+    )
+
+
+def process_url(url: str, *, source_type: str = "url") -> DocumentResult:
+    """Fetch a URL and return a normalised :class:`DocumentResult`.
+
+    Raises nothing — errors are captured into the result text.
+    """
+    try:
+        html_text = fetch_url(url)
+    except FetchError as exc:
+        logger.warning("URL fetch failed for %s: %s", url, exc)
+        return normalise(
+            source_type=source_type,
+            document_type="html",
+            text=f"[Failed to fetch URL: {exc}]",
+            url=url,
+        )
+    except Exception as exc:
+        logger.warning("Unexpected error fetching %s: %s", url, exc, exc_info=True)
+        return normalise(
+            source_type=source_type,
+            document_type="html",
+            text=f"[Failed to fetch URL: {exc}]",
+            url=url,
+        )
+
+    title, text, links = parse_html(html_text)
+    return normalise(
+        source_type=source_type,
+        document_type="html",
+        title=title,
+        text=text,
+        links=links,
+        url=url,
+        size=len(html_text.encode("utf-8", errors="replace")),
+    )

agent/document_processing/types.py

@@ -0,0 +1,51 @@
+"""Canonical types for the Document Intelligence Layer."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Dict, List, Optional
+
+
+@dataclass
+class DocumentResult:
+    """Standardised output produced by every parser in the pipeline.
+
+    All fields follow the schema specified in the design doc (section C).
+    """
+
+    source_type: str  # "telegram_file" | "url" | "local_file"
+    document_type: str  # "html" | "pdf" | "docx" | "txt" | "md" | "json" | "csv"
+    title: str = ""
+    text: str = ""
+    links: List[str] = field(default_factory=list)
+    metadata: Dict[str, object] = field(default_factory=dict)
+
+    # Convenience helpers --------------------------------------------------
+
+    def to_dict(self) -> dict:
+        return {
+            "source_type": self.source_type,
+            "document_type": self.document_type,
+            "title": self.title,
+            "text": self.text,
+            "links": self.links,
+            "metadata": self.metadata,
+        }
+
+    def to_injection_text(self, max_chars: int = 100_000) -> str:
+        """Return a compact text representation for injecting into the LLM context."""
+        parts: List[str] = []
+        source_label = self.metadata.get("filename") or self.metadata.get("url") or self.source_type
+        parts.append(f"[Document: {source_label} ({self.document_type})]")
+        if self.title:
+            parts.append(f"Title: {self.title}")
+        if self.text:
+            text = self.text[:max_chars]
+            if len(self.text) > max_chars:
+                text += f"\n… (truncated, {len(self.text):,} chars total)"
+            parts.append(text)
+        if self.links:
+            parts.append(f"\nLinks ({len(self.links)}):")
+            for link in self.links[:50]:  # cap at 50 links
+                parts.append(f"  - {link}")
+        return "\n".join(parts)