fix: support large Telegram documents

gateway/platforms/telegram.py

@@ -11,6 +11,7 @@
 import json
 import logging
 import os
+import shutil
 import tempfile
 import html as _html
 import re
@@ -73,6 +74,7 @@ class _MockContextTypes:
     cache_audio_from_bytes,
     cache_video_from_bytes,
     cache_document_from_bytes,
+    get_document_cache_dir,
     resolve_proxy_url,
     SUPPORTED_VIDEO_TYPES,
     SUPPORTED_DOCUMENT_TYPES,
@@ -4127,15 +4129,43 @@ async def _handle_media_message(self, update: Update, context: ContextTypes.DEFA
                         ext = mime_to_ext.get(doc_mime, "")
 
                 # Check file size early so image documents cannot bypass the
-                # document size limit by taking the image path.
-                MAX_DOC_BYTES = 20 * 1024 * 1024
-                if not doc.file_size or doc.file_size > MAX_DOC_BYTES:
-                    event.text = (
-                        "The document is too large or its size could not be verified. "
-                        "Maximum: 20 MB."
+                # document size limit by taking the image path. The limit is
+                # profile-configurable (platforms.telegram.extra.max_document_mb)
+                # because self-hosted/local Bot API deployments can safely handle
+                # files far above Telegram Cloud's 20 MB download limit.
+                max_document_mb_raw = self.config.extra.get("max_document_mb", 20)
+                try:
+                    max_document_mb = float(max_document_mb_raw)
+                except (TypeError, ValueError):
+                    logger.warning(
+                        "[Telegram] Invalid max_document_mb=%r; falling back to 20 MB",
+                        max_document_mb_raw,
+                    )
+                    max_document_mb = 20.0
+                max_doc_bytes = int(max_document_mb * 1024 * 1024)
+
+                if not doc.file_size:
+                    await self.send(
+                        event.source.chat_id,
+                        "Не смогла проверить размер документа, поэтому не скачиваю его. "
+                        "Перешлите PDF ещё раз; если повторится — пришлите файл меньшего размера.",
+                        reply_to=event.message_id,
+                    )
+                    logger.info("[Telegram] Document size could not be verified")
+                    return
+
+                if doc.file_size > max_doc_bytes:
+                    limit_label = f"{max_document_mb:g} MB"
+                    await self.send(
+                        event.source.chat_id,
+                        f"Документ слишком большой: {doc.file_size:,} bytes. Лимит этого бота: {limit_label}.",
+                        reply_to=event.message_id,
+                    )
+                    logger.info(
+                        "[Telegram] Document too large: %s bytes (limit=%s bytes)",
+                        doc.file_size,
+                        max_doc_bytes,
                     )
-                    logger.info("[Telegram] Document too large: %s bytes", doc.file_size)
-                    await self.handle_message(event)
                     return
 
                 # Telegram may deliver screenshots/photos as documents. If the
@@ -4195,20 +4225,59 @@ async def _handle_media_message(self, update: Update, context: ContextTypes.DEFA
                     await self.handle_message(event)
                     return
 
-                # Download and cache
-                file_obj = await doc.get_file()
-                doc_bytes = await file_obj.download_as_bytearray()
-                raw_bytes = bytes(doc_bytes)
-                cached_path = cache_document_from_bytes(raw_bytes, original_filename or f"document{ext}")
+                # Download/cache. With a self-hosted Bot API in local_mode,
+                # get_file() returns a server-local file_path; copying it is faster
+                # and avoids large in-memory bytearrays/timeouts for 90MB+ PDFs.
+                file_obj = await doc.get_file(
+                    read_timeout=300,
+                    write_timeout=300,
+                    connect_timeout=30,
+                    pool_timeout=30,
+                )
+                raw_bytes: Optional[bytes] = None
+                local_file_path = getattr(file_obj, "file_path", None)
+                if local_file_path and not str(local_file_path).startswith(("http://", "https://")):
+                    source_path = _Path(str(local_file_path))
+                    if source_path.is_file():
+                        cache_dir = get_document_cache_dir()
+                        safe_name = _Path(original_filename or f"document{ext}").name.replace("\x00", "").strip()
+                        if not safe_name or safe_name in {".", ".."}:
+                            safe_name = f"document{ext}"
+                        cached_path_obj = cache_dir / f"doc_{file_obj.file_unique_id}_{safe_name}"
+                        if not cached_path_obj.resolve().is_relative_to(cache_dir.resolve()):
+                            raise ValueError(f"Path traversal rejected: {original_filename!r}")
+                        shutil.copyfile(source_path, cached_path_obj)
+                        cached_path = str(cached_path_obj)
+                    else:
+                        logger.warning("[Telegram] Local Bot API file path is not readable: %s", local_file_path)
+                        doc_bytes = await file_obj.download_as_bytearray(
+                            read_timeout=300,
+                            write_timeout=300,
+                            connect_timeout=30,
+                            pool_timeout=30,
+                        )
+                        raw_bytes = bytes(doc_bytes)
+                        cached_path = cache_document_from_bytes(raw_bytes, original_filename or f"document{ext}")
+                else:
+                    doc_bytes = await file_obj.download_as_bytearray(
+                        read_timeout=300,
+                        write_timeout=300,
+                        connect_timeout=30,
+                        pool_timeout=30,
+                    )
+                    raw_bytes = bytes(doc_bytes)
+                    cached_path = cache_document_from_bytes(raw_bytes, original_filename or f"document{ext}")
                 mime_type = SUPPORTED_DOCUMENT_TYPES[ext]
                 event.media_urls = [cached_path]
                 event.media_types = [mime_type]
                 logger.info("[Telegram] Cached user document at %s", cached_path)
 
                 # For text files, inject content into event.text (capped at 100 KB)
                 MAX_TEXT_INJECT_BYTES = 100 * 1024
-                if ext in {".md", ".txt"} and len(raw_bytes) <= MAX_TEXT_INJECT_BYTES:
+                if ext in {".md", ".txt"} and doc.file_size <= MAX_TEXT_INJECT_BYTES:
                     try:
+                        if raw_bytes is None:
+                            raw_bytes = _Path(cached_path).read_bytes()
                         text_content = raw_bytes.decode("utf-8")
                         display_name = original_filename or f"document{ext}"
                         display_name = re.sub(r'[^\w.\- ]', '_', display_name)
@@ -4225,6 +4294,13 @@ async def _handle_media_message(self, update: Update, context: ContextTypes.DEFA
 
             except Exception as e:
                 logger.warning("[Telegram] Failed to cache document: %s", e, exc_info=True)
+                await self.send(
+                    event.source.chat_id,
+                    "Не смогла получить файл из Telegram. Файл не передан агенту, поэтому сжатие не запускала. "
+                    "Перешлите PDF ещё раз; если ошибка повторится — попробуем уменьшить лимит/размер или проверим Bot API.",
+                    reply_to=event.message_id,
+                )
+                return
 
         media_group_id = getattr(msg, "media_group_id", None)
         if media_group_id:

hermes_cli/gateway.py

@@ -2132,6 +2132,7 @@ def generate_systemd_unit(system: bool = False, run_as_user: str | None = None)
         username, group_name, home_dir = _system_service_identity(run_as_user)
         hermes_home = _hermes_home_for_target_user(home_dir)
         profile_arg = _profile_arg(hermes_home)
+        profile_env = f'Environment="HERMES_PROFILE={profile_arg.split(" ", 1)[1]}"\n' if profile_arg else ""
         # Remap all paths that may resolve under the calling user's home
         # (e.g. /root/) to the target user's home so the service can
         # actually access them.
@@ -2163,7 +2164,7 @@ def generate_systemd_unit(system: bool = False, run_as_user: str | None = None)
 Environment="PATH={sane_path}"
 Environment="VIRTUAL_ENV={venv_dir}"
 Environment="HERMES_HOME={hermes_home}"
-Restart=always
+{profile_env}Restart=always
 RestartSec=60
 RestartMaxDelaySec=300
 RestartSteps=5
@@ -2181,6 +2182,7 @@ def generate_systemd_unit(system: bool = False, run_as_user: str | None = None)
 
     hermes_home = str(get_hermes_home().resolve())
     profile_arg = _profile_arg(hermes_home)
+    profile_env = f'Environment="HERMES_PROFILE={profile_arg.split(" ", 1)[1]}"\n' if profile_arg else ""
     path_entries.extend(_build_user_local_paths(Path.home(), path_entries))
     path_entries.extend(_build_wsl_interop_paths(path_entries))
     path_entries.extend(common_bin_paths)
@@ -2198,7 +2200,7 @@ def generate_systemd_unit(system: bool = False, run_as_user: str | None = None)
 Environment="PATH={sane_path}"
 Environment="VIRTUAL_ENV={venv_dir}"
 Environment="HERMES_HOME={hermes_home}"
-Restart=always
+{profile_env}Restart=always
 RestartSec=60
 RestartMaxDelaySec=300
 RestartSteps=5

pyproject.toml

@@ -71,6 +71,9 @@ anthropic = ["anthropic==0.86.0"]
 exa = ["exa-py==2.10.2"]
 firecrawl = ["firecrawl-py==4.17.0"]
 parallel-web = ["parallel-web==0.4.2"]
+# Microsoft Office document extraction (.docx/.pptx/.xlsx) — optional so
+# restricted profiles can enable it without broad shell/code_execution access.
+office = ["python-docx==1.2.0", "python-pptx==1.0.2", "openpyxl==3.1.5"]
 # Image generation backends
 fal = ["fal-client==0.13.1"]
 # Edge TTS — default TTS provider but still optional (users can pick
@@ -192,6 +195,7 @@ all = [
   "hermes-agent[exa]",
   "hermes-agent[firecrawl]",
   "hermes-agent[parallel-web]",
+  "hermes-agent[office]",
   "hermes-agent[fal]",
   "hermes-agent[edge-tts]",
   "hermes-agent[modal]",

tests/tools/test_office_extract_tool.py

@@ -0,0 +1,92 @@
+import json
+
+import pytest
+
+from tools.office_extract_tool import office_extract, check_office_extract_requirements
+from toolsets import get_toolset, resolve_toolset
+
+
+def _load(result: str) -> dict:
+    return json.loads(result)
+
+
+def test_office_extract_reads_docx_text_and_tables(tmp_path):
+    docx = pytest.importorskip("docx")
+    path = tmp_path / "brief.docx"
+    doc = docx.Document()
+    doc.add_heading("Client Brief", level=1)
+    doc.add_paragraph("Launch a summer grill campaign for Miratorg.")
+    table = doc.add_table(rows=2, cols=2)
+    table.cell(0, 0).text = "Audience"
+    table.cell(0, 1).text = "Families"
+    table.cell(1, 0).text = "Tone"
+    table.cell(1, 1).text = "Warm"
+    doc.save(path)
+
+    result = _load(office_extract(str(path)))
+
+    assert result["success"] is True
+    assert result["format"] == "docx"
+    assert result["file_path"] == str(path)
+    assert "# Client Brief" in result["markdown"]
+    assert "Launch a summer grill campaign" in result["markdown"]
+    assert "Audience | Families" in result["markdown"]
+    assert result["metadata"]["paragraph_count"] >= 2
+    assert result["metadata"]["table_count"] == 1
+
+
+def test_office_extract_reads_pptx_slide_text(tmp_path):
+    pptx = pytest.importorskip("pptx")
+    path = tmp_path / "deck.pptx"
+    presentation = pptx.Presentation()
+    slide = presentation.slides.add_slide(presentation.slide_layouts[1])
+    slide.shapes.title.text = "Big idea"
+    slide.placeholders[1].text = "Fire + family + weekend ritual"
+    presentation.save(path)
+
+    result = _load(office_extract(str(path)))
+
+    assert result["success"] is True
+    assert result["format"] == "pptx"
+    assert "## Slide 1" in result["markdown"]
+    assert "Big idea" in result["markdown"]
+    assert "weekend ritual" in result["markdown"]
+    assert result["metadata"]["slide_count"] == 1
+
+
+def test_office_extract_reads_xlsx_sheets(tmp_path):
+    openpyxl = pytest.importorskip("openpyxl")
+    path = tmp_path / "budget.xlsx"
+    workbook = openpyxl.Workbook()
+    sheet = workbook.active
+    sheet.title = "Plan"
+    sheet.append(["Channel", "Budget"])
+    sheet.append(["Telegram", 100000])
+    sheet.append(["Outdoor", 250000])
+    workbook.save(path)
+
+    result = _load(office_extract(str(path)))
+
+    assert result["success"] is True
+    assert result["format"] == "xlsx"
+    assert "## Sheet: Plan" in result["markdown"]
+    assert "Channel | Budget" in result["markdown"]
+    assert "Telegram | 100000" in result["markdown"]
+    assert result["metadata"]["sheet_count"] == 1
+
+
+def test_office_extract_rejects_legacy_binary_office_formats(tmp_path):
+    path = tmp_path / "legacy.doc"
+    path.write_bytes(b"not really a doc")
+
+    result = _load(office_extract(str(path)))
+
+    assert result["success"] is False
+    assert result["error_code"] == "unsupported_legacy_format"
+    assert ".docx" in result["error"]
+
+
+def test_office_extract_toolset_is_registered_when_requirements_are_available():
+    assert check_office_extract_requirements() is True
+    assert "office_extract" in get_toolset("office")["tools"]
+    assert "office_extract" in resolve_toolset("office")