From 318653de5d87991869147c814096165a0d0af45b Mon Sep 17 00:00:00 2001 From: Hermes Agent Date: Sat, 27 Jun 2026 15:31:43 +0000 Subject: [PATCH 1/3] fix(photon): harden standalone send retries --- plugins/platforms/photon/adapter.py | 136 ++++++++++++++++++++++------ 1 file changed, 106 insertions(+), 30 deletions(-) diff --git a/plugins/platforms/photon/adapter.py b/plugins/platforms/photon/adapter.py index 27df34ecf2c4d..b5e9ab5011632 100644 --- a/plugins/platforms/photon/adapter.py +++ b/plugins/platforms/photon/adapter.py @@ -261,6 +261,24 @@ def _markdown_enabled() -> bool: } +def _standalone_retry_count() -> int: + raw = os.getenv("PHOTON_STANDALONE_SEND_RETRIES", "1").strip() + try: + value = int(raw) + except ValueError: + return 1 + return max(0, min(value, 3)) + + +def _standalone_retry_delay() -> float: + raw = os.getenv("PHOTON_STANDALONE_RETRY_BASE_DELAY_SECONDS", "2").strip() + try: + value = float(raw) + except ValueError: + return 2.0 + return max(0.0, min(value, 30.0)) + + # --------------------------------------------------------------------------- # Adapter @@ -593,6 +611,9 @@ async def _on_inbound_line(self, line: str) -> None: await self._dispatch_inbound(event) except Exception: logger.exception("[photon] inbound dispatch failed") + return + if msg_id: + self._mark_seen(msg_id) def _is_duplicate(self, msg_id: str) -> bool: now = time.time() @@ -603,13 +624,18 @@ def _is_duplicate(self, msg_id: str) -> bool: # New or expired: record and enforce a HARD size bound (evict oldest, # insertion-order) so a burst of unique ids within the window can't grow # the dict without limit — not just the expired-only prune. + self._mark_seen(msg_id) + return False + + def _mark_seen(self, msg_id: str) -> None: + now = time.time() + seen = self._seen_messages if msg_id in seen: del seen[msg_id] # refresh insertion order seen[msg_id] = now if len(seen) > _DEDUP_MAX_SIZE: for old in list(seen.keys())[: len(seen) - _DEDUP_MAX_SIZE]: del seen[old] - return False async def _dispatch_inbound(self, event: Dict[str, Any]) -> None: """Normalize a sidecar inbound event and dispatch it to the gateway. @@ -1458,17 +1484,25 @@ async def _send_with_retry( "[photon] Send failed: %s - retrying plain-text message", error_str, ) - fallback_result = await self.send( - chat_id=chat_id, - content=text[: self.MAX_MESSAGE_LENGTH], - reply_to=reply_to, - metadata=metadata, + plain_text = strip_markdown(text)[: self.MAX_MESSAGE_LENGTH] or text[ + : self.MAX_MESSAGE_LENGTH + ] + fallback_result = await self._sidecar_send( + chat_id, + plain_text, + markdown=False, ) if not fallback_result.success: logger.error("[photon] Plain-text retry also failed: %s", fallback_result.error) return fallback_result - async def _sidecar_send(self, space_id: str, text: str) -> SendResult: + async def _sidecar_send( + self, + space_id: str, + text: str, + *, + markdown: Optional[bool] = None, + ) -> SendResult: if len(text) > self.MAX_MESSAGE_LENGTH: logger.warning( "[photon] truncating outbound from %d to %d chars", @@ -1478,7 +1512,9 @@ async def _sidecar_send(self, space_id: str, text: str) -> SendResult: body: Dict[str, Any] = {"spaceId": space_id, "text": text} # Omit the key when disabled so an older sidecar (pre-`format`) # keeps accepting the body during a half-upgraded restart. - if _markdown_enabled(): + if markdown is None: + markdown = _markdown_enabled() + if markdown: body["format"] = "markdown" try: data = await self._sidecar_call("/send", body) @@ -1686,24 +1722,69 @@ async def _standalone_send( base = f"http://{_DEFAULT_SIDECAR_BIND}:{port}" headers = {"X-Hermes-Sidecar-Token": token} last_message_id: Optional[str] = None + + async def post_sidecar( + client: "httpx.AsyncClient", + path: str, + body: Dict[str, Any], + ) -> tuple[Optional[Dict[str, Any]], Optional[str]]: + resp = await client.post(f"{base}{path}", json=body, headers=headers) + if resp.status_code != 200: + return None, f"sidecar returned {resp.status_code}: {resp.text[:200]}" + data = resp.json() or {} + if not data.get("ok"): + return None, data.get("error") or "sidecar reported failure" + return data, None + + async def post_text_with_retry( + client: "httpx.AsyncClient", + text: str, + ) -> tuple[Optional[Dict[str, Any]], Optional[str]]: + retries = _standalone_retry_count() + base_delay = _standalone_retry_delay() + use_markdown = _markdown_enabled() + body: Dict[str, Any] = { + "spaceId": chat_id, + "text": text[:_MAX_MESSAGE_LENGTH], + } + if use_markdown: + body["format"] = "markdown" + + last_error: Optional[str] = None + for attempt in range(retries + 1): + data, error = await post_sidecar(client, "/send", body) + if error is None: + return data, None + last_error = error + if attempt >= retries or not PhotonAdapter._is_retryable_error(error): + break + delay = base_delay * (2 ** attempt) + logger.warning( + "[photon] Standalone send failed (attempt %d/%d, retrying in %.1fs): %s", + attempt + 1, + retries + 1, + delay, + error, + ) + await asyncio.sleep(delay) + + if use_markdown: + plain = strip_markdown(body["text"])[:_MAX_MESSAGE_LENGTH] or body["text"] + fallback_body = {"spaceId": chat_id, "text": plain} + data, fallback_error = await post_sidecar(client, "/send", fallback_body) + if fallback_error is None: + return data, None + return None, fallback_error + + return None, last_error or "sidecar reported failure" + try: async with httpx.AsyncClient(timeout=30.0) as client: # 1. Text body first (if any), so it leads the conversation. if message: - send_body: Dict[str, Any] = { - "spaceId": chat_id, - "text": message[:_MAX_MESSAGE_LENGTH], - } - if _markdown_enabled(): - send_body["format"] = "markdown" - resp = await client.post( - f"{base}/send", json=send_body, headers=headers, - ) - if resp.status_code != 200: - return {"error": f"sidecar returned {resp.status_code}: {resp.text[:200]}"} - data = resp.json() or {} - if not data.get("ok"): - return {"error": data.get("error") or "sidecar reported failure"} + data, error = await post_text_with_retry(client, message) + if error is not None: + return {"error": error} last_message_id = data.get("messageId") # 2. Each attachment as a separate /send-attachment call. @@ -1724,14 +1805,9 @@ async def _standalone_send( } if guessed: att_body["mimeType"] = guessed - resp = await client.post( - f"{base}/send-attachment", json=att_body, headers=headers, - ) - if resp.status_code != 200: - return {"error": f"sidecar returned {resp.status_code}: {resp.text[:200]}"} - data = resp.json() or {} - if not data.get("ok"): - return {"error": data.get("error") or "sidecar reported failure"} + data, error = await post_sidecar(client, "/send-attachment", att_body) + if error is not None: + return {"error": error} last_message_id = data.get("messageId") or last_message_id return {"success": True, "message_id": last_message_id} From bd69f5dbdd932bd5b1caafb834927664d7ea9aff Mon Sep 17 00:00:00 2001 From: Codex Date: Sat, 27 Jun 2026 21:21:47 +0000 Subject: [PATCH 2/3] test(photon): cover standalone send retry fallback --- .../plugins/platforms/photon/test_markdown.py | 98 +++++++++++++++++++ 1 file changed, 98 insertions(+) diff --git a/tests/plugins/platforms/photon/test_markdown.py b/tests/plugins/platforms/photon/test_markdown.py index 6e803d6531791..48c854cd87072 100644 --- a/tests/plugins/platforms/photon/test_markdown.py +++ b/tests/plugins/platforms/photon/test_markdown.py @@ -127,3 +127,101 @@ async def post(self, url: str, json: Dict[str, Any], headers=None): assert result.get("success") is True assert posted[0][1]["format"] == "markdown" + + +@pytest.mark.asyncio +async def test_standalone_send_retries_retryable_sidecar_error( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("PHOTON_MARKDOWN", raising=False) + monkeypatch.setenv("PHOTON_SIDECAR_TOKEN", "tok") + monkeypatch.setenv("PHOTON_STANDALONE_RETRY_BASE_DELAY_SECONDS", "0") + + posted: List[Tuple[str, Dict[str, Any]]] = [] + + class _Resp: + status_code = 200 + + def __init__(self, payload: Dict[str, Any]) -> None: + self._payload = payload + + def json(self) -> Dict[str, Any]: + return self._payload + + class _FakeClient: + def __init__(self, *a, **k): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + return False + + async def post(self, url: str, json: Dict[str, Any], headers=None): + posted.append((url, json)) + if len(posted) == 1: + return _Resp({"ok": False, "error": "internal sidecar error"}) + return _Resp({"ok": True, "messageId": "m-retry"}) + + monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _FakeClient) + + cfg = PlatformConfig(enabled=True, token="", extra={}) + result = await photon_adapter._standalone_send(cfg, "+15551234567", _MD) + + assert result.get("success") is True + assert result.get("message_id") == "m-retry" + assert len(posted) == 2 + assert posted[0][0].endswith("/send") + assert posted[1][0].endswith("/send") + assert posted[0][1]["format"] == "markdown" + assert posted[1][1]["format"] == "markdown" + + +@pytest.mark.asyncio +async def test_standalone_send_falls_back_to_plain_text_after_markdown_failures( + monkeypatch: pytest.MonkeyPatch, +) -> None: + monkeypatch.delenv("PHOTON_MARKDOWN", raising=False) + monkeypatch.setenv("PHOTON_SIDECAR_TOKEN", "tok") + monkeypatch.setenv("PHOTON_STANDALONE_RETRY_BASE_DELAY_SECONDS", "0") + + posted: List[Tuple[str, Dict[str, Any]]] = [] + + class _Resp: + status_code = 200 + + def __init__(self, payload: Dict[str, Any]) -> None: + self._payload = payload + + def json(self) -> Dict[str, Any]: + return self._payload + + class _FakeClient: + def __init__(self, *a, **k): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + return False + + async def post(self, url: str, json: Dict[str, Any], headers=None): + posted.append((url, json)) + if len(posted) <= 2: + return _Resp({"ok": False, "error": "internal sidecar error"}) + return _Resp({"ok": True, "messageId": "m-plain"}) + + monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _FakeClient) + + cfg = PlatformConfig(enabled=True, token="", extra={}) + result = await photon_adapter._standalone_send(cfg, "+15551234567", _MD) + + assert result.get("success") is True + assert result.get("message_id") == "m-plain" + assert len(posted) == 3 + assert posted[0][1]["format"] == "markdown" + assert posted[1][1]["format"] == "markdown" + assert "format" not in posted[2][1] + assert posted[2][1]["text"] == "bold and code" From f27e9b85a8cf9f536277e02a87a1885ba529fdaf Mon Sep 17 00:00:00 2001 From: Codex Date: Wed, 15 Jul 2026 20:43:05 +0000 Subject: [PATCH 3/3] fix(photon): address standalone retry review --- plugins/platforms/photon/README.md | 2 + plugins/platforms/photon/adapter.py | 26 +++--- plugins/platforms/photon/plugin.yaml | 8 ++ .../plugins/platforms/photon/test_markdown.py | 91 +++++++++++++++++++ 4 files changed, 113 insertions(+), 14 deletions(-) diff --git a/plugins/platforms/photon/README.md b/plugins/platforms/photon/README.md index 91680ebd1a6cb..682cbe7953c21 100644 --- a/plugins/platforms/photon/README.md +++ b/plugins/platforms/photon/README.md @@ -123,6 +123,8 @@ All env vars are documented in `plugin.yaml`. The most important: | `PHOTON_MAX_INLINE_ATTACHMENT_BYTES` | 20 MB | Max inbound attachment size the sidecar reads & inlines | | `PHOTON_TELEMETRY` | false | Spectrum SDK telemetry — toggle with `hermes photon telemetry on\|off` (restart the gateway to apply) | | `PHOTON_MARKDOWN` | true | Send agent replies as markdown (iMessage renders natively). `false` strips formatting to plain text | +| `PHOTON_STANDALONE_SEND_RETRIES` | 1 (bounded 0-3) | Retry standalone text sends after safe transient failures | +| `PHOTON_STANDALONE_RETRY_BASE_DELAY_SECONDS` | 2 (bounded 0-30) | Base seconds for exponential retry backoff | | `PHOTON_REACTIONS` | false | Tapback 👀/👍/👎 as processing status; tapbacks on bot messages reach the agent as `reaction:added:` | ## Attachments & limitations diff --git a/plugins/platforms/photon/adapter.py b/plugins/platforms/photon/adapter.py index b5e9ab5011632..e6867913f3146 100644 --- a/plugins/platforms/photon/adapter.py +++ b/plugins/platforms/photon/adapter.py @@ -611,9 +611,6 @@ async def _on_inbound_line(self, line: str) -> None: await self._dispatch_inbound(event) except Exception: logger.exception("[photon] inbound dispatch failed") - return - if msg_id: - self._mark_seen(msg_id) def _is_duplicate(self, msg_id: str) -> bool: now = time.time() @@ -624,18 +621,13 @@ def _is_duplicate(self, msg_id: str) -> bool: # New or expired: record and enforce a HARD size bound (evict oldest, # insertion-order) so a burst of unique ids within the window can't grow # the dict without limit — not just the expired-only prune. - self._mark_seen(msg_id) - return False - - def _mark_seen(self, msg_id: str) -> None: - now = time.time() - seen = self._seen_messages if msg_id in seen: del seen[msg_id] # refresh insertion order seen[msg_id] = now if len(seen) > _DEDUP_MAX_SIZE: for old in list(seen.keys())[: len(seen) - _DEDUP_MAX_SIZE]: del seen[old] + return False async def _dispatch_inbound(self, event: Dict[str, Any]) -> None: """Normalize a sidecar inbound event and dispatch it to the gateway. @@ -1728,7 +1720,13 @@ async def post_sidecar( path: str, body: Dict[str, Any], ) -> tuple[Optional[Dict[str, Any]], Optional[str]]: - resp = await client.post(f"{base}{path}", json=body, headers=headers) + try: + resp = await client.post(f"{base}{path}", json=body, headers=headers) + except (httpx.ConnectError, httpx.ConnectTimeout) as exc: + # The connection was never established, so retrying cannot + # duplicate a delivered message. Read/write timeouts deliberately + # escape to the outer handler because delivery is ambiguous. + return None, f"{type(exc).__name__}: {exc}" if resp.status_code != 200: return None, f"sidecar returned {resp.status_code}: {resp.text[:200]}" data = resp.json() or {} @@ -1783,8 +1781,8 @@ async def post_text_with_retry( # 1. Text body first (if any), so it leads the conversation. if message: data, error = await post_text_with_retry(client, message) - if error is not None: - return {"error": error} + if error is not None or data is None: + return {"error": error or "sidecar reported failure"} last_message_id = data.get("messageId") # 2. Each attachment as a separate /send-attachment call. @@ -1806,8 +1804,8 @@ async def post_text_with_retry( if guessed: att_body["mimeType"] = guessed data, error = await post_sidecar(client, "/send-attachment", att_body) - if error is not None: - return {"error": error} + if error is not None or data is None: + return {"error": error or "sidecar reported failure"} last_message_id = data.get("messageId") or last_message_id return {"success": True, "message_id": last_message_id} diff --git a/plugins/platforms/photon/plugin.yaml b/plugins/platforms/photon/plugin.yaml index a39193a81bf4d..2648f211218b9 100644 --- a/plugins/platforms/photon/plugin.yaml +++ b/plugins/platforms/photon/plugin.yaml @@ -82,6 +82,14 @@ optional_env: description: "Send agent replies as markdown — iMessage renders it natively, other Spectrum platforms degrade to plain text (true/false, default true)" prompt: "Render replies as markdown? (true/false)" password: false + - name: PHOTON_STANDALONE_SEND_RETRIES + description: "Retries for standalone text sends after safe transient failures (0-3, default 1)" + prompt: "Standalone send retry count" + password: false + - name: PHOTON_STANDALONE_RETRY_BASE_DELAY_SECONDS + description: "Base seconds for standalone-send exponential backoff (0-30, default 2)" + prompt: "Standalone send retry base delay (seconds)" + password: false - name: PHOTON_REACTIONS description: "Tapback 👀/👍/👎 on messages as processing status and route tapbacks on bot messages to the agent (true/false, default false)" prompt: "Enable reaction tapbacks? (true/false)" diff --git a/tests/plugins/platforms/photon/test_markdown.py b/tests/plugins/platforms/photon/test_markdown.py index 48c854cd87072..eb556c7fb7e78 100644 --- a/tests/plugins/platforms/photon/test_markdown.py +++ b/tests/plugins/platforms/photon/test_markdown.py @@ -178,6 +178,97 @@ async def post(self, url: str, json: Dict[str, Any], headers=None): assert posted[1][1]["format"] == "markdown" +@pytest.mark.asyncio +@pytest.mark.parametrize("connect_kind", ["error", "timeout"]) +async def test_standalone_send_retries_connect_error( + monkeypatch: pytest.MonkeyPatch, + connect_kind: str, +) -> None: + monkeypatch.delenv("PHOTON_MARKDOWN", raising=False) + monkeypatch.setenv("PHOTON_SIDECAR_TOKEN", "tok") + monkeypatch.setenv("PHOTON_STANDALONE_RETRY_BASE_DELAY_SECONDS", "0") + + posted: List[Tuple[str, Dict[str, Any]]] = [] + + class _Resp: + status_code = 200 + + @staticmethod + def json() -> Dict[str, Any]: + return {"ok": True, "messageId": "m-connected"} + + class _FakeClient: + def __init__(self, *a, **k): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + return False + + async def post(self, url: str, json: Dict[str, Any], headers=None): + posted.append((url, json)) + if len(posted) == 1: + request = photon_adapter.httpx.Request("POST", url) + if connect_kind == "timeout": + raise photon_adapter.httpx.ConnectTimeout( + "connection timed out", request=request + ) + raise photon_adapter.httpx.ConnectError("connection refused", request=request) + return _Resp() + + monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _FakeClient) + + cfg = PlatformConfig(enabled=True, token="", extra={}) + result = await photon_adapter._standalone_send(cfg, "+155****4567", _MD) + + assert result == {"success": True, "message_id": "m-connected"} + assert len(posted) == 2 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("timeout_kind", ["read", "write"]) +async def test_standalone_send_does_not_retry_ambiguous_timeout( + monkeypatch: pytest.MonkeyPatch, + timeout_kind: str, +) -> None: + monkeypatch.delenv("PHOTON_MARKDOWN", raising=False) + monkeypatch.setenv("PHOTON_SIDECAR_TOKEN", "tok") + monkeypatch.setenv("PHOTON_STANDALONE_RETRY_BASE_DELAY_SECONDS", "0") + + posted: List[Tuple[str, Dict[str, Any]]] = [] + + class _FakeClient: + def __init__(self, *a, **k): + pass + + async def __aenter__(self): + return self + + async def __aexit__(self, *a): + return False + + async def post(self, url: str, json: Dict[str, Any], headers=None): + posted.append((url, json)) + request = photon_adapter.httpx.Request("POST", url) + if timeout_kind == "read": + raise photon_adapter.httpx.ReadTimeout( + "send timed out", request=request + ) + raise photon_adapter.httpx.WriteTimeout( + "send timed out", request=request + ) + + monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _FakeClient) + + cfg = PlatformConfig(enabled=True, token="", extra={}) + result = await photon_adapter._standalone_send(cfg, "+155****4567", _MD) + + assert result == {"error": "Photon standalone send failed: send timed out"} + assert len(posted) == 1 + + @pytest.mark.asyncio async def test_standalone_send_falls_back_to_plain_text_after_markdown_failures( monkeypatch: pytest.MonkeyPatch,