fix(photon): harden standalone send retries

plugins/platforms/photon/README.md

@@ -123,6 +123,8 @@ All env vars are documented in `plugin.yaml`. The most important:
 | `PHOTON_MAX_INLINE_ATTACHMENT_BYTES` | 20 MB           | Max inbound attachment size the sidecar reads & inlines |
 | `PHOTON_TELEMETRY`        | false                      | Spectrum SDK telemetry — toggle with `hermes photon telemetry on\|off` (restart the gateway to apply) |
 | `PHOTON_MARKDOWN`         | true                       | Send agent replies as markdown (iMessage renders natively). `false` strips formatting to plain text |
+| `PHOTON_STANDALONE_SEND_RETRIES` | 1 (bounded 0-3)      | Retry standalone text sends after safe transient failures |
+| `PHOTON_STANDALONE_RETRY_BASE_DELAY_SECONDS` | 2 (bounded 0-30) | Base seconds for exponential retry backoff |
 | `PHOTON_REACTIONS`        | false                      | Tapback 👀/👍/👎 as processing status; tapbacks on bot messages reach the agent as `reaction:added:<emoji>` |
 
 ## Attachments & limitations

plugins/platforms/photon/adapter.py

@@ -261,6 +261,24 @@ def _markdown_enabled() -> bool:
     }
 
 
+def _standalone_retry_count() -> int:
+    raw = os.getenv("PHOTON_STANDALONE_SEND_RETRIES", "1").strip()
+    try:
+        value = int(raw)
+    except ValueError:
+        return 1
+    return max(0, min(value, 3))
+
+
+def _standalone_retry_delay() -> float:
+    raw = os.getenv("PHOTON_STANDALONE_RETRY_BASE_DELAY_SECONDS", "2").strip()
+    try:
+        value = float(raw)
+    except ValueError:
+        return 2.0
+    return max(0.0, min(value, 30.0))
+
+
 # ---------------------------------------------------------------------------
 # Adapter
 
@@ -1458,17 +1476,25 @@ async def _send_with_retry(
             "[photon] Send failed: %s - retrying plain-text message",
             error_str,
         )
-        fallback_result = await self.send(
-            chat_id=chat_id,
-            content=text[: self.MAX_MESSAGE_LENGTH],
-            reply_to=reply_to,
-            metadata=metadata,
+        plain_text = strip_markdown(text)[: self.MAX_MESSAGE_LENGTH] or text[
+            : self.MAX_MESSAGE_LENGTH
+        ]
+        fallback_result = await self._sidecar_send(
+            chat_id,
+            plain_text,
+            markdown=False,
         )
         if not fallback_result.success:
             logger.error("[photon] Plain-text retry also failed: %s", fallback_result.error)
         return fallback_result
 
-    async def _sidecar_send(self, space_id: str, text: str) -> SendResult:
+    async def _sidecar_send(
+        self,
+        space_id: str,
+        text: str,
+        *,
+        markdown: Optional[bool] = None,
+    ) -> SendResult:
         if len(text) > self.MAX_MESSAGE_LENGTH:
             logger.warning(
                 "[photon] truncating outbound from %d to %d chars",
@@ -1478,7 +1504,9 @@ async def _sidecar_send(self, space_id: str, text: str) -> SendResult:
         body: Dict[str, Any] = {"spaceId": space_id, "text": text}
         # Omit the key when disabled so an older sidecar (pre-`format`)
         # keeps accepting the body during a half-upgraded restart.
-        if _markdown_enabled():
+        if markdown is None:
+            markdown = _markdown_enabled()
+        if markdown:
             body["format"] = "markdown"
         try:
             data = await self._sidecar_call("/send", body)
@@ -1686,24 +1714,75 @@ async def _standalone_send(
     base = f"http://{_DEFAULT_SIDECAR_BIND}:{port}"
     headers = {"X-Hermes-Sidecar-Token": token}
     last_message_id: Optional[str] = None
+
+    async def post_sidecar(
+        client: "httpx.AsyncClient",
+        path: str,
+        body: Dict[str, Any],
+    ) -> tuple[Optional[Dict[str, Any]], Optional[str]]:
+        try:
+            resp = await client.post(f"{base}{path}", json=body, headers=headers)
+        except (httpx.ConnectError, httpx.ConnectTimeout) as exc:
+            # The connection was never established, so retrying cannot
+            # duplicate a delivered message. Read/write timeouts deliberately
+            # escape to the outer handler because delivery is ambiguous.
+            return None, f"{type(exc).__name__}: {exc}"
+        if resp.status_code != 200:
+            return None, f"sidecar returned {resp.status_code}: {resp.text[:200]}"
+        data = resp.json() or {}
+        if not data.get("ok"):
+            return None, data.get("error") or "sidecar reported failure"
+        return data, None
+
+    async def post_text_with_retry(
+        client: "httpx.AsyncClient",
+        text: str,
+    ) -> tuple[Optional[Dict[str, Any]], Optional[str]]:
+        retries = _standalone_retry_count()
+        base_delay = _standalone_retry_delay()
+        use_markdown = _markdown_enabled()
+        body: Dict[str, Any] = {
+            "spaceId": chat_id,
+            "text": text[:_MAX_MESSAGE_LENGTH],
+        }
+        if use_markdown:
+            body["format"] = "markdown"
+
+        last_error: Optional[str] = None
+        for attempt in range(retries + 1):
+            data, error = await post_sidecar(client, "/send", body)
+            if error is None:
+                return data, None
+            last_error = error
+            if attempt >= retries or not PhotonAdapter._is_retryable_error(error):
+                break
+            delay = base_delay * (2 ** attempt)
+            logger.warning(
+                "[photon] Standalone send failed (attempt %d/%d, retrying in %.1fs): %s",
+                attempt + 1,
+                retries + 1,
+                delay,
+                error,
+            )
+            await asyncio.sleep(delay)
+
+        if use_markdown:
+            plain = strip_markdown(body["text"])[:_MAX_MESSAGE_LENGTH] or body["text"]
+            fallback_body = {"spaceId": chat_id, "text": plain}
+            data, fallback_error = await post_sidecar(client, "/send", fallback_body)
+            if fallback_error is None:
+                return data, None
+            return None, fallback_error
+
+        return None, last_error or "sidecar reported failure"
+
     try:
         async with httpx.AsyncClient(timeout=30.0) as client:
             # 1. Text body first (if any), so it leads the conversation.
             if message:
-                send_body: Dict[str, Any] = {
-                    "spaceId": chat_id,
-                    "text": message[:_MAX_MESSAGE_LENGTH],
-                }
-                if _markdown_enabled():
-                    send_body["format"] = "markdown"
-                resp = await client.post(
-                    f"{base}/send", json=send_body, headers=headers,
-                )
-                if resp.status_code != 200:
-                    return {"error": f"sidecar returned {resp.status_code}: {resp.text[:200]}"}
-                data = resp.json() or {}
-                if not data.get("ok"):
-                    return {"error": data.get("error") or "sidecar reported failure"}
+                data, error = await post_text_with_retry(client, message)
+                if error is not None or data is None:
+                    return {"error": error or "sidecar reported failure"}
                 last_message_id = data.get("messageId")
 
             # 2. Each attachment as a separate /send-attachment call.
@@ -1724,14 +1803,9 @@ async def _standalone_send(
                 }
                 if guessed:
                     att_body["mimeType"] = guessed
-                resp = await client.post(
-                    f"{base}/send-attachment", json=att_body, headers=headers,
-                )
-                if resp.status_code != 200:
-                    return {"error": f"sidecar returned {resp.status_code}: {resp.text[:200]}"}
-                data = resp.json() or {}
-                if not data.get("ok"):
-                    return {"error": data.get("error") or "sidecar reported failure"}
+                data, error = await post_sidecar(client, "/send-attachment", att_body)
+                if error is not None or data is None:
+                    return {"error": error or "sidecar reported failure"}
                 last_message_id = data.get("messageId") or last_message_id
 
         return {"success": True, "message_id": last_message_id}

plugins/platforms/photon/plugin.yaml

@@ -82,6 +82,14 @@ optional_env:
     description: "Send agent replies as markdown — iMessage renders it natively, other Spectrum platforms degrade to plain text (true/false, default true)"
     prompt: "Render replies as markdown? (true/false)"
     password: false
+  - name: PHOTON_STANDALONE_SEND_RETRIES
+    description: "Retries for standalone text sends after safe transient failures (0-3, default 1)"
+    prompt: "Standalone send retry count"
+    password: false
+  - name: PHOTON_STANDALONE_RETRY_BASE_DELAY_SECONDS
+    description: "Base seconds for standalone-send exponential backoff (0-30, default 2)"
+    prompt: "Standalone send retry base delay (seconds)"
+    password: false
   - name: PHOTON_REACTIONS
     description: "Tapback 👀/👍/👎 on messages as processing status and route tapbacks on bot messages to the agent (true/false, default false)"
     prompt: "Enable reaction tapbacks? (true/false)"

tests/plugins/platforms/photon/test_markdown.py

@@ -127,3 +127,192 @@ async def post(self, url: str, json: Dict[str, Any], headers=None):
 
     assert result.get("success") is True
     assert posted[0][1]["format"] == "markdown"
+
+
+@pytest.mark.asyncio
+async def test_standalone_send_retries_retryable_sidecar_error(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.delenv("PHOTON_MARKDOWN", raising=False)
+    monkeypatch.setenv("PHOTON_SIDECAR_TOKEN", "tok")
+    monkeypatch.setenv("PHOTON_STANDALONE_RETRY_BASE_DELAY_SECONDS", "0")
+
+    posted: List[Tuple[str, Dict[str, Any]]] = []
+
+    class _Resp:
+        status_code = 200
+
+        def __init__(self, payload: Dict[str, Any]) -> None:
+            self._payload = payload
+
+        def json(self) -> Dict[str, Any]:
+            return self._payload
+
+    class _FakeClient:
+        def __init__(self, *a, **k):
+            pass
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *a):
+            return False
+
+        async def post(self, url: str, json: Dict[str, Any], headers=None):
+            posted.append((url, json))
+            if len(posted) == 1:
+                return _Resp({"ok": False, "error": "internal sidecar error"})
+            return _Resp({"ok": True, "messageId": "m-retry"})
+
+    monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _FakeClient)
+
+    cfg = PlatformConfig(enabled=True, token="", extra={})
+    result = await photon_adapter._standalone_send(cfg, "+15551234567", _MD)
+
+    assert result.get("success") is True
+    assert result.get("message_id") == "m-retry"
+    assert len(posted) == 2
+    assert posted[0][0].endswith("/send")
+    assert posted[1][0].endswith("/send")
+    assert posted[0][1]["format"] == "markdown"
+    assert posted[1][1]["format"] == "markdown"
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("connect_kind", ["error", "timeout"])
+async def test_standalone_send_retries_connect_error(
+    monkeypatch: pytest.MonkeyPatch,
+    connect_kind: str,
+) -> None:
+    monkeypatch.delenv("PHOTON_MARKDOWN", raising=False)
+    monkeypatch.setenv("PHOTON_SIDECAR_TOKEN", "tok")
+    monkeypatch.setenv("PHOTON_STANDALONE_RETRY_BASE_DELAY_SECONDS", "0")
+
+    posted: List[Tuple[str, Dict[str, Any]]] = []
+
+    class _Resp:
+        status_code = 200
+
+        @staticmethod
+        def json() -> Dict[str, Any]:
+            return {"ok": True, "messageId": "m-connected"}
+
+    class _FakeClient:
+        def __init__(self, *a, **k):
+            pass
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *a):
+            return False
+
+        async def post(self, url: str, json: Dict[str, Any], headers=None):
+            posted.append((url, json))
+            if len(posted) == 1:
+                request = photon_adapter.httpx.Request("POST", url)
+                if connect_kind == "timeout":
+                    raise photon_adapter.httpx.ConnectTimeout(
+                        "connection timed out", request=request
+                    )
+                raise photon_adapter.httpx.ConnectError("connection refused", request=request)
+            return _Resp()
+
+    monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _FakeClient)
+
+    cfg = PlatformConfig(enabled=True, token="", extra={})
+    result = await photon_adapter._standalone_send(cfg, "+155****4567", _MD)
+
+    assert result == {"success": True, "message_id": "m-connected"}
+    assert len(posted) == 2
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize("timeout_kind", ["read", "write"])
+async def test_standalone_send_does_not_retry_ambiguous_timeout(
+    monkeypatch: pytest.MonkeyPatch,
+    timeout_kind: str,
+) -> None:
+    monkeypatch.delenv("PHOTON_MARKDOWN", raising=False)
+    monkeypatch.setenv("PHOTON_SIDECAR_TOKEN", "tok")
+    monkeypatch.setenv("PHOTON_STANDALONE_RETRY_BASE_DELAY_SECONDS", "0")
+
+    posted: List[Tuple[str, Dict[str, Any]]] = []
+
+    class _FakeClient:
+        def __init__(self, *a, **k):
+            pass
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *a):
+            return False
+
+        async def post(self, url: str, json: Dict[str, Any], headers=None):
+            posted.append((url, json))
+            request = photon_adapter.httpx.Request("POST", url)
+            if timeout_kind == "read":
+                raise photon_adapter.httpx.ReadTimeout(
+                    "send timed out", request=request
+                )
+            raise photon_adapter.httpx.WriteTimeout(
+                "send timed out", request=request
+            )
+
+    monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _FakeClient)
+
+    cfg = PlatformConfig(enabled=True, token="", extra={})
+    result = await photon_adapter._standalone_send(cfg, "+155****4567", _MD)
+
+    assert result == {"error": "Photon standalone send failed: send timed out"}
+    assert len(posted) == 1
+
+
+@pytest.mark.asyncio
+async def test_standalone_send_falls_back_to_plain_text_after_markdown_failures(
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    monkeypatch.delenv("PHOTON_MARKDOWN", raising=False)
+    monkeypatch.setenv("PHOTON_SIDECAR_TOKEN", "tok")
+    monkeypatch.setenv("PHOTON_STANDALONE_RETRY_BASE_DELAY_SECONDS", "0")
+
+    posted: List[Tuple[str, Dict[str, Any]]] = []
+
+    class _Resp:
+        status_code = 200
+
+        def __init__(self, payload: Dict[str, Any]) -> None:
+            self._payload = payload
+
+        def json(self) -> Dict[str, Any]:
+            return self._payload
+
+    class _FakeClient:
+        def __init__(self, *a, **k):
+            pass
+
+        async def __aenter__(self):
+            return self
+
+        async def __aexit__(self, *a):
+            return False
+
+        async def post(self, url: str, json: Dict[str, Any], headers=None):
+            posted.append((url, json))
+            if len(posted) <= 2:
+                return _Resp({"ok": False, "error": "internal sidecar error"})
+            return _Resp({"ok": True, "messageId": "m-plain"})
+
+    monkeypatch.setattr(photon_adapter.httpx, "AsyncClient", _FakeClient)
+
+    cfg = PlatformConfig(enabled=True, token="", extra={})
+    result = await photon_adapter._standalone_send(cfg, "+15551234567", _MD)
+
+    assert result.get("success") is True
+    assert result.get("message_id") == "m-plain"
+    assert len(posted) == 3
+    assert posted[0][1]["format"] == "markdown"
+    assert posted[1][1]["format"] == "markdown"
+    assert "format" not in posted[2][1]
+    assert posted[2][1]["text"] == "bold and code"