feat(agent): add curl_cffi TLS impersonation transport for Cloudflare-protected endpoints

agent/curl_cffi_transport.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""httpx.Client subclass that routes send() through curl_cffi for TLS impersonation.
+
+The OpenAI SDK calls build_request() + send(request, stream=...), and checks
+isinstance(client, httpx.Client). By subclassing httpx.Client and overriding
+only send(), we pass all checks while curl_cffi handles the actual TLS.
+"""
+
+from __future__ import annotations
+
+import httpx
+from curl_cffi import requests as curl_requests
+
+# httpx auto-headers that identify us as a Python bot — we strip these and
+# let curl_cffi (with browser impersonation) set its own headers instead.
+_CFFI_STRIP_REQUEST_HEADERS = {
+    "user-agent",       # httpx sets "python-httpx/X.Y.Z" — instant Cloudflare 403
+    "accept-encoding",  # curl_cffi handles its own encoding
+    "connection",       # curl manages keep-alive natively
+    "accept",           # let curl set its browser-like Accept header
+}
+
+
+class _CurlCffiByteStream(httpx.SyncByteStream):
+    """Expose a curl_cffi streaming response through httpx's stream API."""
+
+    def __init__(self, curl_response) -> None:
+        self._curl_response = curl_response
+
+    def __iter__(self):
+        yield from self._curl_response.iter_content(chunk_size=65536)
+
+    def close(self) -> None:
+        self._curl_response.close()
+
+
+class CurlCffiClient(httpx.Client):
+    """httpx.Client that routes actual HTTP requests through curl_cffi.
+
+    build_request() still creates normal httpx.Request objects (using httpx's
+    URL normalization and header merging). send() converts the request to a
+    curl_cffi call and returns an httpx.Response.
+
+    This means:
+    - isinstance(curl_client, httpx.Client) → True (OpenAI SDK gate passes)
+    - The TLS handshake uses Chrome 124's fingerprint → Cloudflare allows it
+    - httpx auto-headers (python-httpx UA, accept-encoding, etc.) are stripped
+      so curl_cffi can send its own browser-like headers
+    """
+
+    def __init__(
+        self,
+        impersonate: str = "chrome124",
+        **kwargs,
+    ) -> None:
+        super().__init__(**kwargs)
+        self._curl_session = curl_requests.Session(impersonate=impersonate)
+
+    def send(self, request: httpx.Request, *, stream: bool = False, **kwargs) -> httpx.Response:
+        """Execute an httpx.Request through curl_cffi.
+
+        Strips httpx's auto-headers (User-Agent, Accept-Encoding, etc.) so
+        curl_cffi sends its own browser-impersonating headers instead.
+        """
+        # Read body
+        content: bytes | None = None
+        if request.content:
+            if isinstance(request.content, bytes):
+                content = request.content
+            else:
+                content = request.content.encode("utf-8")
+
+        # Filter headers: pass through everything EXCEPT httpx auto-headers
+        filtered_headers: list[tuple[str, str]] = []
+        for k, v in request.headers.items():
+            if k.lower() not in _CFFI_STRIP_REQUEST_HEADERS:
+                filtered_headers.append((k, v))
+
+        # Make the curl_cffi request with browser impersonation
+        curl_resp = self._curl_session.request(
+            method=request.method,
+            url=str(request.url),
+            headers=filtered_headers,
+            data=content,
+            stream=stream,
+        )
+        if curl_resp is None:
+            raise RuntimeError("curl_cffi returned no response")
+
+        # Convert response headers — strip Content-Encoding since
+        # curl_cffi already decompressed the response body
+        resp_headers: list[tuple[str, str]] = []
+        for k, v in curl_resp.headers.multi_items():
+            if k.lower() == "content-encoding":
+                continue
+            resp_headers.append((k, v))
+
+        if stream:
+            return httpx.Response(
+                status_code=curl_resp.status_code,
+                headers=resp_headers,
+                stream=_CurlCffiByteStream(curl_resp),
+                request=request,
+            )
+
+        return httpx.Response(
+            status_code=curl_resp.status_code,
+            headers=resp_headers,
+            content=curl_resp.content,
+            request=request,
+        )
+
+    def close(self) -> None:
+        self._curl_session.close()
+        super().close()
+
+
+def build_curl_cffi_http_client(
+    impersonate: str = "chrome124",
+    proxy: str | None = None,
+) -> CurlCffiClient:
+    """Build a CurlCffiClient with browser TLS impersonation.
+
+    Can be passed as http_client= to the OpenAI SDK directly.
+    """
+    client = CurlCffiClient(
+        impersonate=impersonate,
+        timeout=httpx.Timeout(connect=15.0, read=600.0, write=30.0, pool=30.0),
+    )
+
+    if proxy:
+        client._curl_session.proxies = {"http": proxy, "https": proxy}
+
+    return client

pyproject.toml

@@ -131,7 +131,7 @@ edge-tts = ["edge-tts==7.2.7"]
 modal = ["modal==1.3.4"]
 daytona = ["daytona==0.155.0"]
 hindsight = ["hindsight-client==0.6.1"]
-dev = ["debugpy==1.8.20", "pytest==9.0.2", "pytest-asyncio==1.3.0", "mcp==1.26.0", "starlette==1.0.1", "ty==0.0.21", "ruff==0.15.10", "setuptools==82.0.1"]  # starlette: CVE-2026-48710
+dev = ["debugpy==1.8.20", "pytest==9.0.2", "pytest-asyncio==1.3.0", "mcp==1.26.0", "starlette==1.0.1", "ty==0.0.21", "ruff==0.15.10", "setuptools==82.0.1", "curl_cffi==0.15.0"]  # starlette: CVE-2026-48710
 messaging = ["python-telegram-bot[webhooks]==22.6", "discord.py[voice]==2.7.1", "aiohttp==3.13.4", "brotlicffi==1.2.0.1", "slack-bolt==1.27.0", "slack-sdk==3.40.1", "qrcode==7.4.2"]  # aiohttp: CVE-2026-34513/34518/34519/34520/34525
 cron = []  # croniter is now a core dependency; this extra kept for back-compat
 slack = ["slack-bolt==1.27.0", "slack-sdk==3.40.1", "aiohttp==3.13.4"]
@@ -192,6 +192,7 @@ acp = ["agent-client-protocol==0.9.0"]
 mistral = ["mistralai==2.4.8"]
 bedrock = ["boto3==1.42.89"]
 azure-identity = ["azure-identity==1.25.3"]
+curl-cffi = ["curl_cffi==0.15.0"]
 termux = [
   # Baseline Android / Termux path for reliable fresh installs.
   "python-telegram-bot[webhooks]==22.6",

run_agent.py

@@ -3439,8 +3439,65 @@ def _is_openai_client_closed(client: Any) -> bool:
             return bool(getattr(http_client, "is_closed", False))
         return False
 
+    @staticmethod
+    def _tls_impersonation_profile() -> "str | None":
+        """Read model.tls_impersonate from config.
+
+        Returns the impersonation profile name (e.g. ``"chrome124"``) or
+        ``None`` when the key is absent, empty, or set to ``"none"``.
+        """
+        try:
+            from hermes_cli.config import load_config
+            cfg = load_config()
+            val = (cfg.get("model") or {}).get("tls_impersonate")
+            if isinstance(val, str) and val.strip().lower() not in ("", "none"):
+                return val.strip().lower()
+        except Exception:
+            pass
+        return None
+
+    @staticmethod
+    def _is_cloudflare_protected(base_url: str) -> bool:
+        """True when *base_url* is known to sit behind Cloudflare WAF."""
+        url_lower = (base_url or "").lower()
+        return any(
+            domain in url_lower
+            for domain in ("chatgpt.com", "api.openai.com")
+        )
+
     @staticmethod
     def _build_keepalive_http_client(base_url: str = "") -> Any:
+        # ── curl_cffi TLS impersonation for Cloudflare-protected endpoints ──
+        impersonate = AIAgent._tls_impersonation_profile()
+        if impersonate and AIAgent._is_cloudflare_protected(base_url):
+            try:
+                try:
+                    from tools.lazy_deps import ensure as _lazy_ensure
+                    _lazy_ensure("provider.curl_cffi", prompt=False)
+                except ImportError:
+                    pass
+
+                from agent.curl_cffi_transport import build_curl_cffi_http_client
+                _proxy = _get_proxy_for_base_url(base_url)
+                logger.info(
+                    "Using curl_cffi TLS impersonation (%s) for %s",
+                    impersonate, base_url[:80],
+                )
+                return build_curl_cffi_http_client(
+                    impersonate=impersonate,
+                    proxy=_proxy,
+                )
+            except ImportError:
+                logger.warning(
+                    "tls_impersonate is set but curl_cffi is not installed; "
+                    "falling back to stock httpx. Install: pip install curl_cffi"
+                )
+            except Exception as exc:
+                logger.warning(
+                    "curl_cffi transport setup failed: %s; falling back to stock httpx", exc
+                )
+
+        # ── Stock httpx with TCP keepalives ──
         try:
             import httpx as _httpx
             import socket as _socket

tests/run_agent/test_curl_cffi_transport.py

@@ -0,0 +1,94 @@
+"""Tests for the optional curl_cffi-backed httpx client."""
+from __future__ import annotations
+
+import httpx
+
+from agent.curl_cffi_transport import CurlCffiClient, build_curl_cffi_http_client
+
+
+class _FakeHeaders(dict):
+    def multi_items(self):
+        return list(self.items())
+
+
+class _FakeCurlResponse:
+    def __init__(self, *, content=b"body", chunks=None):
+        self.status_code = 200
+        self.headers = _FakeHeaders({"content-encoding": "gzip", "x-test": "ok"})
+        self.content = content
+        self._chunks = chunks or [b"a", b"b"]
+        self.closed = False
+
+    def iter_content(self, chunk_size=65536):
+        yield from self._chunks
+
+    def close(self):
+        self.closed = True
+
+
+class _FakeCurlSession:
+    def __init__(self, response):
+        self.response = response
+        self.calls = []
+        self.closed = False
+
+    def request(self, **kwargs):
+        self.calls.append(kwargs)
+        return self.response
+
+    def close(self):
+        self.closed = True
+
+
+def test_curl_cffi_client_is_httpx_client():
+    client = build_curl_cffi_http_client("chrome124")
+    try:
+        assert isinstance(client, httpx.Client)
+    finally:
+        client.close()
+
+
+def test_send_strips_httpx_bot_headers_and_content_encoding():
+    response = _FakeCurlResponse(content=b"ok")
+    session = _FakeCurlSession(response)
+    client = CurlCffiClient("chrome124")
+    object.__setattr__(client, "_curl_session", session)
+    request = client.build_request(
+        "POST",
+        "https://example.test/path",
+        headers={"User-Agent": "python-httpx/0.28.1", "Accept": "*/*", "X-Keep": "1"},
+        content=b"payload",
+    )
+
+    try:
+        got = client.send(request)
+    finally:
+        client.close()
+
+    sent_headers = {k.lower(): v for k, v in session.calls[0]["headers"]}
+    assert "user-agent" not in sent_headers
+    assert "accept" not in sent_headers
+    assert sent_headers["x-keep"] == "1"
+    assert got.status_code == 200
+    assert got.content == b"ok"
+    assert got.headers["x-test"] == "ok"
+    assert "content-encoding" not in got.headers
+
+
+def test_streaming_response_preserves_chunks_and_closes_curl_response():
+    response = _FakeCurlResponse(chunks=[b"chunk-1", b"chunk-2"])
+    session = _FakeCurlSession(response)
+    client = CurlCffiClient("chrome124")
+    object.__setattr__(client, "_curl_session", session)
+    request = client.build_request("GET", "https://example.test/stream")
+
+    try:
+        got = client.send(request, stream=True)
+        assert got.status_code == 200
+        assert list(got.iter_bytes()) == [b"chunk-1", b"chunk-2"]
+        got.close()
+    finally:
+        client.close()
+
+    assert session.calls[0]["stream"] is True
+    assert response.closed is True

tools/lazy_deps.py

@@ -86,6 +86,8 @@
     # when model.auth_mode=entra_id is selected; key-based azure-foundry
     # users never pay this import.
     "provider.azure_identity": ("azure-identity==1.25.3",),
+    # Optional TLS fingerprint impersonation for Cloudflare-protected OpenAI/Codex endpoints.
+    "provider.curl_cffi": ("curl_cffi==0.15.0",),
 
     # ─── Web search backends ───────────────────────────────────────────────
     "search.exa": ("exa-py==2.10.2",),