From 73f6b29bb343ecc873a690447f9ed722f36853fc Mon Sep 17 00:00:00 2001 From: Midnight-Kyo Date: Mon, 8 Jun 2026 11:06:37 +0400 Subject: [PATCH 1/2] feat(agent): add curl_cffi TLS impersonation transport for Cloudflare-protected endpoints MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add CurlCffiClient — an httpx.Client subclass that routes send() through curl_cffi with browser TLS fingerprint impersonation. Config-gated via model.tls_impersonate in config.yaml. When set to 'chrome124' (the profile confirmed to bypass chatgpt.com Cloudflare on this droplet), the Codex/OpenAI transport impersonates Chrome 124's JA3/JA4 TLS fingerprint, preventing the 403 + cf-mitigated: challenge that blocks httpx from datacenter IPs. Also strips httpx auto-headers (User-Agent: python-httpx/..., etc.) that otherwise flag requests as bot traffic regardless of TLS. Refs: #30480 Co-Authored-By: Wahab (@Midnight-Kyo) --- agent/curl_cffi_transport.py | 113 +++++++++++++++++++++++++++++++++++ run_agent.py | 51 ++++++++++++++++ 2 files changed, 164 insertions(+) create mode 100644 agent/curl_cffi_transport.py diff --git a/agent/curl_cffi_transport.py b/agent/curl_cffi_transport.py new file mode 100644 index 000000000000..2fb486f5c8ae --- /dev/null +++ b/agent/curl_cffi_transport.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""httpx.Client subclass that routes send() through curl_cffi for TLS impersonation. + +The OpenAI SDK calls build_request() + send(request, stream=...), and checks +isinstance(client, httpx.Client). By subclassing httpx.Client and overriding +only send(), we pass all checks while curl_cffi handles the actual TLS. +""" + +from __future__ import annotations + +import httpx +from curl_cffi import requests as curl_requests + +# httpx auto-headers that identify us as a Python bot — we strip these and +# let curl_cffi (with browser impersonation) set its own headers instead. +_CFFI_STRIP_REQUEST_HEADERS = { + "user-agent", # httpx sets "python-httpx/X.Y.Z" — instant Cloudflare 403 + "accept-encoding", # curl_cffi handles its own encoding + "connection", # curl manages keep-alive natively + "accept", # let curl set its browser-like Accept header +} + + +class CurlCffiClient(httpx.Client): + """httpx.Client that routes actual HTTP requests through curl_cffi. + + build_request() still creates normal httpx.Request objects (using httpx's + URL normalization and header merging). send() converts the request to a + curl_cffi call and returns an httpx.Response. + + This means: + - isinstance(curl_client, httpx.Client) → True (OpenAI SDK gate passes) + - The TLS handshake uses Chrome 124's fingerprint → Cloudflare allows it + - httpx auto-headers (python-httpx UA, accept-encoding, etc.) are stripped + so curl_cffi can send its own browser-like headers + """ + + def __init__( + self, + impersonate: str = "chrome124", + **kwargs, + ) -> None: + super().__init__(**kwargs) + self._curl_session = curl_requests.Session(impersonate=impersonate) + + def send(self, request: httpx.Request, *, stream: bool = False, **kwargs) -> httpx.Response: + """Execute an httpx.Request through curl_cffi. + + Strips httpx's auto-headers (User-Agent, Accept-Encoding, etc.) so + curl_cffi sends its own browser-impersonating headers instead. + """ + # Read body + content: bytes | None = None + if request.content: + if isinstance(request.content, bytes): + content = request.content + else: + content = request.content.encode("utf-8") + + # Filter headers: pass through everything EXCEPT httpx auto-headers + filtered_headers: list[tuple[str, str]] = [] + for k, v in request.headers.items(): + if k.lower() not in _CFFI_STRIP_REQUEST_HEADERS: + filtered_headers.append((k, v)) + + # Make the curl_cffi request with browser impersonation + curl_resp = self._curl_session.request( + method=request.method, + url=str(request.url), + headers=filtered_headers, + data=content, + stream=stream, + ) + + # Convert response headers — strip Content-Encoding since + # curl_cffi already decompressed the response body + resp_headers: list[tuple[str, str]] = [] + for k, v in curl_resp.headers.multi_items(): + if k.lower() == "content-encoding": + continue + resp_headers.append((k, v)) + + resp_body = curl_resp.content if not stream else b"" + + return httpx.Response( + status_code=curl_resp.status_code, + headers=resp_headers, + content=resp_body, + request=request, + ) + + def close(self) -> None: + self._curl_session.close() + super().close() + + +def build_curl_cffi_http_client( + impersonate: str = "chrome124", + proxy: str | None = None, +) -> CurlCffiClient: + """Build a CurlCffiClient with browser TLS impersonation. + + Can be passed as http_client= to the OpenAI SDK directly. + """ + client = CurlCffiClient( + impersonate=impersonate, + timeout=httpx.Timeout(connect=15.0, read=600.0, write=30.0, pool=30.0), + ) + + if proxy: + client._curl_session.proxies = {"http": proxy, "https": proxy} + + return client diff --git a/run_agent.py b/run_agent.py index 0be8b1763fa4..6bada4a7fae8 100644 --- a/run_agent.py +++ b/run_agent.py @@ -3439,8 +3439,59 @@ def _is_openai_client_closed(client: Any) -> bool: return bool(getattr(http_client, "is_closed", False)) return False + @staticmethod + def _tls_impersonation_profile() -> "str | None": + """Read model.tls_impersonate from config. + + Returns the impersonation profile name (e.g. ``"chrome124"``) or + ``None`` when the key is absent, empty, or set to ``"none"``. + """ + try: + from hermes_cli.config import load_config + cfg = load_config() + val = (cfg.get("model") or {}).get("tls_impersonate") + if isinstance(val, str) and val.strip().lower() not in ("", "none"): + return val.strip().lower() + except Exception: + pass + return None + + @staticmethod + def _is_cloudflare_protected(base_url: str) -> bool: + """True when *base_url* is known to sit behind Cloudflare WAF.""" + url_lower = (base_url or "").lower() + return any( + domain in url_lower + for domain in ("chatgpt.com", "api.openai.com") + ) + @staticmethod def _build_keepalive_http_client(base_url: str = "") -> Any: + # ── curl_cffi TLS impersonation for Cloudflare-protected endpoints ── + impersonate = AIAgent._tls_impersonation_profile() + if impersonate and AIAgent._is_cloudflare_protected(base_url): + try: + from agent.curl_cffi_transport import build_curl_cffi_http_client + _proxy = _get_proxy_for_base_url(base_url) + logger.info( + "Using curl_cffi TLS impersonation (%s) for %s", + impersonate, base_url[:80], + ) + return build_curl_cffi_http_client( + impersonate=impersonate, + proxy=_proxy, + ) + except ImportError: + logger.warning( + "tls_impersonate is set but curl_cffi is not installed; " + "falling back to stock httpx. Install: pip install curl_cffi" + ) + except Exception as exc: + logger.warning( + "curl_cffi transport setup failed: %s; falling back to stock httpx", exc + ) + + # ── Stock httpx with TCP keepalives ── try: import httpx as _httpx import socket as _socket From c6c8c24915c2777dd35af3f95f55420a730d39a7 Mon Sep 17 00:00:00 2001 From: Midnight-Kyo Date: Sat, 13 Jun 2026 11:13:53 +0400 Subject: [PATCH 2/2] fix(agent): harden curl_cffi transport integration --- agent/curl_cffi_transport.py | 25 +++++- pyproject.toml | 3 +- run_agent.py | 6 ++ tests/run_agent/test_curl_cffi_transport.py | 94 +++++++++++++++++++++ tools/lazy_deps.py | 2 + uv.lock | 33 +++++++- 6 files changed, 159 insertions(+), 4 deletions(-) create mode 100644 tests/run_agent/test_curl_cffi_transport.py diff --git a/agent/curl_cffi_transport.py b/agent/curl_cffi_transport.py index 2fb486f5c8ae..464008ac358b 100644 --- a/agent/curl_cffi_transport.py +++ b/agent/curl_cffi_transport.py @@ -21,6 +21,19 @@ } +class _CurlCffiByteStream(httpx.SyncByteStream): + """Expose a curl_cffi streaming response through httpx's stream API.""" + + def __init__(self, curl_response) -> None: + self._curl_response = curl_response + + def __iter__(self): + yield from self._curl_response.iter_content(chunk_size=65536) + + def close(self) -> None: + self._curl_response.close() + + class CurlCffiClient(httpx.Client): """httpx.Client that routes actual HTTP requests through curl_cffi. @@ -71,6 +84,8 @@ def send(self, request: httpx.Request, *, stream: bool = False, **kwargs) -> htt data=content, stream=stream, ) + if curl_resp is None: + raise RuntimeError("curl_cffi returned no response") # Convert response headers — strip Content-Encoding since # curl_cffi already decompressed the response body @@ -80,12 +95,18 @@ def send(self, request: httpx.Request, *, stream: bool = False, **kwargs) -> htt continue resp_headers.append((k, v)) - resp_body = curl_resp.content if not stream else b"" + if stream: + return httpx.Response( + status_code=curl_resp.status_code, + headers=resp_headers, + stream=_CurlCffiByteStream(curl_resp), + request=request, + ) return httpx.Response( status_code=curl_resp.status_code, headers=resp_headers, - content=resp_body, + content=curl_resp.content, request=request, ) diff --git a/pyproject.toml b/pyproject.toml index 5f645e129482..5d2171abb85b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -131,7 +131,7 @@ edge-tts = ["edge-tts==7.2.7"] modal = ["modal==1.3.4"] daytona = ["daytona==0.155.0"] hindsight = ["hindsight-client==0.6.1"] -dev = ["debugpy==1.8.20", "pytest==9.0.2", "pytest-asyncio==1.3.0", "mcp==1.26.0", "starlette==1.0.1", "ty==0.0.21", "ruff==0.15.10", "setuptools==82.0.1"] # starlette: CVE-2026-48710 +dev = ["debugpy==1.8.20", "pytest==9.0.2", "pytest-asyncio==1.3.0", "mcp==1.26.0", "starlette==1.0.1", "ty==0.0.21", "ruff==0.15.10", "setuptools==82.0.1", "curl_cffi==0.15.0"] # starlette: CVE-2026-48710 messaging = ["python-telegram-bot[webhooks]==22.6", "discord.py[voice]==2.7.1", "aiohttp==3.13.4", "brotlicffi==1.2.0.1", "slack-bolt==1.27.0", "slack-sdk==3.40.1", "qrcode==7.4.2"] # aiohttp: CVE-2026-34513/34518/34519/34520/34525 cron = [] # croniter is now a core dependency; this extra kept for back-compat slack = ["slack-bolt==1.27.0", "slack-sdk==3.40.1", "aiohttp==3.13.4"] @@ -192,6 +192,7 @@ acp = ["agent-client-protocol==0.9.0"] mistral = ["mistralai==2.4.8"] bedrock = ["boto3==1.42.89"] azure-identity = ["azure-identity==1.25.3"] +curl-cffi = ["curl_cffi==0.15.0"] termux = [ # Baseline Android / Termux path for reliable fresh installs. "python-telegram-bot[webhooks]==22.6", diff --git a/run_agent.py b/run_agent.py index 6bada4a7fae8..1c18735e2d34 100644 --- a/run_agent.py +++ b/run_agent.py @@ -3471,6 +3471,12 @@ def _build_keepalive_http_client(base_url: str = "") -> Any: impersonate = AIAgent._tls_impersonation_profile() if impersonate and AIAgent._is_cloudflare_protected(base_url): try: + try: + from tools.lazy_deps import ensure as _lazy_ensure + _lazy_ensure("provider.curl_cffi", prompt=False) + except ImportError: + pass + from agent.curl_cffi_transport import build_curl_cffi_http_client _proxy = _get_proxy_for_base_url(base_url) logger.info( diff --git a/tests/run_agent/test_curl_cffi_transport.py b/tests/run_agent/test_curl_cffi_transport.py new file mode 100644 index 000000000000..3cc85341de41 --- /dev/null +++ b/tests/run_agent/test_curl_cffi_transport.py @@ -0,0 +1,94 @@ +"""Tests for the optional curl_cffi-backed httpx client.""" +from __future__ import annotations + +import httpx + +from agent.curl_cffi_transport import CurlCffiClient, build_curl_cffi_http_client + + +class _FakeHeaders(dict): + def multi_items(self): + return list(self.items()) + + +class _FakeCurlResponse: + def __init__(self, *, content=b"body", chunks=None): + self.status_code = 200 + self.headers = _FakeHeaders({"content-encoding": "gzip", "x-test": "ok"}) + self.content = content + self._chunks = chunks or [b"a", b"b"] + self.closed = False + + def iter_content(self, chunk_size=65536): + yield from self._chunks + + def close(self): + self.closed = True + + +class _FakeCurlSession: + def __init__(self, response): + self.response = response + self.calls = [] + self.closed = False + + def request(self, **kwargs): + self.calls.append(kwargs) + return self.response + + def close(self): + self.closed = True + + +def test_curl_cffi_client_is_httpx_client(): + client = build_curl_cffi_http_client("chrome124") + try: + assert isinstance(client, httpx.Client) + finally: + client.close() + + +def test_send_strips_httpx_bot_headers_and_content_encoding(): + response = _FakeCurlResponse(content=b"ok") + session = _FakeCurlSession(response) + client = CurlCffiClient("chrome124") + object.__setattr__(client, "_curl_session", session) + request = client.build_request( + "POST", + "https://example.test/path", + headers={"User-Agent": "python-httpx/0.28.1", "Accept": "*/*", "X-Keep": "1"}, + content=b"payload", + ) + + try: + got = client.send(request) + finally: + client.close() + + sent_headers = {k.lower(): v for k, v in session.calls[0]["headers"]} + assert "user-agent" not in sent_headers + assert "accept" not in sent_headers + assert sent_headers["x-keep"] == "1" + assert got.status_code == 200 + assert got.content == b"ok" + assert got.headers["x-test"] == "ok" + assert "content-encoding" not in got.headers + + +def test_streaming_response_preserves_chunks_and_closes_curl_response(): + response = _FakeCurlResponse(chunks=[b"chunk-1", b"chunk-2"]) + session = _FakeCurlSession(response) + client = CurlCffiClient("chrome124") + object.__setattr__(client, "_curl_session", session) + request = client.build_request("GET", "https://example.test/stream") + + try: + got = client.send(request, stream=True) + assert got.status_code == 200 + assert list(got.iter_bytes()) == [b"chunk-1", b"chunk-2"] + got.close() + finally: + client.close() + + assert session.calls[0]["stream"] is True + assert response.closed is True diff --git a/tools/lazy_deps.py b/tools/lazy_deps.py index 76f146c7869c..7046cd1ad7cc 100644 --- a/tools/lazy_deps.py +++ b/tools/lazy_deps.py @@ -86,6 +86,8 @@ # when model.auth_mode=entra_id is selected; key-based azure-foundry # users never pay this import. "provider.azure_identity": ("azure-identity==1.25.3",), + # Optional TLS fingerprint impersonation for Cloudflare-protected OpenAI/Codex endpoints. + "provider.curl_cffi": ("curl_cffi==0.15.0",), # ─── Web search backends ─────────────────────────────────────────────── "search.exa": ("exa-py==2.10.2",), diff --git a/uv.lock b/uv.lock index d2786cc37543..c405c5d8d906 100644 --- a/uv.lock +++ b/uv.lock @@ -760,6 +760,31 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/9b/15/6e8e87c6a201d69803a79ac2e29623ce7c2cc9cd1df9db99810cca714373/ctranslate2-4.7.1-cp313-cp313-win_amd64.whl", hash = "sha256:baa6d2b10f57933d8c11791e8522659217918722d07bbef2389a443801125fe7", size = 18844953, upload-time = "2026-02-04T06:11:58.519Z" }, ] +[[package]] +name = "curl-cffi" +version = "0.15.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "cffi" }, + { name = "rich" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/48/5b/89fcfebd3e5e85134147ac99e9f2b2271165fd4d71984fc65da5f17819b7/curl_cffi-0.15.0.tar.gz", hash = "sha256:ea0c67652bf6893d34ee0f82c944f37e488f6147e9421bef1771cc6545b02ded", size = 196437, upload-time = "2026-04-03T11:12:31.525Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5e/42/54ddd442c795f30ce5dd4e49f87ce77505958d3777cd96a91567a3975d2a/curl_cffi-0.15.0-cp310-abi3-macosx_10_9_x86_64.whl", hash = "sha256:bda66404010e9ed743b1b83c20c86f24fe21a9a6873e17479d6e67e29d8ded28", size = 2795267, upload-time = "2026-04-03T11:11:46.48Z" }, + { url = "https://files.pythonhosted.org/packages/83/2d/3915e238579b3c5a92cead5c79130c3b8d20caaba7616cc4d894650e1d6b/curl_cffi-0.15.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:a25620d9bf989c9c029a7d1642999c4c265abb0bad811deb2f77b0b5b2b12e5b", size = 2573544, upload-time = "2026-04-03T11:11:47.951Z" }, + { url = "https://files.pythonhosted.org/packages/2a/b3/9d2f1057749a1b07ba1989db3c1503ce8bed998310bae9aea2c43aa64f20/curl_cffi-0.15.0-cp310-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:582e570aa2586b96ed47cf4a17586b9a3c462cbe43f780487c3dc245c6ef1527", size = 10515369, upload-time = "2026-04-03T11:11:50.126Z" }, + { url = "https://files.pythonhosted.org/packages/b5/1d/6d10dded5ce3fd8157e558ebd97d09e551b77a62cdc1c31e93d0a633cee5/curl_cffi-0.15.0-cp310-abi3-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:838e48212447d9c81364b04707a5c861daf08f8320f9ecb3406a8919d1d5c3b3", size = 10160045, upload-time = "2026-04-03T11:11:52.664Z" }, + { url = "https://files.pythonhosted.org/packages/5c/12/c70b835487ace3b9ba1502631912e3440082b8ae3a162f60b59cb0b6444d/curl_cffi-0.15.0-cp310-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2b6c847d86283b07ae69bb72c82eb8a59242277142aa35b89850f89e792a02fc", size = 11090433, upload-time = "2026-04-03T11:11:55.049Z" }, + { url = "https://files.pythonhosted.org/packages/ea/0d/78edcc4f71934225db99df68197a107386d59080742fc7bf6bb4d007924f/curl_cffi-0.15.0-cp310-abi3-manylinux_2_28_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:9e5e69eee735f659287e2c84444319d68a1fa68dd37abf228943a4074864283a", size = 10479178, upload-time = "2026-04-03T11:11:57.685Z" }, + { url = "https://files.pythonhosted.org/packages/5b/84/1e101c1acb1ea2f0b4992f5c3024f596d8e21db0d53540b9d583f673c4e7/curl_cffi-0.15.0-cp310-abi3-manylinux_2_34_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:aa1323950224db24f4c510d010b3affa02196ca853fb424191fa917a513d3f4b", size = 10317051, upload-time = "2026-04-03T11:12:00.295Z" }, + { url = "https://files.pythonhosted.org/packages/28/42/8ef236b22a6c23d096c85a1dc507efe37bfdfc7a2f8a4b34efb590197369/curl_cffi-0.15.0-cp310-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:41f80170ba844009273b2660da1964ec31e99e5719d16b3422ada87177e32e13", size = 11299660, upload-time = "2026-04-03T11:12:02.791Z" }, + { url = "https://files.pythonhosted.org/packages/1d/01/56aeb055d962da87a1be0d74c6c644e251c7e88129b5471dc44ac724e678/curl_cffi-0.15.0-cp310-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1977e1e12cfb5c11352cbb74acef1bed24eb7d226dab61ca57c168c21acd4d61", size = 11945049, upload-time = "2026-04-03T11:12:05.912Z" }, + { url = "https://files.pythonhosted.org/packages/d8/8c/2abf99a38d6340d66cf0557e0c750ef3f8883dfc5d450087e01c85861343/curl_cffi-0.15.0-cp310-abi3-win_amd64.whl", hash = "sha256:5a0c1896a0d5a5ac1eb89cd24b008d2b718dd1df6fd2f75451b59ca66e49e572", size = 1661649, upload-time = "2026-04-03T11:12:07.948Z" }, + { url = "https://files.pythonhosted.org/packages/3d/39/dfd54f2240d3a9b96d77bacc62b97813b35e2aa8ecf5cd5013c683f1ba96/curl_cffi-0.15.0-cp310-abi3-win_arm64.whl", hash = "sha256:a6d57f8389273a3a1f94370473c74897467bcc36af0a17336989780c507fa43d", size = 1410741, upload-time = "2026-04-03T11:12:10.073Z" }, + { url = "https://files.pythonhosted.org/packages/19/6a/c24df8a4fc22fa84070dcd94abeba43c15e08cc09e35869565c0bad196fd/curl_cffi-0.15.0-cp313-abi3-android_24_arm64_v8a.whl", hash = "sha256:4682dc38d4336e0eb0b185374db90a760efde63cbea994b4e63f3521d44c4c92", size = 7190427, upload-time = "2026-04-03T11:12:12.142Z" }, +] + [[package]] name = "darabonba-core" version = "1.0.5" @@ -1453,10 +1478,14 @@ computer-use = [ { name = "mcp" }, { name = "starlette" }, ] +curl-cffi = [ + { name = "curl-cffi" }, +] daytona = [ { name = "daytona" }, ] dev = [ + { name = "curl-cffi" }, { name = "debugpy" }, { name = "mcp" }, { name = "pytest" }, @@ -1598,6 +1627,8 @@ requires-dist = [ { name = "boto3", marker = "extra == 'bedrock'", specifier = "==1.42.89" }, { name = "brotlicffi", marker = "extra == 'messaging'", specifier = "==1.2.0.1" }, { name = "croniter", specifier = "==6.0.0" }, + { name = "curl-cffi", marker = "extra == 'curl-cffi'", specifier = "==0.15.0" }, + { name = "curl-cffi", marker = "extra == 'dev'", specifier = "==0.15.0" }, { name = "daytona", marker = "extra == 'daytona'", specifier = "==0.155.0" }, { name = "debugpy", marker = "extra == 'dev'", specifier = "==1.8.20" }, { name = "defusedxml", marker = "extra == 'wecom'", specifier = "==0.7.1" }, @@ -1693,7 +1724,7 @@ requires-dist = [ { name = "uvicorn", extras = ["standard"], marker = "extra == 'web'", specifier = "==0.41.0" }, { name = "youtube-transcript-api", marker = "extra == 'youtube'", specifier = "==1.2.4" }, ] -provides-extras = ["anthropic", "exa", "firecrawl", "parallel-web", "fal", "edge-tts", "modal", "daytona", "hindsight", "dev", "messaging", "cron", "slack", "matrix", "wecom", "cli", "tts-premium", "voice", "pty", "honcho", "vision", "mcp", "nemo-relay", "homeassistant", "sms", "computer-use", "acp", "mistral", "bedrock", "azure-identity", "termux", "termux-all", "dingtalk", "feishu", "google", "youtube", "web", "all"] +provides-extras = ["anthropic", "exa", "firecrawl", "parallel-web", "fal", "edge-tts", "modal", "daytona", "hindsight", "dev", "messaging", "cron", "slack", "matrix", "wecom", "cli", "tts-premium", "voice", "pty", "honcho", "vision", "mcp", "nemo-relay", "homeassistant", "sms", "computer-use", "acp", "mistral", "bedrock", "azure-identity", "curl-cffi", "termux", "termux-all", "dingtalk", "feishu", "google", "youtube", "web", "all"] [[package]] name = "hf-xet"