Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 104 additions & 2 deletions agent/auxiliary_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -1010,6 +1010,7 @@ class AsyncCodexAuxiliaryClient:
"""Async-compatible wrapper matching AsyncOpenAI.chat.completions.create()."""

def __init__(self, sync_wrapper: "CodexAuxiliaryClient"):
self._sync_wrapper = sync_wrapper
sync_adapter = sync_wrapper.chat.completions
async_adapter = _AsyncCodexCompletionsAdapter(sync_adapter)
self.chat = _AsyncCodexChatShim(async_adapter)
Expand Down Expand Up @@ -4487,6 +4488,73 @@ def _force_close_async_httpx(client: Any) -> None:
pass


def _is_client_closed(client: Any) -> bool:
"""Return True when a cached auxiliary client is no longer usable.

The OpenAI SDK reports requests through a closed underlying httpx client as
``APIConnectionError('Connection error.')``. The Codex Responses timeout
watchdog intentionally closes the underlying client to unblock a stuck
stream, so the cache must not hand that wrapper back to later auxiliary
calls.

This is deliberately best-effort and side-effect free: if a provider
wrapper does not expose a closed flag, treat it as open.
"""
if client is None:
return False

seen: set[int] = set()

def _check(obj: Any) -> bool:
if obj is None:
return False
oid = id(obj)
if oid in seen:
return False
seen.add(oid)

try:
closed_flag = getattr(obj, "is_closed", False)
if isinstance(closed_flag, bool) and closed_flag:
return True
if callable(closed_flag):
try:
result = closed_flag()
except TypeError:
result = False
if isinstance(result, bool) and result:
return True
except Exception:
pass

try:
state = getattr(obj, "_state", None)
if state is not None and str(state).endswith(".CLOSED"):
return True
except Exception:
pass

# Wrapper chain examples:
# CodexAuxiliaryClient._real_client -> OpenAI._client -> httpx.Client
# AsyncCodexAuxiliaryClient._sync_wrapper -> CodexAuxiliaryClient
# Anthropic/other adapters may expose _client directly.
# Only inspect explicitly stored attributes. unittest.mock.MagicMock
# fabricates arbitrary attributes on access; recursing into those would
# make every mock look like a nested client and can produce false
# closed detections on cache-hit tests.
try:
obj_vars = vars(obj)
except TypeError:
obj_vars = {}
for attr in ("_sync_wrapper", "_real_client", "_client", "client"):
inner = obj_vars.get(attr)
if inner is not None and inner is not obj and _check(inner):
return True
return False

return _check(client)


def shutdown_cached_clients() -> None:
"""Close all cached clients (sync and async) to prevent event-loop errors.

Expand Down Expand Up @@ -4607,7 +4675,18 @@ def _get_cached_client(
with _client_cache_lock:
if cache_key in _client_cache:
cached_client, cached_default, cached_loop = _client_cache[cache_key]
if async_mode:
if _is_client_closed(cached_client):
# The Codex Responses timeout watchdog closes the underlying
# OpenAI/httpx client to unblock a stuck stream. Do not reuse
# that wrapper: the next request would surface only as the
# misleading OpenAI SDK message "Connection error.".
logger.debug(
"Auxiliary %s client cache entry is closed; evicting and rebuilding",
provider,
)
_force_close_async_httpx(cached_client)
del _client_cache[cache_key]
elif async_mode:
# Validate: the cached client must be bound to the CURRENT,
# OPEN loop. If the loop changed or was closed, the httpx
# transport inside is dead — force-close and replace.
Expand Down Expand Up @@ -4662,7 +4741,30 @@ def _get_cached_client(
del _client_cache[evict_key]
_client_cache[cache_key] = (client, default_model, bound_loop)
else:
client, default_model, _ = _client_cache[cache_key]
cached_client, cached_default, cached_loop = _client_cache[cache_key]
reuse_cached = not _is_client_closed(cached_client)
if reuse_cached and async_mode:
reuse_cached = (
cached_loop is not None
and cached_loop is current_loop
and not cached_loop.is_closed()
)

if reuse_cached:
# Another thread populated an equivalent usable entry while
# we were building outside the lock. Use the winner and
# close the loser so we do not leak connection pools.
_force_close_async_httpx(client)
try:
close_fn = getattr(client, "close", None)
if callable(close_fn):
close_fn()
except Exception:
pass
client, default_model = cached_client, cached_default
else:
_force_close_async_httpx(cached_client)
_client_cache[cache_key] = (client, default_model, bound_loop)
return client, model or default_model


Expand Down
25 changes: 16 additions & 9 deletions agent/codex_responses_adapter.py
Original file line number Diff line number Diff line change
Expand Up @@ -1074,8 +1074,8 @@ def _normalize_codex_response(
message_items_raw: List[Dict[str, Any]] = []
tool_calls: List[Any] = []
has_incomplete_items = response_status in {"queued", "in_progress", "incomplete"}
saw_commentary_phase = False
saw_final_answer_phase = False
saw_hidden_phase_message_text = False
saw_unknown_phase_message_text = False
saw_reasoning_item = False

for item in output:
Expand All @@ -1094,13 +1094,20 @@ def _normalize_codex_response(
normalized_phase = None
if isinstance(item_phase, str):
normalized_phase = item_phase.strip().lower()
if normalized_phase in {"commentary", "analysis"}:
saw_commentary_phase = True
elif normalized_phase in {"final_answer", "final"}:
saw_final_answer_phase = True
message_text = _extract_responses_message_text(item)
if message_text:
content_parts.append(message_text)
is_hidden_phase = normalized_phase in {"commentary", "analysis"}
is_visible_phase = normalized_phase in {None, "", "final_answer", "final"}
if is_hidden_phase:
saw_hidden_phase_message_text = True
elif not is_visible_phase:
saw_unknown_phase_message_text = True
logger.warning(
"Codex response message has unknown phase %r; hiding it and treating response as incomplete.",
normalized_phase,
)
if is_visible_phase:
content_parts.append(message_text)
raw_message_item: Dict[str, Any] = {
"type": "message",
"role": "assistant",
Expand Down Expand Up @@ -1195,7 +1202,7 @@ def _normalize_codex_response(
))

final_text = "\n".join([p for p in content_parts if p]).strip()
if not final_text and hasattr(response, "output_text"):
if not final_text and not (saw_hidden_phase_message_text or saw_unknown_phase_message_text) and hasattr(response, "output_text"):
out_text = getattr(response, "output_text", "")
if isinstance(out_text, str):
final_text = out_text.strip()
Expand Down Expand Up @@ -1245,7 +1252,7 @@ def _normalize_codex_response(
finish_reason = "tool_calls"
elif leaked_tool_call_text:
finish_reason = "incomplete"
elif has_incomplete_items or (saw_commentary_phase and not saw_final_answer_phase):
elif has_incomplete_items or ((saw_hidden_phase_message_text or saw_unknown_phase_message_text) and not final_text):
finish_reason = "incomplete"
elif (reasoning_items_raw or reasoning_parts or saw_reasoning_item) and not final_text:
# Response contains only reasoning (encrypted thinking state and/or
Expand Down
27 changes: 27 additions & 0 deletions agent/codex_runtime.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,33 @@
logger = logging.getLogger(__name__)


try:
import openai.lib._parsing._responses as _openai_parse_responses

if not getattr(_openai_parse_responses.parse_response, "_hermes_none_guard", False):
_hermes_orig_parse_response = _openai_parse_responses.parse_response

def _hermes_patched_parse_response(*, text_format, input_tools, response):
# The ChatGPT Codex backend can close after response.output_item.done
# without response.completed, leaving SDK snapshots with output=None.
if getattr(response, "output", None) is None:
response.output = []
return _hermes_orig_parse_response(
text_format=text_format,
input_tools=input_tools,
response=response,
)

_hermes_patched_parse_response._hermes_none_guard = True
_openai_parse_responses.parse_response = _hermes_patched_parse_response
except Exception as _e:
logger.warning(
"Hermes: failed to apply openai SDK parse_response None-guard (%s); "
"Codex streams that close without response.completed may crash with TypeError.",
_e,
)


def run_codex_app_server_turn(
agent,
*,
Expand Down
13 changes: 10 additions & 3 deletions cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -5052,7 +5052,11 @@ def _resolve_turn_agent_config(self, user_message: str) -> dict:
return route

try:
overrides = resolve_fast_mode_overrides(route["model"])
overrides = resolve_fast_mode_overrides(
route["model"],
provider=runtime.get("provider"),
api_mode=runtime.get("api_mode"),
)
except Exception:
overrides = None
route["request_overrides"] = overrides
Expand Down Expand Up @@ -6307,12 +6311,15 @@ def _show_session_status(self):

def _fast_command_available(self) -> bool:
try:
from hermes_cli.models import model_supports_fast_mode
from hermes_cli.models import model_supports_fast_mode, runtime_supports_priority_processing
except Exception:
return False
agent = getattr(self, "agent", None)
model = getattr(agent, "model", None) or getattr(self, "model", None)
return model_supports_fast_mode(model)
return model_supports_fast_mode(model) or runtime_supports_priority_processing(
getattr(self, "provider", None) or getattr(self, "requested_provider", None),
getattr(self, "api_mode", None),
)

def _command_available(self, slash_command: str) -> bool:
if slash_command == "/fast":
Expand Down
18 changes: 16 additions & 2 deletions cron/scheduler.py
Original file line number Diff line number Diff line change
Expand Up @@ -721,6 +721,20 @@ def _send_media_via_adapter(
logger.warning("Job '%s': failed to send media %s: %s", job.get("id", "?"), media_path, e)


def _raw_response_get(raw_response, key: str):
"""Safely read optional metadata from adapter raw_response objects.

Some gateway adapters return SDK response objects as ``SendResult.raw_response``
(for example Feishu/Lark ``ReplyMessageResponse``) rather than dictionaries.
Cron delivery uses this metadata only for best-effort diagnostics, so a
non-mapping raw response must not turn an already-successful live-adapter
send into a fallback send that duplicates the message.
"""
if isinstance(raw_response, dict):
return raw_response.get(key)
return None


def _deliver_result(job: dict, content: str, adapters=None, loop=None) -> Optional[str]:
"""
Deliver job output to the configured target(s) (origin chat, specific platform, etc.).
Expand Down Expand Up @@ -852,9 +866,9 @@ def _deliver_result(job: dict, content: str, adapters=None, loop=None) -> Option
send_result
and thread_id
and getattr(send_result, "raw_response", None)
and send_result.raw_response.get("thread_fallback")
and _raw_response_get(send_result.raw_response, "thread_fallback")
):
requested_thread_id = send_result.raw_response.get("requested_thread_id") or thread_id
requested_thread_id = _raw_response_get(send_result.raw_response, "requested_thread_id") or thread_id
msg = (
f"configured thread_id {requested_thread_id} for "
f"{platform_name}:{chat_id} was not found; delivered without thread_id"
Expand Down
2 changes: 1 addition & 1 deletion gateway/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -912,7 +912,7 @@ def _merge_platform_map(source_platforms: Any) -> None:
bridged["group_allow_admin_from"] = platform_cfg["group_allow_admin_from"]
if "group_user_allowed_commands" in platform_cfg:
bridged["group_user_allowed_commands"] = platform_cfg["group_user_allowed_commands"]
if plat in {Platform.DISCORD, Platform.SLACK} and "channel_skill_bindings" in platform_cfg:
if plat in {Platform.DISCORD, Platform.SLACK, Platform.FEISHU} and "channel_skill_bindings" in platform_cfg:
bridged["channel_skill_bindings"] = platform_cfg["channel_skill_bindings"]
if "channel_prompts" in platform_cfg:
channel_prompts = platform_cfg["channel_prompts"]
Expand Down
Loading