Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
106 changes: 90 additions & 16 deletions litellm/litellm_core_utils/litellm_logging.py
Original file line number Diff line number Diff line change
Expand Up @@ -1603,6 +1603,90 @@ async def _response_cost_calculator_async(
) -> Optional[float]:
return self._response_cost_calculator(result=result, cache_hit=cache_hit)

@staticmethod
def _is_sync_litellm_request(litellm_params: dict) -> bool:
"""True for sync SDK entrypoints (``completion``), false for async (``acompletion``, etc.)."""
return (
litellm_params.get(CallTypes.acompletion.value, False) is not True
and litellm_params.get(CallTypes.aresponses.value, False) is not True
and litellm_params.get(CallTypes.aembedding.value, False) is not True
and litellm_params.get(CallTypes.aimage_generation.value, False) is not True
and litellm_params.get(CallTypes.atranscription.value, False) is not True
)

def _is_assembled_stream_success(self, result=None) -> bool:
"""Final assembled stream export (not a per-chunk success call).

Per-chunk callers pass a ``ModelResponseStream`` (or ``None``); the
final assembled response is any other non-``None`` value (typically a
``ModelResponse``). Treating a chunk as the assembled response would
prematurely set the ``has_dispatched_final_stream_success`` dedup
guard and silently suppress the real final stream log.
"""
if self.stream is not True:
return False
if result is not None and not isinstance(result, ModelResponseStream):
return True
return (
"async_complete_streaming_response" in self.model_call_details
or self.model_call_details.get("complete_streaming_response") is not None
)

async def dispatch_success_handlers(
self,
result=None,
start_time=None,
end_time=None,
cache_hit=None,
prefer_async_handlers: bool = False,
**kwargs,
) -> None:
"""Route success logging to async and/or sync handlers for this request.

``prefer_async_handlers`` only bypasses the sync-SDK-only shortcut (e.g.
``async for`` on a stream from ``completion()``). Legacy string callbacks
still run via ``executor.submit(success_handler)`` when configured.
"""
from litellm.litellm_core_utils.thread_pool_executor import executor

if self._is_assembled_stream_success(result):
if self.model_call_details.get("has_dispatched_final_stream_success"):
return
self.model_call_details["has_dispatched_final_stream_success"] = True

litellm_params = self.model_call_details.get("litellm_params", {}) or {}
sync_sdk = self._is_sync_litellm_request(litellm_params)
passthrough = self.call_type == CallTypes.pass_through.value
if sync_sdk and not prefer_async_handlers and not passthrough:
self.success_handler(
result,
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
**kwargs,
)
return

await self.async_success_handler(
result,
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
**kwargs,
)

if not self._should_run_sync_callbacks_for_async_calls():
return

executor.submit(
self.success_handler,
result,
start_time=start_time,
end_time=end_time,
cache_hit=cache_hit,
**kwargs,
)

def should_run_logging(
self,
event_type: Literal[
Expand Down Expand Up @@ -2022,13 +2106,7 @@ def success_handler( # noqa: PLR0915
standard_logging_object=kwargs.get("standard_logging_object", None),
)
litellm_params = self.model_call_details.get("litellm_params", {})
is_sync_request = (
litellm_params.get(CallTypes.acompletion.value, False) is not True
and litellm_params.get(CallTypes.aresponses.value, False) is not True
and litellm_params.get(CallTypes.aembedding.value, False) is not True
and litellm_params.get(CallTypes.aimage_generation.value, False) is not True
and litellm_params.get(CallTypes.atranscription.value, False) is not True
)
is_sync_request = self._is_sync_litellm_request(litellm_params)
try:
## BUILD COMPLETE STREAMED RESPONSE
complete_streaming_response: Optional[
Expand Down Expand Up @@ -2484,9 +2562,11 @@ async def async_success_handler( # noqa: PLR0915
print_verbose(
"Logging Details LiteLLM-Async Success Call, cache_hit={}".format(cache_hit)
)
if not self.should_run_logging(
if not self._is_assembled_stream_success(
result
) and not self.should_run_logging(
event_type="async_success"
): # prevent double logging
): # prevent double logging (non-streaming)
return
Comment on lines +2565 to 2570

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2 should_run_logging guard bypassed for all assembled stream calls to async_success_handler

async_success_handler previously checked should_run_logging unconditionally to prevent double logging. The new condition skips that guard whenever _is_assembled_stream_success returns True, relying on the has_dispatched_final_stream_success flag set by dispatch_success_handlers. Any call site that invokes async_success_handler directly with an assembled stream result (bypassing dispatch_success_handlers) will now skip the double-log guard with no fallback. The comment should at minimum make this assumption explicit so future callers don't inadvertently create double-logging regressions.

Suggested change
if not self._is_assembled_stream_success(
result
) and not self.should_run_logging(
event_type="async_success"
): # prevent double logging
): # prevent double logging (non-streaming)
return
# For assembled stream results, dedup is enforced by the
# ``has_dispatched_final_stream_success`` guard in
# ``dispatch_success_handlers``. Always route assembled stream
# responses through ``dispatch_success_handlers`` — calling
# ``async_success_handler`` directly with an assembled result
# bypasses that guard and may produce duplicate log events.
if not self._is_assembled_stream_success(
result
) and not self.should_run_logging(
event_type="async_success"
): # prevent double logging (non-streaming)
return

Note: If this suggestion doesn't match your team's coding style, reply to this and let me know. I'll remember it for next time!


## CALCULATE COST FOR BATCH JOBS
Expand Down Expand Up @@ -2936,13 +3016,7 @@ def failure_handler( # noqa: PLR0915
): # prevent double logging
return
litellm_params = self.model_call_details.get("litellm_params", {})
is_sync_request = (
litellm_params.get(CallTypes.acompletion.value, False) is not True
and litellm_params.get(CallTypes.aresponses.value, False) is not True
and litellm_params.get(CallTypes.aembedding.value, False) is not True
and litellm_params.get(CallTypes.aimage_generation.value, False) is not True
and litellm_params.get(CallTypes.atranscription.value, False) is not True
)
is_sync_request = self._is_sync_litellm_request(litellm_params)

try:
start_time, end_time = self._failure_handler_helper_fn(
Expand Down
20 changes: 9 additions & 11 deletions litellm/litellm_core_utils/streaming_handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -1808,8 +1808,10 @@ def run_success_logging_and_cache_storage(self, processed_chunk, cache_hit: bool
processed_chunk, None, None, cache_hit
)
)
## SYNC LOGGING
self.logging_obj.success_handler(processed_chunk, None, None, cache_hit)
## SYNC LOGGING — only for sync SDK entrypoints; async proxy paths export via async_success_handler
litellm_params = self.logging_obj.model_call_details.get("litellm_params", {})
if self.logging_obj._is_sync_litellm_request(litellm_params):
self.logging_obj.success_handler(processed_chunk, None, None, cache_hit)

def finish_reason_handler(self):
model_response = self.model_response_creator()
Expand Down Expand Up @@ -2206,23 +2208,19 @@ async def __anext__(self) -> "ModelResponseStream": # noqa: PLR0915
cache_hit,
)
else:
# prefer_async_handlers routes CustomLogger to async_success_handler
# when consumers use ``async for`` on sync-SDK streams. Legacy string
# callbacks still run via executor.submit inside dispatch_success_handlers.
asyncio.create_task(
self.logging_obj.async_success_handler(
self.logging_obj.dispatch_success_handlers(
complete_streaming_response,
cache_hit=cache_hit,
start_time=None,
end_time=None,
prefer_async_handlers=True,
)
)

executor.submit(
self.logging_obj.success_handler,
complete_streaming_response,
cache_hit=cache_hit,
start_time=None,
end_time=None,
)

raise StopAsyncIteration # Re-raise StopIteration
else:
self.sent_last_chunk = True
Expand Down
13 changes: 8 additions & 5 deletions litellm/proxy/_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -2713,13 +2713,16 @@ def _safe_hash_litellm_api_key(cls, api_key: str) -> str:
1. Regular API keys from LiteLLM DB
2. JWT tokens used for connecting to LiteLLM API
"""
if api_key.startswith("sk-"):
return hash_token(api_key)
normalized = api_key
if normalized[:7].lower() == "bearer ":
normalized = normalized[7:]
if normalized.startswith("sk-"):
return hash_token(normalized)
from litellm.proxy.auth.handle_jwt import JWTHandler

if JWTHandler.is_jwt(token=api_key):
return f"hashed-jwt-{hash_token(token=api_key)}"
return api_key
if JWTHandler.is_jwt(token=normalized):
return f"hashed-jwt-{hash_token(token=normalized)}"
return normalized

@classmethod
def get_litellm_internal_health_check_user_api_key_auth(cls) -> "UserAPIKeyAuth":
Expand Down
49 changes: 12 additions & 37 deletions litellm/proxy/common_request_processing.py
Original file line number Diff line number Diff line change
Expand Up @@ -1266,7 +1266,7 @@ async def base_process_llm_request( # noqa: PLR0915
# (ProxyLogging._fire_deferred_stream_logging) fires the
# closure after the full streaming pipeline finishes.
# The closure runs non-apply_guardrail hooks on the
# assembled response, then fires both logging handlers.
# assembled response, then fires success logging.
# Only for CustomStreamWrapper — raw async generators from
# passthrough routes bypass CSW and would orphan the closure.
from litellm.litellm_core_utils.streaming_handler import (
Expand Down Expand Up @@ -1387,33 +1387,18 @@ async def _on_deferred_stream_complete(
logging_obj._on_deferred_stream_complete = None # type: ignore[union-attr]
try:
asyncio.create_task(
logging_obj.async_success_handler(
logging_obj.dispatch_success_handlers(
response,
cache_hit=None,
start_time=None,
end_time=None,
prefer_async_handlers=True,
)
)
except Exception as e:
verbose_proxy_logger.exception(
"Error in orphaned streaming async logging: %s", e
)
try:
from litellm.litellm_core_utils.thread_pool_executor import (
executor as _exc,
)

_exc.submit(
logging_obj.success_handler,
response,
cache_hit=None,
start_time=None,
end_time=None,
)
except Exception as e:
verbose_proxy_logger.exception(
"Error in orphaned streaming sync logging: %s", e
)

# Always return the client-requested model name (not provider-prefixed internal identifiers)
# for OpenAI-compatible responses.
Expand Down Expand Up @@ -1615,7 +1600,7 @@ async def _run_deferred_stream_guardrails(
) -> None:
"""
Run non-streaming post-call guardrail hooks on an assembled streaming
response, then fire both async and sync logging handlers.
response, then fire success logging via ``dispatch_success_handlers``.

Called by ProxyLogging._fire_deferred_stream_logging after the full
streaming pipeline (including unified_guardrail end-of-stream blocks)
Expand All @@ -1631,8 +1616,6 @@ async def _run_deferred_stream_guardrails(
Extracted as a static method so tests can call the production
implementation directly rather than reimplementing the closure.
"""
from litellm.litellm_core_utils.thread_pool_executor import executor

_response = assembled_response
try:
from litellm.proxy.proxy_server import llm_router as _global_llm_router
Expand Down Expand Up @@ -1691,31 +1674,23 @@ async def _run_deferred_stream_guardrails(
)
finally:
try:
# Proxy streaming always runs in async context and proxy spend
# logging is async-only; force async dispatch so DB/spend
# callbacks fire regardless of the call-type heuristic in
# _is_sync_litellm_request (which only recognizes a subset of
# async markers stored in litellm_params).
asyncio.create_task(
captured_logging_obj.async_success_handler(
captured_logging_obj.dispatch_success_handlers(
_response,
cache_hit=cache_hit,
start_time=None,
end_time=None,
prefer_async_handlers=True,
)
)
except Exception as e:
verbose_proxy_logger.exception(
"Error in deferred streaming async logging: %s",
e,
)

try:
executor.submit(
captured_logging_obj.success_handler,
_response,
cache_hit=cache_hit,
start_time=None,
end_time=None,
)
except Exception as e:
verbose_proxy_logger.exception(
"Error in deferred streaming sync logging: %s",
"Error in deferred streaming success logging: %s",
e,
)

Expand Down
Loading
Loading