diff --git a/litellm/constants.py b/litellm/constants.py index c1bb7da1b73..2ae365300ef 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1242,6 +1242,11 @@ LITELLM_METADATA_FIELD = "litellm_metadata" OLD_LITELLM_METADATA_FIELD = "metadata" LITELLM_TRUNCATED_PAYLOAD_FIELD = "litellm_truncated" +LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE = ( + "Truncation is a DB storage safeguard. " + "Full, untruncated data is logged to logging callbacks (OTEL, Datadog, etc.). " + "To increase the truncation limit, set `MAX_STRING_LENGTH_PROMPT_IN_DB` in your env." +) ########################### LiteLLM Proxy Specific Constants ########################### ######################################################################################## diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 131841f7b59..f381432a089 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -11,6 +11,10 @@ import litellm from litellm._logging import verbose_proxy_logger +from litellm.constants import ( + LITELLM_TRUNCATED_PAYLOAD_FIELD, + LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, +) from litellm.constants import \ MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB from litellm.constants import REDACTED_BY_LITELM_STRING @@ -628,7 +632,10 @@ def _sanitize_request_body_for_spend_logs_payload( Recursively sanitize request body to prevent logging large base64 strings or other large values. Truncates strings longer than MAX_STRING_LENGTH_PROMPT_IN_DB characters and handles nested dictionaries. """ - from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD + from litellm.constants import ( + LITELLM_TRUNCATED_PAYLOAD_FIELD, + LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, + ) if visited is None: visited = set() @@ -674,7 +681,8 @@ def _sanitize_value(value: Any) -> Any: # Build the truncated string: beginning + truncation marker + end truncated_value = ( f"{value[:start_chars]}" - f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..." + f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. " + f"{LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..." f"{value[-end_chars:]}" ) return truncated_value @@ -791,6 +799,11 @@ def _get_proxy_server_request_for_spend_logs_payload( _request_body = _sanitize_request_body_for_spend_logs_payload(_request_body) _request_body_json_str = json.dumps(_request_body, default=str) + if LITELLM_TRUNCATED_PAYLOAD_FIELD in _request_body_json_str: + verbose_proxy_logger.info( + "Spend Log: request body was truncated before storing in DB. %s", + LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, + ) return _request_body_json_str return "{}" @@ -866,8 +879,15 @@ def _get_response_for_spend_logs_payload( if sanitized_response is None: return "{}" if isinstance(sanitized_response, str): - return sanitized_response - return safe_dumps(sanitized_response) + result_str = sanitized_response + else: + result_str = safe_dumps(sanitized_response) + if LITELLM_TRUNCATED_PAYLOAD_FIELD in result_str: + verbose_proxy_logger.info( + "Spend Log: response was truncated before storing in DB. %s", + LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, + ) + return result_str return "{}" diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 24f45cc5c91..9a64e641b5e 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -16,7 +16,11 @@ from unittest.mock import AsyncMock, MagicMock, patch import litellm -from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD, REDACTED_BY_LITELM_STRING +from litellm.constants import ( + LITELLM_TRUNCATED_PAYLOAD_FIELD, + LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, + REDACTED_BY_LITELM_STRING, +) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy.spend_tracking.spend_tracking_utils import ( _get_messages_for_spend_logs_payload, @@ -60,7 +64,7 @@ def test_sanitize_request_body_for_spend_logs_payload_long_string(): end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars skipped_chars = len(long_string) - (start_chars + end_chars) - expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..." + expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..." expected_length = start_chars + len(expected_truncation_message) + end_chars assert len(sanitized["text"]) == expected_length @@ -86,7 +90,7 @@ def test_sanitize_request_body_for_spend_logs_payload_nested_dict(): end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars skipped_chars = len(long_string) - total_keep - expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..." + expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..." expected_length = start_chars + len(expected_truncation_message) + end_chars assert len(sanitized["outer"]["inner"]["text"]) == expected_length @@ -111,7 +115,7 @@ def test_sanitize_request_body_for_spend_logs_payload_nested_list(): end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars skipped_chars = len(long_string) - total_keep - expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..." + expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..." expected_length = start_chars + len(expected_truncation_message) + end_chars assert len(sanitized["items"][0]["text"]) == expected_length @@ -151,7 +155,7 @@ def test_sanitize_request_body_for_spend_logs_payload_mixed_types(): end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars skipped_chars = len(long_string) - total_keep - expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..." + expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..." expected_length = start_chars + len(expected_truncation_message) + end_chars assert len(sanitized["text"]) == expected_length @@ -396,6 +400,78 @@ def test_get_response_for_spend_logs_payload_truncates_large_embedding(mock_shou assert parsed["data"][0]["other_field"] == "value" +def test_truncation_includes_db_safeguard_note(): + """ + Test that truncated content includes the DB safeguard note explaining + that full data is available in OTEL/other logging integrations. + """ + from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB + + large_error = "Error: " + "x" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 1000) + request_body = {"error_trace": large_error} + sanitized = _sanitize_request_body_for_spend_logs_payload(request_body) + + truncated = sanitized["error_trace"] + assert LITELLM_TRUNCATED_PAYLOAD_FIELD in truncated + assert LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE in truncated + assert "DB storage safeguard" in truncated + assert "logging callbacks" in truncated.lower() or "logging integrations" in truncated.lower() or "logging callbacks" in truncated + + +@patch( + "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" +) +def test_response_truncation_logs_info_message(mock_should_store): + """ + Test that when response is truncated before DB storage, an info log is emitted + noting that full data is available in OTEL/other integrations. + """ + from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB + + mock_should_store.return_value = True + large_text = "B" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500) + payload = cast( + StandardLoggingPayload, + {"response": {"data": [{"content": large_text}]}}, + ) + + with patch( + "litellm.proxy.spend_tracking.spend_tracking_utils.verbose_proxy_logger" + ) as mock_logger: + _get_response_for_spend_logs_payload(payload) + mock_logger.info.assert_called_once() + log_msg = mock_logger.info.call_args[0][0] + assert "response was truncated" in log_msg + + +@patch( + "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" +) +def test_request_body_truncation_logs_info_message(mock_should_store): + """ + Test that when request body is truncated before DB storage, an info log is emitted. + """ + from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB + + mock_should_store.return_value = True + large_prompt = "C" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500) + litellm_params = { + "proxy_server_request": { + "body": {"messages": [{"role": "user", "content": large_prompt}]} + } + } + + with patch( + "litellm.proxy.spend_tracking.spend_tracking_utils.verbose_proxy_logger" + ) as mock_logger: + _get_proxy_server_request_for_spend_logs_payload( + metadata={}, litellm_params=litellm_params, kwargs={} + ) + mock_logger.info.assert_called_once() + log_msg = mock_logger.info.call_args[0][0] + assert "request body was truncated" in log_msg + + def test_safe_dumps_handles_circular_references(): """Test that safe_dumps can handle circular references without raising exceptions"""