From b21140775db3256cdacab55ee6fc247c39702948 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 5 Mar 2026 23:43:44 +0000 Subject: [PATCH] feat(spend-logs): add truncation note when error logs are truncated for DB storage When the messages or response JSON fields in spend logs are truncated before being written to the database, the truncation marker now includes a note explaining: - This is a DB storage safeguard - Full, untruncated data is still sent to logging callbacks (OTEL, Datadog, etc.) - The MAX_STRING_LENGTH_PROMPT_IN_DB env var can be used to increase the limit Also emits a verbose_proxy_logger.info message when truncation occurs in the request body or response spend log paths. Adds 3 new tests: - test_truncation_includes_db_safeguard_note - test_response_truncation_logs_info_message - test_request_body_truncation_logs_info_message Co-authored-by: Krish Dholakia --- litellm/constants.py | 5 ++ .../spend_tracking/spend_tracking_utils.py | 28 +++++- .../test_spend_tracking_utils.py | 86 +++++++++++++++++-- 3 files changed, 110 insertions(+), 9 deletions(-) diff --git a/litellm/constants.py b/litellm/constants.py index c1bb7da1b73..2ae365300ef 100644 --- a/litellm/constants.py +++ b/litellm/constants.py @@ -1242,6 +1242,11 @@ LITELLM_METADATA_FIELD = "litellm_metadata" OLD_LITELLM_METADATA_FIELD = "metadata" LITELLM_TRUNCATED_PAYLOAD_FIELD = "litellm_truncated" +LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE = ( + "Truncation is a DB storage safeguard. " + "Full, untruncated data is logged to logging callbacks (OTEL, Datadog, etc.). " + "To increase the truncation limit, set `MAX_STRING_LENGTH_PROMPT_IN_DB` in your env." +) ########################### LiteLLM Proxy Specific Constants ########################### ######################################################################################## diff --git a/litellm/proxy/spend_tracking/spend_tracking_utils.py b/litellm/proxy/spend_tracking/spend_tracking_utils.py index 131841f7b59..f381432a089 100644 --- a/litellm/proxy/spend_tracking/spend_tracking_utils.py +++ b/litellm/proxy/spend_tracking/spend_tracking_utils.py @@ -11,6 +11,10 @@ import litellm from litellm._logging import verbose_proxy_logger +from litellm.constants import ( + LITELLM_TRUNCATED_PAYLOAD_FIELD, + LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, +) from litellm.constants import \ MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB from litellm.constants import REDACTED_BY_LITELM_STRING @@ -628,7 +632,10 @@ def _sanitize_request_body_for_spend_logs_payload( Recursively sanitize request body to prevent logging large base64 strings or other large values. Truncates strings longer than MAX_STRING_LENGTH_PROMPT_IN_DB characters and handles nested dictionaries. """ - from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD + from litellm.constants import ( + LITELLM_TRUNCATED_PAYLOAD_FIELD, + LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, + ) if visited is None: visited = set() @@ -674,7 +681,8 @@ def _sanitize_value(value: Any) -> Any: # Build the truncated string: beginning + truncation marker + end truncated_value = ( f"{value[:start_chars]}" - f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..." + f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. " + f"{LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..." f"{value[-end_chars:]}" ) return truncated_value @@ -791,6 +799,11 @@ def _get_proxy_server_request_for_spend_logs_payload( _request_body = _sanitize_request_body_for_spend_logs_payload(_request_body) _request_body_json_str = json.dumps(_request_body, default=str) + if LITELLM_TRUNCATED_PAYLOAD_FIELD in _request_body_json_str: + verbose_proxy_logger.info( + "Spend Log: request body was truncated before storing in DB. %s", + LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, + ) return _request_body_json_str return "{}" @@ -866,8 +879,15 @@ def _get_response_for_spend_logs_payload( if sanitized_response is None: return "{}" if isinstance(sanitized_response, str): - return sanitized_response - return safe_dumps(sanitized_response) + result_str = sanitized_response + else: + result_str = safe_dumps(sanitized_response) + if LITELLM_TRUNCATED_PAYLOAD_FIELD in result_str: + verbose_proxy_logger.info( + "Spend Log: response was truncated before storing in DB. %s", + LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, + ) + return result_str return "{}" diff --git a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py index 24f45cc5c91..9a64e641b5e 100644 --- a/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py +++ b/tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py @@ -16,7 +16,11 @@ from unittest.mock import AsyncMock, MagicMock, patch import litellm -from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD, REDACTED_BY_LITELM_STRING +from litellm.constants import ( + LITELLM_TRUNCATED_PAYLOAD_FIELD, + LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE, + REDACTED_BY_LITELM_STRING, +) from litellm.litellm_core_utils.safe_json_dumps import safe_dumps from litellm.proxy.spend_tracking.spend_tracking_utils import ( _get_messages_for_spend_logs_payload, @@ -60,7 +64,7 @@ def test_sanitize_request_body_for_spend_logs_payload_long_string(): end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars skipped_chars = len(long_string) - (start_chars + end_chars) - expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..." + expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..." expected_length = start_chars + len(expected_truncation_message) + end_chars assert len(sanitized["text"]) == expected_length @@ -86,7 +90,7 @@ def test_sanitize_request_body_for_spend_logs_payload_nested_dict(): end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars skipped_chars = len(long_string) - total_keep - expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..." + expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..." expected_length = start_chars + len(expected_truncation_message) + end_chars assert len(sanitized["outer"]["inner"]["text"]) == expected_length @@ -111,7 +115,7 @@ def test_sanitize_request_body_for_spend_logs_payload_nested_list(): end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars skipped_chars = len(long_string) - total_keep - expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..." + expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..." expected_length = start_chars + len(expected_truncation_message) + end_chars assert len(sanitized["items"][0]["text"]) == expected_length @@ -151,7 +155,7 @@ def test_sanitize_request_body_for_spend_logs_payload_mixed_types(): end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars skipped_chars = len(long_string) - total_keep - expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..." + expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..." expected_length = start_chars + len(expected_truncation_message) + end_chars assert len(sanitized["text"]) == expected_length @@ -396,6 +400,78 @@ def test_get_response_for_spend_logs_payload_truncates_large_embedding(mock_shou assert parsed["data"][0]["other_field"] == "value" +def test_truncation_includes_db_safeguard_note(): + """ + Test that truncated content includes the DB safeguard note explaining + that full data is available in OTEL/other logging integrations. + """ + from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB + + large_error = "Error: " + "x" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 1000) + request_body = {"error_trace": large_error} + sanitized = _sanitize_request_body_for_spend_logs_payload(request_body) + + truncated = sanitized["error_trace"] + assert LITELLM_TRUNCATED_PAYLOAD_FIELD in truncated + assert LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE in truncated + assert "DB storage safeguard" in truncated + assert "logging callbacks" in truncated.lower() or "logging integrations" in truncated.lower() or "logging callbacks" in truncated + + +@patch( + "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" +) +def test_response_truncation_logs_info_message(mock_should_store): + """ + Test that when response is truncated before DB storage, an info log is emitted + noting that full data is available in OTEL/other integrations. + """ + from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB + + mock_should_store.return_value = True + large_text = "B" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500) + payload = cast( + StandardLoggingPayload, + {"response": {"data": [{"content": large_text}]}}, + ) + + with patch( + "litellm.proxy.spend_tracking.spend_tracking_utils.verbose_proxy_logger" + ) as mock_logger: + _get_response_for_spend_logs_payload(payload) + mock_logger.info.assert_called_once() + log_msg = mock_logger.info.call_args[0][0] + assert "response was truncated" in log_msg + + +@patch( + "litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs" +) +def test_request_body_truncation_logs_info_message(mock_should_store): + """ + Test that when request body is truncated before DB storage, an info log is emitted. + """ + from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB + + mock_should_store.return_value = True + large_prompt = "C" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500) + litellm_params = { + "proxy_server_request": { + "body": {"messages": [{"role": "user", "content": large_prompt}]} + } + } + + with patch( + "litellm.proxy.spend_tracking.spend_tracking_utils.verbose_proxy_logger" + ) as mock_logger: + _get_proxy_server_request_for_spend_logs_payload( + metadata={}, litellm_params=litellm_params, kwargs={} + ) + mock_logger.info.assert_called_once() + log_msg = mock_logger.info.call_args[0][0] + assert "request body was truncated" in log_msg + + def test_safe_dumps_handles_circular_references(): """Test that safe_dumps can handle circular references without raising exceptions"""