Skip to content

LLM error truncation note #22936

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
krrishdholakia merged 1 commit into from
Mar 6, 2026
Merged

LLM error truncation note #22936

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions litellm/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1242,6 +1242,11 @@
LITELLM_METADATA_FIELD = "litellm_metadata"
OLD_LITELLM_METADATA_FIELD = "metadata"
LITELLM_TRUNCATED_PAYLOAD_FIELD = "litellm_truncated"
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE = (
"Truncation is a DB storage safeguard. "
"Full, untruncated data is logged to logging callbacks (OTEL, Datadog, etc.). "
"To increase the truncation limit, set `MAX_STRING_LENGTH_PROMPT_IN_DB` in your env."
)

########################### LiteLLM Proxy Specific Constants ###########################
########################################################################################
Expand Down
28 changes: 24 additions & 4 deletions litellm/proxy/spend_tracking/spend_tracking_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,10 @@

import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import (
LITELLM_TRUNCATED_PAYLOAD_FIELD,
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE,
)
from litellm.constants import \
MAX_STRING_LENGTH_PROMPT_IN_DB as DEFAULT_MAX_STRING_LENGTH_PROMPT_IN_DB
from litellm.constants import REDACTED_BY_LITELM_STRING
Expand Down Expand Up @@ -628,7 +632,10 @@ def _sanitize_request_body_for_spend_logs_payload(
Recursively sanitize request body to prevent logging large base64 strings or other large values.
Truncates strings longer than MAX_STRING_LENGTH_PROMPT_IN_DB characters and handles nested dictionaries.
"""
from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD
from litellm.constants import (
LITELLM_TRUNCATED_PAYLOAD_FIELD,
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE,
)
Comment on lines +635 to +638
Copy link
Contributor

@greptile-apps greptile-apps bot Mar 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These constants are already imported at the module level (lines 14–17). This local import is now redundant and can be removed.

Suggested change
from litellm.constants import (
LITELLM_TRUNCATED_PAYLOAD_FIELD,
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE,
)
if visited is None:


if visited is None:
visited = set()
Expand Down Expand Up @@ -674,7 +681,8 @@ def _sanitize_value(value: Any) -> Any:
# Build the truncated string: beginning + truncation marker + end
truncated_value = (
f"{value[:start_chars]}"
f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..."
f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. "
f"{LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..."
f"{value[-end_chars:]}"
)
return truncated_value
Comment on lines 682 to 688
Copy link
Contributor

@greptile-apps greptile-apps bot Mar 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The truncation marker now includes LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE (~201 characters), which means every truncated field stored in the database will exceed the intended MAX_STRING_LENGTH_PROMPT_IN_DB limit by approximately 270+ characters. With a default limit of 2048, this represents a ~13% size increase per truncated field—the opposite of the intended safeguard.

The safeguard note is already surfaced in logs via the new verbose_proxy_logger.info() calls (lines 803–806 and 886–889), making the embedded text redundant. Consider either:

  • Storing the note in a separate metadata field
  • Keeping the in-value marker minimal and relying on the log-level info messages

Expand Down Expand Up @@ -791,6 +799,11 @@ def _get_proxy_server_request_for_spend_logs_payload(

_request_body = _sanitize_request_body_for_spend_logs_payload(_request_body)
_request_body_json_str = json.dumps(_request_body, default=str)
if LITELLM_TRUNCATED_PAYLOAD_FIELD in _request_body_json_str:
verbose_proxy_logger.info(
"Spend Log: request body was truncated before storing in DB. %s",
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE,
)
return _request_body_json_str
return "{}"

Expand Down Expand Up @@ -866,8 +879,15 @@ def _get_response_for_spend_logs_payload(
if sanitized_response is None:
return "{}"
if isinstance(sanitized_response, str):
return sanitized_response
return safe_dumps(sanitized_response)
result_str = sanitized_response
else:
result_str = safe_dumps(sanitized_response)
if LITELLM_TRUNCATED_PAYLOAD_FIELD in result_str:
verbose_proxy_logger.info(
"Spend Log: response was truncated before storing in DB. %s",
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE,
)
return result_str
return "{}"


Expand Down
86 changes: 81 additions & 5 deletions tests/test_litellm/proxy/spend_tracking/test_spend_tracking_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,11 @@
from unittest.mock import AsyncMock, MagicMock, patch

import litellm
from litellm.constants import LITELLM_TRUNCATED_PAYLOAD_FIELD, REDACTED_BY_LITELM_STRING
from litellm.constants import (
LITELLM_TRUNCATED_PAYLOAD_FIELD,
LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE,
REDACTED_BY_LITELM_STRING,
)
from litellm.litellm_core_utils.safe_json_dumps import safe_dumps
from litellm.proxy.spend_tracking.spend_tracking_utils import (
_get_messages_for_spend_logs_payload,
Expand Down Expand Up @@ -60,7 +64,7 @@ def test_sanitize_request_body_for_spend_logs_payload_long_string():
end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars

skipped_chars = len(long_string) - (start_chars + end_chars)
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..."
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..."
expected_length = start_chars + len(expected_truncation_message) + end_chars

assert len(sanitized["text"]) == expected_length
Expand All @@ -86,7 +90,7 @@ def test_sanitize_request_body_for_spend_logs_payload_nested_dict():
end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars

skipped_chars = len(long_string) - total_keep
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..."
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..."
expected_length = start_chars + len(expected_truncation_message) + end_chars

assert len(sanitized["outer"]["inner"]["text"]) == expected_length
Expand All @@ -111,7 +115,7 @@ def test_sanitize_request_body_for_spend_logs_payload_nested_list():
end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars

skipped_chars = len(long_string) - total_keep
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..."
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..."
expected_length = start_chars + len(expected_truncation_message) + end_chars

assert len(sanitized["items"][0]["text"]) == expected_length
Expand Down Expand Up @@ -151,7 +155,7 @@ def test_sanitize_request_body_for_spend_logs_payload_mixed_types():
end_chars = MAX_STRING_LENGTH_PROMPT_IN_DB - start_chars

skipped_chars = len(long_string) - total_keep
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars) ..."
expected_truncation_message = f"... ({LITELLM_TRUNCATED_PAYLOAD_FIELD} skipped {skipped_chars} chars. {LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE}) ..."
expected_length = start_chars + len(expected_truncation_message) + end_chars

assert len(sanitized["text"]) == expected_length
Expand Down Expand Up @@ -396,6 +400,78 @@ def test_get_response_for_spend_logs_payload_truncates_large_embedding(mock_shou
assert parsed["data"][0]["other_field"] == "value"


def test_truncation_includes_db_safeguard_note():
"""
Test that truncated content includes the DB safeguard note explaining
that full data is available in OTEL/other logging integrations.
"""
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB

large_error = "Error: " + "x" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 1000)
request_body = {"error_trace": large_error}
sanitized = _sanitize_request_body_for_spend_logs_payload(request_body)

truncated = sanitized["error_trace"]
assert LITELLM_TRUNCATED_PAYLOAD_FIELD in truncated
assert LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE in truncated
assert "DB storage safeguard" in truncated
assert "logging callbacks" in truncated.lower() or "logging integrations" in truncated.lower() or "logging callbacks" in truncated
Copy link
Contributor

@greptile-apps greptile-apps bot Mar 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The third or condition is redundant. Since LITELLM_TRUNCATION_DB_SAFEGUARD_NOTE already contains the lowercase phrase "logging callbacks", the expression "logging callbacks" in truncated will always be True if "logging callbacks" in truncated.lower() is True.

Suggested change
assert "logging callbacks" in truncated.lower() or "logging integrations" in truncated.lower() or "logging callbacks" in truncated
assert "logging callbacks" in truncated.lower() or "logging integrations" in truncated.lower()



@patch(
"litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
)
def test_response_truncation_logs_info_message(mock_should_store):
"""
Test that when response is truncated before DB storage, an info log is emitted
noting that full data is available in OTEL/other integrations.
"""
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB

mock_should_store.return_value = True
large_text = "B" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500)
payload = cast(
StandardLoggingPayload,
{"response": {"data": [{"content": large_text}]}},
)

with patch(
"litellm.proxy.spend_tracking.spend_tracking_utils.verbose_proxy_logger"
) as mock_logger:
_get_response_for_spend_logs_payload(payload)
mock_logger.info.assert_called_once()
log_msg = mock_logger.info.call_args[0][0]
assert "response was truncated" in log_msg


@patch(
"litellm.proxy.spend_tracking.spend_tracking_utils._should_store_prompts_and_responses_in_spend_logs"
)
def test_request_body_truncation_logs_info_message(mock_should_store):
"""
Test that when request body is truncated before DB storage, an info log is emitted.
"""
from litellm.constants import MAX_STRING_LENGTH_PROMPT_IN_DB

mock_should_store.return_value = True
large_prompt = "C" * (MAX_STRING_LENGTH_PROMPT_IN_DB + 500)
litellm_params = {
"proxy_server_request": {
"body": {"messages": [{"role": "user", "content": large_prompt}]}
}
}

with patch(
"litellm.proxy.spend_tracking.spend_tracking_utils.verbose_proxy_logger"
) as mock_logger:
_get_proxy_server_request_for_spend_logs_payload(
metadata={}, litellm_params=litellm_params, kwargs={}
)
mock_logger.info.assert_called_once()
log_msg = mock_logger.info.call_args[0][0]
assert "request body was truncated" in log_msg


def test_safe_dumps_handles_circular_references():
"""Test that safe_dumps can handle circular references without raising exceptions"""

Expand Down
Loading