Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions litellm/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,8 @@
# Set to 0 to disable truncation.
MAX_BASE64_LENGTH_FOR_LOGGING: Final = int(os.getenv("MAX_BASE64_LENGTH_FOR_LOGGING", 64))
REDACTED_BY_LITELLM: Final = "redacted-by-litellm"
# arguments fields contractually hold JSON (consumers json.loads them), so redact to valid JSON
REDACTED_TOOL_CALL_ARGUMENTS: Final = "{}"

MAX_STRING_LENGTH_STDOUT_LOG: Final = get_env_int("MAX_STRING_LENGTH_STDOUT_LOG", 4096)

Expand Down
7 changes: 6 additions & 1 deletion litellm/integrations/langfuse/langfuse_otel.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
LangfuseLLMObsOTELAttributes,
)
from litellm.integrations.opentelemetry import OpenTelemetry, OpenTelemetryConfig
from litellm.litellm_core_utils.safe_json_loads import safe_json_loads
from litellm.types.integrations.langfuse_otel import (
LangfuseSpanAttributes,
)
Expand Down Expand Up @@ -197,7 +198,11 @@ def _set_observation_output(span: Span, response_obj):
)
elif item_type == "function_call":
arguments_str = getattr(item, "arguments", "{}")
arguments_obj = json.loads(arguments_str) if isinstance(arguments_str, str) else arguments_str
arguments_obj = (
safe_json_loads(arguments_str, default={})
if isinstance(arguments_str, str)
else arguments_str
)
langfuse_tool_call = {
"id": getattr(item, "id", ""),
"name": getattr(item, "name", ""),
Expand Down
40 changes: 22 additions & 18 deletions litellm/litellm_core_utils/redact_messages.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
from typing import TYPE_CHECKING, Any, Final

import litellm
from litellm.constants import REDACTED_BY_LITELLM
from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import (
get_metadata_variable_name_from_kwargs,
Expand Down Expand Up @@ -85,28 +85,30 @@ def _redact_tool_calls(tool_calls) -> None:
for tool_call in tool_calls:
function = getattr(tool_call, "function", None)
if function is not None and hasattr(function, "arguments"):
function.arguments = REDACTED_BY_LITELLM
function.arguments = REDACTED_TOOL_CALL_ARGUMENTS


def _redact_function_call(function_call) -> None:
"""Redact legacy assistant function_call arguments."""
if function_call is not None and hasattr(function_call, "arguments"):
function_call.arguments = REDACTED_BY_LITELLM
function_call.arguments = REDACTED_TOOL_CALL_ARGUMENTS


def _redact_choice_content(choice):
"""Helper to redact content in a choice (message or delta)."""
if isinstance(choice, litellm.Choices):
choice.message.content = REDACTED_BY_LITELLM
if hasattr(choice.message, "reasoning_content"):
if choice.message.content is not None:
choice.message.content = REDACTED_BY_LITELLM
if getattr(choice.message, "reasoning_content", None) is not None:
choice.message.reasoning_content = REDACTED_BY_LITELLM
if hasattr(choice.message, "thinking_blocks"):
choice.message.thinking_blocks = None
_redact_tool_calls(getattr(choice.message, "tool_calls", None))
_redact_function_call(getattr(choice.message, "function_call", None))
elif isinstance(choice, litellm.utils.StreamingChoices):
choice.delta.content = REDACTED_BY_LITELLM
if hasattr(choice.delta, "reasoning_content"):
if choice.delta.content is not None:
choice.delta.content = REDACTED_BY_LITELLM
if getattr(choice.delta, "reasoning_content", None) is not None:
choice.delta.reasoning_content = REDACTED_BY_LITELLM
if hasattr(choice.delta, "thinking_blocks"):
choice.delta.thinking_blocks = None
Expand All @@ -133,7 +135,7 @@ def _redact_responses_api_output(output_items):
summary_item.text = REDACTED_BY_LITELLM

if hasattr(output_item, "type") and output_item.type == "function_call" and hasattr(output_item, "arguments"):
output_item.arguments = REDACTED_BY_LITELLM
output_item.arguments = REDACTED_TOOL_CALL_ARGUMENTS


def _redact_responses_api_output_dict(output_items, redacted_str: str):
Expand All @@ -156,7 +158,7 @@ def _redact_responses_api_output_dict(output_items, redacted_str: str):
summary_item["text"] = redacted_str

if output_item.get("type") == "function_call" and "arguments" in output_item:
output_item["arguments"] = redacted_str
output_item["arguments"] = REDACTED_TOOL_CALL_ARGUMENTS


def _redact_standard_logging_object(model_call_details: dict):
Expand Down Expand Up @@ -189,40 +191,42 @@ def _redact_standard_logging_object(model_call_details: dict):
standard_logging_object["response"] = {"text": redacted_str}


def _redact_tool_calls_dict(message: dict, redacted_str: str) -> None:
def _redact_tool_calls_dict(message: dict) -> None:
"""Redact tool call / function_call arguments in a dict-form message or delta."""
tool_calls: Final = message.get("tool_calls")
if isinstance(tool_calls, list):
for tool_call in tool_calls:
if isinstance(tool_call, dict) and isinstance(tool_call.get("function"), dict):
tool_call["function"]["arguments"] = redacted_str
tool_call["function"]["arguments"] = REDACTED_TOOL_CALL_ARGUMENTS

function_call: Final = message.get("function_call")
if isinstance(function_call, dict) and "arguments" in function_call:
function_call["arguments"] = redacted_str
function_call["arguments"] = REDACTED_TOOL_CALL_ARGUMENTS


def _redact_model_response_dict_choices(choices, redacted_str: str):
for choice in choices:
if isinstance(choice, dict):
if "message" in choice and isinstance(choice["message"], dict):
choice["message"]["content"] = redacted_str
if "reasoning_content" in choice["message"]:
if choice["message"].get("content") is not None:
choice["message"]["content"] = redacted_str
if choice["message"].get("reasoning_content") is not None:
choice["message"]["reasoning_content"] = redacted_str
if "thinking_blocks" in choice["message"]:
choice["message"]["thinking_blocks"] = None
if "audio" in choice["message"]:
choice["message"]["audio"] = None
_redact_tool_calls_dict(choice["message"], redacted_str)
_redact_tool_calls_dict(choice["message"])
elif "delta" in choice and isinstance(choice["delta"], dict):
choice["delta"]["content"] = redacted_str
if "reasoning_content" in choice["delta"]:
if choice["delta"].get("content") is not None:
choice["delta"]["content"] = redacted_str
if choice["delta"].get("reasoning_content") is not None:
choice["delta"]["reasoning_content"] = redacted_str
if "thinking_blocks" in choice["delta"]:
choice["delta"]["thinking_blocks"] = None
if "audio" in choice["delta"]:
choice["delta"]["audio"] = None
_redact_tool_calls_dict(choice["delta"], redacted_str)
_redact_tool_calls_dict(choice["delta"])
else:
_redact_choice_content(choice)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import litellm
from litellm._logging import verbose_proxy_logger
from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS
from litellm.proxy._types import SpendLogsPayload
from litellm.proxy.spend_tracking.cold_storage_handler import ColdStorageHandler
from litellm.responses.utils import ResponsesAPIRequestUtils
Expand All @@ -29,6 +30,17 @@
########################################################


def _normalize_redacted_tool_call_arguments(message: Message) -> None:
"""Older releases redacted tool-call arguments to the bare sentinel (invalid JSON);
normalize replayed history to "{}" so provider converters can parse it."""
for tool_call in message.tool_calls or []:
if (function := getattr(tool_call, "function", None)) is not None and function.arguments == REDACTED_BY_LITELLM:
function.arguments = REDACTED_TOOL_CALL_ARGUMENTS
function_call: Final = message.function_call
if function_call is not None and function_call.arguments == REDACTED_BY_LITELLM:
function_call.arguments = REDACTED_TOOL_CALL_ARGUMENTS


class ResponsesSessionHandler:
@staticmethod
async def get_chat_completion_message_history_for_previous_response_id(
Expand Down Expand Up @@ -143,7 +155,8 @@ async def extend_chat_completion_message_with_spend_log_payload(
model_response: Final = ModelResponse(**_response_output)
for choice in model_response.choices:
if hasattr(choice, "message"):
chat_completion_message_history.append(getattr(choice, "message"))
_normalize_redacted_tool_call_arguments(message := getattr(choice, "message"))
chat_completion_message_history.append(message)
return chat_completion_message_history

@staticmethod
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@

from litellm._logging import verbose_logger
from litellm.caching import InMemoryCache
from litellm.constants import REDACTED_BY_LITELLM, REDACTED_TOOL_CALL_ARGUMENTS
from litellm.litellm_core_utils.get_supported_openai_params import (
get_supported_openai_params,
)
Expand Down Expand Up @@ -1551,6 +1552,9 @@ def _transform_responses_api_function_call_to_chat_completion_message(
# store their payload in "input" (raw string) rather than
# "arguments" (JSON string), so normalize to arguments here.
raw_arguments = function_call.get("arguments")
if raw_arguments == REDACTED_BY_LITELLM:
# older releases stored the bare sentinel (invalid JSON) in arguments
raw_arguments = REDACTED_TOOL_CALL_ARGUMENTS
if not raw_arguments and function_call.get("type") == "custom_tool_call":
raw_input: Final = function_call.get("input") or ""
raw_arguments = json.dumps({"content": raw_input}) if raw_input else ""
Expand Down
46 changes: 46 additions & 0 deletions tests/test_litellm/integrations/test_langfuse_otel.py
Original file line number Diff line number Diff line change
Expand Up @@ -933,6 +933,52 @@ def test_responses_api_with_function_calls(self):
assert output_data[0]["arguments"]["location"] == "San Francisco"
assert output_data[0]["arguments"]["unit"] == "celsius"

def test_responses_api_function_call_with_redacted_arguments(self):
"""Sentinel arguments (invalid JSON) must not kill the whole observation output."""
from openai.types.responses import ResponseFunctionToolCall

from litellm.types.integrations.langfuse_otel import LangfuseSpanAttributes

response_obj = ResponsesAPIResponse(
id="response-redacted",
created_at=1625247700,
output=[
ResponseFunctionToolCall(
id="fc-redacted",
type="function_call",
name="get_weather",
call_id="call-redacted",
arguments="redacted-by-litellm",
status="completed",
)
],
)

kwargs = {
"call_type": "responses",
"messages": [{"role": "user", "content": "What's the weather?"}],
"model": "gpt-4o",
"optional_params": {},
}

mock_span = MagicMock()

with patch( # test-quality-ok: the span attribute sink is the observable boundary; sibling tests in this class stub the same seam
"litellm.integrations.arize._utils.safe_set_attribute"
) as mock_safe_set_attribute:
LangfuseOtelLogger._set_langfuse_specific_attributes(mock_span, kwargs, response_obj)

output_calls = [
call
for call in mock_safe_set_attribute.call_args_list
if call.args[1] == LangfuseSpanAttributes.OBSERVATION_OUTPUT.value
]

assert len(output_calls) > 0, "observation.output should still be set"
output_data = json.loads(output_calls[0].args[2])
assert output_data[0]["name"] == "get_weather"
assert output_data[0]["arguments"] == {}


if __name__ == "__main__":
pytest.main([__file__])
86 changes: 78 additions & 8 deletions tests/test_litellm/litellm_core_utils/test_redact_messages.py
Original file line number Diff line number Diff line change
Expand Up @@ -350,11 +350,11 @@ def test_redacts_tool_call_arguments_in_model_response_dict(self):
redacted = perform_redaction({}, result)

message = redacted["choices"][0]["message"]
assert message["content"] == "redacted-by-litellm"
assert message["content"] is None
tool_call = message["tool_calls"][0]
assert tool_call["function"]["arguments"] == "redacted-by-litellm"
assert tool_call["function"]["arguments"] == "{}"
assert tool_call["function"]["name"] == "get_weather"
assert message["function_call"]["arguments"] == "redacted-by-litellm"
assert message["function_call"]["arguments"] == "{}"

def test_redacts_tool_call_arguments_in_streaming_delta_dict(self):
result = {
Expand All @@ -379,7 +379,7 @@ def test_redacts_tool_call_arguments_in_streaming_delta_dict(self):
redacted = perform_redaction({}, result)

delta = redacted["choices"][0]["delta"]
assert delta["tool_calls"][0]["function"]["arguments"] == "redacted-by-litellm"
assert delta["tool_calls"][0]["function"]["arguments"] == "{}"

def test_redacts_tool_call_arguments_on_model_response_object(self):
result = litellm.ModelResponse(
Expand Down Expand Up @@ -408,7 +408,7 @@ def test_redacts_tool_call_arguments_on_model_response_object(self):
redacted = perform_redaction({}, result)

tool_call = redacted.choices[0].message.tool_calls[0]
assert tool_call.function.arguments == "redacted-by-litellm"
assert tool_call.function.arguments == "{}"
assert tool_call.function.name == "get_weather"
assert result.choices[0].message.tool_calls[0].function.arguments == (
'{"city": "sensitive-city"}'
Expand Down Expand Up @@ -442,7 +442,7 @@ def test_redacts_tool_call_arguments_on_streaming_response_object(self):
perform_redaction(details, None)

tool_call = streaming_response.choices[0].delta.tool_calls[0]
assert tool_call.function.arguments == "redacted-by-litellm"
assert tool_call.function.arguments == "{}"

def test_redacts_tool_call_arguments_in_standard_logging_object(self):
details = {
Expand Down Expand Up @@ -472,7 +472,7 @@ def test_redacts_tool_call_arguments_in_standard_logging_object(self):
perform_redaction(details, None)

message = details["standard_logging_object"]["response"]["choices"][0]["message"]
assert message["tool_calls"][0]["function"]["arguments"] == "redacted-by-litellm"
assert message["tool_calls"][0]["function"]["arguments"] == "{}"

def test_redacts_responses_api_function_call_arguments_dict(self):
result = {
Expand All @@ -488,9 +488,79 @@ def test_redacts_responses_api_function_call_arguments_dict(self):

redacted = perform_redaction({}, result)

assert redacted["output"][0]["arguments"] == "redacted-by-litellm"
assert redacted["output"][0]["arguments"] == "{}"
assert redacted["output"][0]["name"] == "get_weather"

def test_redacts_every_tool_call_in_multi_element_list(self):
result = litellm.ModelResponse(
id="resp-multi",
choices=[
litellm.Choices(
message=litellm.Message(
content=None,
role="assistant",
tool_calls=[
{
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": '{"city": "a"}'},
},
{
"id": "call_2",
"type": "function",
"function": {"name": "get_time", "arguments": '{"tz": "b"}'},
},
],
)
)
],
model="gpt-4o",
)

redacted = perform_redaction({}, result)

tool_calls = redacted.choices[0].message.tool_calls
assert tool_calls[0].function.arguments == "{}"
assert tool_calls[1].function.arguments == "{}"

def test_preserves_none_content_on_tool_call_only_message(self):
result = litellm.ModelResponse(
id="resp-none",
choices=[
litellm.Choices(
message=litellm.Message(
content=None,
role="assistant",
tool_calls=[
{
"id": "call_1",
"type": "function",
"function": {"name": "get_weather", "arguments": '{"city": "a"}'},
}
],
)
)
],
model="gpt-4o",
)

redacted = perform_redaction({}, result)

assert redacted.choices[0].message.content is None

def test_redacts_responses_api_function_call_arguments_object(self):
output_item = SimpleNamespace(
type="function_call",
name="get_weather",
arguments='{"city": "sensitive-city"}',
call_id="call_1",
)

_redact_responses_api_output([output_item])

assert output_item.arguments == "{}"
assert output_item.name == "get_weather"

def test_redacts_response_output_objects_with_top_level_text(self):
output_items = [
SimpleNamespace(text="top-level output"),
Expand Down
Loading
Loading