Skip to content
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions litellm/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -1945,6 +1945,15 @@
"auth_token",
"jwt_token",
"private_key",
"authorization",
"api-key",
"x-api-key",
"x-goog-api-key",
"ocp-apim-subscription-key",
"x-litellm-api-key",
"x-mcp-auth",
"cookie",
"set-cookie",
"SLACK_WEBHOOK_URL",
"ALERTING_WEBHOOK_URL",
"webhook_url",
Expand All @@ -1967,6 +1976,12 @@
]
SENTRY_PII_DENYLIST: Final = [
"user_id",
"user_email",
"end_user_id",
"user_api_key_hash",
"user_api_key_user_id",
"user_api_key_user_email",
"user_api_key_end_user_id",
"email",
"phone",
"address",
Expand Down
17 changes: 2 additions & 15 deletions litellm/litellm_core_utils/litellm_logging.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,6 @@
DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT,
EMPTY_MAPPING,
PROVIDER_REQUEST_ID_HEADERS,
SENTRY_DENYLIST,
SENTRY_PII_DENYLIST,
)
from litellm.cost_calculator import (
RealtimeAPITokenUsageProcessor,
Expand Down Expand Up @@ -4471,21 +4469,10 @@ def set_callbacks(callback_list, function_id=None):
print_verbose("Package 'sentry_sdk' is missing. Installing it...")
subprocess.check_call([sys.executable, "-m", "pip", "install", "sentry_sdk"])
import sentry_sdk
from sentry_sdk.scrubber import EventScrubber
from litellm.litellm_core_utils.sentry_scrubbing import build_sentry_init_options

sentry_sdk_instance = sentry_sdk
sentry_trace_rate = os.environ.get("SENTRY_API_TRACE_RATE", "1.0")
sentry_sample_rate = (
os.environ.get("SENTRY_API_SAMPLE_RATE") if "SENTRY_API_SAMPLE_RATE" in os.environ else "1.0"
)
sentry_sdk_instance.init(
dsn=os.environ.get("SENTRY_DSN"),
traces_sample_rate=float(sentry_trace_rate),
sample_rate=float(sentry_sample_rate if sentry_sample_rate else 1.0),
send_default_pii=False, # Prevent sending Personal Identifiable Information
event_scrubber=EventScrubber(denylist=SENTRY_DENYLIST, pii_denylist=SENTRY_PII_DENYLIST),
environment=os.environ.get("SENTRY_ENVIRONMENT", "production"),
)
sentry_sdk_instance.init(**build_sentry_init_options(os.environ))
capture_exception = sentry_sdk_instance.capture_exception
add_breadcrumb = sentry_sdk_instance.add_breadcrumb
elif callback == "slack":
Expand Down
129 changes: 129 additions & 0 deletions litellm/litellm_core_utils/sentry_scrubbing.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
from __future__ import annotations

import re
from collections.abc import Callable, Mapping, Sequence
from functools import reduce
from typing import TYPE_CHECKING, Final, TypeAlias, cast

from pydantic import JsonValue
from sentry_sdk.scrubber import DEFAULT_DENYLIST, DEFAULT_PII_DENYLIST, EventScrubber
from typing_extensions import ReadOnly, TypedDict

from litellm.constants import SENTRY_DENYLIST, SENTRY_PII_DENYLIST
from litellm.secret_managers.main import str_to_bool

if TYPE_CHECKING:
from sentry_sdk.types import Event, Hint

EventScrubFn: TypeAlias = "Callable[[Event, Hint], Event]"

FILTERED: Final = "[Filtered]"
SEND_DEFAULT_PII_ENV: Final = "SENTRY_SEND_DEFAULT_PII"
SECRET_FIELD_NAMES: Final = tuple(DEFAULT_DENYLIST) + tuple(SENTRY_DENYLIST)
PII_FIELD_NAMES: Final = tuple(DEFAULT_PII_DENYLIST) + tuple(SENTRY_PII_DENYLIST)

LITELLM_KEY_PATTERN: Final = re.compile(r"sk-[A-Za-z0-9_-]{16,}")
Comment thread
cursor[bot] marked this conversation as resolved.
Resolved
SOURCE_CONTEXT_KEYS: Final = frozenset({"pre_context", "context_line", "post_context"})
MAX_SCRUB_DEPTH: Final = 64
EMAIL_PATTERN: Final = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}")
SHA256_HEX_PATTERN: Final = re.compile(r"(?<![0-9A-Za-z])[0-9a-f]{64}(?![0-9A-Za-z])")
QUOTED_VALUE: Final = r"'(?:[^'\\]|\\.)*'|\"(?:[^\"\\]|\\.)*\""
BRACKET_ATOM: Final = rf"(?:{QUOTED_VALUE})|[^\[\]{{}}()'\"]"
NESTED_BRACKET_LEVELS: Final = 3
BRACKETED_VALUE: Final = reduce(
lambda inner, _: rf"[\[{{(](?:{BRACKET_ATOM}|{inner})*[\]}})]",
range(NESTED_BRACKET_LEVELS),
rf"[\[{{(](?:{BRACKET_ATOM})*[\]}})]",
)
BARE_VALUE: Final = r"(?!None(?![0-9A-Za-z_]))[^,)\]}\s]+"
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved


class SentryInitOptions(TypedDict):
dsn: ReadOnly[str | None]
traces_sample_rate: ReadOnly[float]
sample_rate: ReadOnly[float]
send_default_pii: ReadOnly[bool]
event_scrubber: ReadOnly[EventScrubber]
before_send: ReadOnly[EventScrubFn]
before_send_transaction: ReadOnly[EventScrubFn]
environment: ReadOnly[str]


def build_repr_field_pattern(field_names: Sequence[str]) -> re.Pattern[str]:
names: Final = "|".join(re.escape(name) for name in field_names)
return re.compile(
rf"(?P<field>(?<![0-9A-Za-z_])(?:{names})=|['\"](?:{names})['\"]:\s*)(?P<value>{QUOTED_VALUE}|{BRACKETED_VALUE}|{BARE_VALUE})",
re.IGNORECASE,
)


def build_string_scrubber(send_default_pii: bool) -> Callable[[str], str]:
field_names: Final = SECRET_FIELD_NAMES if send_default_pii else SECRET_FIELD_NAMES + PII_FIELD_NAMES
field_pattern: Final = build_repr_field_pattern(field_names)
value_patterns: Final = (
(LITELLM_KEY_PATTERN,) if send_default_pii else (LITELLM_KEY_PATTERN, EMAIL_PATTERN, SHA256_HEX_PATTERN)
)

def scrub(text: str) -> str:
fields_scrubbed: Final = field_pattern.sub(_filtered_field, text)
return _substitute_all(value_patterns, fields_scrubbed)

return scrub


def _filtered_field(match: re.Match[str]) -> str:
quote: Final = '"' if match.group("value").startswith('"') else "'"
return f"{match.group('field')}{quote}{FILTERED}{quote}"


def _substitute_all(patterns: Sequence[re.Pattern[str]], text: str) -> str:
return reduce(lambda scrubbed, pattern: pattern.sub(FILTERED, scrubbed), patterns, text)


def scrub_json_strings(value: JsonValue, scrub: Callable[[str], str], depth: int = 0) -> JsonValue:
if depth > MAX_SCRUB_DEPTH:
return FILTERED
if isinstance(value, str):
return scrub(value)
if isinstance(value, dict):
return { # mutable-ok: JSON object
key: item if key in SOURCE_CONTEXT_KEYS else scrub_json_strings(item, scrub, depth + 1)
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
for key, item in value.items()
}
if isinstance(value, list):
return [scrub_json_strings(item, scrub, depth + 1) for item in value] # mutable-ok: JSON array
return value


def build_event_scrubber(send_default_pii: bool) -> EventScrubFn:
scrub: Final = build_string_scrubber(send_default_pii)

def scrub_event(event: Event, _hint: Hint) -> Event:
json_event: Final = cast("JsonValue", event) # cast-ok: [LIT006] the SDK serialized the event to JSON already
return cast("Event", scrub_json_strings(json_event, scrub)) # cast-ok: [LIT006] same JSON shape going back

return scrub_event


def send_default_pii_from_env(env: Mapping[str, str]) -> bool:
return str_to_bool(env.get(SEND_DEFAULT_PII_ENV)) is True


def build_sentry_init_options(env: Mapping[str, str]) -> SentryInitOptions:
send_default_pii: Final = send_default_pii_from_env(env)
scrub_event: Final = build_event_scrubber(send_default_pii)
return SentryInitOptions(
dsn=env.get("SENTRY_DSN"),
traces_sample_rate=float(env.get("SENTRY_API_TRACE_RATE") or "1.0"),
sample_rate=float(env.get("SENTRY_API_SAMPLE_RATE") or "1.0"),
send_default_pii=send_default_pii,
event_scrubber=EventScrubber(
denylist=list(SECRET_FIELD_NAMES), # mutable-ok: EventScrubber appends pii_denylist onto denylist in place
pii_denylist=list(PII_FIELD_NAMES), # mutable-ok: EventScrubber takes List[str]
recursive=True,
send_default_pii=send_default_pii,
),
before_send=scrub_event,
before_send_transaction=scrub_event,
environment=env.get("SENTRY_ENVIRONMENT", "production"),
)
1 change: 1 addition & 0 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -249,6 +249,7 @@ proxy-dev = [
"prisma==0.11.0",
"hypercorn==0.17.3",
"prometheus-client==0.20.0",
"sentry-sdk==2.21.0",
"opentelemetry-api==1.28.0",
"opentelemetry-sdk==1.28.0",
"opentelemetry-exporter-otlp==1.28.0",
Expand Down
1 change: 1 addition & 0 deletions tests/code_coverage_tests/recursive_detector.py
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@
"_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible).
"_replace_string_leaves", # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible).
"_sort_processed_sets", # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible).
"scrub_json_strings", # max depth set (MAX_SCRUB_DEPTH); fails closed by returning "[Filtered]" for anything nested past the cap.
]


Expand Down
148 changes: 30 additions & 118 deletions tests/test_litellm/litellm_core_utils/test_litellm_logging.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@

import litellm
from litellm._logging import session_id_var, trace_id_var
from litellm.constants import SENTRY_DENYLIST, SENTRY_PII_DENYLIST
from litellm.constants import SENTRY_PII_DENYLIST
from litellm.cost_calculator import ocr_batch_cost
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
Expand Down Expand Up @@ -356,108 +356,23 @@ def test_post_call_serializes_dict_with_datetime(logging_obj):
assert "2026-05-11" in serialized


def test_sentry_sample_rate(monkeypatch):
existing_sample_rate = os.getenv("SENTRY_API_SAMPLE_RATE")
try:
# test with default value by removing the environment variable
if existing_sample_rate:
del os.environ["SENTRY_API_SAMPLE_RATE"]

set_callbacks(["sentry"])
# Check if the default sample rate is set to 1.0
assert os.environ.get("SENTRY_API_SAMPLE_RATE") == "1.0"

# test with custom value
monkeypatch.setenv("SENTRY_API_SAMPLE_RATE", "0.5")

set_callbacks(["sentry"])
# Check if the custom sample rate is set correctly
assert os.environ.get("SENTRY_API_SAMPLE_RATE") == "0.5"
except Exception as e:
print(f"Error: {e}")
finally:
# Restore the original environment variable
if existing_sample_rate:
monkeypatch.setenv("SENTRY_API_SAMPLE_RATE", existing_sample_rate)
else:
if "SENTRY_API_SAMPLE_RATE" in os.environ:
del os.environ["SENTRY_API_SAMPLE_RATE"]


def test_sentry_environment(monkeypatch):
"""Test that SENTRY_ENVIRONMENT is properly handled during Sentry initialization"""
existing_environment = os.getenv("SENTRY_ENVIRONMENT")
existing_dsn = os.getenv("SENTRY_DSN")

# Create mock sentry_sdk module
mock_event_scrubber_instance = MagicMock()
mock_event_scrubber_cls = MagicMock(return_value=mock_event_scrubber_instance)

mock_scrubber_module = MagicMock()
mock_scrubber_module.EventScrubber = mock_event_scrubber_cls
import sentry_sdk

mock_sentry_sdk = MagicMock()
mock_sentry_sdk.scrubber = mock_scrubber_module
mock_init = MagicMock()
mock_sentry_sdk.init = mock_init
monkeypatch.setattr(sentry_sdk, "init", mock_init)
monkeypatch.setenv("SENTRY_DSN", "https://test@sentry.io/123456")
monkeypatch.delenv("SENTRY_ENVIRONMENT", raising=False)

# Inject mocks into sys.modules
sys.modules["sentry_sdk"] = mock_sentry_sdk
sys.modules["sentry_sdk.scrubber"] = mock_scrubber_module

try:
# Set a mock DSN to allow Sentry initialization
monkeypatch.setenv("SENTRY_DSN", "https://test@sentry.io/123456")

# Test with default value (no environment set)
if existing_environment:
del os.environ["SENTRY_ENVIRONMENT"]

mock_init.reset_mock()
set_callbacks(["sentry"])
# Check that init was called with default environment "production"
mock_init.assert_called_once()
call_kwargs = mock_init.call_args[1]
assert call_kwargs["environment"] == "production"

# Test with custom environment value
monkeypatch.setenv("SENTRY_ENVIRONMENT", "development")

mock_init.reset_mock()
set_callbacks(["sentry"])
# Check that init was called with custom environment "development"
mock_init.assert_called_once()
call_kwargs = mock_init.call_args[1]
assert call_kwargs["environment"] == "development"

# Test with staging environment
monkeypatch.setenv("SENTRY_ENVIRONMENT", "staging")
set_callbacks(["sentry"])
assert mock_init.call_args[1]["environment"] == "production"

for environment in ("development", "staging"):
monkeypatch.setenv("SENTRY_ENVIRONMENT", environment)
mock_init.reset_mock()
set_callbacks(["sentry"])
# Check that init was called with custom environment "staging"
mock_init.assert_called_once()
call_kwargs = mock_init.call_args[1]
assert call_kwargs["environment"] == "staging"

except Exception as e:
print(f"Error: {e}")
raise
finally:
# Restore the original environment variables
if existing_environment:
monkeypatch.setenv("SENTRY_ENVIRONMENT", existing_environment)
else:
if "SENTRY_ENVIRONMENT" in os.environ:
del os.environ["SENTRY_ENVIRONMENT"]

if existing_dsn:
monkeypatch.setenv("SENTRY_DSN", existing_dsn)
else:
if "SENTRY_DSN" in os.environ:
del os.environ["SENTRY_DSN"]


assert mock_init.call_args[1]["environment"] == environment
def test_use_custom_pricing_for_model():
from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model

Expand Down Expand Up @@ -3099,37 +3014,34 @@ def test_speech_call_is_still_priced_from_input_characters(call_type):


def test_sentry_event_scrubber_initialization(monkeypatch):
# Step 1: Create a fake sentry_sdk.scrubber module
mock_event_scrubber_instance = MagicMock()
mock_event_scrubber_cls = MagicMock(return_value=mock_event_scrubber_instance)

mock_scrubber_module = MagicMock()
mock_scrubber_module.EventScrubber = mock_event_scrubber_cls
import sentry_sdk

# Step 2: Create a fake sentry_sdk module and insert into sys.modules
mock_sentry_sdk = MagicMock()
mock_sentry_sdk.scrubber = mock_scrubber_module
mock_init = MagicMock()
mock_sentry_sdk.init = mock_init
monkeypatch.setattr(sentry_sdk, "init", mock_init)
monkeypatch.delenv("SENTRY_SEND_DEFAULT_PII", raising=False)

# Step 3: Inject both into sys.modules BEFORE import occurs
sys.modules["sentry_sdk"] = mock_sentry_sdk
sys.modules["sentry_sdk.scrubber"] = mock_scrubber_module

# Step 4: Run the actual sentry setup code
set_callbacks(["sentry"])

# Step 5: Assert the EventScrubber was constructed correctly
mock_event_scrubber_cls.assert_called_once_with(
denylist=SENTRY_DENYLIST,
pii_denylist=SENTRY_PII_DENYLIST,
)

# Step 6: Assert the event_scrubber and PII args were passed
mock_init.assert_called_once()
call_args = mock_init.call_args[1]
assert call_args["event_scrubber"] == mock_event_scrubber_instance
assert call_args["send_default_pii"] is False
assert call_args["event_scrubber"].recursive is True
assert {name.lower() for name in SENTRY_PII_DENYLIST} <= {name.lower() for name in call_args["event_scrubber"].denylist}
assert call_args["before_send"] is call_args["before_send_transaction"]


def test_sentry_send_default_pii_opt_in(monkeypatch):
import sentry_sdk

mock_init = MagicMock()
monkeypatch.setattr(sentry_sdk, "init", mock_init)
monkeypatch.setenv("SENTRY_SEND_DEFAULT_PII", "true")

set_callbacks(["sentry"])

call_args = mock_init.call_args[1]
assert call_args["send_default_pii"] is True
assert not {name.lower() for name in SENTRY_PII_DENYLIST} & {name.lower() for name in call_args["event_scrubber"].denylist}


def test_get_masked_values():
Expand Down
Loading
Loading