fix(sentry): scrub PII and secrets inside object reprs and nested locals, add SENTRY_SEND_DEFAULT_PII opt-in

litellm/constants.py

@@ -1967,6 +1967,12 @@
 ]
 SENTRY_PII_DENYLIST: Final = [
     "user_id",
+    "user_email",
+    "end_user_id",
+    "user_api_key_hash",
+    "user_api_key_user_id",
+    "user_api_key_user_email",
+    "user_api_key_end_user_id",
     "email",
     "phone",
     "address",

litellm/litellm_core_utils/litellm_logging.py

@@ -43,8 +43,6 @@
     DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT,
     EMPTY_MAPPING,
     PROVIDER_REQUEST_ID_HEADERS,
-    SENTRY_DENYLIST,
-    SENTRY_PII_DENYLIST,
 )
 from litellm.cost_calculator import (
     RealtimeAPITokenUsageProcessor,
@@ -4471,21 +4469,10 @@ def set_callbacks(callback_list, function_id=None):
                     print_verbose("Package 'sentry_sdk' is missing. Installing it...")
                     subprocess.check_call([sys.executable, "-m", "pip", "install", "sentry_sdk"])
                     import sentry_sdk
-                from sentry_sdk.scrubber import EventScrubber
+                from litellm.litellm_core_utils.sentry_scrubbing import build_sentry_init_options
 
                 sentry_sdk_instance = sentry_sdk
-                sentry_trace_rate = os.environ.get("SENTRY_API_TRACE_RATE", "1.0")
-                sentry_sample_rate = (
-                    os.environ.get("SENTRY_API_SAMPLE_RATE") if "SENTRY_API_SAMPLE_RATE" in os.environ else "1.0"
-                )
-                sentry_sdk_instance.init(
-                    dsn=os.environ.get("SENTRY_DSN"),
-                    traces_sample_rate=float(sentry_trace_rate),
-                    sample_rate=float(sentry_sample_rate if sentry_sample_rate else 1.0),
-                    send_default_pii=False,  # Prevent sending Personal Identifiable Information
-                    event_scrubber=EventScrubber(denylist=SENTRY_DENYLIST, pii_denylist=SENTRY_PII_DENYLIST),
-                    environment=os.environ.get("SENTRY_ENVIRONMENT", "production"),
-                )
+                sentry_sdk_instance.init(**build_sentry_init_options(os.environ))
                 capture_exception = sentry_sdk_instance.capture_exception
                 add_breadcrumb = sentry_sdk_instance.add_breadcrumb
             elif callback == "slack":

litellm/litellm_core_utils/sentry_scrubbing.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import re
+from collections.abc import Callable, Mapping, Sequence
+from typing import TYPE_CHECKING, Final, TypeAlias, cast
+
+from pydantic import JsonValue
+from sentry_sdk.scrubber import EventScrubber
+from typing_extensions import ReadOnly, TypedDict
+
+from litellm.constants import SENTRY_DENYLIST, SENTRY_PII_DENYLIST
+from litellm.secret_managers.main import str_to_bool
+
+if TYPE_CHECKING:
+    from sentry_sdk.types import Event, Hint
+
+EventScrubFn: TypeAlias = "Callable[[Event, Hint], Event]"
+
+FILTERED: Final = "[Filtered]"
+SEND_DEFAULT_PII_ENV: Final = "SENTRY_SEND_DEFAULT_PII"
+
+EMAIL_PATTERN: Final = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}")
+SHA256_HEX_PATTERN: Final = re.compile(r"(?<![0-9A-Za-z])[0-9a-f]{64}(?![0-9A-Za-z])")
+QUOTED_VALUE: Final = r"'(?:[^'\\]|\\.)*'|\"(?:[^\"\\]|\\.)*\""
+BARE_VALUE: Final = r"(?!None(?![0-9A-Za-z_]))[^,)\]}\s]+"
+
+
+class SentryInitOptions(TypedDict):
+    dsn: ReadOnly[str | None]
+    traces_sample_rate: ReadOnly[float]
+    sample_rate: ReadOnly[float]
+    send_default_pii: ReadOnly[bool]
+    event_scrubber: ReadOnly[EventScrubber]
+    before_send: ReadOnly[EventScrubFn]
+    before_send_transaction: ReadOnly[EventScrubFn]
+    environment: ReadOnly[str]
+
+
+def build_repr_field_pattern(field_names: Sequence[str]) -> re.Pattern[str]:
+    names: Final = "|".join(re.escape(name) for name in field_names)
+    return re.compile(
+        rf"(?P<field>(?<![0-9A-Za-z_])(?:{names})=|['\"](?:{names})['\"]:\s*)(?P<value>{QUOTED_VALUE}|{BARE_VALUE})",
+        re.IGNORECASE,
+    )
+
+
+def build_string_scrubber(send_default_pii: bool) -> Callable[[str], str]:
+    field_names: Final = (
+        tuple(SENTRY_DENYLIST) if send_default_pii else tuple(SENTRY_DENYLIST) + tuple(SENTRY_PII_DENYLIST)
+    )
+    field_pattern: Final = build_repr_field_pattern(field_names)
+    value_patterns: Final = () if send_default_pii else (EMAIL_PATTERN, SHA256_HEX_PATTERN)
+
+    def scrub(text: str) -> str:
+        fields_scrubbed: Final = field_pattern.sub(_filtered_field, text)
+        return _substitute_all(value_patterns, fields_scrubbed)
+
+    return scrub
+
+
+def _filtered_field(match: re.Match[str]) -> str:
+    quote: Final = '"' if match.group("value").startswith('"') else "'"
+    return f"{match.group('field')}{quote}{FILTERED}{quote}"
+
+
+def _substitute_all(patterns: Sequence[re.Pattern[str]], text: str) -> str:
+    if not patterns:
+        return text
+    return _substitute_all(patterns[1:], patterns[0].sub(FILTERED, text))
+
+
+def scrub_json_strings(value: JsonValue, scrub: Callable[[str], str]) -> JsonValue:
+    if isinstance(value, str):
+        return scrub(value)
+    if isinstance(value, dict):
+        return {key: scrub_json_strings(item, scrub) for key, item in value.items()}  # mutable-ok: JSON object
+    if isinstance(value, list):
+        return [scrub_json_strings(item, scrub) for item in value]  # mutable-ok: JSON array
+    return value
+
+
+def build_event_scrubber(send_default_pii: bool) -> EventScrubFn:
+    scrub: Final = build_string_scrubber(send_default_pii)
+
+    def scrub_event(event: Event, _hint: Hint) -> Event:
+        json_event: Final = cast("JsonValue", event)  # cast-ok: [LIT006] the SDK serialized the event to JSON already
+        return cast("Event", scrub_json_strings(json_event, scrub))  # cast-ok: [LIT006] same JSON shape going back
+
+    return scrub_event
+
+
+def send_default_pii_from_env(env: Mapping[str, str]) -> bool:
+    return str_to_bool(env.get(SEND_DEFAULT_PII_ENV)) is True
+
+
+def build_sentry_init_options(env: Mapping[str, str]) -> SentryInitOptions:
+    send_default_pii: Final = send_default_pii_from_env(env)
+    scrub_event: Final = build_event_scrubber(send_default_pii)
+    return SentryInitOptions(
+        dsn=env.get("SENTRY_DSN"),
+        traces_sample_rate=float(env.get("SENTRY_API_TRACE_RATE") or "1.0"),
+        sample_rate=float(env.get("SENTRY_API_SAMPLE_RATE") or "1.0"),
+        send_default_pii=send_default_pii,
+        event_scrubber=EventScrubber(
+            denylist=list(SENTRY_DENYLIST),  # mutable-ok: EventScrubber appends pii_denylist onto denylist in place
+            pii_denylist=list(SENTRY_PII_DENYLIST),  # mutable-ok: EventScrubber takes List[str]
+            recursive=True,
+            send_default_pii=send_default_pii,
+        ),
+        before_send=scrub_event,
+        before_send_transaction=scrub_event,
+        environment=env.get("SENTRY_ENVIRONMENT", "production"),
+    )

tests/test_litellm/litellm_core_utils/test_litellm_logging.py

@@ -19,7 +19,7 @@
 
 import litellm
 from litellm._logging import session_id_var, trace_id_var
-from litellm.constants import SENTRY_DENYLIST, SENTRY_PII_DENYLIST
+from litellm.constants import SENTRY_PII_DENYLIST
 from litellm.cost_calculator import ocr_batch_cost
 from litellm.integrations.custom_logger import CustomLogger
 from litellm.litellm_core_utils.litellm_logging import Logging as LitellmLogging
@@ -356,108 +356,23 @@ def test_post_call_serializes_dict_with_datetime(logging_obj):
     assert "2026-05-11" in serialized
 
 
-def test_sentry_sample_rate(monkeypatch):
-    existing_sample_rate = os.getenv("SENTRY_API_SAMPLE_RATE")
-    try:
-        # test with default value by removing the environment variable
-        if existing_sample_rate:
-            del os.environ["SENTRY_API_SAMPLE_RATE"]
-
-        set_callbacks(["sentry"])
-        # Check if the default sample rate is set to 1.0
-        assert os.environ.get("SENTRY_API_SAMPLE_RATE") == "1.0"
-
-        # test with custom value
-        monkeypatch.setenv("SENTRY_API_SAMPLE_RATE", "0.5")
-
-        set_callbacks(["sentry"])
-        # Check if the custom sample rate is set correctly
-        assert os.environ.get("SENTRY_API_SAMPLE_RATE") == "0.5"
-    except Exception as e:
-        print(f"Error: {e}")
-    finally:
-        # Restore the original environment variable
-        if existing_sample_rate:
-            monkeypatch.setenv("SENTRY_API_SAMPLE_RATE", existing_sample_rate)
-        else:
-            if "SENTRY_API_SAMPLE_RATE" in os.environ:
-                del os.environ["SENTRY_API_SAMPLE_RATE"]
-
-
 def test_sentry_environment(monkeypatch):
-    """Test that SENTRY_ENVIRONMENT is properly handled during Sentry initialization"""
-    existing_environment = os.getenv("SENTRY_ENVIRONMENT")
-    existing_dsn = os.getenv("SENTRY_DSN")
-
-    # Create mock sentry_sdk module
-    mock_event_scrubber_instance = MagicMock()
-    mock_event_scrubber_cls = MagicMock(return_value=mock_event_scrubber_instance)
-
-    mock_scrubber_module = MagicMock()
-    mock_scrubber_module.EventScrubber = mock_event_scrubber_cls
+    import sentry_sdk
 
-    mock_sentry_sdk = MagicMock()
-    mock_sentry_sdk.scrubber = mock_scrubber_module
     mock_init = MagicMock()
-    mock_sentry_sdk.init = mock_init
+    monkeypatch.setattr(sentry_sdk, "init", mock_init)
+    monkeypatch.setenv("SENTRY_DSN", "https://test@sentry.io/123456")
+    monkeypatch.delenv("SENTRY_ENVIRONMENT", raising=False)
 
-    # Inject mocks into sys.modules
-    sys.modules["sentry_sdk"] = mock_sentry_sdk
-    sys.modules["sentry_sdk.scrubber"] = mock_scrubber_module
-
-    try:
-        # Set a mock DSN to allow Sentry initialization
-        monkeypatch.setenv("SENTRY_DSN", "https://test@sentry.io/123456")
-
-        # Test with default value (no environment set)
-        if existing_environment:
-            del os.environ["SENTRY_ENVIRONMENT"]
-
-        mock_init.reset_mock()
-        set_callbacks(["sentry"])
-        # Check that init was called with default environment "production"
-        mock_init.assert_called_once()
-        call_kwargs = mock_init.call_args[1]
-        assert call_kwargs["environment"] == "production"
-
-        # Test with custom environment value
-        monkeypatch.setenv("SENTRY_ENVIRONMENT", "development")
-
-        mock_init.reset_mock()
-        set_callbacks(["sentry"])
-        # Check that init was called with custom environment "development"
-        mock_init.assert_called_once()
-        call_kwargs = mock_init.call_args[1]
-        assert call_kwargs["environment"] == "development"
-
-        # Test with staging environment
-        monkeypatch.setenv("SENTRY_ENVIRONMENT", "staging")
+    set_callbacks(["sentry"])
+    assert mock_init.call_args[1]["environment"] == "production"
 
+    for environment in ("development", "staging"):
+        monkeypatch.setenv("SENTRY_ENVIRONMENT", environment)
         mock_init.reset_mock()
         set_callbacks(["sentry"])
-        # Check that init was called with custom environment "staging"
         mock_init.assert_called_once()
-        call_kwargs = mock_init.call_args[1]
-        assert call_kwargs["environment"] == "staging"
-
-    except Exception as e:
-        print(f"Error: {e}")
-        raise
-    finally:
-        # Restore the original environment variables
-        if existing_environment:
-            monkeypatch.setenv("SENTRY_ENVIRONMENT", existing_environment)
-        else:
-            if "SENTRY_ENVIRONMENT" in os.environ:
-                del os.environ["SENTRY_ENVIRONMENT"]
-
-        if existing_dsn:
-            monkeypatch.setenv("SENTRY_DSN", existing_dsn)
-        else:
-            if "SENTRY_DSN" in os.environ:
-                del os.environ["SENTRY_DSN"]
-
-
+        assert mock_init.call_args[1]["environment"] == environment
 def test_use_custom_pricing_for_model():
     from litellm.litellm_core_utils.litellm_logging import use_custom_pricing_for_model
 
@@ -3099,37 +3014,34 @@ def test_speech_call_is_still_priced_from_input_characters(call_type):
 
 
 def test_sentry_event_scrubber_initialization(monkeypatch):
-    # Step 1: Create a fake sentry_sdk.scrubber module
-    mock_event_scrubber_instance = MagicMock()
-    mock_event_scrubber_cls = MagicMock(return_value=mock_event_scrubber_instance)
-
-    mock_scrubber_module = MagicMock()
-    mock_scrubber_module.EventScrubber = mock_event_scrubber_cls
+    import sentry_sdk
 
-    # Step 2: Create a fake sentry_sdk module and insert into sys.modules
-    mock_sentry_sdk = MagicMock()
-    mock_sentry_sdk.scrubber = mock_scrubber_module
     mock_init = MagicMock()
-    mock_sentry_sdk.init = mock_init
+    monkeypatch.setattr(sentry_sdk, "init", mock_init)
+    monkeypatch.delenv("SENTRY_SEND_DEFAULT_PII", raising=False)
 
-    # Step 3: Inject both into sys.modules BEFORE import occurs
-    sys.modules["sentry_sdk"] = mock_sentry_sdk
-    sys.modules["sentry_sdk.scrubber"] = mock_scrubber_module
-
-    # Step 4: Run the actual sentry setup code
     set_callbacks(["sentry"])
 
-    # Step 5: Assert the EventScrubber was constructed correctly
-    mock_event_scrubber_cls.assert_called_once_with(
-        denylist=SENTRY_DENYLIST,
-        pii_denylist=SENTRY_PII_DENYLIST,
-    )
-
-    # Step 6: Assert the event_scrubber and PII args were passed
     mock_init.assert_called_once()
     call_args = mock_init.call_args[1]
-    assert call_args["event_scrubber"] == mock_event_scrubber_instance
     assert call_args["send_default_pii"] is False
+    assert call_args["event_scrubber"].recursive is True
+    assert {name.lower() for name in SENTRY_PII_DENYLIST} <= {name.lower() for name in call_args["event_scrubber"].denylist}
+    assert call_args["before_send"] is call_args["before_send_transaction"]
+
+
+def test_sentry_send_default_pii_opt_in(monkeypatch):
+    import sentry_sdk
+
+    mock_init = MagicMock()
+    monkeypatch.setattr(sentry_sdk, "init", mock_init)
+    monkeypatch.setenv("SENTRY_SEND_DEFAULT_PII", "true")
+
+    set_callbacks(["sentry"])
+
+    call_args = mock_init.call_args[1]
+    assert call_args["send_default_pii"] is True
+    assert not {name.lower() for name in SENTRY_PII_DENYLIST} & {name.lower() for name in call_args["event_scrubber"].denylist}
 
 
 def test_get_masked_values():