fix(sentry): scrub PII and secrets inside object reprs and nested locals, add SENTRY_SEND_DEFAULT_PII opt-in

litellm/constants.py

@@ -1945,6 +1945,15 @@
     "auth_token",
     "jwt_token",
     "private_key",
+    "authorization",
+    "api-key",
+    "x-api-key",
+    "x-goog-api-key",
+    "ocp-apim-subscription-key",
+    "x-litellm-api-key",
+    "x-mcp-auth",
+    "cookie",
+    "set-cookie",
     "SLACK_WEBHOOK_URL",
     "ALERTING_WEBHOOK_URL",
     "webhook_url",
@@ -1967,6 +1976,12 @@
 ]
 SENTRY_PII_DENYLIST: Final = [
     "user_id",
+    "user_email",
+    "end_user_id",
+    "user_api_key_hash",
+    "user_api_key_user_id",
+    "user_api_key_user_email",
+    "user_api_key_end_user_id",
     "email",
     "phone",
     "address",

litellm/litellm_core_utils/litellm_logging.py

@@ -43,8 +43,6 @@
     DEFAULT_MOCK_RESPONSE_PROMPT_TOKEN_COUNT,
     EMPTY_MAPPING,
     PROVIDER_REQUEST_ID_HEADERS,
-    SENTRY_DENYLIST,
-    SENTRY_PII_DENYLIST,
 )
 from litellm.cost_calculator import (
     RealtimeAPITokenUsageProcessor,
@@ -4471,21 +4469,10 @@ def set_callbacks(callback_list, function_id=None):
                     print_verbose("Package 'sentry_sdk' is missing. Installing it...")
                     subprocess.check_call([sys.executable, "-m", "pip", "install", "sentry_sdk"])
                     import sentry_sdk
-                from sentry_sdk.scrubber import EventScrubber
+                from litellm.litellm_core_utils.sentry_scrubbing import build_sentry_init_options
 
                 sentry_sdk_instance = sentry_sdk
-                sentry_trace_rate = os.environ.get("SENTRY_API_TRACE_RATE", "1.0")
-                sentry_sample_rate = (
-                    os.environ.get("SENTRY_API_SAMPLE_RATE") if "SENTRY_API_SAMPLE_RATE" in os.environ else "1.0"
-                )
-                sentry_sdk_instance.init(
-                    dsn=os.environ.get("SENTRY_DSN"),
-                    traces_sample_rate=float(sentry_trace_rate),
-                    sample_rate=float(sentry_sample_rate if sentry_sample_rate else 1.0),
-                    send_default_pii=False,  # Prevent sending Personal Identifiable Information
-                    event_scrubber=EventScrubber(denylist=SENTRY_DENYLIST, pii_denylist=SENTRY_PII_DENYLIST),
-                    environment=os.environ.get("SENTRY_ENVIRONMENT", "production"),
-                )
+                sentry_sdk_instance.init(**build_sentry_init_options(os.environ))
                 capture_exception = sentry_sdk_instance.capture_exception
                 add_breadcrumb = sentry_sdk_instance.add_breadcrumb
             elif callback == "slack":

litellm/litellm_core_utils/sentry_scrubbing.py

@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+import re
+from collections.abc import Callable, Mapping, Sequence
+from functools import reduce
+from typing import TYPE_CHECKING, Final, TypeAlias, cast
+
+from pydantic import JsonValue
+from sentry_sdk.scrubber import DEFAULT_DENYLIST, DEFAULT_PII_DENYLIST, EventScrubber
+from typing_extensions import ReadOnly, TypedDict
+
+from litellm.constants import MINIMUM_CUSTOM_KEY_LENGTH, SENTRY_DENYLIST, SENTRY_PII_DENYLIST
+from litellm.secret_managers.main import str_to_bool
+
+if TYPE_CHECKING:
+    from sentry_sdk.types import Event, Hint
+
+EventScrubFn: TypeAlias = "Callable[[Event, Hint], Event]"
+JsonPath: TypeAlias = tuple[str, ...]
+
+FILTERED: Final = "[Filtered]"
+SEND_DEFAULT_PII_ENV: Final = "SENTRY_SEND_DEFAULT_PII"
+SECRET_FIELD_NAMES: Final = tuple(DEFAULT_DENYLIST) + tuple(SENTRY_DENYLIST)
+PII_FIELD_NAMES: Final = tuple(DEFAULT_PII_DENYLIST) + tuple(SENTRY_PII_DENYLIST)
+
+LITELLM_KEY_PATTERN: Final = re.compile(rf"sk-[A-Za-z0-9_-]{{{MINIMUM_CUSTOM_KEY_LENGTH - len('sk-')},}}")
+SOURCE_CONTEXT_KEYS: Final = frozenset({"pre_context", "context_line", "post_context"})
+STACK_FRAME_PATHS: Final = frozenset(
+    {
+        ("exception", "values", "*", "stacktrace", "frames", "*"),
+        ("threads", "values", "*", "stacktrace", "frames", "*"),
+        ("stacktrace", "frames", "*"),
+    }
+)
+MAX_SCRUB_DEPTH: Final = 64
+EMAIL_PATTERN: Final = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}")
+SHA256_HEX_PATTERN: Final = re.compile(r"(?<![0-9A-Za-z])[0-9a-f]{64}(?![0-9A-Za-z])")
+QUOTED_VALUE: Final = r"'(?:[^'\\]|\\.)*'|\"(?:[^\"\\]|\\.)*\""
+BRACKET_ATOM: Final = rf"(?:{QUOTED_VALUE})|[^\[\]{{}}()'\"]"
+NESTED_BRACKET_LEVELS: Final = 3
+BRACKETED_VALUE: Final = reduce(
+    lambda inner, _: rf"[\[{{(](?:{BRACKET_ATOM}|{inner})*[\]}})]",
+    range(NESTED_BRACKET_LEVELS),
+    rf"[\[{{(](?:{BRACKET_ATOM})*[\]}})]",
+)
+BARE_VALUE: Final = r"(?!None(?![0-9A-Za-z_]))[^,)\]}\s]+"
+
+
+class SentryInitOptions(TypedDict):
+    dsn: ReadOnly[str | None]
+    traces_sample_rate: ReadOnly[float]
+    sample_rate: ReadOnly[float]
+    send_default_pii: ReadOnly[bool]
+    event_scrubber: ReadOnly[EventScrubber]
+    before_send: ReadOnly[EventScrubFn]
+    before_send_transaction: ReadOnly[EventScrubFn]
+    environment: ReadOnly[str]
+
+
+def build_repr_field_pattern(field_names: Sequence[str]) -> re.Pattern[str]:
+    names: Final = "|".join(re.escape(name) for name in field_names)
+    return re.compile(
+        rf"(?P<field>(?<![0-9A-Za-z_])(?:{names})=|['\"](?:{names})['\"]:\s*)(?P<value>{QUOTED_VALUE}|{BRACKETED_VALUE}|{BARE_VALUE})",
+        re.IGNORECASE,
+    )
+
+
+def build_string_scrubber(send_default_pii: bool) -> Callable[[str], str]:
+    field_names: Final = SECRET_FIELD_NAMES if send_default_pii else SECRET_FIELD_NAMES + PII_FIELD_NAMES
+    field_pattern: Final = build_repr_field_pattern(field_names)
+    value_patterns: Final = (
+        (LITELLM_KEY_PATTERN,) if send_default_pii else (LITELLM_KEY_PATTERN, EMAIL_PATTERN, SHA256_HEX_PATTERN)
+    )
+
+    def scrub(text: str) -> str:
+        fields_scrubbed: Final = field_pattern.sub(_filtered_field, text)
+        return _substitute_all(value_patterns, fields_scrubbed)
+
+    return scrub
+
+
+def _filtered_field(match: re.Match[str]) -> str:
+    quote: Final = '"' if match.group("value").startswith('"') else "'"
+    return f"{match.group('field')}{quote}{FILTERED}{quote}"
+
+
+def _substitute_all(patterns: Sequence[re.Pattern[str]], text: str) -> str:
+    return reduce(lambda scrubbed, pattern: pattern.sub(FILTERED, scrubbed), patterns, text)
+
+
+def scrub_json_strings(value: JsonValue, scrub: Callable[[str], str], path: JsonPath = ()) -> JsonValue:
+    if len(path) > MAX_SCRUB_DEPTH:
+        return FILTERED
+    if isinstance(value, str):
+        return scrub(value)
+    if isinstance(value, dict):
+        unscrubbed_keys: Final = SOURCE_CONTEXT_KEYS if path in STACK_FRAME_PATHS else frozenset[str]()
+        return {  # mutable-ok: JSON object
+            key: item if key in unscrubbed_keys else scrub_json_strings(item, scrub, (*path, key))
+            for key, item in value.items()
+        }
+    if isinstance(value, list):
+        return [scrub_json_strings(item, scrub, (*path, "*")) for item in value]  # mutable-ok: JSON array
+    return value
+
+
+def build_event_scrubber(send_default_pii: bool) -> EventScrubFn:
+    scrub: Final = build_string_scrubber(send_default_pii)
+
+    def scrub_event(event: Event, _hint: Hint) -> Event:
+        json_event: Final = cast("JsonValue", event)  # cast-ok: [LIT006] the SDK serialized the event to JSON already
+        return cast("Event", scrub_json_strings(json_event, scrub))  # cast-ok: [LIT006] same JSON shape going back
+
+    return scrub_event
+
+
+def send_default_pii_from_env(env: Mapping[str, str]) -> bool:
+    return str_to_bool(env.get(SEND_DEFAULT_PII_ENV)) is True
+
+
+def build_sentry_init_options(env: Mapping[str, str]) -> SentryInitOptions:
+    send_default_pii: Final = send_default_pii_from_env(env)
+    scrub_event: Final = build_event_scrubber(send_default_pii)
+    return SentryInitOptions(
+        dsn=env.get("SENTRY_DSN"),
+        traces_sample_rate=float(env.get("SENTRY_API_TRACE_RATE") or "1.0"),
+        sample_rate=float(env.get("SENTRY_API_SAMPLE_RATE") or "1.0"),
+        send_default_pii=send_default_pii,
+        event_scrubber=EventScrubber(
+            denylist=list(SECRET_FIELD_NAMES),  # mutable-ok: EventScrubber appends pii_denylist onto denylist in place
+            pii_denylist=list(PII_FIELD_NAMES),  # mutable-ok: EventScrubber takes List[str]
+            recursive=True,
+            send_default_pii=send_default_pii,
+        ),
+        before_send=scrub_event,
+        before_send_transaction=scrub_event,
+        environment=env.get("SENTRY_ENVIRONMENT", "production"),
+    )

pyproject.toml

@@ -249,6 +249,7 @@ proxy-dev = [
     "prisma==0.11.0",
     "hypercorn==0.17.3",
     "prometheus-client==0.20.0",
+    "sentry-sdk==2.21.0",
     "opentelemetry-api==1.28.0",
     "opentelemetry-sdk==1.28.0",
     "opentelemetry-exporter-otlp==1.28.0",

tests/code_coverage_tests/recursive_detector.py

@@ -72,6 +72,7 @@
     "_string_leaves",  # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible).
     "_replace_string_leaves",  # bounded by the nesting depth of a safe_json_structure output (a finite JSON tree, no cycles possible).
     "_sort_processed_sets",  # bounded by the nesting depth of the log-record extra it walks (a finite JSON tree, no cycles possible).
+    "scrub_json_strings",  # max depth set (MAX_SCRUB_DEPTH); fails closed by returning "[Filtered]" for anything nested past the cap.
 ]