From f02f4c1813cc838f1eda694b6d15c2bc44e4a325 Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Sat, 1 Aug 2026 14:45:57 -0700 Subject: [PATCH 1/3] fix(team-callbacks): report API-registered callbacks from GET /team/{team_id}/callback POST /team/{team_id}/callback writes metadata["logging"] while the GET read metadata["callback_settings"], so every team configured through the API or the Admin UI got back an empty list. c620d76fe4 migrated the writer to the new key and left this reader on the old one. Resolve the read the same way request-time resolution does in _get_dynamic_logging_metadata: a logging slot that is present wins outright and callback_settings stays as the deprecated fallback, so the endpoint reports what a request would really do rather than the union of both shapes. An empty logging list therefore reports no callbacks, matching a request that fires none. Decrypt callback_vars for the response and mask the credential keys. Ciphertext would be unusable to the caller, and a value encrypted under a key that is no longer classified as sensitive would otherwise come back as a raw blob. Resolves LIT-5093 --- .../team_callback_endpoints.py | 92 +++++++- .../test_team_callback_endpoints.py | 217 ++++++++++++++++++ ui/litellm-dashboard/src/lib/http/schema.d.ts | 6 + 3 files changed, 308 insertions(+), 7 deletions(-) diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 025b7c4210e1..792fa0807393 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -27,7 +27,15 @@ UserAPIKeyAuth, ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth -from litellm.proxy.common_utils.callback_utils import encrypt_callback_vars +from litellm.proxy.common_utils.callback_utils import ( + decrypt_callback_vars, + encrypt_callback_vars, + is_sensitive_callback_key, +) +from litellm.proxy.litellm_pre_call_utils import ( + _get_validated_callback_metadata, + convert_key_logging_metadata_to_callback, +) from litellm.proxy.management_endpoints.team_endpoints import _verify_team_access from litellm.proxy.management_helpers.utils import management_endpoint_wrapper from litellm.repositories.team_repository import TeamRepository @@ -64,6 +72,75 @@ def _redact_callback_secrets(metadata: Any) -> Any: return redacted +def _mask_sensitive_callback_vars(callbacks: TeamCallbackMetadata) -> None: + """Mask credential-bearing callback vars in place, keeping the rest readable. + + ``callback_vars`` mixes credentials (``langsmith_api_key``, + ``langfuse_secret_key``, ``gcs_path_service_account``) with plain + configuration (project names, bucket names, hosts). The configuration is + what makes a read of this endpoint useful, so only the sensitive keys are + replaced, using the same marker as the audit-log redaction above. + + Masking in place rather than rebuilding the mapping keeps this under the + LIT002 mutable-collection-construction budget. It is safe because the only + caller passes an object it just built from a decrypted deep copy of the + row, so nothing here is reachable from the team's stored metadata. + """ + if not callbacks.callback_vars: + return + for key in tuple(callbacks.callback_vars): + if is_sensitive_callback_key(key): + callbacks.callback_vars[key] = _CALLBACK_VARS_REDACTED + + +def _resolve_team_callbacks(team_metadata: object) -> TeamCallbackMetadata: + """Report the callbacks that are actually in effect for a team. + + A team's callback config can live in either of two metadata slots. + ``metadata["logging"]`` holds the ``AddTeamCallback`` entries written by + ``POST /team/{team_id}/callback`` and by the Admin UI, while + ``metadata["callback_settings"]`` holds the older ``TeamCallbackMetadata`` + shape. Request-time resolution in ``_get_dynamic_logging_metadata`` treats + the two as mutually exclusive: a populated ``logging`` slot wins outright + and ``callback_settings`` is consulted only as the deprecated fallback. + This reader applies the same precedence so it reports what a request would + really do. Merging the two instead would report a ``callback_settings`` + entry as active for a team whose requests never fire it. + + Credential ``callback_vars`` are stored encrypted, so they are decrypted + before being masked by key; a value encrypted under a key that is no longer + classified as sensitive would otherwise come back as raw ciphertext. + """ + if not isinstance(team_metadata, dict): + return TeamCallbackMetadata() + + decrypted = decrypt_callback_vars(team_metadata) + logging_entries = decrypted.get("logging") + + if logging_entries is not None: + # Same predicate the request path uses, so an empty logging slot reports + # no callbacks instead of falling through to the deprecated shape that a + # request would already be ignoring. A non-list value is a row that fails + # at request time; reporting nothing beats reporting a config that never + # fires. + resolved = TeamCallbackMetadata() + for entry in logging_entries if isinstance(logging_entries, list) else (): + if not isinstance(entry, dict): + continue + callback = _get_validated_callback_metadata(item=entry, source="team-level read") + if callback is None: + continue + resolved = convert_key_logging_metadata_to_callback(data=callback, team_callback_settings_obj=resolved) + else: + callback_settings = decrypted.get("callback_settings") + resolved = ( + TeamCallbackMetadata(**callback_settings) if isinstance(callback_settings, dict) else TeamCallbackMetadata() + ) + + _mask_sensitive_callback_vars(resolved) + return resolved + + def _log_audit_task_exception(task: "asyncio.Task[None]") -> None: """Surface a fire-and-forget audit-log task failure. @@ -410,6 +487,12 @@ async def get_team_callbacks( This will return the callback settings for the team with id dbe2f686-a686-4896-864a-4c3924458709 + Covers callbacks registered through POST /team/{team_id}/callback and the Admin UI as well as + teams still on the deprecated callback_settings shape, resolved from the team's stored metadata + with the same precedence used at request time. A key-level logging config overrides the team's + at request time and is not reflected here. Credential-bearing callback_vars are returned masked + as `***REDACTED***` + Returns { "status": "success", "data": { @@ -442,12 +525,7 @@ async def get_team_callbacks( user_api_key_dict=user_api_key_dict, ) - # Retrieve team callback settings from metadata - team_metadata = _existing_team.metadata - team_callback_settings = team_metadata.get("callback_settings", {}) - - # Convert to TeamCallbackMetadata object for consistent structure - team_callback_settings_obj = TeamCallbackMetadata(**team_callback_settings) + team_callback_settings_obj = _resolve_team_callbacks(_existing_team.metadata) return { "status": "success", diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py index d43bf3a3bd8e..50e34a4455a8 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py @@ -459,3 +459,220 @@ async def test_add_team_callbacks_writes_encrypted_callback_vars(monkeypatch): recovered = decrypt_callback_vars(written)["logging"][0]["callback_vars"] assert recovered["langfuse_secret_key"] == "sk-lf-real-secret" assert recovered["langfuse_public_key"] == "pk-lf-real-public" + + +@pytest.mark.asyncio +async def test_get_team_callbacks_returns_callbacks_registered_via_post(monkeypatch): + """POST then GET must round-trip. + + add_team_callbacks writes metadata["logging"]; a GET that reads only + metadata["callback_settings"] reports an empty list for every team + configured through the API. The row handed to the GET here is the exact + payload the POST wrote, so the two code paths cannot drift apart again. + """ + monkeypatch.setenv("LITELLM_SALT_KEY", "test-salt-32-bytes-aaaaaaaaaaaaaa") + row = _team_row(team_id="team-1", metadata={}) + mock_prisma = _patch_prisma(row) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.litellm_proxy_admin_name", "admin"), + patch("litellm.proxy.proxy_server.master_key", None), + ): + await add_team_callbacks( + data=AddTeamCallback( + callback_name="langsmith", + callback_type="success", + callback_vars={ + "langsmith_api_key": "lsv2-real-secret", + "langsmith_project": "tenant-project", + }, + ), + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + litellm_changed_by=None, + ) + + # Feed the GET exactly what the POST persisted. + row.metadata = json.loads(mock_prisma.db.litellm_teamtable.update.await_args.kwargs["data"]["metadata"]) + row.model_dump.return_value["metadata"] = row.metadata + + response = await get_team_callbacks( + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + ) + + assert response["data"]["success_callbacks"] == ["langsmith"] + assert response["data"]["failure_callbacks"] == [] + # Non-secret vars come back usable, the credential is masked, and the + # ciphertext that is stored on the row never reaches the response. + assert response["data"]["callback_vars"]["langsmith_project"] == "tenant-project" + assert response["data"]["callback_vars"]["langsmith_api_key"] == "***REDACTED***" + assert "lsv2-real-secret" not in json.dumps(response) + assert "litellm_enc::" not in json.dumps(response) + + +@pytest.mark.asyncio +async def test_get_team_callbacks_prefers_logging_over_deprecated_callback_settings(): + """A team carrying both shapes must report only the one that actually fires. + + Request-time resolution in _get_dynamic_logging_metadata stops at the + first populated slot: metadata["logging"] wins and callback_settings is + never consulted. Reporting the union here would tell an operator that + gcs_bucket is active on a team whose requests never send to it. + """ + metadata = { + "callback_settings": { + "success_callback": ["gcs_bucket"], + "failure_callback": [], + "callback_vars": {"gcs_bucket_name": "legacy-bucket"}, + }, + "logging": [ + { + "callback_name": "langsmith", + "callback_type": "success_and_failure", + "callback_vars": {"langsmith_project": "tenant-project"}, + }, + {"callback_name": "missing-required-fields"}, + ], + } + mock_prisma = _patch_prisma(_team_row(team_id="team-1", metadata=metadata)) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma): + response = await get_team_callbacks( + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + ) + + assert response["data"]["success_callbacks"] == ["langsmith"] + assert response["data"]["failure_callbacks"] == ["langsmith"] + # The deprecated slot contributes nothing, and the malformed logging entry + # is skipped rather than failing the whole read. + assert response["data"]["callback_vars"] == {"langsmith_project": "tenant-project"} + + +@pytest.mark.asyncio +async def test_get_team_callbacks_reports_nothing_when_logging_slot_is_empty(): + """An empty logging slot must not fall through to callback_settings. + + Request-time resolution stops at the first non-None slot, so a team whose + logging list is empty fires no callbacks at all even when the deprecated + shape is still populated next to it. Reporting gcs_bucket here would tell + an operator a destination is live when nothing is being sent to it. + """ + metadata = { + "logging": [], + "callback_settings": { + "success_callback": ["gcs_bucket"], + "failure_callback": [], + "callback_vars": {"gcs_bucket_name": "legacy-bucket"}, + }, + } + mock_prisma = _patch_prisma(_team_row(team_id="team-1", metadata=metadata)) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma): + response = await get_team_callbacks( + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + ) + + assert response["data"]["success_callbacks"] == [] + assert response["data"]["failure_callbacks"] == [] + assert response["data"]["callback_vars"] == {} + + +@pytest.mark.asyncio +async def test_get_team_callbacks_decrypts_vars_stored_under_non_sensitive_keys(monkeypatch): + """Stored ciphertext must be decrypted, not handed back raw. + + Which keys count as sensitive is a moving classification, so a value can be + encrypted at rest under a key that later stops being masked on read. Without + the decrypt step that value comes back as an unusable litellm_enc:: blob. + """ + from litellm.proxy.common_utils.callback_utils import _CALLBACK_VAR_ENCRYPTED_PREFIX, is_sensitive_callback_key + from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper + + monkeypatch.setenv("LITELLM_SALT_KEY", "test-salt-32-bytes-aaaaaaaaaaaaaa") + assert not is_sensitive_callback_key("langsmith_project"), "test needs a key that is not masked on read" + + metadata = { + "logging": [ + { + "callback_name": "langsmith", + "callback_type": "success", + "callback_vars": { + "langsmith_project": _CALLBACK_VAR_ENCRYPTED_PREFIX + encrypt_value_helper("tenant-project"), + }, + } + ] + } + mock_prisma = _patch_prisma(_team_row(team_id="team-1", metadata=metadata)) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.master_key", None), + ): + response = await get_team_callbacks( + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + ) + + assert response["data"]["callback_vars"]["langsmith_project"] == "tenant-project" + + +@pytest.mark.asyncio +async def test_get_team_callbacks_falls_back_to_deprecated_callback_settings(): + """Teams that never used the API keep working: with no logging slot, the + deprecated callback_settings shape is still reported, exactly as the + request-time fallback would use it.""" + metadata = { + "callback_settings": { + "success_callback": ["gcs_bucket"], + "failure_callback": ["langfuse"], + "callback_vars": { + "gcs_bucket_name": "legacy-bucket", + "langfuse_secret_key": "sk-lf-legacy-plaintext", + }, + } + } + mock_prisma = _patch_prisma(_team_row(team_id="team-1", metadata=metadata)) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma): + response = await get_team_callbacks( + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + ) + + assert response["data"]["success_callbacks"] == ["gcs_bucket"] + assert response["data"]["failure_callbacks"] == ["langfuse"] + assert response["data"]["callback_vars"]["gcs_bucket_name"] == "legacy-bucket" + # Legacy rows predate encryption at rest, so this endpoint is where the + # plaintext secret would otherwise escape. + assert response["data"]["callback_vars"]["langfuse_secret_key"] == "***REDACTED***" + assert "sk-lf-legacy-plaintext" not in json.dumps(response) + + +@pytest.mark.asyncio +async def test_get_team_callbacks_reports_empty_for_team_without_callbacks(): + """A team with no callback config still returns the documented empty shape.""" + mock_prisma = _patch_prisma(_team_row(team_id="team-1", metadata={})) + + with patch("litellm.proxy.proxy_server.prisma_client", mock_prisma): + response = await get_team_callbacks( + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + ) + + assert response["data"] == { + "team_id": "team-1", + "success_callbacks": [], + "failure_callbacks": [], + "callback_vars": {}, + } diff --git a/ui/litellm-dashboard/src/lib/http/schema.d.ts b/ui/litellm-dashboard/src/lib/http/schema.d.ts index 109d638fb9c0..c286c5a98a0a 100644 --- a/ui/litellm-dashboard/src/lib/http/schema.d.ts +++ b/ui/litellm-dashboard/src/lib/http/schema.d.ts @@ -14019,6 +14019,12 @@ export interface paths { * * This will return the callback settings for the team with id dbe2f686-a686-4896-864a-4c3924458709 * + * Covers callbacks registered through POST /team/{team_id}/callback and the Admin UI as well as + * teams still on the deprecated callback_settings shape, resolved from the team's stored metadata + * with the same precedence used at request time. A key-level logging config overrides the team's + * at request time and is not reflected here. Credential-bearing callback_vars are returned masked + * as `***REDACTED***` + * * Returns { * "status": "success", * "data": { From 82dc8fa1b5f77faaf041085351d9ded70e7d5c82 Mon Sep 17 00:00:00 2001 From: yucheng-berri Date: Sat, 1 Aug 2026 15:11:18 -0700 Subject: [PATCH 2/3] Update litellm/proxy/management_endpoints/team_callback_endpoints.py Co-authored-by: devin-ai-integration[bot] <158243242+devin-ai-integration[bot]@users.noreply.github.com> --- .../proxy/management_endpoints/team_callback_endpoints.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index 792fa0807393..c3bcd93ed709 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -118,11 +118,6 @@ def _resolve_team_callbacks(team_metadata: object) -> TeamCallbackMetadata: logging_entries = decrypted.get("logging") if logging_entries is not None: - # Same predicate the request path uses, so an empty logging slot reports - # no callbacks instead of falling through to the deprecated shape that a - # request would already be ignoring. A non-list value is a row that fails - # at request time; reporting nothing beats reporting a config that never - # fires. resolved = TeamCallbackMetadata() for entry in logging_entries if isinstance(logging_entries, list) else (): if not isinstance(entry, dict): From 582587d7b632d8b2e1c0d2f6e49afb87c0a21598 Mon Sep 17 00:00:00 2001 From: Yucheng Zhu Date: Sat, 1 Aug 2026 15:18:43 -0700 Subject: [PATCH 3/3] fix(team-callbacks): mask callback vars that fail to decrypt decrypt_callback_vars passes a value through untouched when it cannot be decrypted, which happens to existing rows after a salt-key rotation. Under a key that is not classified as sensitive that blob reached the caller as opaque ciphertext it could not use or tell apart from a real value, so mask anything still carrying the encrypted prefix. Raised by Greptile on the first commit. --- .../team_callback_endpoints.py | 9 +++- .../test_team_callback_endpoints.py | 42 +++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/management_endpoints/team_callback_endpoints.py b/litellm/proxy/management_endpoints/team_callback_endpoints.py index c3bcd93ed709..d6ace44cabc0 100644 --- a/litellm/proxy/management_endpoints/team_callback_endpoints.py +++ b/litellm/proxy/management_endpoints/team_callback_endpoints.py @@ -28,6 +28,7 @@ ) from litellm.proxy.auth.user_api_key_auth import user_api_key_auth from litellm.proxy.common_utils.callback_utils import ( + _CALLBACK_VAR_ENCRYPTED_PREFIX, decrypt_callback_vars, encrypt_callback_vars, is_sensitive_callback_key, @@ -81,6 +82,11 @@ def _mask_sensitive_callback_vars(callbacks: TeamCallbackMetadata) -> None: what makes a read of this endpoint useful, so only the sensitive keys are replaced, using the same marker as the audit-log redaction above. + A value that still carries the encrypted prefix here failed to decrypt, so + it is masked too. Handing back a ciphertext blob under a key that is not + classified as sensitive would give the caller something it cannot use and + cannot tell apart from a real value. + Masking in place rather than rebuilding the mapping keeps this under the LIT002 mutable-collection-construction budget. It is safe because the only caller passes an object it just built from a decrypted deep copy of the @@ -89,7 +95,8 @@ def _mask_sensitive_callback_vars(callbacks: TeamCallbackMetadata) -> None: if not callbacks.callback_vars: return for key in tuple(callbacks.callback_vars): - if is_sensitive_callback_key(key): + value = callbacks.callback_vars[key] + if is_sensitive_callback_key(key) or str(value).startswith(_CALLBACK_VAR_ENCRYPTED_PREFIX): callbacks.callback_vars[key] = _CALLBACK_VARS_REDACTED diff --git a/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py b/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py index 50e34a4455a8..3b2b1ccb7931 100644 --- a/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py +++ b/tests/test_litellm/proxy/management_endpoints/test_team_callback_endpoints.py @@ -625,6 +625,48 @@ async def test_get_team_callbacks_decrypts_vars_stored_under_non_sensitive_keys( assert response["data"]["callback_vars"]["langsmith_project"] == "tenant-project" +@pytest.mark.asyncio +async def test_get_team_callbacks_masks_values_that_fail_to_decrypt(monkeypatch): + """A value that cannot be decrypted must never leave as ciphertext. + + After a salt-key rotation an existing value no longer decrypts, and the + shared helper passes it through untouched. Under a key that is not + classified as sensitive it would otherwise reach the caller as an opaque + blob that is indistinguishable from a real value. + """ + from litellm.proxy.common_utils.callback_utils import _CALLBACK_VAR_ENCRYPTED_PREFIX + from litellm.proxy.common_utils.encrypt_decrypt_utils import encrypt_value_helper + + monkeypatch.setenv("LITELLM_SALT_KEY", "test-salt-32-bytes-aaaaaaaaaaaaaa") + stale = _CALLBACK_VAR_ENCRYPTED_PREFIX + encrypt_value_helper("tenant-project") + monkeypatch.setenv("LITELLM_SALT_KEY", "test-salt-32-bytes-bbbbbbbbbbbbbb") + + metadata = { + "logging": [ + { + "callback_name": "langsmith", + "callback_type": "success", + "callback_vars": {"langsmith_project": stale}, + } + ] + } + mock_prisma = _patch_prisma(_team_row(team_id="team-1", metadata=metadata)) + + with ( + patch("litellm.proxy.proxy_server.prisma_client", mock_prisma), + patch("litellm.proxy.proxy_server.master_key", None), + ): + response = await get_team_callbacks( + http_request=MagicMock(spec=Request), + team_id="team-1", + user_api_key_dict=_admin_auth(), + ) + + assert response["data"]["success_callbacks"] == ["langsmith"] + assert response["data"]["callback_vars"]["langsmith_project"] == "***REDACTED***" + assert _CALLBACK_VAR_ENCRYPTED_PREFIX not in json.dumps(response) + + @pytest.mark.asyncio async def test_get_team_callbacks_falls_back_to_deprecated_callback_settings(): """Teams that never used the API keep working: with no logging slot, the