Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions litellm/integrations/otel/model/baggage.py
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
from typing import Final

from litellm.integrations.otel.model.metadata import RequestIdentity
from litellm.integrations.otel.model.semconv import GenAI, LiteLLM
from litellm.integrations.otel.model.semconv import LiteLLM

# Attribute key -> value extractor over (identity, request_model,
# team_metadata_keys). The single definition of what may be promoted and under
Expand All @@ -32,7 +32,7 @@
),
LiteLLM.KEY_HASH: lambda identity, model, team_metadata_keys: identity.key_hash,
LiteLLM.END_USER: lambda identity, model, team_metadata_keys: identity.end_user,
GenAI.REQUEST_MODEL: lambda identity, model, team_metadata_keys: model,
LiteLLM.REQUEST_MODEL: lambda identity, model, team_metadata_keys: model,

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Legacy allowlists silently lose models

When an existing deployment explicitly configures baggage_promoted_keys with gen_ai.request.model, promoted_baggage no longer has an extractor for that configured key and silently omits it, causing downstream non-LLM span processors and correlations to lose the request model after upgrading. Please retain compatibility for the old configured value or reject it with an actionable configuration error.

Rule Used: What: avoid backwards-incompatible changes without... (source)

LiteLLM.PROVIDER_MODEL: lambda identity, model, team_metadata_keys: identity.provider_model,
}

Expand All @@ -44,7 +44,7 @@
LiteLLM.TEAM_ALIAS,
LiteLLM.TEAM_METADATA,
LiteLLM.KEY_HASH,
GenAI.REQUEST_MODEL,
LiteLLM.REQUEST_MODEL,
LiteLLM.PROVIDER_MODEL,
)

Expand Down
6 changes: 5 additions & 1 deletion litellm/integrations/otel/model/semconv.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,8 +229,12 @@ class LiteLLM:
TEAM_METADATA: Final = "litellm.team.metadata"
KEY_HASH: Final = "litellm.api_key.hash"
END_USER: Final = "litellm.end_user.id"
# User-facing model from the inbound request. Promoted via Baggage for
# correlation on non-GenAI spans; ``gen_ai.request.model`` is stamped only
# on LLM-call spans by the GenAI mapper.
REQUEST_MODEL: Final = "litellm.request.model"
# The model string litellm actually sent to the provider (the deployment's
# ``litellm_params.model``), distinct from the user-facing ``gen_ai.request.model``.
# ``litellm_params.model``), distinct from the user-facing request model.
PROVIDER_MODEL: Final = "litellm.provider.model"
REQUEST_STREAMING: Final = "litellm.request.streaming"
GUARDRAIL_NAME: Final = "litellm.guardrail.name"
Expand Down
31 changes: 29 additions & 2 deletions tests/test_litellm/integrations/otel/test_otel_v2_baggage.py
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,33 @@ def test_identity_promoted_onto_every_span():
for span in spans:
assert span.attributes.get(LiteLLM.TEAM_ID) == "t1"
assert span.attributes.get(LiteLLM.TEAM_ALIAS) == "team one"
assert span.attributes.get(GenAI.REQUEST_MODEL) == "gpt-4o"
assert span.attributes.get(LiteLLM.REQUEST_MODEL) == "gpt-4o"
for span in spans:
if span.name == "chat gpt-4o":
assert span.attributes.get(GenAI.REQUEST_MODEL) == "gpt-4o"
else:
assert GenAI.REQUEST_MODEL not in span.attributes


def test_gen_ai_request_model_not_promoted_via_baggage():
"""Baggage must not stamp canonical gen_ai.request.model on non-LLM spans."""
engine, exporter = _engine_and_exporter()
data = LLMCallSpanData.from_standard_logging_payload(_payload())
bag = promoted_baggage(data.identity, data.request_model, BAGGAGE_PROMOTED_KEYS)
ctx = ctx_mod.set_request_baggage(bag)

root = engine.start_span(SpanRole.PROXY_REQUEST, "POST /chat/completions", ctx)
root_ctx = ctx_mod.context_from_span(root, ctx)
engine.emit(SpanRole.SERVICE, ServiceSpanData("redis", call_type="set"), root_ctx)
root.end()

spans = exporter.get_finished_spans()
service_span = next(s for s in spans if s.name != "POST /chat/completions")
assert service_span.attributes.get(LiteLLM.REQUEST_MODEL) == "gpt-4o"
assert GenAI.REQUEST_MODEL not in service_span.attributes
root_span = next(s for s in spans if s.name == "POST /chat/completions")
assert root_span.attributes.get(LiteLLM.REQUEST_MODEL) == "gpt-4o"
assert GenAI.REQUEST_MODEL not in root_span.attributes


def test_team_metadata_promoted_only_for_allowlisted_subkeys():
Expand All @@ -99,7 +125,8 @@ def test_team_metadata_promoted_only_for_allowlisted_subkeys():
assert json.loads(span.attributes[LiteLLM.TEAM_METADATA]) == {"tier": "gold"}
# provider model is distinct from the user-facing request model
assert span.attributes.get(LiteLLM.PROVIDER_MODEL) == "azure/my-deployment"
assert span.attributes.get(GenAI.REQUEST_MODEL) == "gpt-4o"
assert span.attributes.get(LiteLLM.REQUEST_MODEL) == "gpt-4o"
assert GenAI.REQUEST_MODEL not in span.attributes


def test_team_metadata_not_promoted_by_default():
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -698,7 +698,7 @@ def test_promoted_baggage_is_bounded_allowlist():
promoted = promoted_baggage(identity, "gpt-4o", BAGGAGE_PROMOTED_KEYS)
assert promoted[LiteLLM.TEAM_ID] == "t1"
assert promoted[LiteLLM.TEAM_ALIAS] == "team one"
assert promoted[GenAI.REQUEST_MODEL] == "gpt-4o"
assert promoted[LiteLLM.REQUEST_MODEL] == "gpt-4o"
# allowlisted metadata sub-key is promoted under the litellm.metadata.* prefix
assert promoted[f"{LiteLLM.METADATA_PREFIX}user_api_key_org_id"] == "org1"
# full metadata blob is NOT promoted
Expand Down
Loading