diff --git a/litellm/integrations/otel/langfuse_logger.py b/litellm/integrations/otel/langfuse_logger.py index f8fd417392f9..c377a0ceeafc 100644 --- a/litellm/integrations/otel/langfuse_logger.py +++ b/litellm/integrations/otel/langfuse_logger.py @@ -1,6 +1,9 @@ from collections.abc import AsyncGenerator, AsyncIterator, Callable, Mapping from typing import TYPE_CHECKING, Final +from opentelemetry.sdk.trace import ReadableSpan +from opentelemetry.trace import Span + from litellm._logging import verbose_logger from litellm.integrations.otel.logger import OpenTelemetryV2 from litellm.integrations.otel.mappers.langfuse import ( @@ -19,15 +22,22 @@ class LangfuseOpenTelemetryV2(OpenTelemetryV2): """Names the trace from the request. Langfuse reads ``langfuse.trace.name`` off the root observation, - and the proxy's root span is still recording when the LLM call starts.""" + and the proxy's root span is still recording when the LLM call starts. Only a root created by this + logger's own provider is stamped; any other root is exported to destinations that are not Langfuse.""" def log_pre_api_call(self, model: str, messages: object, kwargs: Mapping[str, object]) -> None: - root: Final = request_root_span() + root: Final = self._owned_recording_root() name: Final = caller_trace_name(kwargs) - if root is not None and root.is_recording() and name is not None: + if root is not None and name is not None: root.set_attribute(LANGFUSE_TRACE_NAME, name) super().log_pre_api_call(model, messages, kwargs) + def _owned_recording_root(self) -> Span | None: + root: Final = request_root_span() + if root is None or not root.is_recording() or not isinstance(root, ReadableSpan): + return None + return root if root.resource is self.tracer_provider.resource else None + class LangfuseContentOpenTelemetryV2(LangfuseOpenTelemetryV2): """Stamps the request's input and output on the root observation while it is still recording. @@ -59,8 +69,8 @@ async def async_post_call_streaming_iterator_hook( self._stamp_root_io(request_data, lambda: stream_output(tuple(relayed), request_data)) def _stamp_root_io(self, data: Mapping[str, object], render_output: Callable[[], str | None]) -> None: - root: Final = request_root_span() - if root is None or not root.is_recording(): + root: Final = self._owned_recording_root() + if root is None: return try: output: Final = render_output() diff --git a/litellm/integrations/otel/logger.py b/litellm/integrations/otel/logger.py index 9ac748b231ce..0efdb7a0c0b6 100644 --- a/litellm/integrations/otel/logger.py +++ b/litellm/integrations/otel/logger.py @@ -206,6 +206,11 @@ def tracer_provider(self) -> TracerProvider: """The provider this logger emits through, read-only to its callers.""" return self._tracer_provider + @property + def serves_generic_collector(self) -> bool: + """Every exporter is the operator's ``OTEL_*`` destination, as for ``otel`` and mapper-only presets.""" + return all(spec.owner is None for spec in self.config.exporters) + def _init_metrics(self, meter_provider: "MeterProvider | None") -> "GenAIMetricRecorder | None": """Create the six GenAI histograms when metrics are enabled, else ``None``. @@ -259,9 +264,19 @@ def _init_otel_logger_on_litellm_proxy(self) -> None: self._register_in_callback_list(litellm._async_failure_callback) except Exception: pass - if getattr(proxy_server, "open_telemetry_logger", None) is None: + if self._outranks_for_proxy_slot(getattr(proxy_server, "open_telemetry_logger", None)): setattr(proxy_server, "open_telemetry_logger", self) + def _outranks_for_proxy_slot(self, holder: object) -> bool: + """The collector's logger owns the slot; a vendor preset holds it only until that logger is built.""" + if holder is None: + return True + return ( + self.serves_generic_collector + and isinstance(holder, OpenTelemetryV2) + and not holder.serves_generic_collector + ) + # ====================================================================== # # LLM-call callbacks — the span is opened at the ``pre_call`` boundary and # closed here. See ``log_pre_api_call``. @@ -842,10 +857,12 @@ def select_global_otel_v2_logger( ``proxy_server.open_telemetry_logger``), and every other v2 entry point — guardrail, identity seeding, phase spans — already routes through that same ``registered`` owner. Reuse it here too so the global provider has one source - of truth instead of a second, independently-derived guess; this is the logger - a preset (arize, langfuse, …) folds the ``OTEL_*`` base exporter and its own - exporter into, so the FastAPI server span and the gen-ai spans share one - provider and one trace. + of truth instead of a second, independently-derived guess. Each logger exports + only to the destination its callback owns (``otel`` serves ``OTEL_*``, a preset + serves its own backend), so the FastAPI server span and the request root land + at the ``otel`` callback's collector when one is configured and at the first + preset's backend otherwise, whatever the callback order; a preset stamps that + root only when its own provider created it. Fall back to ``in_memory_loggers`` for the SDK path, where no proxy global is set (selecting from there, not ``service_callback``, which a preset logger does @@ -855,8 +872,11 @@ def select_global_otel_v2_logger( """ if registered is not None: return registered - existing: Final = next((cb for cb in in_memory_loggers if isinstance(cb, OpenTelemetryV2)), None) - return existing if existing is not None else OpenTelemetryV2() + v2_loggers: Final = tuple(cb for cb in in_memory_loggers if isinstance(cb, OpenTelemetryV2)) + generic: Final = next((cb for cb in v2_loggers if cb.serves_generic_collector), None) + if generic is not None: + return generic + return v2_loggers[0] if v2_loggers else OpenTelemetryV2() def publish_global_otel_v2_provider( diff --git a/litellm/integrations/otel/presets/agentops.py b/litellm/integrations/otel/presets/agentops.py index 965213f2ee41..f6d85dd80d06 100644 --- a/litellm/integrations/otel/presets/agentops.py +++ b/litellm/integrations/otel/presets/agentops.py @@ -53,7 +53,6 @@ def agentops_preset( return base.model_copy( update={ "exporters": [ - *base.exporters, ExporterSpec( kind=_AGENTOPS_EXPORTER_KIND, endpoint=_AGENTOPS_ENDPOINT, diff --git a/litellm/integrations/otel/presets/arize.py b/litellm/integrations/otel/presets/arize.py index d7ce87f5552b..6ac2c1a2d201 100644 --- a/litellm/integrations/otel/presets/arize.py +++ b/litellm/integrations/otel/presets/arize.py @@ -35,7 +35,6 @@ def arize_preset( return base.model_copy( update={ "exporters": [ - *base.exporters, ExporterSpec( kind=arize_cfg.protocol or "otlp_grpc", endpoint=arize_cfg.endpoint or "https://otlp.arize.com/v1", diff --git a/litellm/integrations/otel/presets/langfuse.py b/litellm/integrations/otel/presets/langfuse.py index 9149e0c0d946..bea050d1d771 100644 --- a/litellm/integrations/otel/presets/langfuse.py +++ b/litellm/integrations/otel/presets/langfuse.py @@ -10,10 +10,7 @@ ExporterSpec, OpenTelemetryV2Config, ) -from litellm.integrations.otel.presets.utils import ( - credential_gated_exporters, - ensure_mappers, -) +from litellm.integrations.otel.presets.utils import ensure_mappers from litellm.types.utils import StandardCallbackDynamicParams @@ -31,7 +28,7 @@ def langfuse_preset( raise return base.model_copy( update={ # mutable-ok: pydantic model_copy takes a plain update mapping - "exporters": credential_gated_exporters(base.exporters, ExporterOwner.LANGFUSE_OTEL), + "exporters": [ExporterSpec(owner=ExporterOwner.LANGFUSE_OTEL, requires_headers=True)], "mapper_names": mappers, } ) @@ -39,7 +36,6 @@ def langfuse_preset( return base.model_copy( update={ "exporters": [ - *base.exporters, ExporterSpec( kind=kind, endpoint=cfg.endpoint, diff --git a/litellm/integrations/otel/presets/levo.py b/litellm/integrations/otel/presets/levo.py index c1580cf7a5b5..e1ea44f585f1 100644 --- a/litellm/integrations/otel/presets/levo.py +++ b/litellm/integrations/otel/presets/levo.py @@ -20,7 +20,6 @@ def levo_preset( return base.model_copy( update={ "exporters": [ - *base.exporters, ExporterSpec( kind="otlp_http", endpoint=cfg.endpoint, diff --git a/litellm/integrations/otel/presets/newrelic.py b/litellm/integrations/otel/presets/newrelic.py index 771b3f643c86..298eee2e86ca 100644 --- a/litellm/integrations/otel/presets/newrelic.py +++ b/litellm/integrations/otel/presets/newrelic.py @@ -54,7 +54,6 @@ def newrelic_preset( return base.model_copy( update={ "exporters": [ - *base.exporters, ExporterSpec( kind="otlp_http", endpoint=endpoint, diff --git a/litellm/integrations/otel/presets/phoenix.py b/litellm/integrations/otel/presets/phoenix.py index f4f34ee75258..e3273c6b9c22 100644 --- a/litellm/integrations/otel/presets/phoenix.py +++ b/litellm/integrations/otel/presets/phoenix.py @@ -69,7 +69,6 @@ def phoenix_preset( return base.model_copy( update={ "exporters": [ - *base.exporters, ExporterSpec( kind=cfg.protocol if hasattr(cfg, "protocol") else "otlp_http", endpoint=cfg.endpoint, diff --git a/litellm/integrations/otel/presets/utils.py b/litellm/integrations/otel/presets/utils.py index 1270c41e77b5..328569d3daf4 100644 --- a/litellm/integrations/otel/presets/utils.py +++ b/litellm/integrations/otel/presets/utils.py @@ -3,8 +3,6 @@ from collections.abc import Iterable from typing import Final -from litellm.integrations.otel.model.config import ExporterOwner, ExporterSpec - def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]: """Return ``mapper_names`` with each of ``names`` appended if not already present. @@ -17,32 +15,3 @@ def ensure_mappers(mapper_names: Iterable[str], *names: str) -> list[str]: if name not in result: result.append(name) return result - - -def credential_gated_exporters( - exporters: "Iterable[ExporterSpec]", owner: "ExporterOwner" -) -> "tuple[ExporterSpec, ...]": - """``exporters`` with the operator's destination replaced by a header-gated one. - - Used when a credential-mandatory backend is asked to build without the operator's - own credentials, so only key/team destinations receive spans. Two things have to - happen for that to mean "export nowhere": the placeholder console spec that - ``OpenTelemetryV2Config`` folds in for an empty exporter list is dropped, or every - span would be printed to stdout, and the gated spec keeps the owner so the - override filter still recognises which backend this provider speaks for. - """ - return ( - *(spec for spec in exporters if not is_unconfigured_placeholder(spec)), - ExporterSpec(owner=owner, requires_headers=True), - ) - - -def is_unconfigured_placeholder(spec: "ExporterSpec") -> bool: - """Whether ``spec`` is the one ``_normalize`` folds in when nothing was configured. - - No field set is what says the operator asked for nothing: an exporter they did - configure survives, even ``OTEL_EXPORTER=console`` whose value matches the default, - and so does the gated spec this module appends, which would otherwise eat itself - when one preset layers onto another. - """ - return not spec.model_fields_set diff --git a/litellm/integrations/otel/presets/weave.py b/litellm/integrations/otel/presets/weave.py index 644cd39ad366..41567758d7e6 100644 --- a/litellm/integrations/otel/presets/weave.py +++ b/litellm/integrations/otel/presets/weave.py @@ -7,13 +7,10 @@ ExporterSpec, OpenTelemetryV2Config, ) -from litellm.integrations.otel.presets.utils import ( - credential_gated_exporters, - ensure_mappers, -) +from litellm.integrations.otel.presets.utils import ensure_mappers from litellm.integrations.weave.weave_otel import ( _get_weave_authorization_header, - get_weave_otel_config, + read_weave_otel_config, ) from litellm.types.utils import StandardCallbackDynamicParams @@ -26,20 +23,19 @@ def weave_preset( base: Final = config_overrides or OpenTelemetryV2Config() mappers: Final = ensure_mappers(base.mapper_names, "openinference", "weave") try: - weave_cfg: Final = get_weave_otel_config() + weave_cfg: Final = read_weave_otel_config() except Exception: if not allow_missing_credentials: raise return base.model_copy( update={ # mutable-ok: pydantic model_copy takes a plain update mapping - "exporters": credential_gated_exporters(base.exporters, ExporterOwner.WEAVE_OTEL), + "exporters": [ExporterSpec(owner=ExporterOwner.WEAVE_OTEL, requires_headers=True)], "mapper_names": mappers, } ) return base.model_copy( update={ "exporters": [ - *base.exporters, ExporterSpec( kind=weave_cfg.protocol or "otlp_http", endpoint=weave_cfg.endpoint, diff --git a/litellm/integrations/weave/weave_otel.py b/litellm/integrations/weave/weave_otel.py index 50289263f389..bb68a867068b 100644 --- a/litellm/integrations/weave/weave_otel.py +++ b/litellm/integrations/weave/weave_otel.py @@ -125,17 +125,8 @@ def weave_otel_endpoint(host: str | None) -> str: return normalized.rstrip("/") + WEAVE_OTEL_ENDPOINT -def get_weave_otel_config() -> WeaveOtelConfig: - """ - Retrieves the Weave OpenTelemetry configuration based on environment variables. - - Environment Variables: - WANDB_API_KEY: Required. W&B API key for authentication. - WANDB_PROJECT_ID: Required. Project ID in format /. - WANDB_HOST: Optional. Custom Weave host URL. Defaults to cloud endpoint. - - Returns: - WeaveOtelConfig: A Pydantic model containing Weave OTEL configuration. +def read_weave_otel_config() -> WeaveOtelConfig: + """Weave OTLP settings from ``WANDB_API_KEY``, ``WANDB_PROJECT_ID`` and optional ``WANDB_HOST``. Raises: ValueError: If required environment variables are missing. @@ -158,10 +149,6 @@ def get_weave_otel_config() -> WeaveOtelConfig: auth_header: Final = _get_weave_authorization_header(api_key=api_key) otlp_auth_headers: Final = f"Authorization={auth_header},project_id={project_id}" - # Set standard OTEL environment variables - os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = endpoint - os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = otlp_auth_headers - return WeaveOtelConfig( otlp_auth_headers=otlp_auth_headers, endpoint=endpoint, @@ -170,6 +157,14 @@ def get_weave_otel_config() -> WeaveOtelConfig: ) +def get_weave_otel_config() -> WeaveOtelConfig: + """``read_weave_otel_config`` plus the v1 side effect of publishing it as the process-wide ``OTEL_*`` env.""" + config: Final = read_weave_otel_config() + os.environ["OTEL_EXPORTER_OTLP_ENDPOINT"] = config.endpoint + os.environ["OTEL_EXPORTER_OTLP_HEADERS"] = config.otlp_auth_headers + return config + + def set_weave_otel_attributes(span: Span, kwargs: Mapping[str, object], response_obj: object): """ Sets OpenTelemetry span attributes for Weave observability. diff --git a/litellm/litellm_core_utils/litellm_logging.py b/litellm/litellm_core_utils/litellm_logging.py index 2d3a99abe819..fe71d030e4e1 100644 --- a/litellm/litellm_core_utils/litellm_logging.py +++ b/litellm/litellm_core_utils/litellm_logging.py @@ -4499,7 +4499,8 @@ def _init_custom_logger_compatible_class( from litellm.integrations.otel.model.config import OpenTelemetryV2Config for callback in _in_memory_loggers: - if isinstance(callback, OpenTelemetryV2): + if isinstance(callback, OpenTelemetryV2) and callback.serves_generic_collector: + _maybe_auto_initialize_arize_phoenix(_in_memory_loggers) return callback otel_settings: Final = _get_custom_logger_settings_from_proxy_server(callback_name=logging_integration) otel_logger_v2: Final = build_otel_v2_logger( @@ -4862,19 +4863,20 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom Returns ``None`` when V2 is off OR when there's no preset registered for ``callback_name`` — callers should then fall through to the legacy path. - A preset that needs operator credentials it cannot find is allowed to build - only when this request has a key/team destination for that backend and another - V2 logger is already registered to carry the fan-out. The resulting logger keeps - only its credential-gated exporter, while the registered logger owns operator - delivery. Without that carrier, a preset that raises or that ends up with nothing - but its gated exporter and the default console placeholder returns ``None``, so the - caller falls through to the legacy path exactly as before V2 landed. + Each preset exports only to the destination it owns; the operator's ``OTEL_*`` + collector is served by the ``otel`` callback alone. A preset that needs operator + credentials it cannot find is allowed to build only when this request has a + key/team destination for that backend and another V2 logger is already registered + to carry the fan-out. Without that carrier, a preset that raises or that has nothing + but its credential-gated exporter returns ``None``, so the caller falls through to + the legacy path exactly as before V2 landed. """ from litellm.integrations.otel.model.config import is_otel_v2_enabled if not is_otel_v2_enabled(): return None from litellm.integrations.otel.logger import OpenTelemetryV2, build_otel_v2_logger + from litellm.integrations.otel.model.config import OpenTelemetryV2Config from litellm.integrations.otel.plumbing.context import destination_backends from litellm.integrations.otel.presets import PRESET_BY_CALLBACK @@ -4887,26 +4889,38 @@ def _maybe_construct_otel_v2(callback_name: str, _in_memory_loggers: list[Custom for callback in _in_memory_loggers: if ( isinstance(callback, OpenTelemetryV2) - and getattr(callback, "callback_name", None) == callback_name + and callback.callback_name == callback_name and (serves_a_destination or not _exports_nowhere(callback.config)) ): return callback try: - built: Final = preset_fn(allow_missing_credentials=carried) + config: Final = preset_fn(allow_missing_credentials=carried) except Exception: # If env vars are missing or the preset raises, defer to the legacy path # so customers get the same error story they had before V2 landed. return None - gated: Final = _is_credential_gated(built) - if gated and not carried and not _has_operator_exporter(built): + exports_nowhere: Final = _exports_nowhere(config) + if exports_nowhere and not carried: return None - config: Final = _only_the_gated_exporter(built) if gated and carried else built - if _exports_nowhere(config): + if exports_nowhere: verbose_logger.warning( "OTel V2: no operator credentials for '%s'; only key/team destinations will receive its traces", callback_name, ) - v2_logger: Final = build_otel_v2_logger(config=config, callback_name=callback_name) + serves_generic_collector: Final = all(spec.owner is None for spec in config.exporters) + otel_settings: Final[Mapping[str, object]] = ( + _get_custom_logger_settings_from_proxy_server(callback_name="otel") + if serves_generic_collector + else MappingProxyType({}) + ) + collector_config: Final = ( + preset_fn(config_overrides=OpenTelemetryV2Config(**otel_settings), allow_missing_credentials=carried) + if otel_settings + else config + ) + v2_logger: Final = build_otel_v2_logger( + config=collector_config, callback_name=callback_name, settings=otel_settings + ) _in_memory_loggers.append(v2_logger) return v2_logger @@ -4916,24 +4930,6 @@ def _exports_nowhere(config: "OpenTelemetryV2Config") -> bool: return all(_is_gated(spec) for spec in config.exporters) -def _is_credential_gated(config: "OpenTelemetryV2Config") -> bool: - """Whether the preset built without the operator's own credentials for its backend.""" - return any(_is_gated(spec) for spec in config.exporters) - - -def _has_operator_exporter(config: "OpenTelemetryV2Config") -> bool: - """Whether the operator configured somewhere real to export, beyond the default console placeholder.""" - from litellm.integrations.otel.presets.utils import is_unconfigured_placeholder - - return any(not _is_gated(spec) and not is_unconfigured_placeholder(spec) for spec in config.exporters) - - -def _only_the_gated_exporter(config: "OpenTelemetryV2Config") -> "OpenTelemetryV2Config": - return config.model_copy( - update={"exporters": [spec for spec in config.exporters if _is_gated(spec)]} # mutable-ok: model_copy update - ) - - def _is_gated(spec: "ExporterSpec") -> bool: return spec.requires_headers and not spec.headers diff --git a/tests/test_litellm/integrations/otel/test_langfuse_logger.py b/tests/test_litellm/integrations/otel/test_langfuse_logger.py index 8db84b090a05..65302963d31a 100644 --- a/tests/test_litellm/integrations/otel/test_langfuse_logger.py +++ b/tests/test_litellm/integrations/otel/test_langfuse_logger.py @@ -374,6 +374,53 @@ def test_unnamed_request_leaves_the_trace_name_off_both_spans(): assert TRACE_NAME_ATTR not in root_attrs and TRACE_NAME_ATTR not in generation_attrs +def test_a_root_owned_by_another_loggers_provider_is_never_stamped_with_langfuse_vocabulary(): + """``callbacks: [otel, langfuse_otel]``: the request root belongs to the ``otel`` logger and exports to the + operator's collector, so Langfuse's trace name and root input/output must not be written on it.""" + generic, generic_exporter = _logger(mappers=("genai",)) + langfuse, langfuse_exporter = _logger() + response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))]) + named: Final = {"proxy_server_request": {"headers": {"langfuse_trace_name": "private-name"}}} + + root = _start_root(generic) + asyncio.run(langfuse.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), CHAT_DATA, "acompletion")) + langfuse.log_pre_api_call( + model="gpt-5.4-mini", messages=[], kwargs={"litellm_call_id": "c1", "litellm_params": named} + ) + asyncio.run( + langfuse.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=response) + ) + root.end() + + root_attrs = _root_attrs(generic_exporter) + assert not any(key.startswith("langfuse.") for key in root_attrs) + assert all(span.name != LITELLM_PROXY_REQUEST_SPAN_NAME for span in langfuse_exporter.get_finished_spans()) + + +def test_a_root_owned_by_the_langfuse_provider_is_still_named_and_given_its_io_next_to_another_logger(): + """``callbacks: [langfuse_otel]`` with a second v2 logger alongside: the root was created by the Langfuse + provider, so the ownership check must not stop the trace name and root input/output from reaching Langfuse.""" + _logger(mappers=("genai",)) + langfuse, langfuse_exporter = _logger() + response = ModelResponse(choices=[Choices(message=Message(role="assistant", content="pong"))]) + named: Final = {"proxy_server_request": {"headers": {"langfuse_trace_name": "private-name"}}} + + root = _start_root(langfuse) + asyncio.run(langfuse.async_pre_call_hook(UserAPIKeyAuth(), DualCache(), CHAT_DATA, "acompletion")) + langfuse.log_pre_api_call( + model="gpt-5.4-mini", messages=[], kwargs={"litellm_call_id": "c1", "litellm_params": named} + ) + asyncio.run( + langfuse.async_post_call_success_hook(data=CHAT_DATA, user_api_key_dict=UserAPIKeyAuth(), response=response) + ) + root.end() + + root_attrs = _root_attrs(langfuse_exporter) + assert root_attrs[TRACE_NAME_ATTR] == "private-name" + assert json.loads(root_attrs[INPUT_ATTR]) == CHAT_DATA["messages"] + assert "pong" in root_attrs[OUTPUT_ATTR] + + @pytest.mark.parametrize( ("capture", "mappers"), [("no_content", ("genai", "langfuse")), ("span_only", ("genai",))], diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py index 67695d5aed8c..fd996b972e8b 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_destinations.py @@ -3,14 +3,14 @@ import contextvars import time from base64 import b64encode -from collections.abc import Mapping +from collections.abc import Callable, Mapping from functools import reduce from types import MappingProxyType import pytest from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export import SimpleSpanProcessor, SpanExporter from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter from opentelemetry.trace import Status, StatusCode @@ -22,6 +22,7 @@ build_otel_v2_logger, fan_out_provider, publish_global_otel_v2_provider, + select_global_otel_v2_logger, ) from litellm.integrations.otel.model.config import ( ExporterOwner, @@ -51,6 +52,11 @@ destination_for, ) from litellm.integrations.otel.presets.langfuse import langfuse_preset +from litellm.litellm_core_utils import litellm_logging +from litellm.litellm_core_utils.litellm_logging import ( + _init_custom_logger_compatible_class, + _maybe_construct_otel_v2, +) from litellm.proxy._types import AddTeamCallback, UserAPIKeyAuth from litellm.proxy.litellm_pre_call_utils import ( convert_key_logging_metadata_to_callback, @@ -92,6 +98,12 @@ def emit(provider: TracerProvider, name: str = "chat gpt-4") -> None: pass +def swap_exporter_factories(monkeypatch, **factories: Callable[[ExporterSpec], SpanExporter]) -> None: + """Swap the exporter built for each ``kind`` for the duration of one test.""" + for kind, factory in factories.items(): + monkeypatch.setitem(otel_providers._EXPORTER_FACTORIES, kind, factory) + + def wired_provider(dest_exporter: InMemorySpanExporter, global_exporter: InMemorySpanExporter) -> TracerProvider: """The operator's provider: one owned exporter plus the tenant fan-out.""" provider = TracerProvider() @@ -1445,8 +1457,6 @@ def test_langfuse_still_raises_for_a_global_callback_with_no_credentials(self, m langfuse_preset() def test_a_credential_less_proxy_builds_the_gated_logger_beside_a_v2_carrier(self, monkeypatch): - from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - credential_less_proxy(monkeypatch) monkeypatch.setenv("LITELLM_OTEL_V2", "true") carrier = build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory")) @@ -1466,8 +1476,6 @@ def test_a_credential_less_proxy_with_no_destinations_falls_back_to_the_legacy_p """Nothing can use a credential-less langfuse here, so the operator has to get the same story as before v2: the legacy integration, not a global provider that exports nowhere.""" - from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - credential_less_proxy(monkeypatch) monkeypatch.setenv("LITELLM_OTEL_V2", "true") @@ -1477,9 +1485,10 @@ def test_a_credential_less_proxy_with_no_destinations_falls_back_to_the_legacy_p assert logger is None - def test_a_valid_newrelic_base_exporter_survives_without_a_license_key(self, monkeypatch): - from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - + def test_a_credentialless_newrelic_does_not_take_over_the_operator_collector(self, monkeypatch): + """The ``OTEL_*`` collector belongs to the ``otel`` callback. A New Relic + entry with no license key has nowhere of its own to export, so it takes + the legacy path instead of shipping New Relic-shaped spans to the collector.""" monkeypatch.delenv("NEW_RELIC_LICENSE_KEY", raising=False) monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318") monkeypatch.setenv("LITELLM_OTEL_V2", "true") @@ -1488,15 +1497,21 @@ def test_a_valid_newrelic_base_exporter_survives_without_a_license_key(self, mon logger = in_fresh_context(_maybe_construct_otel_v2, "newrelic", []) is_otel_v2_enabled.cache_clear() + assert logger is None + + def test_a_credentialed_newrelic_exports_only_to_new_relic(self, monkeypatch): + monkeypatch.setenv("NEW_RELIC_LICENSE_KEY", "nr-license") + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318") + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + + is_otel_v2_enabled.cache_clear() + logger = in_fresh_context(_maybe_construct_otel_v2, "newrelic", []) + is_otel_v2_enabled.cache_clear() + assert logger is not None - assert [spec.endpoint for spec in logger.config.exporters] == [ - "http://collector.local:4318", - "https://otlp.nr-data.net", - ] + assert [spec.endpoint for spec in logger.config.exporters] == ["https://otlp.nr-data.net"] def test_a_credentialless_newrelic_without_a_base_exporter_falls_back(self, monkeypatch): - from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - monkeypatch.delenv("NEW_RELIC_LICENSE_KEY", raising=False) for name in _OTEL_SHORTHAND_ENV: monkeypatch.delenv(name, raising=False) @@ -1508,12 +1523,10 @@ def test_a_credentialless_newrelic_without_a_base_exporter_falls_back(self, monk assert logger is None - def test_an_explicit_console_exporter_keeps_a_credentialless_preset_on_v2(self, monkeypatch, capfd): - """``OTEL_EXPORTER=console`` reads exactly like the placeholder ``_normalize`` - folds in, but the operator asked for it, so a credential-less New Relic keeps - the V2 logger and its spans reach stdout instead of the legacy path.""" - from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - + def test_an_explicit_console_exporter_is_served_by_otel_not_by_a_credentialless_preset(self, monkeypatch): + """``OTEL_EXPORTER=console`` is the operator's generic destination. Only the + ``otel`` callback prints there; a credential-less New Relic has nothing of + its own to export to and takes the legacy path.""" monkeypatch.delenv("NEW_RELIC_LICENSE_KEY", raising=False) for name in _OTEL_SHORTHAND_ENV: monkeypatch.delenv(name, raising=False) @@ -1522,15 +1535,13 @@ def test_an_explicit_console_exporter_keeps_a_credentialless_preset_on_v2(self, is_otel_v2_enabled.cache_clear() logger = in_fresh_context(_maybe_construct_otel_v2, "newrelic", []) + generic = build_otel_v2_logger(OpenTelemetryV2Config()) is_otel_v2_enabled.cache_clear() - assert logger is not None - assert logger.config.exporters[0].kind == "console" - assert not logger.config.exporters[0].requires_headers + assert logger is None + assert [spec.kind for spec in generic.config.exporters] == ["console"] def test_a_destination_for_one_backend_does_not_degrade_another(self, monkeypatch): - from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - credential_less_proxy(monkeypatch) monkeypatch.delenv("WANDB_API_KEY", raising=False) monkeypatch.setenv("LITELLM_OTEL_V2", "true") @@ -1548,8 +1559,6 @@ def run(): def test_the_exporter_less_logger_is_not_reused_by_a_request_without_destinations(self, monkeypatch): """Reusing it would let one team's destination decide how every later request without one is logged, long after the degrade was justified.""" - from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - credential_less_proxy(monkeypatch) monkeypatch.setenv("LITELLM_OTEL_V2", "true") loggers = [build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory"))] @@ -1567,8 +1576,6 @@ def with_destination(): assert plain is None def test_a_credentialed_logger_is_still_reused_across_requests(self, monkeypatch): - from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-1") monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-1") monkeypatch.setenv("LITELLM_OTEL_V2", "true") @@ -1584,8 +1591,6 @@ def test_a_credentialed_logger_is_still_reused_across_requests(self, monkeypatch @staticmethod def _degraded_langfuse_beside(loggers, monkeypatch): - from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - credential_less_proxy(monkeypatch) monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318") monkeypatch.setenv("LITELLM_OTEL_V2", "true") @@ -1615,8 +1620,6 @@ def test_a_credential_less_proxy_with_a_destination_but_no_v2_carrier_falls_back """Only a V2 logger publishes the provider the fan-out rides on, so a legacy callback beside this one leaves the destination just as unreachable as no callback at all, and the operator keeps the pre-V2 story.""" - from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - credential_less_proxy(monkeypatch) monkeypatch.setenv("LITELLM_OTEL_V2", "true") @@ -1630,28 +1633,24 @@ def run(): assert logger is None - def test_a_credentialed_logger_beside_another_v2_logger_keeps_every_exporter(self, monkeypatch): - """Only a degraded preset gives the collector up; an operator who configured - both the backend and the collector still exports to both, as on base.""" - from litellm.litellm_core_utils.litellm_logging import _maybe_construct_otel_v2 - + @pytest.mark.parametrize("beside", [(), ("collector",)]) + def test_a_credentialed_langfuse_exports_only_to_langfuse(self, monkeypatch, beside): + """The operator's collector is the ``otel`` callback's destination whether or + not that callback is registered; Langfuse never inherits it.""" monkeypatch.setenv("LANGFUSE_PUBLIC_KEY", "pk-lf-1") monkeypatch.setenv("LANGFUSE_SECRET_KEY", "sk-lf-1") monkeypatch.setenv("LANGFUSE_HOST", "https://cloud.langfuse.com") monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318") monkeypatch.setenv("LITELLM_OTEL_V2", "true") - collector_logger = build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory")) + loggers = [build_otel_v2_logger(OpenTelemetryV2Config(exporter="in_memory")) for _ in beside] is_otel_v2_enabled.cache_clear() - logger = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", [collector_logger]) + logger = in_fresh_context(_maybe_construct_otel_v2, "langfuse_otel", loggers) is_otel_v2_enabled.cache_clear() assert logger is not None - assert [spec.endpoint for spec in logger.config.exporters] == [ - "http://collector.local:4318", - "https://cloud.langfuse.com/api/public/otel", - ] - assert all(spec.headers for spec in logger.config.exporters if spec.requires_headers) + assert [spec.endpoint for spec in logger.config.exporters] == ["https://cloud.langfuse.com/api/public/otel"] + assert all(spec.headers for spec in logger.config.exporters) class TestContextIsolation: @@ -1664,26 +1663,254 @@ def first(): assert in_fresh_context(request_destinations) == () -class TestOperatorShorthandSurvivesDegradation: - def test_a_generic_otlp_collector_keeps_receiving_when_langfuse_has_no_credentials(self, monkeypatch): - """Only the stdout placeholder is dropped. An operator who set the standard - OTLP env vars configured a real destination and must keep it.""" +class TestOperatorShorthandStaysWithOtel: + def test_a_credential_less_langfuse_leaves_the_operator_collector_to_the_otel_callback(self, monkeypatch): monkeypatch.delenv("LANGFUSE_PUBLIC_KEY", raising=False) monkeypatch.delenv("LANGFUSE_SECRET_KEY", raising=False) monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", "http://collector.local:4318") config = langfuse_preset(allow_missing_credentials=True) - assert [spec.endpoint for spec in config.exporters] == ["http://collector.local:4318", None] - assert [spec.kind for spec in config.exporters] == ["otlp_http", "console"] + assert [spec.endpoint for spec in config.exporters] == [None] + assert [spec.owner for spec in config.exporters] == [ExporterOwner.LANGFUSE_OTEL] - def test_the_stdout_placeholder_is_still_dropped_when_it_is_the_only_exporter(self, monkeypatch): + def test_the_stdout_placeholder_is_never_inherited_by_a_credential_less_preset(self, monkeypatch): credential_less_proxy(monkeypatch) config = langfuse_preset(allow_missing_credentials=True) assert all(spec.requires_headers and not spec.headers for spec in config.exporters) + def test_a_lone_credentialed_preset_without_otel_env_prints_nothing_to_stdout(self, monkeypatch): + """``callbacks: [langfuse_otel]`` and no ``OTEL_*`` at all: the base config's stdout placeholder + belongs to ``otel``, so the Langfuse logger must not dump every span as JSON on the proxy log.""" + for name in _OTEL_SHORTHAND_ENV: + monkeypatch.delenv(name, raising=False) + for name, value in _VENDOR_ENV["langfuse_otel"].items(): + monkeypatch.setenv(name, value) + langfuse_sink, stdout = InMemorySpanExporter(), InMemorySpanExporter() + swap_exporter_factories(monkeypatch, otlp_http=lambda spec: langfuse_sink, console=lambda spec: stdout) + + logger = build_otel_v2_logger(langfuse_preset(), callback_name="langfuse_otel") + emit(logger.tracer_provider, "vendor only") + logger.tracer_provider.force_flush() + + assert [s.name for s in langfuse_sink.get_finished_spans()] == ["vendor only"] + assert stdout.get_finished_spans() == () + + +_COLLECTOR = "http://collector.local:4318" + +#: Env that gives each endpoint-owning preset its own credentials. +_VENDOR_ENV = MappingProxyType( + { + "langfuse_otel": { + "LANGFUSE_PUBLIC_KEY": "pk", + "LANGFUSE_SECRET_KEY": "sk", + "LANGFUSE_HOST": "https://lf.local", + }, + "arize": {"ARIZE_SPACE_ID": "space", "ARIZE_API_KEY": "key"}, + "arize_phoenix": { + "PHOENIX_API_KEY": "key", + "PHOENIX_COLLECTOR_HTTP_ENDPOINT": "https://phoenix.local/v1/traces", + }, + "newrelic": {"NEW_RELIC_LICENSE_KEY": "license"}, + "agentops": {"AGENTOPS_API_KEY": "ao-key"}, + "levo": { + "LEVOAI_API_KEY": "k", + "LEVOAI_ORG_ID": "org", + "LEVOAI_WORKSPACE_ID": "ws", + "LEVOAI_COLLECTOR_URL": "https://levo.local/v1/traces", + }, + "weave_otel": {"WANDB_API_KEY": "wandb", "WANDB_PROJECT_ID": "entity/project"}, + } +) + + +class TestDestinationOwnership: + """A callback exports only to the destination it owns: ``otel`` serves the operator's + ``OTEL_*`` collector and a preset serves its own backend. The leak this closes was every + preset inheriting the collector from the base config, so its vendor-vocabulary spans + (``langfuse.*``, OpenInference, ...) landed on the operator's collector too.""" + + @staticmethod + def _capture_exporters(monkeypatch) -> dict[str, InMemorySpanExporter]: + sinks: dict[str, InMemorySpanExporter] = {} + + def factory(spec: ExporterSpec) -> InMemorySpanExporter: + return sinks.setdefault(spec.endpoint or "", InMemorySpanExporter()) + + swap_exporter_factories(monkeypatch, otlp_http=factory, otlp_grpc=factory, agentops=factory) + return sinks + + @staticmethod + def _operator_with(monkeypatch, vendor: str) -> None: + for name in ("LANGFUSE_PUBLIC_KEY", "LANGFUSE_SECRET_KEY", *_OTEL_SHORTHAND_ENV): + monkeypatch.delenv(name, raising=False) + for name, value in _VENDOR_ENV[vendor].items(): + monkeypatch.setenv(name, value) + monkeypatch.setenv("OTEL_EXPORTER_OTLP_ENDPOINT", _COLLECTOR) + monkeypatch.setenv("LITELLM_OTEL_V2", "true") + is_otel_v2_enabled.cache_clear() + + @pytest.mark.parametrize("vendor", sorted(_VENDOR_ENV)) + def test_a_presets_spans_never_reach_the_operator_collector(self, monkeypatch, vendor): + sinks = self._capture_exporters(monkeypatch) + self._operator_with(monkeypatch, vendor) + generic = build_otel_v2_logger(OpenTelemetryV2Config()) + preset = in_fresh_context(_maybe_construct_otel_v2, vendor, [generic]) + is_otel_v2_enabled.cache_clear() + assert preset is not None + + emit(generic.tracer_provider, "from otel") + emit(preset.tracer_provider, "from preset") + generic.tracer_provider.force_flush() + preset.tracer_provider.force_flush() + + assert [s.name for s in sinks[_COLLECTOR].get_finished_spans()] == ["from otel"] + vendor_sinks = {endpoint: sink for endpoint, sink in sinks.items() if endpoint != _COLLECTOR} + assert [[s.name for s in sink.get_finished_spans()] for sink in vendor_sinks.values()] == [["from preset"]] + + @pytest.mark.parametrize("vendor", sorted(_VENDOR_ENV)) + def test_a_preset_built_first_does_not_point_otel_at_its_own_backend(self, monkeypatch, vendor): + """``callbacks: [, otel]``: reading the vendor's credentials must not rewrite the + process-wide ``OTEL_*`` env, or the ``otel`` built next inherits the vendor's endpoint and auth.""" + sinks = self._capture_exporters(monkeypatch) + self._operator_with(monkeypatch, vendor) + preset = in_fresh_context(_maybe_construct_otel_v2, vendor, []) + generic = build_otel_v2_logger(OpenTelemetryV2Config()) + is_otel_v2_enabled.cache_clear() + assert preset is not None + + emit(generic.tracer_provider, "from otel") + generic.tracer_provider.force_flush() + + assert [(spec.endpoint, spec.headers) for spec in generic.config.exporters] == [(_COLLECTOR, None)] + assert [s.name for s in sinks[_COLLECTOR].get_finished_spans()] == ["from otel"] + + @pytest.mark.parametrize("otel_first", [True, False]) + def test_the_otel_entry_never_reuses_a_preset_logger_whatever_the_order(self, monkeypatch, otel_first): + from litellm.proxy import proxy_server + + self._operator_with(monkeypatch, "langfuse_otel") + monkeypatch.setattr(proxy_server, "open_telemetry_logger", None) + loggers: list[CustomLogger] = [] + monkeypatch.setattr(litellm_logging, "_in_memory_loggers", loggers) + + def build(name): + if name == "otel": + return _init_custom_logger_compatible_class("otel", None, None) + return _maybe_construct_otel_v2(name, loggers) + + order = ("otel", "langfuse_otel") if otel_first else ("langfuse_otel", "otel") + built = {name: in_fresh_context(build, name) for name in order} + is_otel_v2_enabled.cache_clear() + + generic, preset = built["otel"], built["langfuse_otel"] + assert isinstance(generic, OpenTelemetryV2) and isinstance(preset, OpenTelemetryV2) + assert generic is not preset + assert generic.callback_name is None and preset.callback_name == "langfuse_otel" + assert [spec.endpoint for spec in generic.config.exporters] == [_COLLECTOR] + assert [spec.endpoint for spec in preset.config.exporters] == ["https://lf.local/api/public/otel"] + assert select_global_otel_v2_logger(loggers) is generic + assert proxy_server.open_telemetry_logger is generic + + def test_the_otel_entry_reuses_a_mapper_only_preset_that_already_serves_the_collector(self, monkeypatch): + """``callbacks: [langtrace, otel]``: Langtrace owns no backend of its own, it is the operator's + collector plus a mapper, so a second generic logger would export every span there twice.""" + sinks = self._capture_exporters(monkeypatch) + self._operator_with(monkeypatch, "langfuse_otel") + loggers: list[CustomLogger] = [] + monkeypatch.setattr(litellm_logging, "_in_memory_loggers", loggers) + + langtrace = in_fresh_context(_maybe_construct_otel_v2, "langtrace", loggers) + generic = in_fresh_context(_init_custom_logger_compatible_class, "otel", None, None) + is_otel_v2_enabled.cache_clear() + + assert isinstance(langtrace, OpenTelemetryV2) and generic is langtrace + assert "langtrace" in langtrace.config.mapper_names + emit(langtrace.tracer_provider, "chat") + langtrace.tracer_provider.force_flush() + assert [s.name for s in sinks[_COLLECTOR].get_finished_spans()] == ["chat"] + + def test_a_mapper_only_preset_serving_the_collector_honors_callback_settings_otel(self, monkeypatch): + """``callbacks: [langtrace, otel]`` plus ``callback_settings.otel``: the one logger that serves the + operator's collector must carry the operator's ``otel`` settings, or reusing it for ``otel`` drops them.""" + sinks = self._capture_exporters(monkeypatch) + self._operator_with(monkeypatch, "langfuse_otel") + monkeypatch.setattr( + litellm, + "callback_settings", + MappingProxyType({"otel": MappingProxyType({"service_name": "ops-collector", "message_logging": False})}), + ) + loggers: list[CustomLogger] = [] # mutable-ok: stands in for the process-wide _in_memory_loggers list + monkeypatch.setattr(litellm_logging, "_in_memory_loggers", loggers) + + langtrace = in_fresh_context(_maybe_construct_otel_v2, "langtrace", loggers) + generic = in_fresh_context(_init_custom_logger_compatible_class, "otel", None, None) + is_otel_v2_enabled.cache_clear() + + assert isinstance(langtrace, OpenTelemetryV2) and generic is langtrace + assert "langtrace" in langtrace.config.mapper_names + assert langtrace.message_logging is False + emit(langtrace.tracer_provider, "chat") + langtrace.tracer_provider.force_flush() + (span,) = sinks[_COLLECTOR].get_finished_spans() + assert span.resource.attributes["service.name"] == "ops-collector" + + def test_the_otel_entry_still_auto_initializes_phoenix_when_it_reuses_a_mapper_only_preset(self, monkeypatch): + """``callbacks: [langtrace, otel]`` with Phoenix env vars: riding the Langtrace logger must not skip the + Phoenix auto-init that a lone ``otel`` entry performs.""" + from litellm.integrations.arize.arize_phoenix import ArizePhoenixLogger + + self._capture_exporters(monkeypatch) + self._operator_with(monkeypatch, "langfuse_otel") + monkeypatch.setenv("PHOENIX_COLLECTOR_HTTP_ENDPOINT", "http://phoenix.invalid/v1/traces") + monkeypatch.setattr(litellm, "callbacks", []) + loggers: list[CustomLogger] = [] # mutable-ok: stands in for the process-wide _in_memory_loggers list + monkeypatch.setattr(litellm_logging, "_in_memory_loggers", loggers) + + langtrace = in_fresh_context(_maybe_construct_otel_v2, "langtrace", loggers) + generic = in_fresh_context(_init_custom_logger_compatible_class, "otel", None, None) + is_otel_v2_enabled.cache_clear() + + assert isinstance(langtrace, OpenTelemetryV2) and generic is langtrace + assert sum(isinstance(cb, ArizePhoenixLogger) for cb in loggers) == 1 + + @pytest.mark.parametrize("otel_first", [True, False]) + def test_the_otel_callback_holds_the_proxy_slot_whatever_the_order(self, monkeypatch, otel_first): + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "open_telemetry_logger", None) + generic = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory")]) + vendor = OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)]) + + def build(name): + if name == "otel": + return build_otel_v2_logger(generic) + return build_otel_v2_logger(vendor, callback_name=name) + + order = ("otel", "langfuse_otel") if otel_first else ("langfuse_otel", "otel") + built = {name: build(name) for name in order} + + assert proxy_server.open_telemetry_logger is built["otel"] + + def test_a_lone_preset_keeps_the_proxy_slot_and_the_global(self, monkeypatch): + from litellm.proxy import proxy_server + + monkeypatch.setattr(proxy_server, "open_telemetry_logger", None) + first = build_otel_v2_logger( + OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.LANGFUSE_OTEL)]), + callback_name="langfuse_otel", + ) + second = build_otel_v2_logger( + OpenTelemetryV2Config(exporters=[ExporterSpec(kind="in_memory", owner=ExporterOwner.ARIZE_AX)]), + callback_name="arize", + ) + + assert proxy_server.open_telemetry_logger is first + assert select_global_otel_v2_logger([first, second]) is first + class TestBackendEndpointParity: def test_arize_follows_its_own_http_endpoint_instead_of_the_grpc_default(self, monkeypatch): @@ -2585,75 +2812,6 @@ def test_a_retired_processor_is_still_closed_after_shutdown(self): assert held.shutdown_calls == 1 -class TestCredentialGatedExporters: - def test_layering_a_second_preset_does_not_eat_the_first_gated_exporter(self, monkeypatch): - """``base.Preset`` advertises ``config_overrides`` layering, and the gated spec - is itself a console exporter with no endpoint.""" - credential_less_proxy(monkeypatch) - from litellm.integrations.otel.presets.utils import credential_gated_exporters - - once = credential_gated_exporters((), ExporterOwner.LANGFUSE_OTEL) - twice = credential_gated_exporters(once, ExporterOwner.WEAVE_OTEL) - - assert [spec.owner for spec in twice] == [ExporterOwner.LANGFUSE_OTEL, ExporterOwner.WEAVE_OTEL] - - def test_an_exporter_the_operator_configured_survives(self): - from litellm.integrations.otel.presets.utils import credential_gated_exporters - - operator_console = ExporterSpec(kind="console", use_simple_processor=True) - - kept = credential_gated_exporters((operator_console,), ExporterOwner.LANGFUSE_OTEL) - - assert kept[0] == operator_console - - def test_an_otlp_exporter_on_its_default_endpoint_survives(self): - """``OTEL_EXPORTER=otlp_http`` with no endpoint is a real collector on the SDK's - default port, not the placeholder, so the transport is what tells them apart.""" - from litellm.integrations.otel.presets.utils import credential_gated_exporters - - operator_otlp = ExporterSpec(kind="otlp_http", endpoint=None, headers=None) - - kept = credential_gated_exporters((operator_otlp,), ExporterOwner.LANGFUSE_OTEL) - - assert kept[0] == operator_otlp - - def test_an_in_memory_exporter_the_operator_asked_for_survives(self): - """``OTEL_EXPORTER=in_memory`` stores spans, so it is a destination the operator - chose, not the placeholder that stands in for choosing nothing.""" - from litellm.integrations.otel.presets.utils import credential_gated_exporters - - operator_memory = ExporterSpec(kind="in_memory", endpoint=None, headers=None) - - kept = credential_gated_exporters((operator_memory,), ExporterOwner.LANGFUSE_OTEL) - - assert kept[0] == operator_memory - - def test_the_synthesized_stdout_placeholder_is_dropped(self, monkeypatch): - from litellm.integrations.otel.presets.utils import credential_gated_exporters - - for name in _OTEL_SHORTHAND_ENV: - monkeypatch.delenv(name, raising=False) - placeholder = OpenTelemetryV2Config().exporters[0] - - kept = credential_gated_exporters((placeholder,), ExporterOwner.LANGFUSE_OTEL) - - assert [spec.owner for spec in kept] == [ExporterOwner.LANGFUSE_OTEL] - - def test_a_console_exporter_the_operator_named_survives(self, monkeypatch): - """Same kind, endpoint and headers as the placeholder; only the fact that the - operator set ``OTEL_EXPORTER`` tells them apart.""" - from litellm.integrations.otel.presets.utils import credential_gated_exporters - - for name in _OTEL_SHORTHAND_ENV: - monkeypatch.delenv(name, raising=False) - monkeypatch.setenv("OTEL_EXPORTER", "console") - operator_console = OpenTelemetryV2Config().exporters[0] - - kept = credential_gated_exporters((operator_console,), ExporterOwner.LANGFUSE_OTEL) - - assert kept[0] is operator_console - - class TestTenantHostSsrfGuard: """Anyone who can mint a key can write ``langfuse_host``, so the host it names has to be one the operator approved.""" diff --git a/tests/test_litellm/integrations/otel/test_otel_v2_multibackend.py b/tests/test_litellm/integrations/otel/test_otel_v2_multibackend.py index e879766c5c79..033835a676b1 100644 --- a/tests/test_litellm/integrations/otel/test_otel_v2_multibackend.py +++ b/tests/test_litellm/integrations/otel/test_otel_v2_multibackend.py @@ -1,9 +1,9 @@ -"""Multi-backend fan-out: one TracerProvider, *N* SpanProcessors. +"""Multi-backend fan-out within one callback: one TracerProvider, *N* SpanProcessors. -V1 needed a separate ``TracerProvider`` per integration to avoid stepping on -the global. V2 attaches a ``SpanProcessor`` per exporter to the *same* -provider, so the same trace ID lights up every backend — no duplicate spans, -no per-integration provider caches. +A single ``OpenTelemetryV2Config`` may list several exporters (say two ``OTEL_*`` +collectors), and its provider attaches one ``SpanProcessor`` per exporter so the same +span reaches each of them. Which exporters a config lists is decided per callback: a +preset only lists the destination it owns, see ``TestDestinationOwnership``. """ import pytest