Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion tests/e2e/coverage_registry/logging.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
- {id: logging.datadog.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, embeddings], source: "integrations/datadog/datadog.py", rationale: "Powers dashboards/alerts; cardinality regressions common"}
- {id: logging.datadog.failure.exports_metric, module: logging, tier: P0, event: failure, assertions: [exports_metric], exercised_on: [chat_completions], source: "integrations/datadog/datadog.py", rationale: "Failure metrics for alerting/SLO"}
- {id: logging.prometheus.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, embeddings], source: "integrations/prometheus.py", rationale: "Standard OSS metrics; per-key cardinality (existing e2e)"}
- {id: logging.otel.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, embeddings], source: "integrations/otel/logger.py", rationale: "OTEL spans on every call path"}
- {id: logging.otel.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses, embeddings], source: "integrations/otel/logger.py", rationale: "OTEL spans on every call path"}
- {id: logging.otel.failure.exports_metric, module: logging, tier: P0, event: failure, assertions: [exports_metric], exercised_on: [chat_completions, messages], source: "integrations/otel/logger.py", rationale: "Error spans for observability continuity"}
- {id: logging.braintrust.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions, messages], source: "integrations/braintrust_logging.py", rationale: "Evals platform spend"}
- {id: logging.langsmith.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions, messages], source: "integrations/langsmith.py", rationale: "LangChain ecosystem"}
Expand Down
1 change: 1 addition & 0 deletions tests/e2e/e2e_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
UI_PASSWORD = os.environ.get("E2E_UI_PASSWORD", MASTER_KEY)

CHEAP_ANTHROPIC_MODEL = os.environ.get("E2E_CHEAP_ANTHROPIC_MODEL", "claude-haiku-4-5")
CHEAP_OPENAI_MODEL = os.environ.get("E2E_CHEAP_OPENAI_MODEL", "gpt-5.5")

# Jaeger query API of the compose stack's OTEL trace destination (the `jaeger`
# service in docker-compose.yml maps it to host 16686). Trace-completeness tests
Expand Down
21 changes: 21 additions & 0 deletions tests/e2e/logging/logging_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,14 @@
)


class ResponsesRequestBody(BaseModel):
"""OpenAI Responses API /v1/responses request (non-streaming)."""

model: str
input: str
max_output_tokens: int


class TeamCallbackBody(BaseModel):
callback_name: Literal["langfuse_otel", "langfuse", "langsmith", "gcs"]
callback_type: Literal["success", "failure", "success_and_failure"]
Expand Down Expand Up @@ -469,6 +477,19 @@ def messages_raw(self, key: str, model: str, text: str, *, max_tokens: int = 16)
),
)

def responses_raw(
self, key: str, model: str, text: str, *, max_output_tokens: int = 64
) -> StreamingResponse:
"""Non-streaming POST /v1/responses (OpenAI Responses API): raw outcome
judged by status/body/headers, for tests that need x-litellm-call-id.
max_output_tokens caps reasoning-model output cost; a capped response is
still a 200 and still exports the trace."""
return self.gateway.transport.send(
"/v1/responses",
headers=self.gateway.transport.bearer(key),
json=ResponsesRequestBody(model=model, input=text, max_output_tokens=max_output_tokens),
)

def scrape_metrics(self) -> str:
return self.gateway.probe("/metrics", params=NoBody()).body

Expand Down
39 changes: 37 additions & 2 deletions tests/e2e/logging/test_otel_trace_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
import pytest
from pydantic import BaseModel, ConfigDict

from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker
from e2e_config import CHEAP_ANTHROPIC_MODEL, CHEAP_OPENAI_MODEL, unique_marker
from e2e_http import NoBody, StreamingResponse, require_successful_call
from lifecycle import ResourceManager
from logging_client import LoggingClient
Expand Down Expand Up @@ -151,7 +151,7 @@ def _settled_names(*, route: str, genai_span: str) -> set[str]:


class TestOtelTraceCompleteness:
@pytest.mark.covers("logging.otel.success.exports_metric")
@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["chat_completions"])
def test_chat_completions_exports_complete_trace(
self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager
) -> None:
Expand Down Expand Up @@ -222,3 +222,38 @@ def test_messages_exports_complete_trace(
settled_prefixes={DB_SPAN_PREFIX},
)
_assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}")

@pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["responses"])
def test_responses_exports_complete_trace(
self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager
) -> None:
"""This test verifies that one successful non-streaming /v1/responses request
produces exactly one complete OTEL trace.

The trace must have a single root span named "POST /v1/responses". The
authentication, database, cost-writing, and model-call spans must all belong to
the same trace and have valid parent relationships leading back to that root.

The model-call span is expected to be named "chat <model>". The test fails if
the request is split across multiple traces, if any span references a missing
parent, or if the model-call span cannot be connected back to the root."""
route = "/v1/responses"
_assert_otel_destination_configured(client)

key = client.key_with_alias(f"otel-trace-responses-{unique_marker()}", models=[CHEAP_OPENAI_MODEL])
resources.defer(lambda: client.delete_key(key))

marker = unique_marker()
outcome = _first_ok(
client,
lambda: client.responses_raw(key, CHEAP_OPENAI_MODEL, f"reply with one word {marker}"),
)
assert outcome.call_id is not None, "success response must carry x-litellm-call-id"

genai_span = f"chat {CHEAP_OPENAI_MODEL}"
hits = otel_reader.poll_traces_for_call(
call_id=outcome.call_id,
settled_names=_settled_names(route=route, genai_span=genai_span),
settled_prefixes={DB_SPAN_PREFIX},
)
_assert_complete_trace(hits, route=route, genai_span=genai_span)
Loading