diff --git a/tests/e2e/coverage_registry/logging.yaml b/tests/e2e/coverage_registry/logging.yaml index 65ab8f0096f..c4b26387c13 100644 --- a/tests/e2e/coverage_registry/logging.yaml +++ b/tests/e2e/coverage_registry/logging.yaml @@ -8,7 +8,7 @@ - {id: logging.datadog.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, embeddings], source: "integrations/datadog/datadog.py", rationale: "Powers dashboards/alerts; cardinality regressions common"} - {id: logging.datadog.failure.exports_metric, module: logging, tier: P0, event: failure, assertions: [exports_metric], exercised_on: [chat_completions], source: "integrations/datadog/datadog.py", rationale: "Failure metrics for alerting/SLO"} - {id: logging.prometheus.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, embeddings], source: "integrations/prometheus.py", rationale: "Standard OSS metrics; per-key cardinality (existing e2e)"} -- {id: logging.otel.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, embeddings], source: "integrations/otel/logger.py", rationale: "OTEL spans on every call path"} +- {id: logging.otel.success.exports_metric, module: logging, tier: P0, event: success, assertions: [exports_metric], exercised_on: [chat_completions, messages, responses, embeddings], source: "integrations/otel/logger.py", rationale: "OTEL spans on every call path"} - {id: logging.otel.failure.exports_metric, module: logging, tier: P0, event: failure, assertions: [exports_metric], exercised_on: [chat_completions, messages], source: "integrations/otel/logger.py", rationale: "Error spans for observability continuity"} - {id: logging.braintrust.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions, messages], source: "integrations/braintrust_logging.py", rationale: "Evals platform spend"} - {id: logging.langsmith.success.logs_spend, module: logging, tier: P1, event: success, assertions: [logs_spend], exercised_on: [chat_completions, messages], source: "integrations/langsmith.py", rationale: "LangChain ecosystem"} diff --git a/tests/e2e/e2e_config.py b/tests/e2e/e2e_config.py index 6bfec2b514e..6e6c30709de 100644 --- a/tests/e2e/e2e_config.py +++ b/tests/e2e/e2e_config.py @@ -25,6 +25,7 @@ UI_PASSWORD = os.environ.get("E2E_UI_PASSWORD", MASTER_KEY) CHEAP_ANTHROPIC_MODEL = os.environ.get("E2E_CHEAP_ANTHROPIC_MODEL", "claude-haiku-4-5") +CHEAP_OPENAI_MODEL = os.environ.get("E2E_CHEAP_OPENAI_MODEL", "gpt-5.5") # Jaeger query API of the compose stack's OTEL trace destination (the `jaeger` # service in docker-compose.yml maps it to host 16686). Trace-completeness tests diff --git a/tests/e2e/logging/logging_client.py b/tests/e2e/logging/logging_client.py index 6071e657bd3..bffdf71ed80 100644 --- a/tests/e2e/logging/logging_client.py +++ b/tests/e2e/logging/logging_client.py @@ -35,6 +35,7 @@ unwrap, ) from models import ( + AnthropicMessagesBody, ChatBody, ChatMessage, ChatResponse, @@ -75,6 +76,14 @@ ) +class ResponsesRequestBody(BaseModel): + """OpenAI Responses API /v1/responses request (non-streaming).""" + + model: str + input: str + max_output_tokens: int + + class TeamCallbackBody(BaseModel): callback_name: Literal["langfuse_otel", "langfuse", "langsmith", "gcs"] callback_type: Literal["success", "failure", "success_and_failure"] @@ -455,6 +464,32 @@ def chat_raw( json=body, ) + def messages_raw(self, key: str, model: str, text: str, *, max_tokens: int = 16) -> StreamingResponse: + """Non-streaming POST /v1/messages (Anthropic-native body): raw outcome + judged by status/body/headers, for tests that need x-litellm-call-id.""" + return self.gateway.transport.send( + "/v1/messages", + headers=self.gateway.transport.bearer(key), + json=AnthropicMessagesBody( + model=model, + max_tokens=max_tokens, + messages=[ChatMessage(role="user", content=text)], + ), + ) + + def responses_raw( + self, key: str, model: str, text: str, *, max_output_tokens: int = 64 + ) -> StreamingResponse: + """Non-streaming POST /v1/responses (OpenAI Responses API): raw outcome + judged by status/body/headers, for tests that need x-litellm-call-id. + max_output_tokens caps reasoning-model output cost; a capped response is + still a 200 and still exports the trace.""" + return self.gateway.transport.send( + "/v1/responses", + headers=self.gateway.transport.bearer(key), + json=ResponsesRequestBody(model=model, input=text, max_output_tokens=max_output_tokens), + ) + def scrape_metrics(self) -> str: return self.gateway.probe("/metrics", params=NoBody()).body diff --git a/tests/e2e/logging/test_otel_trace_e2e.py b/tests/e2e/logging/test_otel_trace_e2e.py index d445da3dff0..90887ea5510 100644 --- a/tests/e2e/logging/test_otel_trace_e2e.py +++ b/tests/e2e/logging/test_otel_trace_e2e.py @@ -23,7 +23,7 @@ import pytest from pydantic import BaseModel, ConfigDict -from e2e_config import CHEAP_ANTHROPIC_MODEL, unique_marker +from e2e_config import CHEAP_ANTHROPIC_MODEL, CHEAP_OPENAI_MODEL, unique_marker from e2e_http import NoBody, StreamingResponse, require_successful_call from lifecycle import ResourceManager from logging_client import LoggingClient @@ -151,7 +151,7 @@ def _settled_names(*, route: str, genai_span: str) -> set[str]: class TestOtelTraceCompleteness: - @pytest.mark.covers("logging.otel.success.exports_metric") + @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["chat_completions"]) def test_chat_completions_exports_complete_trace( self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager ) -> None: @@ -189,3 +189,71 @@ def test_chat_completions_exports_complete_trace( settled_prefixes={DB_SPAN_PREFIX}, ) _assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}") + + @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["messages"]) + def test_messages_exports_complete_trace( + self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager + ) -> None: + """This test verifies that one successful non-streaming /v1/messages request + produces exactly one complete OTEL trace. + + The trace must have a single root span named "POST /v1/messages". The + authentication, database, cost-writing, and model-call spans must all belong to + the same trace and have valid parent relationships leading back to that root. + + The model-call span is expected to be named "chat ". The test fails if + the request is split across multiple traces, if any span references a missing + parent, or if the model-call span cannot be connected back to the root.""" + route = "/v1/messages" + _assert_otel_destination_configured(client) + + key = client.key_with_alias(f"otel-trace-messages-{unique_marker()}", models=[MODEL]) + resources.defer(lambda: client.delete_key(key)) + + marker = unique_marker() + outcome = _first_ok( + client, lambda: client.messages_raw(key, MODEL, f"reply with one word {marker}", max_tokens=16) + ) + assert outcome.call_id is not None, "success response must carry x-litellm-call-id" + + hits = otel_reader.poll_traces_for_call( + call_id=outcome.call_id, + settled_names=_settled_names(route=route, genai_span=f"chat {MODEL}"), + settled_prefixes={DB_SPAN_PREFIX}, + ) + _assert_complete_trace(hits, route=route, genai_span=f"chat {MODEL}") + + @pytest.mark.covers("logging.otel.success.exports_metric", exercised_on=["responses"]) + def test_responses_exports_complete_trace( + self, client: LoggingClient, otel_reader: OtelReader, resources: ResourceManager + ) -> None: + """This test verifies that one successful non-streaming /v1/responses request + produces exactly one complete OTEL trace. + + The trace must have a single root span named "POST /v1/responses". The + authentication, database, cost-writing, and model-call spans must all belong to + the same trace and have valid parent relationships leading back to that root. + + The model-call span is expected to be named "chat ". The test fails if + the request is split across multiple traces, if any span references a missing + parent, or if the model-call span cannot be connected back to the root.""" + route = "/v1/responses" + _assert_otel_destination_configured(client) + + key = client.key_with_alias(f"otel-trace-responses-{unique_marker()}", models=[CHEAP_OPENAI_MODEL]) + resources.defer(lambda: client.delete_key(key)) + + marker = unique_marker() + outcome = _first_ok( + client, + lambda: client.responses_raw(key, CHEAP_OPENAI_MODEL, f"reply with one word {marker}"), + ) + assert outcome.call_id is not None, "success response must carry x-litellm-call-id" + + genai_span = f"chat {CHEAP_OPENAI_MODEL}" + hits = otel_reader.poll_traces_for_call( + call_id=outcome.call_id, + settled_names=_settled_names(route=route, genai_span=genai_span), + settled_prefixes={DB_SPAN_PREFIX}, + ) + _assert_complete_trace(hits, route=route, genai_span=genai_span)