test(e2e): vendor API strategy coverage across endpoints

tests/e2e/access_control/test_chat_auth_headers_e2e.py

@@ -0,0 +1,106 @@
+"""Chat Authorization header matrix on LLM routes (LIT-4778).
+
+Virtual-key chat must reject missing and malformed Authorization headers before
+any provider call. These cases sit next to the existing valid/invalid key check
+and pin the bearer-token failure matrix.
+"""
+
+from __future__ import annotations
+
+import pytest
+from pydantic import BaseModel
+
+from e2e_config import unique_marker
+from e2e_http import AuthHeaders, NoBody, StreamingResponse
+from lifecycle import ResourceManager
+from models import ChatBody, ChatMessage, LiteLLMParamsBody
+from proxy_client import ProxyClient
+
+pytestmark = pytest.mark.e2e
+
+OPENAI_BACKEND = "openai/gpt-4o-mini"
+CHAT_PATH = "/chat/completions"
+
+
+class RawAuthorizationHeaders(BaseModel):
+    Authorization: str
+
+
+def _register_model(proxy: ProxyClient, resources: ResourceManager) -> str:
+    model = f"e2e-auth-headers-{unique_marker()}"
+    model_id = proxy.create_model(
+        model,
+        LiteLLMParamsBody(model=OPENAI_BACKEND, api_key="os.environ/OPENAI_API_KEY"),
+    )
+    resources.defer(lambda: proxy.delete_model(model_id))
+    return model
+
+
+def _chat_with_headers(
+    proxy: ProxyClient, headers: BaseModel, model: str
+) -> StreamingResponse:
+    return proxy.transport.send(
+        CHAT_PATH,
+        headers=headers,
+        json=ChatBody(
+            model=model,
+            messages=[ChatMessage(role="user", content="should not run")],
+            max_tokens=8,
+        ),
+    )
+
+
+def _assert_auth_denied(result: StreamingResponse, context: str) -> None:
+    assert result.status_code in (401, 403), (
+        f"{context}: expected 401/403, got {result.status_code}: {result.body[:300]}"
+    )
+
+
+class TestChatAuthHeaders:
+    @pytest.mark.covers("other.auth.llm_chat.missing_header_denied")
+    def test_missing_authorization_header_is_denied(
+        self, proxy: ProxyClient, resources: ResourceManager
+    ) -> None:
+        model = _register_model(proxy, resources)
+        result = _chat_with_headers(proxy, NoBody(), model)
+        _assert_auth_denied(result, "missing Authorization")
+
+    @pytest.mark.covers("other.auth.llm_chat.invalid_bearer_denied")
+    def test_bearer_invalid_token_is_denied(
+        self, proxy: ProxyClient, resources: ResourceManager
+    ) -> None:
+        model = _register_model(proxy, resources)
+        result = _chat_with_headers(
+            proxy, AuthHeaders(authorization="Bearer invalid_token"), model
+        )
+        _assert_auth_denied(result, "Bearer invalid_token")
+
+    @pytest.mark.covers("other.auth.llm_chat.no_bearer_prefix_denied")
+    def test_token_without_bearer_prefix_is_denied(
+        self, proxy: ProxyClient, resources: ResourceManager
+    ) -> None:
+        model = _register_model(proxy, resources)
+        result = _chat_with_headers(
+            proxy, RawAuthorizationHeaders(Authorization="invalid_token"), model
+        )
+        _assert_auth_denied(result, "token without Bearer prefix")
+
+    @pytest.mark.covers("other.auth.llm_chat.empty_bearer_denied")
+    def test_empty_bearer_token_is_denied(
+        self, proxy: ProxyClient, resources: ResourceManager
+    ) -> None:
+        model = _register_model(proxy, resources)
+        result = _chat_with_headers(
+            proxy, AuthHeaders(authorization="Bearer "), model
+        )
+        _assert_auth_denied(result, "empty Bearer token")
+
+    @pytest.mark.covers("other.auth.llm_chat.not_bearer_scheme_denied")
+    def test_not_bearer_scheme_is_denied(
+        self, proxy: ProxyClient, resources: ResourceManager
+    ) -> None:
+        model = _register_model(proxy, resources)
+        result = _chat_with_headers(
+            proxy, RawAuthorizationHeaders(Authorization="NotBearer validtoken123"), model
+        )
+        _assert_auth_denied(result, "NotBearer scheme")

tests/e2e/coverage_registry/guardrail.yaml

@@ -12,7 +12,7 @@
 - {id: guardrail.bedrock.post_call.blocks, module: guardrail, tier: P0, hook_point: post_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/bedrock_guardrails.py", rationale: "Block harmful output"}
 - {id: guardrail.lakera.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/lakera_ai_v2.py", rationale: "Prompt-injection block pre-execution"}
 - {id: guardrail.lakera.post_call.blocks, module: guardrail, tier: P0, hook_point: post_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/lakera_ai_v2.py", rationale: "Post-call injection on multi-turn chains"}
-- {id: guardrail.openai_moderations.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/openai/moderations.py", rationale: "Content policy for regulated industries"}
+- {id: guardrail.openai_moderations.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions, messages, responses], source: "guardrail_hooks/openai/moderations.py", rationale: "Content policy for regulated industries; vendor §10 category matrix across chat/messages/responses (LIT-4778)"}
 - {id: guardrail.aim.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/aim/aim.py", rationale: "Security guardrail malicious-input"}
 - {id: guardrail.aim.post_call.blocks, module: guardrail, tier: P1, hook_point: post_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/aim/aim.py", rationale: "Output security check"}
 - {id: guardrail.ibm_guardrails.pre_call.blocks, module: guardrail, tier: P1, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/ibm_guardrails/ibm_detector.py", rationale: "Enterprise multi-policy"}

tests/e2e/coverage_registry/llm_conversational.yaml

@@ -1,5 +1,8 @@
 # LLM conversational endpoints (chat_completions, messages, responses). Grounded in proxy handlers + model_prices json.
 - {id: llm.chat_completions.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "Core endpoint/route/capability"}
+- {id: llm.chat_completions.openai.multi_turn.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: multi_turn, streaming: nonstream, assertions: [works], source: "vendor testing strategy §16.2 / LIT-4778", rationale: "Multi-turn history is forwarded so turn 2 can use turn 1 answer"}
+- {id: llm.chat_completions.openai.input_validation.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor testing strategy §9.2 / LIT-4778", rationale: "Missing/invalid chat fields return client errors, not silent success"}
+- {id: llm.chat_completions.openai.input_sanitization.nonstream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: input_sanitization, streaming: nonstream, assertions: [works], source: "vendor testing strategy §11.3 / LIT-4778", rationale: "SQL injection and XSS payloads must not 5xx the proxy"}
 - {id: llm.chat_completions.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: basic, streaming: stream, assertions: [works], source: "proxy_server.py:8455", rationale: "Core streaming"}
 - {id: llm.chat_completions.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: chat_completions, route: openai, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "proxy_server.py:8455", rationale: "Cost logging regression catch"}
 - {id: llm.chat_completions.openai.passthrough.nonstream.cost_logged, module: llm, tier: P1, subject_endpoint: chat_completions, route: openai, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "test_passthrough_e2e.py", rationale: "OpenAI-format chat via the raw /openai/{endpoint} passthrough (/openai/v1/chat/completions); proxy swaps in OPENAI_API_KEY and still logs a costed pass_through_endpoint row (LIT-4752)"}
@@ -42,6 +45,7 @@
 - {id: llm.chat_completions.azure_foundry.basic.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: azure_foundry, capability: basic, streaming: nonstream, assertions: [works], source: "proxy_server.py:8455", rationale: "Azure Foundry (azure_ai); newer, smoke"}
 - {id: llm.chat_completions.hosted_vllm.passthrough.nonstream.works, module: llm, tier: P1, subject_endpoint: chat_completions, route: hosted_vllm, capability: basic, streaming: nonstream, assertions: [works], source: "test_vllm_passthrough_e2e.py", rationale: "OpenAI-format chat via the raw /vllm/{endpoint} passthrough (/vllm/v1/chat/completions), forwarded to a self-hosted vLLM-compatible backend (VLLM_API_BASE); LIT-4751. Batch/file passthrough is not coverable on self-hosted vLLM, which serves no OpenAI Batch API"}
 - {id: llm.messages.anthropic.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: basic, streaming: nonstream, assertions: [works], source: "anthropic_endpoints/endpoints.py:64", rationale: "Core endpoint; Anthropic Messages native"}
+- {id: llm.messages.anthropic.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: messages, route: anthropic, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.10 / LIT-4778", rationale: "Messages missing messages/max_tokens/model rejected"}
 - {id: llm.messages.anthropic.basic.stream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: basic, streaming: stream, assertions: [works], source: "anthropic_endpoints/endpoints.py:64", rationale: "Streaming Messages API"}
 - {id: llm.messages.anthropic.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "anthropic_endpoints/endpoints.py:64", rationale: "Cost logged on passthrough"}
 - {id: llm.messages.anthropic.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: anthropic, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Tool calls via Messages API"}
@@ -56,6 +60,7 @@
 - {id: llm.messages.vertex.mid_conversation_system.nonstream.cache_hit, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works, cache_hit], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex serves Claude on the native Anthropic contract, so flagged 4.8+/5 must keep mid-conversation system reminders in messages; hoisting mutates the system prefix and collapses the prompt cache (customer RCA gap)", fail_before_fix: proven}
 - {id: llm.messages.vertex.mid_conversation_system.nonstream.works, module: llm, tier: P0, subject_endpoint: messages, route: vertex, capability: mid_conversation_system, streaming: nonstream, assertions: [works], source: "llms/vertex_ai/vertex_ai_partner_models/anthropic/experimental_pass_through/transformation.py", rationale: "Vertex Claude <= 4.7 rejects role system inside messages; unflagged models must hoist reminders into top-level system or every Claude Code session 400s (customer RCA gap)", fail_before_fix: proven}
 - {id: llm.responses.openai.basic.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Core endpoint; OpenAI Responses native"}
+- {id: llm.responses.openai.input_validation.nonstream.works, module: llm, tier: P1, subject_endpoint: responses, route: openai, capability: input_validation, streaming: nonstream, assertions: [works], source: "vendor strategy §9.9 / LIT-4778", rationale: "Responses missing/empty input, missing model, invalid max_output_tokens"}
 - {id: llm.responses.openai.basic.stream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: stream, assertions: [works], source: "response_api_endpoints/endpoints.py:26", rationale: "Streaming via /v1/responses"}
 - {id: llm.responses.openai.basic.nonstream.cost_logged, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: basic, streaming: nonstream, assertions: [works, cost_logged], source: "response_api_endpoints/endpoints.py:26", rationale: "Cost logged on responses"}
 - {id: llm.responses.openai.tool_use.nonstream.works, module: llm, tier: P0, subject_endpoint: responses, route: openai, capability: tool_use, streaming: nonstream, assertions: [works], source: "model_prices json", rationale: "Tool calls via Responses API"}