Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
35 changes: 0 additions & 35 deletions tests/audio_tests/test_audio_speech.py
Original file line number Diff line number Diff line change
Expand Up @@ -667,38 +667,3 @@ async def test_aws_polly_tts_with_ssml():
assert request_body["VoiceId"] == "Joanna"


@pytest.mark.asyncio
async def test_aws_polly_tts_real_api():
"""
Test AWS Polly TTS with real API request.
Requires AWS credentials to be configured.
"""
speech_file_path = Path(__file__).parent / "aws_polly_speech_generative.mp3"

response = await litellm.aspeech(
model="aws_polly/generative",
voice="Joanna",
input="Hello, this is a test of AWS Polly text to speech integration with LiteLLM.",
aws_region_name="us-east-1",
)

from litellm.types.llms.openai import HttpxBinaryResponseContent

assert isinstance(response, HttpxBinaryResponseContent)

binary_content = response.content
assert len(binary_content) > 0

# MP3 files start with ID3 tag or MPEG sync word
assert (
binary_content[:3] == b"ID3"
or binary_content[:2] == b"\xff\xfb"
or binary_content[:2] == b"\xff\xf3"
)

response.stream_to_file(speech_file_path)

assert speech_file_path.exists()
assert speech_file_path.stat().st_size > 0

print(f"AWS Polly TTS audio saved to: {speech_file_path}")
16 changes: 0 additions & 16 deletions tests/audio_tests/test_whisper.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,22 +61,6 @@ async def _run_transcription(
assert transcript.text is not None


@pytest.mark.parametrize(
"response_format, timestamp_granularities",
[("json", None), ("vtt", None), ("verbose_json", ["word"])],
)
@pytest.mark.asyncio
@pytest.mark.flaky(retries=3, delay=1)
async def test_transcription_openai_whisper(response_format, timestamp_granularities):
await _run_transcription(
model="whisper-1",
api_key=None,
api_base=None,
response_format=response_format,
timestamp_granularities=timestamp_granularities,
)


@pytest.mark.parametrize(
"response_format, timestamp_granularities",
[("json", None), ("vtt", None), ("verbose_json", ["word"])],
Expand Down
78 changes: 0 additions & 78 deletions tests/batches_tests/test_openai_batches_and_files.py
Original file line number Diff line number Diff line change
Expand Up @@ -118,84 +118,6 @@ async def cancel_batch_unless_already_terminal(batch_id: str, provider: str) ->
print("cancel_batch_response=", cancel_batch_response)


@pytest.mark.parametrize("provider", ["openai"]) # , "azure"
@pytest.mark.asyncio
@skip_if_no_openai_network
async def test_create_batch(provider, tmp_path):
"""
1. Create File for Batch completion
2. Create Batch Request
3. Retrieve the specific batch
"""
if provider == "azure":
# Don't have anymore Azure Quota
return
file_name = "openai_batch_completions.jsonl"
_current_dir = os.path.dirname(os.path.abspath(__file__))
file_path = os.path.join(_current_dir, file_name)

with open(file_path, "rb") as batch_file:
file_obj = await litellm.acreate_file(
file=batch_file,
purpose="batch",
custom_llm_provider=provider,
)
print("Response from creating file=", file_obj)

batch_input_file_id = file_obj.id
assert (
batch_input_file_id is not None
), "Failed to create file, expected a non null file_id but got {batch_input_file_id}"

await asyncio.sleep(1)
create_batch_response = await litellm.acreate_batch(
completion_window="24h",
endpoint="/v1/chat/completions",
input_file_id=batch_input_file_id,
custom_llm_provider=provider,
metadata={"key1": "value1", "key2": "value2"},
)

print("response from litellm.create_batch=", create_batch_response)
await asyncio.sleep(6)

assert (
create_batch_response.id is not None
), f"Failed to create batch, expected a non null batch_id but got {create_batch_response.id}"
assert (
create_batch_response.endpoint == "/v1/chat/completions"
or create_batch_response.endpoint == "/chat/completions"
), f"Failed to create batch, expected endpoint to be /v1/chat/completions but got {create_batch_response.endpoint}"
assert (
create_batch_response.input_file_id == batch_input_file_id
), f"Failed to create batch, expected input_file_id to be {batch_input_file_id} but got {create_batch_response.input_file_id}"

retrieved_batch = await litellm.aretrieve_batch(
batch_id=create_batch_response.id, custom_llm_provider=provider
)
print("retrieved batch=", retrieved_batch)
# just assert that we retrieved a non None batch

assert retrieved_batch.id == create_batch_response.id

# list all batches
list_batches = await litellm.alist_batches(custom_llm_provider=provider, limit=2)
print("list_batches=", list_batches)

file_content = await litellm.afile_content(
file_id=batch_input_file_id, custom_llm_provider=provider
)

result = file_content.content

result_file_path = tmp_path / "batch_job_results_furniture.jsonl"
result_file_path.write_bytes(result)

await cancel_batch_unless_already_terminal(batch_id=create_batch_response.id, provider=provider)

pass


class TestCustomLogger(CustomLogger):
def __init__(self):
super().__init__()
Expand Down
2 changes: 1 addition & 1 deletion tests/e2e/coverage_registry/guardrail.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
- {id: guardrail.presidio.post_call.spend_log_stores_masked_output, module: guardrail, tier: P0, hook_point: post_call, assertions: [masks], exercised_on: [chat_completions, chat_completions_stream, messages, anthropic_messages_stream, responses], source: "guardrail_hooks/presidio.py", fail_before_fix: proven, rationale: "When an output guardrail masks the response, the spend log stores the masked text the caller received rather than the raw model output, on every endpoint and both stream modes (LIT-8325)"}
- {id: guardrail.presidio.logging_only.masks, module: guardrail, tier: P0, hook_point: logging_only, assertions: [masks], exercised_on: [chat_completions, messages], source: "guardrail_hooks/presidio.py", rationale: "Redact in logs without blocking"}
- {id: guardrail.presidio.pre_call.logs_masked_entities, module: guardrail, tier: P0, hook_point: pre_call, assertions: [logs_masked_entities], exercised_on: [chat_completions], source: "guardrail_hooks/presidio.py", rationale: "A masking run must record itself on the spend log: the dashboard's guardrail panel renders the masked-entity counts and per-entity scores straight off metadata.guardrail_information, so a run that masks but records nothing leaves an operator unable to audit it"}
- {id: guardrail.bedrock.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "guardrail_hooks/bedrock_guardrails.py", rationale: "AWS content guardrail blocks harmful input"}
- {id: guardrail.bedrock.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions, messages, responses], source: "guardrail_hooks/bedrock_guardrails.py", rationale: "AWS content guardrail blocks harmful input"}
- {id: guardrail.litellm_content_filter.pre_call.blocks, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [chat_completions], source: "test_team_disable_global_guardrail_e2e.py", rationale: "Local content-filter default-on blocks banned keyword pre-call"}
- {id: guardrail.litellm_content_filter.pre_call.blocks_video, module: guardrail, tier: P0, hook_point: pre_call, assertions: [blocks], exercised_on: [videos], source: "test_key_guardrail_video_e2e.py", fail_before_fix: proven, rationale: "A content-filter guardrail attached to a key (metadata.guardrails) blocks a banned prompt on POST /v1/videos before the provider is called; before the fix the route's call type was unknown to the unified guardrail hook and the prompt went to the provider unscanned (LIT-6685)"}
- {id: guardrail.litellm_content_filter.pre_call.allows, module: guardrail, tier: P0, hook_point: pre_call, assertions: [allows], exercised_on: [chat_completions], source: "test_team_disable_global_guardrail_e2e.py", rationale: "Team disable_global_guardrails bypasses default-on content filter"}
Expand Down
88 changes: 87 additions & 1 deletion tests/e2e/guardrails/test_bedrock_guardrail_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,12 @@

import pytest
from e2e_config import unique_marker
from e2e_http import UnknownApiError
from e2e_http import StreamingResponse, UnknownApiError
from guardrails_client import (
BedrockGuardrailParamsBody,
GuardrailsClient,
poll_until_blocked,
poll_until_blocked_stream,
)
from lifecycle import ResourceManager
from pydantic import JsonValue, TypeAdapter
Expand Down Expand Up @@ -135,3 +136,88 @@ def test_bedrock_post_call_blocks_denied_model_output(
)
case _:
pytest.fail(f"bedrock post_call guardrail did not block denied model output; got {result}")

@pytest.mark.covers("guardrail.bedrock.pre_call.blocks", exercised_on=["messages"])
def test_bedrock_pre_call_blocks_on_messages(
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
) -> None:
name = _register_pre_call(client, resources, "e2e-bedrock-messages")

result = poll_until_blocked_stream(
lambda: client.messages_raw(scoped_key, MODEL, BLOCKED_PROMPT, guardrails=[name])
)
_assert_policy_block(result, "/v1/messages")

@pytest.mark.covers("guardrail.bedrock.pre_call.blocks", exercised_on=["responses"])
def test_bedrock_pre_call_blocks_on_responses(
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
) -> None:
name = _register_pre_call(client, resources, "e2e-bedrock-responses")

result = poll_until_blocked_stream(
lambda: client.responses(scoped_key, MODEL, BLOCKED_PROMPT, guardrails=[name])
)
_assert_policy_block(result, "/v1/responses")

@pytest.mark.covers("guardrail.bedrock.post_call.blocks", exercised_on=["chat_completions"])
def test_bedrock_post_call_blocks_denied_streamed_output_and_passes_clean_streams(
self, client: GuardrailsClient, resources: ResourceManager, scoped_key: str
) -> None:
blocked_word = os.environ.get("BEDROCK_GUARDRAIL_BLOCKED_WORD", "FORBIDDENWORD")
name = f"e2e-bedrock-post-stream-{unique_marker()}"
guardrail_id = client.register(
name,
BedrockGuardrailParamsBody(
mode="post_call",
default_on=False,
guardrailIdentifier=os.environ["BEDROCK_GUARDRAIL_IDENTIFIER"],
guardrailVersion=os.environ["BEDROCK_GUARDRAIL_VERSION"],
),
)
resources.defer(lambda: client.delete_guardrail(guardrail_id))

prompt = f"Reply with exactly this one word and nothing else: {blocked_word}"
blocked = poll_until_blocked_stream(
lambda: client.chat_stream_raw(scoped_key, MODEL, prompt, guardrails=[name], max_tokens=128)
)
_assert_policy_block(blocked, "streamed /chat/completions")
frames = _sse_data_frames(blocked.body)
assert len(frames) == 1, f"a blocked stream must carry exactly one error frame, got: {blocked.body[:400]}"
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
error_frame = _JSON.validate_json(frames[0])
assert isinstance(error_frame, dict) and set(error_frame) == {"error"}, (
f"the only frame of a blocked stream must be an error, not model content: {frames[0][:400]}"
)
assert blocked_word not in json.dumps(_without_assessments(error_frame)), (
f"the blocked model output must not leak into the error frame; got: {frames[0][:400]}"
)

clean = client.chat_stream_raw(
scoped_key, MODEL, "Reply with exactly this one word and nothing else: hello", guardrails=[name]
)
assert clean.ok and clean.is_streaming, f"a clean output must stream through the guardrail: {clean.body[:400]}"
assert clean.stream_events and clean.stream_error is None, f"clean stream carried no content: {clean!r}"


def _register_pre_call(client: GuardrailsClient, resources: ResourceManager, prefix: str) -> str:
name = f"{prefix}-{unique_marker()}"
guardrail_id = client.create_bedrock_guardrail(
name,
identifier=os.environ["BEDROCK_GUARDRAIL_IDENTIFIER"],
version=os.environ["BEDROCK_GUARDRAIL_VERSION"],
)
resources.defer(lambda: client.delete_guardrail(guardrail_id))
return name


def _sse_data_frames(body: str) -> tuple[str, ...]:
payloads = (line.removeprefix("data:").strip() for line in body.splitlines() if line.startswith("data:"))
return tuple(payload for payload in payloads if payload != "[DONE]")


def _assert_policy_block(result: StreamingResponse, surface: str) -> None:
assert result.status_code == 400, (
f"{surface}: a Bedrock policy block must be HTTP 400, got {result.status_code}: {result.body[:400]}"
)
assert "violated guardrail policy" in result.body.lower(), (
f"{surface}: block body must name the guardrail verdict; got: {result.body[:400]}"
)
24 changes: 24 additions & 0 deletions tests/e2e/llm_translation/structured_output.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,24 @@
from __future__ import annotations

from typing import Final

from pydantic import TypeAdapter

SENTIMENT_PROMPT: Final = "Classify the sentiment of this review: 'The battery died after two days.'"
SENTIMENT_LABELS: Final = frozenset({"positive", "negative", "neutral"})
SENTIMENT_OUTPUT_FORMAT: Final[dict[str, object]] = {
"type": "json_schema",
"schema": {
"type": "object",
"properties": {"sentiment": {"type": "string", "enum": sorted(SENTIMENT_LABELS)}},
"required": ["sentiment"],
"additionalProperties": False,
},
}
_SENTIMENT_JSON: Final = TypeAdapter(dict[str, str])


def assert_sentiment_json(text: str) -> None:
parsed = _SENTIMENT_JSON.validate_json(text)
assert set(parsed) == {"sentiment"}, f"output_format schema not enforced, extra or missing keys: {parsed}"
assert parsed["sentiment"] in SENTIMENT_LABELS, f"sentiment outside the schema enum: {parsed}"
30 changes: 30 additions & 0 deletions tests/e2e/llm_translation/test_audio_speech_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,3 +139,33 @@ def test_empty_input_returns_error(
json=_OptionalSpeechBody(model=model, input="", voice="alloy"),
)
assert_client_error(result, "speech empty input")


MP3_PREFIXES = (b"ID3", b"\xff\xfb", b"\xff\xf3", b"\xff\xf2")


class TestAwsPollySpeech:
def test_polly_generative_voice_returns_mp3(
self, proxy: ProxyClient, resources: ResourceManager, sdk: SdkClients
) -> None:
model = f"e2e-speech-polly-{unique_marker()}"
model_id = proxy.create_model(
model,
LiteLLMParamsBody(
model="aws_polly/generative",
aws_access_key_id="os.environ/AWS_ACCESS_KEY_ID",
aws_secret_access_key="os.environ/AWS_SECRET_ACCESS_KEY",
aws_region_name="os.environ/AWS_REGION",
),
)
resources.defer(lambda: proxy.delete_model(model_id))
client = sdk.openai(resources.key())

response = client.audio.speech.with_raw_response.create(
model=model, voice="alloy", input="Hello from the gateway.", response_format="mp3"
)
content_type = response_header(response.headers, "content-type")
assert "audio" in (content_type or ""), f"polly speech content-type is not audio: {content_type!r}"
assert response.content.startswith(MP3_PREFIXES), (
f"polly speech body is not MP3 audio: {response.content[:16]!r}"
)
61 changes: 59 additions & 2 deletions tests/e2e/llm_translation/test_audio_transcriptions_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,11 +17,11 @@

import pytest
from e2e_config import unique_marker
from e2e_http import UnknownApiError
from e2e_http import UnknownApiError, unwrap
from lifecycle import ResourceManager
from models import LiteLLMParamsBody
from proxy_client import ProxyClient
from pydantic import BaseModel
from pydantic import BaseModel, Field
from sdk_clients import SdkClients

pytestmark = pytest.mark.e2e
Expand Down Expand Up @@ -119,3 +119,60 @@ def test_missing_model_returns_error(
)
case other:
pytest.fail(f"missing model expected a model-specific 400, got {other!r}")


class _WhisperForm(BaseModel):
model: str
response_format: str
timestamp_granularities: str | None = Field(default=None, serialization_alias="timestamp_granularities[]")


class _TranscriptWord(BaseModel):
word: str
start: float
end: float


class _VerboseTranscription(BaseModel):
text: str
words: list[_TranscriptWord] = []


class TestWhisperTranscriptionFormats:
def _upload[R: BaseModel](
self, proxy: ProxyClient, resources: ResourceManager, form: _WhisperForm, response_type: type[R]
) -> R:
model_id = proxy.create_model(
form.model, LiteLLMParamsBody(model="openai/whisper-1", api_key="os.environ/OPENAI_API_KEY")
)
resources.defer(lambda: proxy.delete_model(model_id))
return unwrap(
proxy.transport.upload(
"/v1/audio/transcriptions",
headers=proxy.transport.bearer(resources.key()),
form=form,
filename=WEATHER_WAV.name,
content=WEATHER_WAV.read_bytes(),
file_content_type="audio/wav",
response_type=response_type,
)
)

def test_vtt_format_returns_webvtt_transcript(self, proxy: ProxyClient, resources: ResourceManager) -> None:
form = _WhisperForm(model=f"e2e-whisper-vtt-{unique_marker()}", response_format="vtt")
transcript = self._upload(proxy, resources, form, _TranscriptionResult)
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
assert transcript.text.lstrip().startswith("WEBVTT"), f"vtt transcript is not WebVTT: {transcript.text[:200]!r}"
assert "weather" in transcript.text.lower(), f"vtt transcript lost the spoken words: {transcript.text!r}"

def test_verbose_json_returns_word_timestamps(self, proxy: ProxyClient, resources: ResourceManager) -> None:
form = _WhisperForm(
model=f"e2e-whisper-verbose-{unique_marker()}",
response_format="verbose_json",
timestamp_granularities="word",
)
transcript = self._upload(proxy, resources, form, _VerboseTranscription)
assert "weather" in transcript.text.lower(), f"verbose transcript lost the spoken words: {transcript.text!r}"
assert transcript.words, f"word timestamps were requested but none came back: {transcript!r}"
assert all(word.start <= word.end for word in transcript.words), (
f"word timings out of order: {transcript.words}"
)
Loading
Loading