diff --git a/litellm/llms/vertex_ai/videos/transformation.py b/litellm/llms/vertex_ai/videos/transformation.py index 25db3a673a1c..e6e3c2739c1a 100644 --- a/litellm/llms/vertex_ai/videos/transformation.py +++ b/litellm/llms/vertex_ai/videos/transformation.py @@ -8,12 +8,14 @@ import base64 import time from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Final, TypedDict, cast +from types import MappingProxyType +from typing import TYPE_CHECKING, Any, ClassVar, Final, TypedDict, cast import httpx from httpx._types import FileContent, RequestFiles from typing_extensions import ReadOnly +import litellm from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS from litellm.images.utils import ImageEditRequestUtils from litellm.llms.base_llm.videos.transformation import BaseVideoConfig @@ -119,6 +121,23 @@ class VertexAIVideoConfig(BaseVideoConfig, VertexBase): 3. Extract video data (base64) from response """ + _OPENAI_VIDEO_SIZE_TO_ASPECT_RATIO: ClassVar[Mapping[str, str]] = MappingProxyType( + { + "1280x720": "16:9", + "1920x1080": "16:9", + "720x1280": "9:16", + "1080x1920": "9:16", + } + ) + _OPENAI_VIDEO_SIZE_TO_RESOLUTION: ClassVar[Mapping[str, str]] = MappingProxyType( + { + "1280x720": "720p", + "1920x1080": "1080p", + "720x1280": "720p", + "1080x1920": "1080p", + } + ) + def __init__(self): BaseVideoConfig.__init__(self) VertexBase.__init__(self) @@ -161,6 +180,9 @@ def map_openai_params( - prompt → prompt (in instances) - input_reference → image (in instances) - size → aspectRatio (e.g., "1280x720" → "16:9") + - size → resolution for models with resolution-tier pricing when inferable + ("1280x720"/"720x1280" → "720p", "1920x1080"/"1080x1920" → "1080p"); + skipped if ``resolution`` is already set - seconds → durationSeconds (defaults to 4 seconds if not provided) """ mapped_params: Final[dict[str, object]] = {} @@ -175,6 +197,9 @@ def map_openai_params( if "parameters" in video_create_optional_params: mapped_params["parameters"] = video_create_optional_params["parameters"] + if "resolution" in video_create_optional_params: + mapped_params["resolution"] = video_create_optional_params["resolution"] + # Map size to aspectRatio if "size" in video_create_optional_params: size: Final = video_create_optional_params["size"] @@ -182,6 +207,15 @@ def map_openai_params( aspect_ratio: Final = self._convert_size_to_aspect_ratio(size) if aspect_ratio: mapped_params["aspectRatio"] = aspect_ratio + nested_params: Final = video_create_optional_params.get("parameters") + has_resolution = "resolution" in mapped_params or ( + isinstance(nested_params, dict) and nested_params.get("resolution") is not None + ) + supports_resolution = self._supports_resolution_inference(model) + if supports_resolution and not has_resolution: + inferred_resolution = self._convert_size_to_resolution(size) + if inferred_resolution is not None: + mapped_params["resolution"] = inferred_resolution # Map seconds to durationSeconds, default to 4 seconds (matching OpenAI) if "seconds" in video_create_optional_params: @@ -205,14 +239,16 @@ def _convert_size_to_aspect_ratio(self, size: str) -> str | None: if not size: return None - aspect_ratio_map: Final = { - "1280x720": "16:9", - "1920x1080": "16:9", - "720x1280": "9:16", - "1080x1920": "9:16", - } + return self._OPENAI_VIDEO_SIZE_TO_ASPECT_RATIO.get(size, "16:9") - return aspect_ratio_map.get(size, "16:9") + def _convert_size_to_resolution(self, size: str) -> str | None: + return self._OPENAI_VIDEO_SIZE_TO_RESOLUTION.get(size) + + @staticmethod + def _supports_resolution_inference(model: str) -> bool: + model_key: Final = model if model.startswith("vertex_ai/") else f"vertex_ai/{model}" + model_info: Final = litellm.model_cost.get(model_key) + return model_info is not None and model_info.get("output_cost_per_second_1080p") is not None def validate_environment( self, diff --git a/litellm/model_prices_and_context_window_backup.json b/litellm/model_prices_and_context_window_backup.json index bebbcc321814..ec175025b42c 100644 --- a/litellm/model_prices_and_context_window_backup.json +++ b/litellm/model_prices_and_context_window_backup.json @@ -43318,6 +43318,22 @@ "video" ] }, + "vertex_ai/veo-3.1-lite-generate-001": { + "litellm_provider": "vertex_ai-video-models", + "max_input_tokens": 1024, + "max_tokens": 1024, + "mode": "video_generation", + "output_cost_per_second": 0.05, + "output_cost_per_second_1080p": 0.08, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#veo", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, "voyage/rerank-2": { "input_cost_per_token": 5e-08, "litellm_provider": "voyage", diff --git a/litellm/types/videos/main.py b/litellm/types/videos/main.py index 3677cec3c8fd..99b08f6caf6b 100644 --- a/litellm/types/videos/main.py +++ b/litellm/types/videos/main.py @@ -2,7 +2,7 @@ from openai.types.audio.transcription_create_params import FileTypes from pydantic import BaseModel -from typing_extensions import TypedDict +from typing_extensions import ReadOnly, TypedDict class VideoObject(BaseModel): @@ -76,6 +76,7 @@ class VideoCreateOptionalRequestParams(TypedDict, total=False): image: Any | None # Image for image-to-video; dict with gcsUri/bytesBase64Encoded, or file-like object parameters: dict[str, Any] | None # Provider-specific parameters block passed directly to the API model: str | None + resolution: ReadOnly[str | None] seconds: str | None size: str | None characters: list[dict[str, str]] | None diff --git a/model_prices_and_context_window.json b/model_prices_and_context_window.json index bebbcc321814..ec175025b42c 100644 --- a/model_prices_and_context_window.json +++ b/model_prices_and_context_window.json @@ -43318,6 +43318,22 @@ "video" ] }, + "vertex_ai/veo-3.1-lite-generate-001": { + "litellm_provider": "vertex_ai-video-models", + "max_input_tokens": 1024, + "max_tokens": 1024, + "mode": "video_generation", + "output_cost_per_second": 0.05, + "output_cost_per_second_1080p": 0.08, + "source": "https://cloud.google.com/gemini-enterprise-agent-platform/generative-ai/pricing#veo", + "supported_modalities": [ + "text", + "image" + ], + "supported_output_modalities": [ + "video" + ] + }, "voyage/rerank-2": { "input_cost_per_token": 5e-08, "litellm_provider": "voyage", diff --git a/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py b/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py index 862969abbc29..6ba8706b0d8a 100644 --- a/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py +++ b/tests/test_litellm/llms/vertex_ai/videos/test_vertex_video_transformation.py @@ -4,13 +4,17 @@ import base64 import json -import os -from unittest.mock import MagicMock, Mock, patch +from collections.abc import Mapping +from pathlib import Path +from typing import cast +from unittest.mock import Mock, patch import httpx import pytest import litellm +from litellm.litellm_core_utils.get_llm_provider_logic import get_llm_provider +from litellm.llms.openai.cost_calculation import video_generation_cost from litellm.llms.vertex_ai.videos.transformation import ( VertexAIVideoConfig, _convert_image_to_vertex_format, @@ -18,6 +22,21 @@ from litellm.types.router import GenericLiteLLMParams from litellm.types.videos.main import VideoObject +VEO_31_LITE_VERTEX_MODEL = "vertex_ai/veo-3.1-lite-generate-001" +ROOT_MODEL_COST_PATH = ( + Path(__file__).parents[5] / "model_prices_and_context_window.json" +) +BACKUP_MODEL_COST_PATH = ( + Path(__file__).parents[5] + / "litellm" + / "model_prices_and_context_window_backup.json" +) +ModelCostMap = Mapping[str, Mapping[str, object]] + + +def _load_model_cost_map(path: Path) -> ModelCostMap: + return cast(ModelCostMap, json.loads(path.read_text())) + class TestVertexAIVideoConfig: """Test VertexAIVideoConfig transformation class.""" @@ -117,6 +136,56 @@ def test_get_complete_url_default_location(self): # Should NOT include endpoint assert not url.endswith(":predictLongRunning") + def test_veo_31_lite_model_cost_entries_match_pricing(self): + for path in (ROOT_MODEL_COST_PATH, BACKUP_MODEL_COST_PATH): + model_cost = _load_model_cost_map(path) + info = model_cost.get(VEO_31_LITE_VERTEX_MODEL) + + assert info is not None, f"{VEO_31_LITE_VERTEX_MODEL} missing from {path}" + assert info["litellm_provider"] == "vertex_ai-video-models" + assert info["mode"] == "video_generation" + assert info["max_input_tokens"] == 1024 + assert info["output_cost_per_second"] == 0.05 + assert info["output_cost_per_second_1080p"] == 0.08 + assert info["supported_modalities"] == ["text", "image"] + + def test_veo_31_lite_provider_routing_from_local_model_map( + self, monkeypatch: pytest.MonkeyPatch + ): + model_cost = _load_model_cost_map(BACKUP_MODEL_COST_PATH) + vertex_video_models = { + model_name.removeprefix("vertex_ai/") + for model_name, info in model_cost.items() + if info.get("litellm_provider") == "vertex_ai-video-models" + } + monkeypatch.setattr(litellm, "vertex_ai_video_models", vertex_video_models) + + model, custom_llm_provider, _, _ = get_llm_provider( + model="veo-3.1-lite-generate-001" + ) + + assert model == "veo-3.1-lite-generate-001" + assert custom_llm_provider == "vertex_ai" + + def test_veo_31_lite_cost_uses_resolution_tiers(self): + model_cost = _load_model_cost_map(BACKUP_MODEL_COST_PATH) + model_info = model_cost[VEO_31_LITE_VERTEX_MODEL] + + assert video_generation_cost( + model=VEO_31_LITE_VERTEX_MODEL, + duration_seconds=10.0, + custom_llm_provider="vertex_ai", + model_info=dict(model_info), + video_resolution="720p", + ) == pytest.approx(0.5) + assert video_generation_cost( + model=VEO_31_LITE_VERTEX_MODEL, + duration_seconds=10.0, + custom_llm_provider="vertex_ai", + model_info=dict(model_info), + video_resolution="1080p", + ) == pytest.approx(0.8) + def test_transform_video_create_request(self): """Test transformation of video creation request.""" prompt = "A cat playing with a ball of yarn" @@ -210,6 +279,95 @@ def test_map_openai_params(self): assert mapped["durationSeconds"] == 8 assert mapped["aspectRatio"] == "16:9" + assert "resolution" not in mapped + + @pytest.mark.parametrize( + ("model", "size", "expected_resolution"), + ( + (VEO_31_LITE_VERTEX_MODEL, "1280x720", "720p"), + ( + VEO_31_LITE_VERTEX_MODEL.removeprefix("vertex_ai/"), + "1920x1080", + "1080p", + ), + ), + ) + def test_map_openai_size_to_resolution_for_resolution_tier_model( + self, + model: str, + size: str, + expected_resolution: str, + monkeypatch: pytest.MonkeyPatch, + ): + model_cost = _load_model_cost_map(BACKUP_MODEL_COST_PATH) + monkeypatch.setitem( + litellm.model_cost, + VEO_31_LITE_VERTEX_MODEL, + dict(model_cost[VEO_31_LITE_VERTEX_MODEL]), + ) + + mapped = self.config.map_openai_params( + video_create_optional_params={"size": size}, + model=model, + drop_params=False, + ) + + assert mapped["aspectRatio"] == "16:9" + assert mapped["resolution"] == expected_resolution + + def test_map_openai_size_does_not_infer_resolution_for_veo_2(self): + mapped = self.config.map_openai_params( + video_create_optional_params={"size": "1920x1080"}, + model="vertex_ai/veo-2.0-generate-001", + drop_params=False, + ) + + assert mapped["aspectRatio"] == "16:9" + assert "resolution" not in mapped + + def test_map_openai_size_does_not_infer_resolution_for_existing_veo_3( + self, monkeypatch: pytest.MonkeyPatch + ): + model = "veo-3.1-generate-001" + model_key = f"vertex_ai/{model}" + model_cost = _load_model_cost_map(BACKUP_MODEL_COST_PATH) + monkeypatch.setitem(litellm.model_cost, model_key, dict(model_cost[model_key])) + + mapped = self.config.map_openai_params( + video_create_optional_params={"size": "1920x1080"}, + model=model, + drop_params=False, + ) + + assert mapped["aspectRatio"] == "16:9" + assert "resolution" not in mapped + + def test_map_openai_size_does_not_override_provider_resolution(self): + mapped = self.config.map_openai_params( + video_create_optional_params={ + "size": "1920x1080", + "parameters": {"resolution": "720p"}, + }, + model=VEO_31_LITE_VERTEX_MODEL, + drop_params=False, + ) + + assert mapped["aspectRatio"] == "16:9" + assert "resolution" not in mapped + assert mapped["parameters"] == {"resolution": "720p"} + + def test_map_openai_size_does_not_override_direct_resolution(self): + mapped = self.config.map_openai_params( + video_create_optional_params={ + "size": "1920x1080", + "resolution": "720p", + }, + model=VEO_31_LITE_VERTEX_MODEL, + drop_params=False, + ) + + assert mapped["aspectRatio"] == "16:9" + assert mapped["resolution"] == "720p" def test_map_openai_params_default_duration(self): """Test that durationSeconds is omitted when not provided."""