From a01248658e498274ae62d8f3edb191e28c74c2f6 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 13 Mar 2026 13:06:18 +0530 Subject: [PATCH 01/23] fix(streaming): preserve upstream custom fields on final chunk Ensure final finish_reason chunks retain non-OpenAI attributes from original provider chunks, including the holding_chunk flush path where delta is non-empty. Add regression tests for both final-chunk branches. Made-with: Cursor --- .../litellm_core_utils/streaming_handler.py | 13 +++- .../test_streaming_handler.py | 73 +++++++++++++++++++ 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/litellm/litellm_core_utils/streaming_handler.py b/litellm/litellm_core_utils/streaming_handler.py index db2369d03d6..6e991e6911b 100644 --- a/litellm/litellm_core_utils/streaming_handler.py +++ b/litellm/litellm_core_utils/streaming_handler.py @@ -31,7 +31,7 @@ ) from litellm.litellm_core_utils.redact_messages import LiteLLMLoggingObject from litellm.litellm_core_utils.thread_pool_executor import executor -from litellm.types.llms.openai import ChatCompletionChunk +from litellm.types.llms.openai import OpenAIChatCompletionChunk from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import ( Delta, @@ -745,7 +745,7 @@ def set_model_id( def copy_model_response_level_provider_specific_fields( self, - original_chunk: Union[ModelResponseStream, ChatCompletionChunk], + original_chunk: Union[ModelResponseStream, OpenAIChatCompletionChunk], model_response: ModelResponseStream, ) -> ModelResponseStream: """ @@ -1012,6 +1012,15 @@ def return_processed_chunk_logic( # noqa # if delta is None _is_delta_empty = self.is_delta_empty(delta=model_response.choices[0].delta) + # Preserve custom attributes from original chunk (applies to both + # empty and non-empty delta final chunks). + _original_chunk = response_obj.get("original_chunk", None) + if _original_chunk is not None: + preserve_upstream_non_openai_attributes( + model_response=model_response, + original_chunk=_original_chunk, + ) + if _is_delta_empty: model_response.choices[0].delta = Delta( content=None diff --git a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py index 6a64e7020b9..5d7b291e7b3 100644 --- a/tests/test_litellm/litellm_core_utils/test_streaming_handler.py +++ b/tests/test_litellm/litellm_core_utils/test_streaming_handler.py @@ -615,6 +615,79 @@ def test_streaming_handler_with_stop_chunk( assert returned_chunk is None +def test_finish_reason_chunk_preserves_non_openai_attributes( + initialized_custom_stream_wrapper: CustomStreamWrapper, +): + """ + Regression test for #23444: + Preserve upstream non-OpenAI attributes on final finish_reason chunk. + """ + initialized_custom_stream_wrapper.received_finish_reason = "stop" + + original_chunk = ModelResponseStream( + id="chatcmpl-test", + created=1742093326, + model=None, + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(content=""), + logprobs=None, + ) + ], + ) + setattr(original_chunk, "custom_field", {"key": "value"}) + + returned_chunk = initialized_custom_stream_wrapper.return_processed_chunk_logic( + completion_obj={"content": ""}, + response_obj={"original_chunk": original_chunk}, + model_response=ModelResponseStream(), + ) + + assert returned_chunk is not None + assert getattr(returned_chunk, "custom_field", None) == {"key": "value"} + + +def test_finish_reason_with_holding_chunk_preserves_non_openai_attributes( + initialized_custom_stream_wrapper: CustomStreamWrapper, +): + """ + Regression test for #23444 holding-chunk path: + preserve custom attributes when _is_delta_empty is False after flushing + holding_chunk. + """ + initialized_custom_stream_wrapper.received_finish_reason = "stop" + initialized_custom_stream_wrapper.holding_chunk = "filtered text" + + original_chunk = ModelResponseStream( + id="chatcmpl-test-2", + created=1742093327, + model=None, + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(content=""), + logprobs=None, + ) + ], + ) + setattr(original_chunk, "custom_field", {"key": "value"}) + + returned_chunk = initialized_custom_stream_wrapper.return_processed_chunk_logic( + completion_obj={"content": ""}, + response_obj={"original_chunk": original_chunk}, + model_response=ModelResponseStream(), + ) + + assert returned_chunk is not None + assert returned_chunk.choices[0].delta.content == "filtered text" + assert getattr(returned_chunk, "custom_field", None) == {"key": "value"} + + def test_set_response_id_propagation_empty_to_valid( initialized_custom_stream_wrapper: CustomStreamWrapper, ): From 8f769ef524a5bf95f3517961e85b6a3867c385ae Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Fri, 13 Mar 2026 17:54:33 +0530 Subject: [PATCH 02/23] docs(blog): add WebRTC blog post link Made-with: Cursor --- litellm/blog_posts.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/litellm/blog_posts.json b/litellm/blog_posts.json index 15340514bcc..fa768b3ec57 100644 --- a/litellm/blog_posts.json +++ b/litellm/blog_posts.json @@ -1,10 +1,10 @@ { "posts": [ { - "title": "Incident Report: SERVER_ROOT_PATH regression broke UI routing", - "description": "How a single line removal caused UI 404s for all deployments using SERVER_ROOT_PATH, and the tests we added to prevent it from happening again.", - "date": "2026-02-21", - "url": "https://docs.litellm.ai/blog/server-root-path-incident" + "title": "Realtime WebRTC HTTP Endpoints", + "description": "Use the LiteLLM proxy to route OpenAI-style WebRTC realtime via HTTP: client_secrets and SDP exchange.", + "date": "2026-03-12", + "url": "https://docs.litellm.ai/blog/realtime_webrtc_http_endpoints" } ] } From 7ed9be55b1a1adc357c3894c3a833487ecb9ef9f Mon Sep 17 00:00:00 2001 From: bbarwik Date: Sat, 14 Mar 2026 20:43:00 +0000 Subject: [PATCH 03/23] fix: merge annotations from all streaming chunks in stream_chunk_builder Previously, stream_chunk_builder only took annotations from the first chunk that contained them, losing any annotations from later chunks. This is a problem because providers like Gemini/Vertex AI send grounding metadata (converted to annotations) in the final streaming chunk, while other providers may spread annotations across multiple chunks. Changes: - Collect and merge annotations from ALL annotation-bearing chunks instead of only using the first one --- litellm/main.py | 11 +- .../test_stream_chunk_builder_annotations.py | 191 ++++++++++++++++++ 2 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 tests/test_litellm/test_stream_chunk_builder_annotations.py diff --git a/litellm/main.py b/litellm/main.py index 781a940ca71..81319bc432f 100644 --- a/litellm/main.py +++ b/litellm/main.py @@ -7528,8 +7528,15 @@ def stream_chunk_builder( # noqa: PLR0915 ] if len(annotation_chunks) > 0: - annotations = annotation_chunks[0]["choices"][0]["delta"]["annotations"] - response["choices"][0]["message"]["annotations"] = annotations + # Merge annotations from ALL chunks — providers may spread + # them across multiple streaming chunks or send them only in + # the final chunk. + all_annotations: list = [] + for ac in annotation_chunks: + all_annotations.extend( + ac["choices"][0]["delta"]["annotations"] + ) + response["choices"][0]["message"]["annotations"] = all_annotations audio_chunks = [ chunk diff --git a/tests/test_litellm/test_stream_chunk_builder_annotations.py b/tests/test_litellm/test_stream_chunk_builder_annotations.py new file mode 100644 index 00000000000..9c7ad4126b0 --- /dev/null +++ b/tests/test_litellm/test_stream_chunk_builder_annotations.py @@ -0,0 +1,191 @@ +""" +Tests for stream_chunk_builder annotation merging. + +Previously, stream_chunk_builder only took annotations from the FIRST +annotation chunk, losing any annotations that arrived in later chunks. +This fix merges annotations from ALL chunks. +""" + +from litellm import stream_chunk_builder +from litellm.types.utils import Delta, ModelResponseStream, StreamingChoices + + +def test_stream_chunk_builder_merges_annotations_from_multiple_chunks(): + """ + stream_chunk_builder must merge annotations from ALL streaming chunks, + not just take them from the first annotation chunk. + + Providers may spread annotations across multiple chunks (e.g. Gemini + sends grounding metadata in the final chunk, while intermediate chunks + may carry different annotations). + """ + annotation_a = { + "type": "url_citation", + "url_citation": { + "url": "https://example.com/a", + "title": "Source A", + "start_index": 0, + "end_index": 10, + }, + } + annotation_b = { + "type": "url_citation", + "url_citation": { + "url": "https://example.com/b", + "title": "Source B", + "start_index": 20, + "end_index": 30, + }, + } + + chunks = [ + ModelResponseStream( + id="chatcmpl-test", + created=1700000000, + model="test-model", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta( + content="Part one. ", + role="assistant", + annotations=[annotation_a], + ), + ) + ], + ), + ModelResponseStream( + id="chatcmpl-test", + created=1700000000, + model="test-model", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta(content="Part two."), + ) + ], + ), + ModelResponseStream( + id="chatcmpl-test", + created=1700000000, + model="test-model", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta( + content=None, + annotations=[annotation_b], + ), + ) + ], + ), + ] + + response = stream_chunk_builder(chunks=chunks) + assert response is not None + + message = response["choices"][0]["message"] + assert message.annotations is not None + assert len(message.annotations) == 2 + assert message.annotations[0] == annotation_a + assert message.annotations[1] == annotation_b + + +def test_stream_chunk_builder_single_annotation_chunk_still_works(): + """ + When annotations come from a single chunk (most common case), + stream_chunk_builder must still work correctly (no regression). + """ + annotation = { + "type": "url_citation", + "url_citation": { + "url": "https://example.com/only", + "title": "Only Source", + "start_index": 0, + "end_index": 5, + }, + } + + chunks = [ + ModelResponseStream( + id="chatcmpl-test", + created=1700000000, + model="test-model", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta(content="Hello", role="assistant"), + ) + ], + ), + ModelResponseStream( + id="chatcmpl-test", + created=1700000000, + model="test-model", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(content=None, annotations=[annotation]), + ) + ], + ), + ] + + response = stream_chunk_builder(chunks=chunks) + assert response is not None + + message = response["choices"][0]["message"] + assert message.annotations is not None + assert len(message.annotations) == 1 + assert message.annotations[0] == annotation + + +def test_stream_chunk_builder_no_annotations(): + """ + When no chunks contain annotations, the message should not have + an annotations key (no regression). + """ + chunks = [ + ModelResponseStream( + id="chatcmpl-test", + created=1700000000, + model="test-model", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason=None, + index=0, + delta=Delta(content="Hello", role="assistant"), + ) + ], + ), + ModelResponseStream( + id="chatcmpl-test", + created=1700000000, + model="test-model", + object="chat.completion.chunk", + choices=[ + StreamingChoices( + finish_reason="stop", + index=0, + delta=Delta(content=None), + ) + ], + ), + ] + + response = stream_chunk_builder(chunks=chunks) + assert response is not None + + message = response["choices"][0]["message"] + assert not hasattr(message, "annotations") or message.annotations is None From dd1ea3d39ef8d2584a31454e35e6130e7a7d4b79 Mon Sep 17 00:00:00 2001 From: brtydse100 <92057527+brtydse100@users.noreply.github.com> Date: Sun, 15 Mar 2026 10:29:59 +0200 Subject: [PATCH 04/23] Support multiple headers mapped to the customer user role (#23664) * added the header mapping feature * added tests * final cleanup * final cleanup * added missing test and logic * fixed header sending bug * Update litellm/proxy/auth/auth_utils.py Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> * added back init file in responses + fixed test_auth_utils.py int local_testing --------- Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- litellm/proxy/auth/auth_utils.py | 27 +++-- tests/local_testing/test_auth_utils.py | 2 +- .../proxy/auth/test_auth_utils.py | 106 ++++++++++++++++++ 3 files changed, 127 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/auth/auth_utils.py b/litellm/proxy/auth/auth_utils.py index 9a24041faad..0d3c627446b 100644 --- a/litellm/proxy/auth/auth_utils.py +++ b/litellm/proxy/auth/auth_utils.py @@ -662,11 +662,12 @@ def _has_user_setup_sso(): return sso_setup -def get_customer_user_header_from_mapping(user_id_mapping) -> Optional[str]: +def get_customer_user_header_from_mapping(user_id_mapping) -> Optional[list]: """Return the header_name mapped to CUSTOMER role, if any (dict-based).""" if not user_id_mapping: return None items = user_id_mapping if isinstance(user_id_mapping, list) else [user_id_mapping] + customer_headers_mappings = [] for item in items: if not isinstance(item, dict): continue @@ -675,7 +676,11 @@ def get_customer_user_header_from_mapping(user_id_mapping) -> Optional[str]: if role is None or not header_name: continue if str(role).lower() == str(LitellmUserRoles.CUSTOMER).lower(): - return header_name + customer_headers_mappings.append(header_name.lower()) + + if customer_headers_mappings: + return customer_headers_mappings + return None @@ -724,7 +729,7 @@ def get_end_user_id_from_request_body( # User query: "system not respecting user_header_name property" # This implies the key in general_settings is 'user_header_name'. if request_headers is not None: - custom_header_name_to_check: Optional[str] = None + custom_header_name_to_check: Optional[Union[list, str]] = None # Prefer user mappings (new behavior) user_id_mapping = general_settings.get("user_header_mappings", None) @@ -741,13 +746,21 @@ def get_end_user_id_from_request_body( custom_header_name_to_check = value # If we have a header name to check, try to read it from request headers - if isinstance(custom_header_name_to_check, str): + if isinstance(custom_header_name_to_check, list): + headers_lower = {k.lower(): v for k, v in request_headers.items()} + for expected_header in custom_header_name_to_check: + header_value = headers_lower.get(expected_header) + if header_value is not None: + user_id_str = str(header_value) + if user_id_str.strip(): + return user_id_str + + elif isinstance(custom_header_name_to_check, str): for header_name, header_value in request_headers.items(): if header_name.lower() == custom_header_name_to_check.lower(): - user_id_from_header = header_value user_id_str = ( - str(user_id_from_header) - if user_id_from_header is not None + str(header_value) + if header_value is not None else "" ) if user_id_str.strip(): diff --git a/tests/local_testing/test_auth_utils.py b/tests/local_testing/test_auth_utils.py index d36f96b1a39..bffcb40baf7 100644 --- a/tests/local_testing/test_auth_utils.py +++ b/tests/local_testing/test_auth_utils.py @@ -268,7 +268,7 @@ def test_get_customer_user_header_from_mapping_returns_customer_header(): {"header_name": "X-OpenWebUI-User-Email", "litellm_user_role": "customer"}, ] result = get_customer_user_header_from_mapping(mappings) - assert result == "X-OpenWebUI-User-Email" + assert result == ["x-openwebui-user-email"] def test_get_customer_user_header_from_mapping_no_customer_returns_none(): diff --git a/tests/test_litellm/proxy/auth/test_auth_utils.py b/tests/test_litellm/proxy/auth/test_auth_utils.py index 82920ce1d80..5e42b110aa0 100644 --- a/tests/test_litellm/proxy/auth/test_auth_utils.py +++ b/tests/test_litellm/proxy/auth/test_auth_utils.py @@ -209,3 +209,109 @@ def test_get_model_from_request_supports_google_model_names_with_slashes(): def test_get_model_from_request_vertex_passthrough_still_works(): route = "/vertex_ai/v1/projects/p/locations/l/publishers/google/models/gemini-1.5-pro:generateContent" assert get_model_from_request(request_data={}, route=route) == "gemini-1.5-pro" + + +def test_get_customer_user_header_returns_none_when_no_customer_role(): + from litellm.proxy.auth.auth_utils import get_customer_user_header_from_mapping + + mappings = [ + {"header_name": "X-OpenWebUI-User-Id", "litellm_user_role": "internal_user"} + ] + result = get_customer_user_header_from_mapping(mappings) + assert result is None + + +def test_get_customer_user_header_returns_none_for_single_non_customer_mapping(): + from litellm.proxy.auth.auth_utils import get_customer_user_header_from_mapping + + mapping = {"header_name": "X-Only-Internal", "litellm_user_role": "internal_user"} + result = get_customer_user_header_from_mapping(mapping) + assert result is None + +def test_get_customer_user_header_from_mapping_returns_customer_header(): + from litellm.proxy.auth.auth_utils import get_customer_user_header_from_mapping + + mappings = [ + {"header_name": "X-OpenWebUI-User-Id", "litellm_user_role": "internal_user"}, + {"header_name": "X-OpenWebUI-User-Email", "litellm_user_role": "customer"}, + ] + result = get_customer_user_header_from_mapping(mappings) + assert result == ["x-openwebui-user-email"] + + +def test_get_customer_user_header_returns_customers_header_in_config_order_when_multiple_exist(): + from litellm.proxy.auth.auth_utils import get_customer_user_header_from_mapping + + mappings = [ + {"header_name": "X-OpenWebUI-User-Id", "litellm_user_role": "internal_user"}, + {"header_name": "X-OpenWebUI-User-Email", "litellm_user_role": "customer"}, + {"header_name": "X-User-Id", "litellm_user_role": "customer"}, + ] + result = get_customer_user_header_from_mapping(mappings) + assert result == ['x-openwebui-user-email', 'x-user-id'] + + +def test_get_end_user_id_returns_id_from_user_header_mappings(): + from litellm.proxy.auth.auth_utils import get_end_user_id_from_request_body + + mappings = [ + {"header_name": "x-openwebui-user-id", "litellm_user_role": "internal_user"}, + {"header_name": "x-openwebui-user-email", "litellm_user_role": "customer"}, + ] + general_settings = {"user_header_mappings": mappings} + headers = {"x-openwebui-user-email": "1234"} + + with patch("litellm.proxy.auth.auth_utils._get_customer_id_from_standard_headers", return_value=None), \ + patch("litellm.proxy.proxy_server.general_settings", general_settings): + result = get_end_user_id_from_request_body(request_body={}, request_headers=headers) + + assert result == "1234" + + +def test_get_end_user_id_returns_first_customer_header_when_multiple_mappings_exist(): + from litellm.proxy.auth.auth_utils import get_end_user_id_from_request_body + + mappings = [ + {"header_name": "x-openwebui-user-id", "litellm_user_role": "internal_user"}, + {"header_name": "x-user-id", "litellm_user_role": "customer"}, + {"header_name": "x-openwebui-user-email", "litellm_user_role": "customer"}, + ] + general_settings = {"user_header_mappings": mappings} + headers = { + "x-user-id": "user-456", + "x-openwebui-user-email": "user@example.com", + } + + with patch("litellm.proxy.auth.auth_utils._get_customer_id_from_standard_headers", return_value=None), \ + patch("litellm.proxy.proxy_server.general_settings", general_settings): + result = get_end_user_id_from_request_body(request_body={}, request_headers=headers) + + assert result == "user-456" + + +def test_get_end_user_id_returns_none_when_no_customer_role_in_mappings(): + from litellm.proxy.auth.auth_utils import get_end_user_id_from_request_body + + mappings = [ + {"header_name": "x-openwebui-user-id", "litellm_user_role": "internal_user"}, + ] + general_settings = {"user_header_mappings": mappings} + headers = {"x-openwebui-user-id": "user-789"} + + with patch("litellm.proxy.auth.auth_utils._get_customer_id_from_standard_headers", return_value=None), \ + patch("litellm.proxy.proxy_server.general_settings", general_settings): + result = get_end_user_id_from_request_body(request_body={}, request_headers=headers) + + assert result is None + +def test_get_end_user_id_falls_back_to_deprecated_user_header_name(): + from litellm.proxy.auth.auth_utils import get_end_user_id_from_request_body + + general_settings = {"user_header_name": "x-custom-user-id"} + headers = {"x-custom-user-id": "user-legacy"} + + with patch("litellm.proxy.auth.auth_utils._get_customer_id_from_standard_headers", return_value=None), \ + patch("litellm.proxy.proxy_server.general_settings", general_settings): + result = get_end_user_id_from_request_body(request_body={}, request_headers=headers) + + assert result == "user-legacy" From 1a8f8c6d5255b7932da5e4d40ba29f365e8d5b43 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 16 Mar 2026 10:47:15 +0530 Subject: [PATCH 05/23] Refactor: Filtering beta header after transformation --- litellm/anthropic_beta_headers_manager.py | 36 +++++++++++++++++++ litellm/llms/anthropic/chat/handler.py | 16 +++++---- .../test_anthropic_beta_headers_filtering.py | 27 ++++++++++++++ 3 files changed, 72 insertions(+), 7 deletions(-) diff --git a/litellm/anthropic_beta_headers_manager.py b/litellm/anthropic_beta_headers_manager.py index efa57ca0586..97d223088fa 100644 --- a/litellm/anthropic_beta_headers_manager.py +++ b/litellm/anthropic_beta_headers_manager.py @@ -367,6 +367,42 @@ def update_headers_with_filtered_beta( return headers +def update_request_with_filtered_beta( + headers: dict, + request_data: dict, + provider: str, +) -> tuple[dict, dict]: + """ + Update both headers and request body beta fields based on provider support. + Modifies both dicts in place and returns them. + + Args: + headers: Request headers dict (will be modified in place) + request_data: Request body dict (will be modified in place) + provider: Provider name + + Returns: + Tuple of (updated headers, updated request_data) + """ + headers = update_headers_with_filtered_beta(headers=headers, provider=provider) + + existing_body_betas = request_data.get("anthropic_beta") + if not existing_body_betas: + return headers, request_data + + filtered_body_betas = filter_and_transform_beta_headers( + beta_headers=existing_body_betas, + provider=provider, + ) + + if filtered_body_betas: + request_data["anthropic_beta"] = filtered_body_betas + else: + request_data.pop("anthropic_beta", None) + + return headers, request_data + + def get_unsupported_headers(provider: str) -> List[str]: """ Get all beta headers that are unsupported by a provider (have null values in mapping). diff --git a/litellm/llms/anthropic/chat/handler.py b/litellm/llms/anthropic/chat/handler.py index 72cc7ecd9cc..5eebebc2e23 100644 --- a/litellm/llms/anthropic/chat/handler.py +++ b/litellm/llms/anthropic/chat/handler.py @@ -23,6 +23,9 @@ import litellm.litellm_core_utils import litellm.types import litellm.types.utils +from litellm.anthropic_beta_headers_manager import ( + update_request_with_filtered_beta, +) from litellm.constants import RESPONSE_FORMAT_TOOL_NAME from litellm.litellm_core_utils.core_helpers import map_finish_reason from litellm.llms.custom_httpx.http_handler import ( @@ -58,9 +61,6 @@ from ...base import BaseLLM from ..common_utils import AnthropicError, process_anthropic_headers -from litellm.anthropic_beta_headers_manager import ( - update_headers_with_filtered_beta, -) from .transformation import AnthropicConfig if TYPE_CHECKING: @@ -339,10 +339,6 @@ def completion( litellm_params=litellm_params, ) - headers = update_headers_with_filtered_beta( - headers=headers, provider=custom_llm_provider - ) - config = ProviderConfigManager.get_provider_chat_config( model=model, provider=LlmProviders(custom_llm_provider), @@ -360,6 +356,12 @@ def completion( headers=headers, ) + headers, data = update_request_with_filtered_beta( + headers=headers, + request_data=data, + provider=custom_llm_provider, + ) + ## LOGGING logging_obj.pre_call( input=messages, diff --git a/tests/test_litellm/test_anthropic_beta_headers_filtering.py b/tests/test_litellm/test_anthropic_beta_headers_filtering.py index a2c5608828a..447419b27d7 100644 --- a/tests/test_litellm/test_anthropic_beta_headers_filtering.py +++ b/tests/test_litellm/test_anthropic_beta_headers_filtering.py @@ -17,6 +17,7 @@ import litellm from litellm.anthropic_beta_headers_manager import ( filter_and_transform_beta_headers, + update_request_with_filtered_beta, ) @@ -116,6 +117,32 @@ def test_unknown_headers_filtered_out(self, provider): unknown not in filtered ), f"Unknown header '{unknown}' should be filtered out for {provider}" + def test_update_request_with_filtered_beta_vertex_ai(self): + """Test combined filtering for both HTTP headers and request body betas.""" + headers = { + "anthropic-beta": "files-api-2025-04-14,context-management-2025-06-27,code-execution-2025-05-22" + } + request_data = { + "anthropic_beta": [ + "files-api-2025-04-14", + "context-management-2025-06-27", + "code-execution-2025-05-22", + ] + } + + filtered_headers, filtered_request_data = update_request_with_filtered_beta( + headers=headers, + request_data=request_data, + provider="vertex_ai", + ) + + assert ( + filtered_headers.get("anthropic-beta") == "context-management-2025-06-27" + ) + assert filtered_request_data.get("anthropic_beta") == [ + "context-management-2025-06-27" + ] + @pytest.mark.asyncio async def test_anthropic_messages_http_headers_filtering(self): """Test that Anthropic messages API filters HTTP headers correctly.""" From 22b333cae6134babbf53b48a31e9332d1a4905a8 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 16 Mar 2026 12:08:06 +0530 Subject: [PATCH 06/23] Fix downloading vertex ai files --- .../proxy/hooks/managed_files.py | 18 ++++- .../llms/vertex_ai/batches/transformation.py | 11 ++- .../proxy/hooks/test_managed_files.py | 77 +++++++++++++++++++ .../test_vertex_ai_batch_transformation.py | 38 +++++++++ 4 files changed, 139 insertions(+), 5 deletions(-) create mode 100644 tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py diff --git a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py index 37ca341fdf2..5530054170c 100644 --- a/enterprise/litellm_enterprise/proxy/hooks/managed_files.py +++ b/enterprise/litellm_enterprise/proxy/hooks/managed_files.py @@ -26,6 +26,7 @@ get_batch_id_from_unified_batch_id, get_content_type_from_file_object, get_model_id_from_unified_batch_id, + get_models_from_unified_file_id, normalize_mime_type_for_provider, ) from litellm.types.llms.openai import ( @@ -904,6 +905,21 @@ async def async_post_call_success_hook( ) # managed batch id model_id = cast(Optional[str], response._hidden_params.get("model_id")) model_name = cast(Optional[str], response._hidden_params.get("model_name")) + resolved_model_name = model_name + + # Some providers (e.g. Vertex batch retrieve) do not set model_name on + # the response. In that case, recover target_model_names from the input + # managed file metadata so unified output IDs preserve routing metadata. + if not resolved_model_name and isinstance(unified_file_id, str): + decoded_unified_file_id = ( + _is_base64_encoded_unified_file_id(unified_file_id) + or unified_file_id + ) + target_model_names = get_models_from_unified_file_id( + decoded_unified_file_id + ) + if target_model_names: + resolved_model_name = ",".join(target_model_names) original_response_id = response.id if (unified_batch_id or unified_file_id) and model_id: @@ -919,7 +935,7 @@ async def async_post_call_success_hook( unified_file_id = self.get_unified_output_file_id( output_file_id=original_file_id, model_id=model_id, - model_name=model_name, + model_name=resolved_model_name, ) setattr(response, file_attr, unified_file_id) diff --git a/litellm/llms/vertex_ai/batches/transformation.py b/litellm/llms/vertex_ai/batches/transformation.py index 7cb06fea9e2..86bdc2c7b5f 100644 --- a/litellm/llms/vertex_ai/batches/transformation.py +++ b/litellm/llms/vertex_ai/batches/transformation.py @@ -1,6 +1,6 @@ -from litellm._uuid import uuid from typing import Any, Dict +from litellm._uuid import uuid from litellm.llms.vertex_ai.common_utils import ( _convert_vertex_datetime_to_openai_datetime, ) @@ -144,9 +144,10 @@ def _get_output_file_id_from_vertex_ai_batch_response( output_file_id: str = ( response.get("outputInfo", OutputInfo()).get("gcsOutputDirectory", "") - + "/predictions.jsonl" ) - if output_file_id != "/predictions.jsonl": + if output_file_id: + output_file_id = output_file_id.rstrip("/") + "/predictions.jsonl" + if output_file_id and output_file_id != "/predictions.jsonl": return output_file_id output_config = response.get("outputConfig") @@ -158,7 +159,9 @@ def _get_output_file_id_from_vertex_ai_batch_response( return output_file_id output_uri_prefix = gcs_destination.get("outputUriPrefix", "") - return output_uri_prefix + if output_uri_prefix.endswith("/predictions.jsonl"): + return output_uri_prefix + return output_uri_prefix.rstrip("/") + "/predictions.jsonl" @classmethod def _get_batch_job_status_from_vertex_ai_batch_response( diff --git a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py index 58fbd9e64ba..9f4ca4ed108 100644 --- a/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py +++ b/tests/enterprise/litellm_enterprise/proxy/hooks/test_managed_files.py @@ -1,4 +1,6 @@ +import base64 import json +from typing import cast from unittest.mock import AsyncMock, MagicMock, patch import pytest @@ -477,6 +479,81 @@ async def test_output_file_id_for_batch_retrieve(): assert not cast(LiteLLMBatch, response).output_file_id.startswith("file-") +@pytest.mark.asyncio +async def test_output_file_id_preserves_target_model_names_when_model_name_missing(): + """ + Regression test: when provider response does not include _hidden_params.model_name + (e.g. Vertex batch retrieve), unified output_file_id should still include + target_model_names from the managed input file ID. + """ + from openai.types.batch import BatchRequestCounts + + from litellm.proxy._types import UserAPIKeyAuth + from litellm.types.llms.openai import OpenAIFileObject + from litellm.types.utils import LiteLLMBatch + + batch = LiteLLMBatch( + id="batch_123", + completion_window="24h", + created_at=1750883933, + endpoint="/v1/chat/completions", + input_file_id="file-input-provider-id", + object="batch", + status="completed", + output_file_id="file-provider-output-id", + request_counts=BatchRequestCounts(completed=1, failed=0, total=1), + usage=None, + ) + + # Build a valid managed input id string and base64 encode it. + managed_input_file_payload = ( + "litellm_proxy:application/octet-stream;" + "unified_id,test-uuid;" + "target_model_names,gemini-2.5-pro;" + "llm_output_file_id,file-input-1;" + "llm_output_file_model_id,model-id-1" + ) + managed_input_file_id = ( + base64.urlsafe_b64encode(managed_input_file_payload.encode()) + .decode() + .rstrip("=") + ) + + batch._hidden_params = { + "model_id": "model-id-1", + "unified_batch_id": "litellm_proxy;model_id:model-id-1;llm_batch_id:batch_123", + "unified_file_id": managed_input_file_id, + # Intentionally omit model_name to mimic Vertex issue. + } + + proxy_managed_files = _PROXY_LiteLLMManagedFiles( + DualCache(), prisma_client=AsyncMock() + ) + + provider_output_file = OpenAIFileObject( + id="file-provider-output-id", + object="file", + bytes=1, + created_at=1, + filename="predictions.jsonl", + purpose="batch_output", + ) + + with patch("litellm.afile_retrieve", new_callable=AsyncMock) as mock_retrieve: + mock_retrieve.return_value = provider_output_file + response = await proxy_managed_files.async_post_call_success_hook( + data={}, + user_api_key_dict=UserAPIKeyAuth(user_id="test-user"), + response=batch, + ) + + decoded_output_file_id = _is_base64_encoded_unified_file_id( + cast(LiteLLMBatch, response).output_file_id + ) + assert decoded_output_file_id + assert "target_model_names,gemini-2.5-pro" in cast(str, decoded_output_file_id) + + @pytest.mark.asyncio async def test_error_file_id_for_failed_batch(): """ diff --git a/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py new file mode 100644 index 00000000000..1aab74ddc26 --- /dev/null +++ b/tests/test_litellm/llms/vertex_ai/test_vertex_ai_batch_transformation.py @@ -0,0 +1,38 @@ +from litellm.llms.vertex_ai.batches.transformation import VertexAIBatchTransformation + + +def test_output_file_id_uses_predictions_jsonl_with_output_info(): + response = { + "outputInfo": { + "gcsOutputDirectory": "gs://test-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-pro/prediction-model-123" + } + } + + output_file_id = VertexAIBatchTransformation._get_output_file_id_from_vertex_ai_batch_response( + response + ) + + assert ( + output_file_id + == "gs://test-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-pro/prediction-model-123/predictions.jsonl" + ) + + +def test_output_file_id_falls_back_to_output_uri_prefix_with_predictions_jsonl(): + response = { + "outputInfo": {}, + "outputConfig": { + "gcsDestination": { + "outputUriPrefix": "gs://test-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-pro/prediction-model-456" + } + }, + } + + output_file_id = VertexAIBatchTransformation._get_output_file_id_from_vertex_ai_batch_response( + response + ) + + assert ( + output_file_id + == "gs://test-bucket/litellm-vertex-files/publishers/google/models/gemini-2.5-pro/prediction-model-456/predictions.jsonl" + ) From 61519d6c6505eea7ffdfd262229a8dce61d8afdc Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 16 Mar 2026 16:11:21 +0530 Subject: [PATCH 07/23] fix(video): decode managed character ids robustly Support missing base64 padding in managed character/video IDs so copied encoded IDs still decode to the original upstream character ID. Made-with: Cursor --- litellm/types/videos/utils.py | 104 ++++++++++++++++++++++++++++++++++ 1 file changed, 104 insertions(+) diff --git a/litellm/types/videos/utils.py b/litellm/types/videos/utils.py index 4916394e7e7..3a100129bcd 100644 --- a/litellm/types/videos/utils.py +++ b/litellm/types/videos/utils.py @@ -12,6 +12,26 @@ from litellm.types.videos.main import DecodedVideoId VIDEO_ID_PREFIX = "video_" +CHARACTER_ID_PREFIX = "character_" +CHARACTER_ID_TEMPLATE = "litellm:custom_llm_provider:{};model_id:{};character_id:{}" + + +class DecodedCharacterId(dict): + """Structure representing a decoded character ID.""" + + custom_llm_provider: Optional[str] + model_id: Optional[str] + character_id: str + + +def _add_base64_padding(value: str) -> str: + """ + Add missing base64 padding when IDs are copied without trailing '=' chars. + """ + missing_padding = len(value) % 4 + if missing_padding: + value += "=" * (4 - missing_padding) + return value def encode_video_id_with_provider( @@ -59,6 +79,7 @@ def decode_video_id_with_provider(encoded_video_id: str) -> DecodedVideoId: try: cleaned_id = encoded_video_id.replace(VIDEO_ID_PREFIX, "") + cleaned_id = _add_base64_padding(cleaned_id) decoded_id = base64.b64decode(cleaned_id.encode("utf-8")).decode("utf-8") if ";" not in decoded_id: @@ -103,3 +124,86 @@ def extract_original_video_id(encoded_video_id: str) -> str: """Extract original video ID without encoding.""" decoded = decode_video_id_with_provider(encoded_video_id) return decoded.get("video_id", encoded_video_id) + + +def encode_character_id_with_provider( + character_id: str, provider: str, model_id: Optional[str] = None +) -> str: + """Encode provider and model_id into character_id using base64.""" + if not provider or not character_id: + return character_id + + decoded = decode_character_id_with_provider(character_id) + if decoded.get("custom_llm_provider") is not None: + return character_id + + assembled_id = CHARACTER_ID_TEMPLATE.format(provider, model_id or "", character_id) + base64_encoded_id: str = base64.b64encode(assembled_id.encode("utf-8")).decode( + "utf-8" + ) + return f"{CHARACTER_ID_PREFIX}{base64_encoded_id}" + + +def decode_character_id_with_provider(encoded_character_id: str) -> DecodedCharacterId: + """Decode provider and model_id from encoded character_id.""" + if not encoded_character_id: + return DecodedCharacterId( + custom_llm_provider=None, + model_id=None, + character_id=encoded_character_id, + ) + + if not encoded_character_id.startswith(CHARACTER_ID_PREFIX): + return DecodedCharacterId( + custom_llm_provider=None, + model_id=None, + character_id=encoded_character_id, + ) + + try: + cleaned_id = encoded_character_id.replace(CHARACTER_ID_PREFIX, "") + cleaned_id = _add_base64_padding(cleaned_id) + decoded_id = base64.b64decode(cleaned_id.encode("utf-8")).decode("utf-8") + + if ";" not in decoded_id: + return DecodedCharacterId( + custom_llm_provider=None, + model_id=None, + character_id=encoded_character_id, + ) + + parts = decoded_id.split(";") + + custom_llm_provider = None + model_id = None + decoded_character_id = encoded_character_id + + if len(parts) >= 3: + custom_llm_provider_part = parts[0] + model_id_part = parts[1] + character_id_part = parts[2] + + custom_llm_provider = custom_llm_provider_part.replace( + "litellm:custom_llm_provider:", "" + ) + model_id = model_id_part.replace("model_id:", "") + decoded_character_id = character_id_part.replace("character_id:", "") + + return DecodedCharacterId( + custom_llm_provider=custom_llm_provider, + model_id=model_id, + character_id=decoded_character_id, + ) + except Exception as e: + verbose_logger.debug(f"Error decoding character_id '{encoded_character_id}': {e}") + return DecodedCharacterId( + custom_llm_provider=None, + model_id=None, + character_id=encoded_character_id, + ) + + +def extract_original_character_id(encoded_character_id: str) -> str: + """Extract original character ID without encoding.""" + decoded = decode_character_id_with_provider(encoded_character_id) + return decoded.get("character_id", encoded_character_id) From 4a7ef7b1d2b407d3cbb0a8066c908540ea2c2ecc Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 16 Mar 2026 16:12:07 +0530 Subject: [PATCH 08/23] fix(video): enforce character endpoint video MIME handling Use typed character response models and video multipart helpers so /videos/characters forwards uploaded MP4 files with video/* content type. Made-with: Cursor --- litellm/llms/openai/videos/transformation.py | 155 ++++++++++++++++++- litellm/types/videos/main.py | 50 +++++- 2 files changed, 202 insertions(+), 3 deletions(-) diff --git a/litellm/llms/openai/videos/transformation.py b/litellm/llms/openai/videos/transformation.py index e224097fb02..0501a67fc16 100644 --- a/litellm/llms/openai/videos/transformation.py +++ b/litellm/llms/openai/videos/transformation.py @@ -1,4 +1,5 @@ -from io import BufferedReader +import mimetypes +from io import BufferedReader, BytesIO from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union, cast import httpx @@ -10,7 +11,11 @@ from litellm.secret_managers.main import get_secret_str from litellm.types.llms.openai import CreateVideoRequest from litellm.types.router import GenericLiteLLMParams -from litellm.types.videos.main import VideoCreateOptionalRequestParams, VideoObject +from litellm.types.videos.main import ( + CharacterObject, + VideoCreateOptionalRequestParams, + VideoObject, +) from litellm.types.videos.utils import ( encode_video_id_with_provider, extract_original_video_id, @@ -430,6 +435,106 @@ def get_error_class( headers=headers, ) + def transform_video_create_character_request( + self, + name: str, + video: Any, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, list]: + url = f"{api_base.rstrip('/')}/characters" + files_list: List[Tuple[str, Any]] = [("name", (None, name))] + self._add_video_to_files(files_list, video, "video") + return url, files_list + + def transform_video_create_character_response( + self, + raw_response: httpx.Response, + logging_obj: Any, + ) -> CharacterObject: + return CharacterObject(**raw_response.json()) + + def transform_video_get_character_request( + self, + character_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + url = f"{api_base.rstrip('/')}/characters/{character_id}" + return url, {} + + def transform_video_get_character_response( + self, + raw_response: httpx.Response, + logging_obj: Any, + ) -> CharacterObject: + return CharacterObject(**raw_response.json()) + + def transform_video_edit_request( + self, + prompt: str, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + extra_body: Optional[Dict[str, Any]] = None, + ) -> Tuple[str, Dict]: + original_video_id = extract_original_video_id(video_id) + url = f"{api_base.rstrip('/')}/edits" + data: Dict[str, Any] = {"prompt": prompt, "video": {"id": original_video_id}} + if extra_body: + data.update(extra_body) + return url, data + + def transform_video_edit_response( + self, + raw_response: httpx.Response, + logging_obj: Any, + custom_llm_provider: Optional[str] = None, + ) -> VideoObject: + video_obj = VideoObject(**raw_response.json()) + if custom_llm_provider and video_obj.id: + video_obj.id = encode_video_id_with_provider( + video_obj.id, custom_llm_provider, None + ) + return video_obj + + def transform_video_extension_request( + self, + prompt: str, + video_id: str, + seconds: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + extra_body: Optional[Dict[str, Any]] = None, + ) -> Tuple[str, Dict]: + original_video_id = extract_original_video_id(video_id) + url = f"{api_base.rstrip('/')}/extensions" + data: Dict[str, Any] = { + "prompt": prompt, + "seconds": seconds, + "video": {"id": original_video_id}, + } + if extra_body: + data.update(extra_body) + return url, data + + def transform_video_extension_response( + self, + raw_response: httpx.Response, + logging_obj: Any, + custom_llm_provider: Optional[str] = None, + ) -> VideoObject: + video_obj = VideoObject(**raw_response.json()) + if custom_llm_provider and video_obj.id: + video_obj.id = encode_video_id_with_provider( + video_obj.id, custom_llm_provider, None + ) + return video_obj + def _add_image_to_files( self, files_list: List[Tuple[str, Any]], @@ -445,3 +550,49 @@ def _add_image_to_files( files_list.append( (field_name, ("input_reference.png", image, image_content_type)) ) + + def _add_video_to_files( + self, + files_list: List[Tuple[str, Any]], + video: Any, + field_name: str, + ) -> None: + """ + Add a video to files with proper video MIME type detection. + + This path is used by POST /videos/characters and must send video/mp4, + not image/* content types. + """ + filename = getattr(video, "name", None) or "input_video.mp4" + content_type = self._get_video_content_type(video=video, filename=filename) + files_list.append((field_name, (filename, video, content_type))) + + def _get_video_content_type(self, video: Any, filename: str) -> str: + guessed_content_type, _ = mimetypes.guess_type(filename) + if guessed_content_type and guessed_content_type.startswith("video/"): + return guessed_content_type + + # Fast-path detection for common MP4 signatures when filename is missing/incorrect. + try: + header_bytes = b"" + if isinstance(video, BytesIO): + current_pos = video.tell() + video.seek(0) + header_bytes = video.read(64) + video.seek(current_pos) + elif isinstance(video, BufferedReader): + current_pos = video.tell() + video.seek(0) + header_bytes = video.read(64) + video.seek(current_pos) + elif isinstance(video, bytes): + header_bytes = video[:64] + + # MP4 typically includes ftyp in first box. + if b"ftyp" in header_bytes: + return "video/mp4" + except Exception: + pass + + # OpenAI create-character currently supports mp4. + return "video/mp4" diff --git a/litellm/types/videos/main.py b/litellm/types/videos/main.py index b6357f3273f..24454890fe3 100644 --- a/litellm/types/videos/main.py +++ b/litellm/types/videos/main.py @@ -3,7 +3,7 @@ from pydantic import BaseModel from typing_extensions import TypedDict -from litellm.types.utils import FileTypes +FileTypes = Any class VideoObject(BaseModel): @@ -104,3 +104,51 @@ class DecodedVideoId(TypedDict, total=False): custom_llm_provider: Optional[str] model_id: Optional[str] video_id: str + + +class DecodedCharacterId(TypedDict, total=False): + """Structure representing a decoded character ID""" + + custom_llm_provider: Optional[str] + model_id: Optional[str] + character_id: str + + +class CharacterObject(BaseModel): + """Represents a character created from a video.""" + + id: str + object: Literal["character"] = "character" + created_at: int + name: str + _hidden_params: Dict[str, Any] = {} + + def __contains__(self, key): + return hasattr(self, key) + + def get(self, key, default=None): + return getattr(self, key, default) + + def __getitem__(self, key): + return getattr(self, key) + + def json(self, **kwargs): # type: ignore + try: + return self.model_dump(**kwargs) + except Exception: + return self.dict() + + +class VideoEditRequestParams(TypedDict, total=False): + """TypedDict for video edit request parameters.""" + + prompt: str + video: Dict[str, str] # {"id": "video_123"} + + +class VideoExtensionRequestParams(TypedDict, total=False): + """TypedDict for video extension request parameters.""" + + prompt: str + seconds: str + video: Dict[str, str] # {"id": "video_123"} From 94405b621812d9c06eaf758ca41b36ecd7b78841 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 16 Mar 2026 16:13:11 +0530 Subject: [PATCH 09/23] fix(types): use direct FileTypes import in video schemas Avoid the temporary Any alias and use a concrete FileTypes import compatible with type checks. Made-with: Cursor --- litellm/types/videos/main.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/litellm/types/videos/main.py b/litellm/types/videos/main.py index 24454890fe3..880fec7640d 100644 --- a/litellm/types/videos/main.py +++ b/litellm/types/videos/main.py @@ -1,10 +1,9 @@ from typing import Any, Dict, List, Literal, Optional +from openai.types.audio.transcription_create_params import FileTypes # type: ignore from pydantic import BaseModel from typing_extensions import TypedDict -FileTypes = Any - class VideoObject(BaseModel): """Represents a generated video object.""" From 79c787b85d4c123366f89adb8d31c2e5b5a5a529 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 16 Mar 2026 17:53:54 +0530 Subject: [PATCH 10/23] Add new videos endpoints --- litellm/proxy/video_endpoints/utils.py | 56 ++++++++++++++++++++++++++ litellm/types/llms/openai.py | 2 + litellm/types/utils.py | 31 ++++++++++++++ litellm/types/videos/main.py | 1 + 4 files changed, 90 insertions(+) create mode 100644 litellm/proxy/video_endpoints/utils.py diff --git a/litellm/proxy/video_endpoints/utils.py b/litellm/proxy/video_endpoints/utils.py new file mode 100644 index 00000000000..36203bdc77e --- /dev/null +++ b/litellm/proxy/video_endpoints/utils.py @@ -0,0 +1,56 @@ +from typing import Any, Dict, Optional + +import orjson + +from litellm.types.videos.utils import encode_character_id_with_provider + + +def extract_model_from_target_model_names(target_model_names: Any) -> Optional[str]: + if isinstance(target_model_names, str): + target_model_names = [m.strip() for m in target_model_names.split(",") if m.strip()] + elif not isinstance(target_model_names, list): + return None + return target_model_names[0] if target_model_names else None + + +def get_custom_provider_from_data(data: Dict[str, Any]) -> Optional[str]: + custom_llm_provider = data.get("custom_llm_provider") + if custom_llm_provider: + return custom_llm_provider + + extra_body = data.get("extra_body") + if isinstance(extra_body, str): + try: + parsed_extra_body = orjson.loads(extra_body) + if isinstance(parsed_extra_body, dict): + extra_body = parsed_extra_body + except Exception: + extra_body = None + + if isinstance(extra_body, dict): + extra_body_custom_llm_provider = extra_body.get("custom_llm_provider") + if isinstance(extra_body_custom_llm_provider, str): + return extra_body_custom_llm_provider + + return None + + +def encode_character_id_in_response( + response: Any, custom_llm_provider: str, model_id: Optional[str] +) -> Any: + if isinstance(response, dict) and response.get("id"): + response["id"] = encode_character_id_with_provider( + character_id=response["id"], + provider=custom_llm_provider, + model_id=model_id, + ) + return response + + character_id = getattr(response, "id", None) + if isinstance(character_id, str) and character_id: + response.id = encode_character_id_with_provider( + character_id=character_id, + provider=custom_llm_provider, + model_id=model_id, + ) + return response diff --git a/litellm/types/llms/openai.py b/litellm/types/llms/openai.py index 0184919b543..a2df3f2e0d6 100644 --- a/litellm/types/llms/openai.py +++ b/litellm/types/llms/openai.py @@ -2187,6 +2187,7 @@ class CreateVideoRequest(TypedDict, total=False): model: Optional[str] - The video generation model to use (defaults to sora-2) seconds: Optional[str] - Clip duration in seconds (defaults to 4 seconds) size: Optional[str] - Output resolution formatted as width x height (defaults to 720x1280) + characters: Optional[List[Dict[str, str]]] - Character references to include in generation user: Optional[str] - A unique identifier representing your end-user extra_headers: Optional[Dict[str, str]] - Additional headers extra_body: Optional[Dict[str, str]] - Additional body parameters @@ -2198,6 +2199,7 @@ class CreateVideoRequest(TypedDict, total=False): model: Optional[str] seconds: Optional[str] size: Optional[str] + characters: Optional[List[Dict[str, str]]] user: Optional[str] extra_headers: Optional[Dict[str, str]] extra_body: Optional[Dict[str, str]] diff --git a/litellm/types/utils.py b/litellm/types/utils.py index 892c9578b94..f20958f3f84 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -358,6 +358,14 @@ class CallTypes(str, Enum): avideo_retrieve_job = "avideo_retrieve_job" video_delete = "video_delete" avideo_delete = "avideo_delete" + video_create_character = "video_create_character" + avideo_create_character = "avideo_create_character" + video_get_character = "video_get_character" + avideo_get_character = "avideo_get_character" + video_edit = "video_edit" + avideo_edit = "avideo_edit" + video_extension = "video_extension" + avideo_extension = "avideo_extension" vector_store_file_create = "vector_store_file_create" avector_store_file_create = "avector_store_file_create" vector_store_file_list = "vector_store_file_list" @@ -700,6 +708,26 @@ class CallTypes(str, Enum): ], "/videos/{video_id}/remix": [CallTypes.avideo_remix, CallTypes.video_remix], "/v1/videos/{video_id}/remix": [CallTypes.avideo_remix, CallTypes.video_remix], + "/videos/characters": [ + CallTypes.avideo_create_character, + CallTypes.video_create_character, + ], + "/v1/videos/characters": [ + CallTypes.avideo_create_character, + CallTypes.video_create_character, + ], + "/videos/characters/{character_id}": [ + CallTypes.avideo_get_character, + CallTypes.video_get_character, + ], + "/v1/videos/characters/{character_id}": [ + CallTypes.avideo_get_character, + CallTypes.video_get_character, + ], + "/videos/edits": [CallTypes.avideo_edit, CallTypes.video_edit], + "/v1/videos/edits": [CallTypes.avideo_edit, CallTypes.video_edit], + "/videos/extensions": [CallTypes.avideo_extension, CallTypes.video_extension], + "/v1/videos/extensions": [CallTypes.avideo_extension, CallTypes.video_extension], # Vector Stores "/vector_stores": [CallTypes.avector_store_create, CallTypes.vector_store_create], "/v1/vector_stores": [ @@ -3465,6 +3493,9 @@ class SpecialEnums(Enum): LITELLM_MANAGED_VIDEO_COMPLETE_STR = ( "litellm:custom_llm_provider:{};model_id:{};video_id:{}" ) + LITELLM_MANAGED_VIDEO_CHARACTER_COMPLETE_STR = ( + "litellm:custom_llm_provider:{};model_id:{};character_id:{}" + ) class ServiceTier(Enum): diff --git a/litellm/types/videos/main.py b/litellm/types/videos/main.py index 880fec7640d..2ee44944bcc 100644 --- a/litellm/types/videos/main.py +++ b/litellm/types/videos/main.py @@ -82,6 +82,7 @@ class VideoCreateOptionalRequestParams(TypedDict, total=False): model: Optional[str] seconds: Optional[str] size: Optional[str] + characters: Optional[List[Dict[str, str]]] user: Optional[str] extra_headers: Optional[Dict[str, str]] extra_body: Optional[Dict[str, str]] From c33889200a68c8e448b07631a00d6a48851de66a Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 16 Mar 2026 17:54:03 +0530 Subject: [PATCH 11/23] Add new videos endpoints --- litellm/proxy/video_endpoints/endpoints.py | 430 ++++++++++++++++++++- 1 file changed, 429 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index f7a71c10339..9e8784df0aa 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -16,7 +16,15 @@ get_custom_llm_provider_from_request_query, ) from litellm.proxy.image_endpoints.endpoints import batch_to_bytesio -from litellm.types.videos.utils import decode_video_id_with_provider +from litellm.proxy.video_endpoints.utils import ( + encode_character_id_in_response, + extract_model_from_target_model_names, + get_custom_provider_from_data, +) +from litellm.types.videos.utils import ( + decode_character_id_with_provider, + decode_video_id_with_provider, +) router = APIRouter() @@ -504,3 +512,423 @@ async def video_remix( proxy_logging_obj=proxy_logging_obj, version=version, ) + + +@router.post( + "/v1/videos/characters", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, + tags=["videos"], +) +@router.post( + "/videos/characters", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, + tags=["videos"], +) +async def video_create_character( + request: Request, + fastapi_response: Response, + video: UploadFile = File(...), + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Create a character from an uploaded video file. + + Follows the OpenAI Videos API spec: + https://platform.openai.com/docs/api-reference/videos/create-character + + Example: + ```bash + curl -X POST "http://localhost:4000/v1/videos/characters" \ + -H "Authorization: Bearer sk-1234" \ + -F "video=@character_video.mp4" \ + -F "name=my_character" + ``` + """ + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + data = await _read_request_body(request=request) + video_file = await batch_to_bytesio([video]) + if video_file: + data["video"] = video_file[0] + + target_model_name = extract_model_from_target_model_names( + data.get("target_model_names") + ) + if target_model_name and not data.get("model"): + data["model"] = target_model_name + + custom_llm_provider = ( + get_custom_llm_provider_from_request_headers(request=request) + or get_custom_llm_provider_from_request_query(request=request) + or get_custom_provider_from_data(data=data) + or "openai" + ) + data["custom_llm_provider"] = custom_llm_provider + + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + response = await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="avideo_create_character", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=None, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + if target_model_name: + hidden_params = getattr(response, "_hidden_params", {}) or {} + provider_for_encoding = ( + hidden_params.get("custom_llm_provider") + or custom_llm_provider + or "openai" + ) + model_id_for_encoding = hidden_params.get("model_id") or data.get("model") + response = encode_character_id_in_response( + response=response, + custom_llm_provider=provider_for_encoding, + model_id=model_id_for_encoding, + ) + return response + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + + +@router.get( + "/v1/videos/characters/{character_id}", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, + tags=["videos"], +) +@router.get( + "/videos/characters/{character_id}", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, + tags=["videos"], +) +async def video_get_character( + character_id: str, + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Retrieve a character by ID. + + Follows the OpenAI Videos API spec: + https://platform.openai.com/docs/api-reference/videos/get-character + + Example: + ```bash + curl -X GET "http://localhost:4000/v1/videos/characters/char_123" \ + -H "Authorization: Bearer sk-1234" + ``` + """ + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + original_requested_character_id = character_id + data: Dict[str, Any] = {"character_id": character_id} + + decoded = decode_character_id_with_provider(character_id) + provider_from_id = decoded.get("custom_llm_provider") + model_id_from_decoded = decoded.get("model_id") + decoded_character_id = decoded.get("character_id") + if decoded_character_id: + data["character_id"] = decoded_character_id + + custom_llm_provider = ( + get_custom_llm_provider_from_request_headers(request=request) + or get_custom_llm_provider_from_request_query(request=request) + or await get_custom_llm_provider_from_request_body(request=request) + or provider_from_id + or "openai" + ) + data["custom_llm_provider"] = custom_llm_provider + + if model_id_from_decoded and llm_router: + resolved_model = llm_router.resolve_model_name_from_model_id( + model_id_from_decoded + ) + if resolved_model: + data["model"] = resolved_model + + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + response = await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="avideo_get_character", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=None, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + if original_requested_character_id.startswith("character_"): + provider_for_encoding = provider_from_id or custom_llm_provider or "openai" + model_id_for_encoding = model_id_from_decoded + response = encode_character_id_in_response( + response=response, + custom_llm_provider=provider_for_encoding, + model_id=model_id_for_encoding, + ) + return response + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + + +@router.post( + "/v1/videos/edits", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, + tags=["videos"], +) +@router.post( + "/videos/edits", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, + tags=["videos"], +) +async def video_edit( + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Create a video edit job. + + Follows the OpenAI Videos API spec: + https://platform.openai.com/docs/api-reference/videos/create-edit + + Example: + ```bash + curl -X POST "http://localhost:4000/v1/videos/edits" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "Make it brighter", "video": {"id": "video_123"}}' + ``` + """ + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + body = await request.body() + data = orjson.loads(body) + + # Extract video_id from nested video object + video_ref = data.pop("video", {}) + video_id = video_ref.get("id", "") if isinstance(video_ref, dict) else "" + data["video_id"] = video_id + + decoded = decode_video_id_with_provider(video_id) + provider_from_id = decoded.get("custom_llm_provider") + model_id_from_decoded = decoded.get("model_id") + + custom_llm_provider = ( + get_custom_llm_provider_from_request_headers(request=request) + or get_custom_llm_provider_from_request_query(request=request) + or get_custom_provider_from_data(data=data) + or provider_from_id + or "openai" + ) + data["custom_llm_provider"] = custom_llm_provider + + if model_id_from_decoded and llm_router: + resolved_model = llm_router.resolve_model_name_from_model_id( + model_id_from_decoded + ) + if resolved_model: + data["model"] = resolved_model + + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="avideo_edit", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=None, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) + + +@router.post( + "/v1/videos/extensions", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, + tags=["videos"], +) +@router.post( + "/videos/extensions", + dependencies=[Depends(user_api_key_auth)], + response_class=ORJSONResponse, + tags=["videos"], +) +async def video_extension( + request: Request, + fastapi_response: Response, + user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), +): + """ + Create a video extension. + + Follows the OpenAI Videos API spec: + https://platform.openai.com/docs/api-reference/videos/create-extension + + Example: + ```bash + curl -X POST "http://localhost:4000/v1/videos/extensions" \ + -H "Authorization: Bearer sk-1234" \ + -H "Content-Type: application/json" \ + -d '{"prompt": "Continue the scene", "seconds": "5", "video": {"id": "video_123"}}' + ``` + """ + from litellm.proxy.proxy_server import ( + general_settings, + llm_router, + proxy_config, + proxy_logging_obj, + select_data_generator, + user_api_base, + user_max_tokens, + user_model, + user_request_timeout, + user_temperature, + version, + ) + + body = await request.body() + data = orjson.loads(body) + + # Extract video_id from nested video object + video_ref = data.pop("video", {}) + video_id = video_ref.get("id", "") if isinstance(video_ref, dict) else "" + data["video_id"] = video_id + + decoded = decode_video_id_with_provider(video_id) + provider_from_id = decoded.get("custom_llm_provider") + model_id_from_decoded = decoded.get("model_id") + + custom_llm_provider = ( + get_custom_llm_provider_from_request_headers(request=request) + or get_custom_llm_provider_from_request_query(request=request) + or get_custom_provider_from_data(data=data) + or provider_from_id + or "openai" + ) + data["custom_llm_provider"] = custom_llm_provider + + if model_id_from_decoded and llm_router: + resolved_model = llm_router.resolve_model_name_from_model_id( + model_id_from_decoded + ) + if resolved_model: + data["model"] = resolved_model + + processor = ProxyBaseLLMRequestProcessing(data=data) + try: + return await processor.base_process_llm_request( + request=request, + fastapi_response=fastapi_response, + user_api_key_dict=user_api_key_dict, + route_type="avideo_extension", + proxy_logging_obj=proxy_logging_obj, + llm_router=llm_router, + general_settings=general_settings, + proxy_config=proxy_config, + select_data_generator=select_data_generator, + model=None, + user_model=user_model, + user_temperature=user_temperature, + user_request_timeout=user_request_timeout, + user_max_tokens=user_max_tokens, + user_api_base=user_api_base, + version=version, + ) + except Exception as e: + raise await processor._handle_llm_api_exception( + e=e, + user_api_key_dict=user_api_key_dict, + proxy_logging_obj=proxy_logging_obj, + version=version, + ) From 8dab5dec886fa7b6714d835b04922185c52e7e55 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 16 Mar 2026 17:54:35 +0530 Subject: [PATCH 12/23] Add new videos endpoints routing and init --- litellm/videos/__init__.py | 26 +- litellm/videos/main.py | 520 +++++++++++++++++++++++++++++++++++++ 2 files changed, 541 insertions(+), 5 deletions(-) diff --git a/litellm/videos/__init__.py b/litellm/videos/__init__.py index 716add5f5d7..9fb66d7557a 100644 --- a/litellm/videos/__init__.py +++ b/litellm/videos/__init__.py @@ -1,16 +1,24 @@ """Video generation and management functions for LiteLLM.""" from .main import ( + avideo_content, + avideo_create_character, + avideo_edit, + avideo_extension, avideo_generation, - video_generation, + avideo_get_character, avideo_list, - video_list, + avideo_remix, avideo_status, - video_status, - avideo_content, video_content, - avideo_remix, + video_create_character, + video_edit, + video_extension, + video_generation, + video_get_character, + video_list, video_remix, + video_status, ) __all__ = [ @@ -24,4 +32,12 @@ "video_content", "avideo_remix", "video_remix", + "avideo_create_character", + "video_create_character", + "avideo_get_character", + "video_get_character", + "avideo_edit", + "video_edit", + "avideo_extension", + "video_extension", ] diff --git a/litellm/videos/main.py b/litellm/videos/main.py index f6c9bb00576..a86f1d7dcea 100644 --- a/litellm/videos/main.py +++ b/litellm/videos/main.py @@ -15,6 +15,7 @@ from litellm.types.router import GenericLiteLLMParams from litellm.types.utils import CallTypes, FileTypes from litellm.types.videos.main import ( + CharacterObject, VideoCreateOptionalRequestParams, VideoObject, ) @@ -1090,3 +1091,522 @@ def video_status( # noqa: PLR0915 completion_kwargs=local_vars, extra_kwargs=kwargs, ) + + +@client +async def avideo_create_character( + name: str, + video: Any, + timeout=600, + custom_llm_provider=None, + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + **kwargs, +) -> CharacterObject: + """ + Asynchronously create a character from an uploaded video file. + Maps to POST /v1/videos/characters + """ + local_vars = locals() + try: + loop = asyncio.get_event_loop() + kwargs["async_call"] = True + + if custom_llm_provider is None: + custom_llm_provider = "openai" + + func = partial( + video_create_character, + name=name, + video=video, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + **kwargs, + ) + + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response + + return response + except Exception as e: + raise litellm.exception_type( + model="", + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + +@client +def video_create_character( + name: str, + video: Any, + timeout=600, + custom_llm_provider=None, + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + **kwargs, +) -> Union[CharacterObject, Coroutine[Any, Any, CharacterObject]]: + """ + Create a character from an uploaded video file. + Maps to POST /v1/videos/characters + """ + local_vars = locals() + try: + litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore + litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) + _is_async = kwargs.pop("async_call", False) is True + + mock_response = kwargs.get("mock_response", None) + if mock_response is not None: + if isinstance(mock_response, str): + mock_response = json.loads(mock_response) + return CharacterObject(**mock_response) + + if custom_llm_provider is None: + custom_llm_provider = "openai" + + litellm_params = GenericLiteLLMParams(**kwargs) + + provider_config: Optional[BaseVideoConfig] = ProviderConfigManager.get_provider_video_config( + model=None, + provider=litellm.LlmProviders(custom_llm_provider), + ) + + if provider_config is None: + raise ValueError(f"video create character is not supported for {custom_llm_provider}") + + local_vars.update(kwargs) + request_params: Dict = {"name": name} + + litellm_logging_obj.update_environment_variables( + model="", + user=kwargs.get("user"), + optional_params=dict(request_params), + litellm_params={"litellm_call_id": litellm_call_id, **request_params}, + custom_llm_provider=custom_llm_provider, + ) + + litellm_logging_obj.call_type = CallTypes.video_create_character.value + + return base_llm_http_handler.video_create_character_handler( + name=name, + video=video, + video_provider_config=provider_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=litellm_logging_obj, + extra_headers=extra_headers, + timeout=timeout or DEFAULT_REQUEST_TIMEOUT, + _is_async=_is_async, + client=kwargs.get("client"), + ) + + except Exception as e: + raise litellm.exception_type( + model="", + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + +@client +async def avideo_get_character( + character_id: str, + timeout=600, + custom_llm_provider=None, + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + **kwargs, +) -> CharacterObject: + """ + Asynchronously retrieve a character by ID. + Maps to GET /v1/videos/characters/{character_id} + """ + local_vars = locals() + try: + loop = asyncio.get_event_loop() + kwargs["async_call"] = True + + func = partial( + video_get_character, + character_id=character_id, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + **kwargs, + ) + + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response + + return response + except Exception as e: + raise litellm.exception_type( + model="", + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + +@client +def video_get_character( + character_id: str, + timeout=600, + custom_llm_provider=None, + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + **kwargs, +) -> Union[CharacterObject, Coroutine[Any, Any, CharacterObject]]: + """ + Retrieve a character by ID. + Maps to GET /v1/videos/characters/{character_id} + """ + local_vars = locals() + try: + litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore + litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) + _is_async = kwargs.pop("async_call", False) is True + + mock_response = kwargs.get("mock_response", None) + if mock_response is not None: + if isinstance(mock_response, str): + mock_response = json.loads(mock_response) + return CharacterObject(**mock_response) + + if custom_llm_provider is None: + custom_llm_provider = "openai" + + litellm_params = GenericLiteLLMParams(**kwargs) + + provider_config: Optional[BaseVideoConfig] = ProviderConfigManager.get_provider_video_config( + model=None, + provider=litellm.LlmProviders(custom_llm_provider), + ) + + if provider_config is None: + raise ValueError(f"video get character is not supported for {custom_llm_provider}") + + local_vars.update(kwargs) + request_params: Dict = {"character_id": character_id} + + litellm_logging_obj.update_environment_variables( + model="", + user=kwargs.get("user"), + optional_params=dict(request_params), + litellm_params={"litellm_call_id": litellm_call_id, **request_params}, + custom_llm_provider=custom_llm_provider, + ) + + litellm_logging_obj.call_type = CallTypes.video_get_character.value + + return base_llm_http_handler.video_get_character_handler( + character_id=character_id, + video_provider_config=provider_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=litellm_logging_obj, + extra_headers=extra_headers, + timeout=timeout or DEFAULT_REQUEST_TIMEOUT, + _is_async=_is_async, + client=kwargs.get("client"), + ) + + except Exception as e: + raise litellm.exception_type( + model="", + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + +@client +async def avideo_edit( + video_id: str, + prompt: str, + timeout=600, + custom_llm_provider=None, + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + **kwargs, +) -> VideoObject: + """ + Asynchronously create a video edit job. + Maps to POST /v1/videos/edits + """ + local_vars = locals() + try: + loop = asyncio.get_event_loop() + kwargs["async_call"] = True + + func = partial( + video_edit, + video_id=video_id, + prompt=prompt, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + **kwargs, + ) + + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response + + return response + except Exception as e: + raise litellm.exception_type( + model="", + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + +@client +def video_edit( + video_id: str, + prompt: str, + timeout=600, + custom_llm_provider=None, + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + **kwargs, +) -> Union[VideoObject, Coroutine[Any, Any, VideoObject]]: + """ + Create a video edit job. + Maps to POST /v1/videos/edits + """ + local_vars = locals() + try: + litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore + litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) + _is_async = kwargs.pop("async_call", False) is True + + mock_response = kwargs.get("mock_response", None) + if mock_response is not None: + if isinstance(mock_response, str): + mock_response = json.loads(mock_response) + return VideoObject(**mock_response) + + if custom_llm_provider is None: + decoded = decode_video_id_with_provider(video_id) + custom_llm_provider = decoded.get("custom_llm_provider") or "openai" + + litellm_params = GenericLiteLLMParams(**kwargs) + + provider_config: Optional[BaseVideoConfig] = ProviderConfigManager.get_provider_video_config( + model=None, + provider=litellm.LlmProviders(custom_llm_provider), + ) + + if provider_config is None: + raise ValueError(f"video edit is not supported for {custom_llm_provider}") + + local_vars.update(kwargs) + request_params: Dict = {"video_id": video_id, "prompt": prompt} + + litellm_logging_obj.update_environment_variables( + model="", + user=kwargs.get("user"), + optional_params=dict(request_params), + litellm_params={"litellm_call_id": litellm_call_id, **request_params}, + custom_llm_provider=custom_llm_provider, + ) + + litellm_logging_obj.call_type = CallTypes.video_edit.value + + return base_llm_http_handler.video_edit_handler( + prompt=prompt, + video_id=video_id, + video_provider_config=provider_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=litellm_logging_obj, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout or DEFAULT_REQUEST_TIMEOUT, + _is_async=_is_async, + client=kwargs.get("client"), + ) + + except Exception as e: + raise litellm.exception_type( + model="", + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + +@client +async def avideo_extension( + video_id: str, + prompt: str, + seconds: str, + timeout=600, + custom_llm_provider=None, + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + **kwargs, +) -> VideoObject: + """ + Asynchronously create a video extension. + Maps to POST /v1/videos/extensions + """ + local_vars = locals() + try: + loop = asyncio.get_event_loop() + kwargs["async_call"] = True + + func = partial( + video_extension, + video_id=video_id, + prompt=prompt, + seconds=seconds, + timeout=timeout, + custom_llm_provider=custom_llm_provider, + extra_headers=extra_headers, + extra_query=extra_query, + extra_body=extra_body, + **kwargs, + ) + + ctx = contextvars.copy_context() + func_with_context = partial(ctx.run, func) + init_response = await loop.run_in_executor(None, func_with_context) + + if asyncio.iscoroutine(init_response): + response = await init_response + else: + response = init_response + + return response + except Exception as e: + raise litellm.exception_type( + model="", + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) + + +@client +def video_extension( + video_id: str, + prompt: str, + seconds: str, + timeout=600, + custom_llm_provider=None, + extra_headers: Optional[Dict[str, Any]] = None, + extra_query: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + **kwargs, +) -> Union[VideoObject, Coroutine[Any, Any, VideoObject]]: + """ + Create a video extension. + Maps to POST /v1/videos/extensions + """ + local_vars = locals() + try: + litellm_logging_obj: LiteLLMLoggingObj = kwargs.pop("litellm_logging_obj") # type: ignore + litellm_call_id: Optional[str] = kwargs.get("litellm_call_id", None) + _is_async = kwargs.pop("async_call", False) is True + + mock_response = kwargs.get("mock_response", None) + if mock_response is not None: + if isinstance(mock_response, str): + mock_response = json.loads(mock_response) + return VideoObject(**mock_response) + + if custom_llm_provider is None: + decoded = decode_video_id_with_provider(video_id) + custom_llm_provider = decoded.get("custom_llm_provider") or "openai" + + litellm_params = GenericLiteLLMParams(**kwargs) + + provider_config: Optional[BaseVideoConfig] = ProviderConfigManager.get_provider_video_config( + model=None, + provider=litellm.LlmProviders(custom_llm_provider), + ) + + if provider_config is None: + raise ValueError(f"video extension is not supported for {custom_llm_provider}") + + local_vars.update(kwargs) + request_params: Dict = {"video_id": video_id, "prompt": prompt, "seconds": seconds} + + litellm_logging_obj.update_environment_variables( + model="", + user=kwargs.get("user"), + optional_params=dict(request_params), + litellm_params={"litellm_call_id": litellm_call_id, **request_params}, + custom_llm_provider=custom_llm_provider, + ) + + litellm_logging_obj.call_type = CallTypes.video_extension.value + + return base_llm_http_handler.video_extension_handler( + prompt=prompt, + video_id=video_id, + seconds=seconds, + video_provider_config=provider_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=litellm_logging_obj, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout or DEFAULT_REQUEST_TIMEOUT, + _is_async=_is_async, + client=kwargs.get("client"), + ) + + except Exception as e: + raise litellm.exception_type( + model="", + custom_llm_provider=custom_llm_provider, + original_exception=e, + completion_kwargs=local_vars, + extra_kwargs=kwargs, + ) From 14a691ffd541dc8c1a8b031a5f4bbda4aa0fb227 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 16 Mar 2026 17:56:21 +0530 Subject: [PATCH 13/23] Add new videos transformation --- .../llms/base_llm/videos/transformation.py | 107 ++++ litellm/llms/custom_httpx/llm_http_handler.py | 600 ++++++++++++++++++ litellm/llms/gemini/videos/transformation.py | 47 +- litellm/llms/openai/videos/transformation.py | 32 + .../llms/runwayml/videos/transformation.py | 24 + .../llms/vertex_ai/videos/transformation.py | 24 + litellm/proxy/common_request_processing.py | 8 + litellm/proxy/route_llm_request.py | 14 + litellm/router.py | 40 ++ tests/test_litellm/test_video_generation.py | 542 ++++++++++++++++ 10 files changed, 1427 insertions(+), 11 deletions(-) diff --git a/litellm/llms/base_llm/videos/transformation.py b/litellm/llms/base_llm/videos/transformation.py index 2201a63363d..6fb6f69a9e8 100644 --- a/litellm/llms/base_llm/videos/transformation.py +++ b/litellm/llms/base_llm/videos/transformation.py @@ -11,6 +11,7 @@ if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from litellm.types.videos.main import CharacterObject as _CharacterObject from litellm.types.videos.main import VideoObject as _VideoObject from ..chat.transformation import BaseLLMException as _BaseLLMException @@ -18,10 +19,12 @@ LiteLLMLoggingObj = _LiteLLMLoggingObj BaseLLMException = _BaseLLMException VideoObject = _VideoObject + CharacterObject = _CharacterObject else: LiteLLMLoggingObj = Any BaseLLMException = Any VideoObject = Any + CharacterObject = Any class BaseVideoConfig(ABC): @@ -265,6 +268,110 @@ def transform_video_status_retrieve_response( ) -> VideoObject: pass + @abstractmethod + def transform_video_create_character_request( + self, + name: str, + video: Any, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, list]: + """ + Transform the video create character request into a URL and files list (multipart). + + Returns: + Tuple[str, list]: (url, files_list) for the multipart POST request + """ + pass + + @abstractmethod + def transform_video_create_character_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> CharacterObject: + pass + + @abstractmethod + def transform_video_get_character_request( + self, + character_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + ) -> Tuple[str, Dict]: + """ + Transform the video get character request into a URL and params. + + Returns: + Tuple[str, Dict]: (url, params) for the GET request + """ + pass + + @abstractmethod + def transform_video_get_character_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + ) -> CharacterObject: + pass + + @abstractmethod + def transform_video_edit_request( + self, + prompt: str, + video_id: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + extra_body: Optional[Dict[str, Any]] = None, + ) -> Tuple[str, Dict]: + """ + Transform the video edit request into a URL and JSON data. + + Returns: + Tuple[str, Dict]: (url, data) for the POST request + """ + pass + + @abstractmethod + def transform_video_edit_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: Optional[str] = None, + ) -> VideoObject: + pass + + @abstractmethod + def transform_video_extension_request( + self, + prompt: str, + video_id: str, + seconds: str, + api_base: str, + litellm_params: GenericLiteLLMParams, + headers: dict, + extra_body: Optional[Dict[str, Any]] = None, + ) -> Tuple[str, Dict]: + """ + Transform the video extension request into a URL and JSON data. + + Returns: + Tuple[str, Dict]: (url, data) for the POST request + """ + pass + + @abstractmethod + def transform_video_extension_response( + self, + raw_response: httpx.Response, + logging_obj: LiteLLMLoggingObj, + custom_llm_provider: Optional[str] = None, + ) -> VideoObject: + pass + def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] ) -> BaseLLMException: diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index 4394343c8e3..28f3b24a95a 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6114,6 +6114,606 @@ async def async_video_remix_handler( provider_config=video_remix_provider_config, ) + def video_create_character_handler( + self, + name: str, + video: Any, + video_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + _is_async: bool = False, + client=None, + api_key: Optional[str] = None, + ): + if _is_async: + return self.async_video_create_character_handler( + name=name, + video=video, + video_provider_config=video_provider_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + timeout=timeout, + client=client, + api_key=api_key, + ) + + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = video_provider_config.validate_environment( + api_key=api_key or litellm_params.get("api_key", None), + headers=extra_headers or {}, + model="", + ) + if extra_headers: + headers.update(extra_headers) + + api_base = video_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + url, files_list = video_provider_config.transform_video_create_character_request( + name=name, + video=video, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + logging_obj.pre_call( + input=name, + api_key="", + additional_args={ + "complete_input_dict": {"name": name}, + "api_base": url, + "headers": headers, + }, + ) + + try: + response = sync_httpx_client.post( + url=url, + headers=headers, + files=files_list, + timeout=timeout, + ) + return video_provider_config.transform_video_create_character_response( + raw_response=response, + logging_obj=logging_obj, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=video_provider_config) + + async def async_video_create_character_handler( + self, + name: str, + video: Any, + video_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + client=None, + api_key: Optional[str] = None, + ): + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = video_provider_config.validate_environment( + api_key=api_key or litellm_params.get("api_key", None), + headers=extra_headers or {}, + model="", + ) + if extra_headers: + headers.update(extra_headers) + + api_base = video_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + url, files_list = video_provider_config.transform_video_create_character_request( + name=name, + video=video, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + logging_obj.pre_call( + input=name, + api_key="", + additional_args={ + "complete_input_dict": {"name": name}, + "api_base": url, + "headers": headers, + }, + ) + + try: + response = await async_httpx_client.post( + url=url, + headers=headers, + files=files_list, + timeout=timeout, + ) + return video_provider_config.transform_video_create_character_response( + raw_response=response, + logging_obj=logging_obj, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=video_provider_config) + + def video_get_character_handler( + self, + character_id: str, + video_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + _is_async: bool = False, + client=None, + api_key: Optional[str] = None, + ): + if _is_async: + return self.async_video_get_character_handler( + character_id=character_id, + video_provider_config=video_provider_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + timeout=timeout, + client=client, + api_key=api_key, + ) + + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = video_provider_config.validate_environment( + api_key=api_key or litellm_params.get("api_key", None), + headers=extra_headers or {}, + model="", + ) + if extra_headers: + headers.update(extra_headers) + + api_base = video_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + url, params = video_provider_config.transform_video_get_character_request( + character_id=character_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + logging_obj.pre_call( + input=character_id, + api_key="", + additional_args={"api_base": url, "headers": headers}, + ) + + try: + response = sync_httpx_client.get( + url=url, + headers=headers, + params=params + ) + return video_provider_config.transform_video_get_character_response( + raw_response=response, + logging_obj=logging_obj, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=video_provider_config) + + async def async_video_get_character_handler( + self, + character_id: str, + video_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + client=None, + api_key: Optional[str] = None, + ): + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = video_provider_config.validate_environment( + api_key=api_key or litellm_params.get("api_key", None), + headers=extra_headers or {}, + model="", + ) + if extra_headers: + headers.update(extra_headers) + + api_base = video_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + url, params = video_provider_config.transform_video_get_character_request( + character_id=character_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + ) + + logging_obj.pre_call( + input=character_id, + api_key="", + additional_args={"api_base": url, "headers": headers}, + ) + + try: + response = await async_httpx_client.get( + url=url, + headers=headers, + params=params + ) + return video_provider_config.transform_video_get_character_response( + raw_response=response, + logging_obj=logging_obj, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=video_provider_config) + + def video_edit_handler( + self, + prompt: str, + video_id: str, + video_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + _is_async: bool = False, + client=None, + api_key: Optional[str] = None, + ): + if _is_async: + return self.async_video_edit_handler( + prompt=prompt, + video_id=video_id, + video_provider_config=video_provider_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout, + client=client, + api_key=api_key, + ) + + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = video_provider_config.validate_environment( + api_key=api_key or litellm_params.get("api_key", None), + headers=extra_headers or {}, + model="", + ) + if extra_headers: + headers.update(extra_headers) + + api_base = video_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + url, data = video_provider_config.transform_video_edit_request( + prompt=prompt, + video_id=video_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + extra_body=extra_body, + ) + + logging_obj.pre_call( + input=prompt, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": url, + "headers": headers, + "video_id": video_id, + }, + ) + + try: + response = sync_httpx_client.post( + url=url, + headers=headers, + json=data, + timeout=timeout, + ) + return video_provider_config.transform_video_edit_response( + raw_response=response, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=video_provider_config) + + async def async_video_edit_handler( + self, + prompt: str, + video_id: str, + video_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + client=None, + api_key: Optional[str] = None, + ): + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = video_provider_config.validate_environment( + api_key=api_key or litellm_params.get("api_key", None), + headers=extra_headers or {}, + model="", + ) + if extra_headers: + headers.update(extra_headers) + + api_base = video_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + url, data = video_provider_config.transform_video_edit_request( + prompt=prompt, + video_id=video_id, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + extra_body=extra_body, + ) + + logging_obj.pre_call( + input=prompt, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": url, + "headers": headers, + "video_id": video_id, + }, + ) + + try: + response = await async_httpx_client.post( + url=url, + headers=headers, + json=data, + timeout=timeout, + ) + return video_provider_config.transform_video_edit_response( + raw_response=response, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=video_provider_config) + + def video_extension_handler( + self, + prompt: str, + video_id: str, + seconds: str, + video_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + _is_async: bool = False, + client=None, + api_key: Optional[str] = None, + ): + if _is_async: + return self.async_video_extension_handler( + prompt=prompt, + video_id=video_id, + seconds=seconds, + video_provider_config=video_provider_config, + custom_llm_provider=custom_llm_provider, + litellm_params=litellm_params, + logging_obj=logging_obj, + extra_headers=extra_headers, + extra_body=extra_body, + timeout=timeout, + client=client, + api_key=api_key, + ) + + if client is None or not isinstance(client, HTTPHandler): + sync_httpx_client = _get_httpx_client( + params={"ssl_verify": litellm_params.get("ssl_verify", None)} + ) + else: + sync_httpx_client = client + + headers = video_provider_config.validate_environment( + api_key=api_key or litellm_params.get("api_key", None), + headers=extra_headers or {}, + model="", + ) + if extra_headers: + headers.update(extra_headers) + + api_base = video_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + url, data = video_provider_config.transform_video_extension_request( + prompt=prompt, + video_id=video_id, + seconds=seconds, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + extra_body=extra_body, + ) + + logging_obj.pre_call( + input=prompt, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": url, + "headers": headers, + "video_id": video_id, + }, + ) + + try: + response = sync_httpx_client.post( + url=url, + headers=headers, + json=data, + timeout=timeout, + ) + return video_provider_config.transform_video_extension_response( + raw_response=response, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=video_provider_config) + + async def async_video_extension_handler( + self, + prompt: str, + video_id: str, + seconds: str, + video_provider_config: BaseVideoConfig, + custom_llm_provider: str, + litellm_params, + logging_obj, + extra_headers: Optional[Dict[str, Any]] = None, + extra_body: Optional[Dict[str, Any]] = None, + timeout: Optional[float] = None, + client=None, + api_key: Optional[str] = None, + ): + if client is None or not isinstance(client, AsyncHTTPHandler): + async_httpx_client = get_async_httpx_client( + llm_provider=litellm.LlmProviders(custom_llm_provider), + params={"ssl_verify": litellm_params.get("ssl_verify", None)}, + ) + else: + async_httpx_client = client + + headers = video_provider_config.validate_environment( + api_key=api_key or litellm_params.get("api_key", None), + headers=extra_headers or {}, + model="", + ) + if extra_headers: + headers.update(extra_headers) + + api_base = video_provider_config.get_complete_url( + model="", + api_base=litellm_params.get("api_base", None), + litellm_params=dict(litellm_params), + ) + + url, data = video_provider_config.transform_video_extension_request( + prompt=prompt, + video_id=video_id, + seconds=seconds, + api_base=api_base, + litellm_params=litellm_params, + headers=headers, + extra_body=extra_body, + ) + + logging_obj.pre_call( + input=prompt, + api_key="", + additional_args={ + "complete_input_dict": data, + "api_base": url, + "headers": headers, + "video_id": video_id, + }, + ) + + try: + response = await async_httpx_client.post( + url=url, + headers=headers, + json=data, + timeout=timeout, + ) + return video_provider_config.transform_video_extension_response( + raw_response=response, + logging_obj=logging_obj, + custom_llm_provider=custom_llm_provider, + ) + except Exception as e: + raise self._handle_error(e=e, provider_config=video_provider_config) + def video_list_handler( self, after: Optional[str], diff --git a/litellm/llms/gemini/videos/transformation.py b/litellm/llms/gemini/videos/transformation.py index c16b20fe579..0798472310e 100644 --- a/litellm/llms/gemini/videos/transformation.py +++ b/litellm/llms/gemini/videos/transformation.py @@ -1,29 +1,30 @@ -from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union import base64 +from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union import httpx from httpx._types import RequestFiles -from litellm.types.videos.main import VideoCreateOptionalRequestParams, VideoObject -from litellm.types.router import GenericLiteLLMParams -from litellm.secret_managers.main import get_secret_str -from litellm.types.videos.utils import ( - encode_video_id_with_provider, - extract_original_video_id, -) -from litellm.images.utils import ImageEditRequestUtils import litellm +from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS +from litellm.images.utils import ImageEditRequestUtils +from litellm.llms.base_llm.videos.transformation import BaseVideoConfig +from litellm.secret_managers.main import get_secret_str from litellm.types.llms.gemini import ( GeminiLongRunningOperationResponse, GeminiVideoGenerationInstance, GeminiVideoGenerationParameters, GeminiVideoGenerationRequest, ) -from litellm.constants import DEFAULT_GOOGLE_VIDEO_DURATION_SECONDS -from litellm.llms.base_llm.videos.transformation import BaseVideoConfig +from litellm.types.router import GenericLiteLLMParams +from litellm.types.videos.main import VideoCreateOptionalRequestParams, VideoObject +from litellm.types.videos.utils import ( + encode_video_id_with_provider, + extract_original_video_id, +) if TYPE_CHECKING: from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj + from ...base_llm.chat.transformation import BaseLLMException as _BaseLLMException LiteLLMLoggingObj = _LiteLLMLoggingObj @@ -524,6 +525,30 @@ def transform_video_delete_response( """Video delete is not supported.""" raise NotImplementedError("Video delete is not supported by Google Veo.") + def transform_video_create_character_request(self, name, video, api_base, litellm_params, headers): + raise NotImplementedError("video create character is not supported for Gemini") + + def transform_video_create_character_response(self, raw_response, logging_obj): + raise NotImplementedError("video create character is not supported for Gemini") + + def transform_video_get_character_request(self, character_id, api_base, litellm_params, headers): + raise NotImplementedError("video get character is not supported for Gemini") + + def transform_video_get_character_response(self, raw_response, logging_obj): + raise NotImplementedError("video get character is not supported for Gemini") + + def transform_video_edit_request(self, prompt, video_id, api_base, litellm_params, headers, extra_body=None): + raise NotImplementedError("video edit is not supported for Gemini") + + def transform_video_edit_response(self, raw_response, logging_obj, custom_llm_provider=None): + raise NotImplementedError("video edit is not supported for Gemini") + + def transform_video_extension_request(self, prompt, video_id, seconds, api_base, litellm_params, headers, extra_body=None): + raise NotImplementedError("video extension is not supported for Gemini") + + def transform_video_extension_response(self, raw_response, logging_obj, custom_llm_provider=None): + raise NotImplementedError("video extension is not supported for Gemini") + def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] ) -> BaseLLMException: diff --git a/litellm/llms/openai/videos/transformation.py b/litellm/llms/openai/videos/transformation.py index 0501a67fc16..61baa56949c 100644 --- a/litellm/llms/openai/videos/transformation.py +++ b/litellm/llms/openai/videos/transformation.py @@ -18,6 +18,7 @@ ) from litellm.types.videos.utils import ( encode_video_id_with_provider, + extract_original_character_id, extract_original_video_id, ) @@ -51,6 +52,7 @@ def get_supported_openai_params(self, model: str) -> list: "input_reference", "seconds", "size", + "characters", "user", "extra_headers", ] @@ -126,6 +128,7 @@ def transform_video_create_request( model=model, prompt=prompt, **video_create_optional_request_params ) request_dict = cast(Dict, video_create_request) + request_dict = self._decode_character_ids_in_create_video_request(request_dict) # Handle input_reference parameter if provided _input_reference = video_create_optional_request_params.get("input_reference") @@ -143,6 +146,35 @@ def transform_video_create_request( ) return data_without_files, files_list, api_base + def _decode_character_ids_in_create_video_request(self, request_dict: Dict) -> Dict: + """ + Decode LiteLLM-managed encoded character ids for provider requests. + + OpenAI expects character ids like `char_...`. If a caller sends + `character_`, convert it back to the + original provider id before forwarding upstream. + """ + raw_characters = request_dict.get("characters") + if not isinstance(raw_characters, list): + return request_dict + + decoded_characters: List[Any] = [] + for character in raw_characters: + if not isinstance(character, dict): + decoded_characters.append(character) + continue + + character_id = character.get("id") + if isinstance(character_id, str): + decoded_character = dict(character) + decoded_character["id"] = extract_original_character_id(character_id) + decoded_characters.append(decoded_character) + else: + decoded_characters.append(character) + + request_dict["characters"] = decoded_characters + return request_dict + def transform_video_create_response( self, model: str, diff --git a/litellm/llms/runwayml/videos/transformation.py b/litellm/llms/runwayml/videos/transformation.py index 3fc656a92bd..2c29c2e21ee 100644 --- a/litellm/llms/runwayml/videos/transformation.py +++ b/litellm/llms/runwayml/videos/transformation.py @@ -592,6 +592,30 @@ def transform_video_status_retrieve_response( return video_obj + def transform_video_create_character_request(self, name, video, api_base, litellm_params, headers): + raise NotImplementedError("video create character is not supported for RunwayML") + + def transform_video_create_character_response(self, raw_response, logging_obj): + raise NotImplementedError("video create character is not supported for RunwayML") + + def transform_video_get_character_request(self, character_id, api_base, litellm_params, headers): + raise NotImplementedError("video get character is not supported for RunwayML") + + def transform_video_get_character_response(self, raw_response, logging_obj): + raise NotImplementedError("video get character is not supported for RunwayML") + + def transform_video_edit_request(self, prompt, video_id, api_base, litellm_params, headers, extra_body=None): + raise NotImplementedError("video edit is not supported for RunwayML") + + def transform_video_edit_response(self, raw_response, logging_obj, custom_llm_provider=None): + raise NotImplementedError("video edit is not supported for RunwayML") + + def transform_video_extension_request(self, prompt, video_id, seconds, api_base, litellm_params, headers, extra_body=None): + raise NotImplementedError("video extension is not supported for RunwayML") + + def transform_video_extension_response(self, raw_response, logging_obj, custom_llm_provider=None): + raise NotImplementedError("video extension is not supported for RunwayML") + def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] ) -> BaseLLMException: diff --git a/litellm/llms/vertex_ai/videos/transformation.py b/litellm/llms/vertex_ai/videos/transformation.py index e61f2f46ec8..07b3d6faf70 100644 --- a/litellm/llms/vertex_ai/videos/transformation.py +++ b/litellm/llms/vertex_ai/videos/transformation.py @@ -624,6 +624,30 @@ def transform_video_delete_response( """Video delete is not supported.""" raise NotImplementedError("Video delete is not supported by Vertex AI Veo.") + def transform_video_create_character_request(self, name, video, api_base, litellm_params, headers): + raise NotImplementedError("video create character is not supported for Vertex AI") + + def transform_video_create_character_response(self, raw_response, logging_obj): + raise NotImplementedError("video create character is not supported for Vertex AI") + + def transform_video_get_character_request(self, character_id, api_base, litellm_params, headers): + raise NotImplementedError("video get character is not supported for Vertex AI") + + def transform_video_get_character_response(self, raw_response, logging_obj): + raise NotImplementedError("video get character is not supported for Vertex AI") + + def transform_video_edit_request(self, prompt, video_id, api_base, litellm_params, headers, extra_body=None): + raise NotImplementedError("video edit is not supported for Vertex AI") + + def transform_video_edit_response(self, raw_response, logging_obj, custom_llm_provider=None): + raise NotImplementedError("video edit is not supported for Vertex AI") + + def transform_video_extension_request(self, prompt, video_id, seconds, api_base, litellm_params, headers, extra_body=None): + raise NotImplementedError("video extension is not supported for Vertex AI") + + def transform_video_extension_response(self, raw_response, logging_obj, custom_llm_provider=None): + raise NotImplementedError("video extension is not supported for Vertex AI") + def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers] ) -> BaseLLMException: diff --git a/litellm/proxy/common_request_processing.py b/litellm/proxy/common_request_processing.py index a9e9d519f6f..72765aab7da 100644 --- a/litellm/proxy/common_request_processing.py +++ b/litellm/proxy/common_request_processing.py @@ -599,6 +599,10 @@ async def common_processing_pre_call_logic( "avideo_status", "avideo_content", "avideo_remix", + "avideo_create_character", + "avideo_get_character", + "avideo_edit", + "avideo_extension", "acreate_container", "alist_containers", "aingest", @@ -850,6 +854,10 @@ async def base_process_llm_request( "avideo_status", "avideo_content", "avideo_remix", + "avideo_create_character", + "avideo_get_character", + "avideo_edit", + "avideo_extension", "acreate_container", "alist_containers", "aingest", diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index 6e02d28b383..a8169ca6d08 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -54,6 +54,10 @@ def _is_a2a_agent_model(model_name: Any) -> bool: "avideo_status": "/videos/{video_id}", "avideo_content": "/videos/{video_id}/content", "avideo_remix": "/videos/{video_id}/remix", + "avideo_create_character": "/videos/characters", + "avideo_get_character": "/videos/characters/{character_id}", + "avideo_edit": "/videos/edits", + "avideo_extension": "/videos/extensions", "acreate_realtime_client_secret": "/realtime/client_secrets", "arealtime_calls": "/realtime/calls", "acreate_container": "/containers", @@ -201,6 +205,10 @@ async def route_request( # noqa: PLR0915 - Complex routing function, refactorin "avideo_status", "avideo_content", "avideo_remix", + "avideo_create_character", + "avideo_get_character", + "avideo_edit", + "avideo_extension", "acreate_container", "alist_containers", "aretrieve_container", @@ -370,6 +378,10 @@ async def route_request( # noqa: PLR0915 - Complex routing function, refactorin "avideo_status", "avideo_content", "avideo_remix", + "avideo_create_character", + "avideo_get_character", + "avideo_edit", + "avideo_extension", "avector_store_file_list", "avector_store_file_retrieve", "avector_store_file_content", @@ -449,6 +461,8 @@ async def route_request( # noqa: PLR0915 - Complex routing function, refactorin "avideo_status", "avideo_content", "avideo_remix", + "avideo_edit", + "avideo_extension", ]: # Video endpoints: If model is provided (e.g., from decoded video_id), try router first try: diff --git a/litellm/router.py b/litellm/router.py index 0fd4ba80c74..f34368172ac 100644 --- a/litellm/router.py +++ b/litellm/router.py @@ -1076,12 +1076,20 @@ def _initialize_video_endpoints(self): """Initialize video endpoints.""" from litellm.videos import ( avideo_content, + avideo_create_character, + avideo_edit, + avideo_extension, avideo_generation, + avideo_get_character, avideo_list, avideo_remix, avideo_status, video_content, + video_create_character, + video_edit, + video_extension, video_generation, + video_get_character, video_list, video_remix, video_status, @@ -1111,6 +1119,26 @@ def _initialize_video_endpoints(self): avideo_remix, call_type="avideo_remix" ) self.video_remix = self.factory_function(video_remix, call_type="video_remix") + self.avideo_create_character = self.factory_function( + avideo_create_character, call_type="avideo_create_character" + ) + self.video_create_character = self.factory_function( + video_create_character, call_type="video_create_character" + ) + self.avideo_get_character = self.factory_function( + avideo_get_character, call_type="avideo_get_character" + ) + self.video_get_character = self.factory_function( + video_get_character, call_type="video_get_character" + ) + self.avideo_edit = self.factory_function(avideo_edit, call_type="avideo_edit") + self.video_edit = self.factory_function(video_edit, call_type="video_edit") + self.avideo_extension = self.factory_function( + avideo_extension, call_type="avideo_extension" + ) + self.video_extension = self.factory_function( + video_extension, call_type="video_extension" + ) def _initialize_container_endpoints(self): """Initialize container endpoints.""" @@ -4828,6 +4856,14 @@ def factory_function( "video_content", "avideo_remix", "video_remix", + "avideo_create_character", + "video_create_character", + "avideo_get_character", + "video_get_character", + "avideo_edit", + "video_edit", + "avideo_extension", + "video_extension", "acreate_container", "create_container", "alist_containers", @@ -4995,6 +5031,10 @@ async def async_wrapper( "avideo_status", "avideo_content", "avideo_remix", + "avideo_create_character", + "avideo_get_character", + "avideo_edit", + "avideo_extension", "acreate_skill", "alist_skills", "aget_skill", diff --git a/tests/test_litellm/test_video_generation.py b/tests/test_litellm/test_video_generation.py index 661cdd87099..b65db466b9f 100644 --- a/tests/test_litellm/test_video_generation.py +++ b/tests/test_litellm/test_video_generation.py @@ -1,4 +1,5 @@ import asyncio +import io import json import os import sys @@ -174,6 +175,34 @@ def test_video_generation_request_transformation(self): assert files == [] assert returned_api_base == "https://api.openai.com/v1/videos" + def test_video_generation_request_decodes_encoded_character_ids(self): + """Encoded character IDs should be decoded before upstream create-video call.""" + from litellm.types.videos.utils import encode_character_id_with_provider + + config = OpenAIVideoConfig() + encoded_character_id = encode_character_id_with_provider( + character_id="char_123", + provider="openai", + model_id="sora-2", + ) + + data, files, returned_api_base = config.transform_video_create_request( + model="sora-2", + prompt="Test video prompt", + api_base="https://api.openai.com/v1/videos", + video_create_optional_request_params={ + "seconds": "8", + "size": "720x1280", + "characters": [{"id": encoded_character_id}], + }, + litellm_params=MagicMock(), + headers={}, + ) + + assert data["characters"] == [{"id": "char_123"}] + assert files == [] + assert returned_api_base == "https://api.openai.com/v1/videos" + def test_video_generation_response_transformation(self): """Test video generation response transformation.""" config = OpenAIVideoConfig() @@ -1623,3 +1652,516 @@ def test_video_remix_handler_prefers_explicit_api_key(): if __name__ == "__main__": pytest.main([__file__]) + + +# ===== Tests for new video endpoints (characters, edits, extensions) ===== + + +class TestVideoCreateCharacter: + """Tests for video_create_character / avideo_create_character.""" + + def test_video_create_character_transform_request(self): + """Verify multipart form construction for POST /videos/characters.""" + config = OpenAIVideoConfig() + fake_video = b"fake_video_bytes" + + url, files_list = config.transform_video_create_character_request( + name="hero", + video=fake_video, + api_base="https://api.openai.com/v1/videos", + litellm_params=MagicMock(), + headers={}, + ) + + assert url == "https://api.openai.com/v1/videos/characters" + # Should have (name field) + (video file field) = 2 entries + assert len(files_list) == 2 + field_names = [f[0] for f in files_list] + assert "name" in field_names + assert "video" in field_names + + def test_video_create_character_sets_video_mimetype(self): + """Ensure character video upload is sent as video/mp4.""" + config = OpenAIVideoConfig() + fake_video = io.BytesIO(b"....ftyp....video-bytes") + fake_video.name = "character.mp4" + + _, files_list = config.transform_video_create_character_request( + name="hero", + video=fake_video, + api_base="https://api.openai.com/v1/videos", + litellm_params=MagicMock(), + headers={}, + ) + + video_parts = [f for f in files_list if f[0] == "video"] + assert len(video_parts) == 1 + video_tuple = video_parts[0][1] + assert video_tuple[0] == "character.mp4" + assert video_tuple[2] == "video/mp4" + + def test_video_create_character_transform_response(self): + """Verify CharacterObject is returned from response.""" + from litellm.types.videos.main import CharacterObject + + config = OpenAIVideoConfig() + mock_response = MagicMock() + mock_response.json.return_value = { + "id": "char_abc123", + "object": "character", + "created_at": 1712697600, + "name": "hero", + } + + result = config.transform_video_create_character_response( + raw_response=mock_response, + logging_obj=MagicMock(), + ) + + assert isinstance(result, CharacterObject) + assert result.id == "char_abc123" + assert result.name == "hero" + + def test_video_create_character_mock_response(self): + """video_create_character returns CharacterObject on mock_response.""" + from litellm.types.videos.main import CharacterObject + from litellm.videos.main import video_create_character + + response = video_create_character( + name="hero", + video=b"fake", + mock_response={ + "id": "char_abc", + "object": "character", + "created_at": 1712697600, + "name": "hero", + }, + ) + assert isinstance(response, CharacterObject) + assert response.id == "char_abc" + + +class TestVideoGetCharacter: + """Tests for video_get_character / avideo_get_character.""" + + def test_video_get_character_transform_request(self): + """Verify URL construction for GET /videos/characters/{character_id}.""" + config = OpenAIVideoConfig() + + url, params = config.transform_video_get_character_request( + character_id="char_xyz", + api_base="https://api.openai.com/v1/videos", + litellm_params=MagicMock(), + headers={}, + ) + + assert url == "https://api.openai.com/v1/videos/characters/char_xyz" + assert params == {} + + def test_video_get_character_transform_response(self): + """Verify CharacterObject is returned from GET response.""" + from litellm.types.videos.main import CharacterObject + + config = OpenAIVideoConfig() + mock_response = MagicMock() + mock_response.json.return_value = { + "id": "char_xyz", + "object": "character", + "created_at": 1712697600, + "name": "villain", + } + + result = config.transform_video_get_character_response( + raw_response=mock_response, + logging_obj=MagicMock(), + ) + + assert isinstance(result, CharacterObject) + assert result.id == "char_xyz" + assert result.name == "villain" + + def test_video_get_character_mock_response(self): + """video_get_character returns CharacterObject on mock_response.""" + from litellm.types.videos.main import CharacterObject + from litellm.videos.main import video_get_character + + response = video_get_character( + character_id="char_xyz", + mock_response={ + "id": "char_xyz", + "object": "character", + "created_at": 1712697600, + "name": "villain", + }, + ) + assert isinstance(response, CharacterObject) + assert response.id == "char_xyz" + + +class TestVideoEdit: + """Tests for video_edit / avideo_edit.""" + + def test_video_edit_transform_request(self): + """Verify JSON body with video.id for POST /videos/edits.""" + config = OpenAIVideoConfig() + + url, data = config.transform_video_edit_request( + prompt="make it brighter", + video_id="video_abc123", + api_base="https://api.openai.com/v1/videos", + litellm_params=MagicMock(), + headers={}, + ) + + assert url == "https://api.openai.com/v1/videos/edits" + assert data["prompt"] == "make it brighter" + assert data["video"]["id"] == "video_abc123" + + def test_video_edit_transform_request_with_extra_body(self): + """Extra body params are merged into request data.""" + config = OpenAIVideoConfig() + + url, data = config.transform_video_edit_request( + prompt="darken it", + video_id="video_abc123", + api_base="https://api.openai.com/v1/videos", + litellm_params=MagicMock(), + headers={}, + extra_body={"resolution": "1080p"}, + ) + + assert data["resolution"] == "1080p" + + def test_video_edit_mock_response(self): + """video_edit returns VideoObject on mock_response.""" + from litellm.videos.main import video_edit + + response = video_edit( + video_id="video_abc123", + prompt="make it brighter", + mock_response={ + "id": "video_edit_001", + "object": "video", + "status": "queued", + "created_at": 1712697600, + }, + ) + assert isinstance(response, VideoObject) + assert response.id == "video_edit_001" + + def test_video_edit_strips_encoded_provider_from_video_id(self): + """Provider-encoded video IDs are decoded before sending to API.""" + from litellm.types.videos.utils import encode_video_id_with_provider + config = OpenAIVideoConfig() + + encoded_id = encode_video_id_with_provider("raw_video_id", "openai", None) + url, data = config.transform_video_edit_request( + prompt="test", + video_id=encoded_id, + api_base="https://api.openai.com/v1/videos", + litellm_params=MagicMock(), + headers={}, + ) + + # The video.id in the request body should be the raw ID, not the encoded one + assert data["video"]["id"] == "raw_video_id" + + +class TestVideoExtension: + """Tests for video_extension / avideo_extension.""" + + def test_video_extension_transform_request(self): + """Verify JSON body with video.id + seconds for POST /videos/extensions.""" + config = OpenAIVideoConfig() + + url, data = config.transform_video_extension_request( + prompt="continue the scene", + video_id="video_abc123", + seconds="5", + api_base="https://api.openai.com/v1/videos", + litellm_params=MagicMock(), + headers={}, + ) + + assert url == "https://api.openai.com/v1/videos/extensions" + assert data["prompt"] == "continue the scene" + assert data["seconds"] == "5" + assert data["video"]["id"] == "video_abc123" + + def test_video_extension_transform_request_with_extra_body(self): + """Extra body params are merged into request data.""" + config = OpenAIVideoConfig() + + url, data = config.transform_video_extension_request( + prompt="extend", + video_id="video_abc123", + seconds="10", + api_base="https://api.openai.com/v1/videos", + litellm_params=MagicMock(), + headers={}, + extra_body={"model": "sora-2"}, + ) + + assert data["model"] == "sora-2" + + def test_video_extension_mock_response(self): + """video_extension returns VideoObject on mock_response.""" + from litellm.videos.main import video_extension + + response = video_extension( + video_id="video_abc123", + prompt="continue the scene", + seconds="5", + mock_response={ + "id": "video_ext_001", + "object": "video", + "status": "queued", + "created_at": 1712697600, + }, + ) + assert isinstance(response, VideoObject) + assert response.id == "video_ext_001" + + def test_video_extension_strips_encoded_provider_from_video_id(self): + """Provider-encoded video IDs are decoded before sending to API.""" + from litellm.types.videos.utils import encode_video_id_with_provider + config = OpenAIVideoConfig() + + encoded_id = encode_video_id_with_provider("raw_video_id", "openai", None) + url, data = config.transform_video_extension_request( + prompt="extend", + video_id=encoded_id, + seconds="5", + api_base="https://api.openai.com/v1/videos", + litellm_params=MagicMock(), + headers={}, + ) + + assert data["video"]["id"] == "raw_video_id" + + +@pytest.fixture +def video_proxy_test_client(): + from fastapi import FastAPI + from fastapi.testclient import TestClient + + from litellm.proxy.auth.user_api_key_auth import user_api_key_auth + from litellm.proxy.video_endpoints.endpoints import router as video_router + + app = FastAPI() + app.include_router(video_router) + app.dependency_overrides[user_api_key_auth] = lambda: MagicMock() + return TestClient(app) + + +def test_character_id_encode_decode_roundtrip(): + from litellm.types.videos.utils import ( + decode_character_id_with_provider, + encode_character_id_with_provider, + ) + + encoded = encode_character_id_with_provider( + character_id="char_raw_123", + provider="vertex_ai", + model_id="veo-2.0-generate-001", + ) + decoded = decode_character_id_with_provider(encoded) + + assert decoded["character_id"] == "char_raw_123" + assert decoded["custom_llm_provider"] == "vertex_ai" + assert decoded["model_id"] == "veo-2.0-generate-001" + + +def test_character_id_decode_handles_missing_base64_padding(): + from litellm.types.videos.utils import ( + decode_character_id_with_provider, + encode_character_id_with_provider, + ) + + encoded = encode_character_id_with_provider( + character_id="id", + provider="openai", + model_id="gpt-4o", + ) + encoded_without_padding = encoded.rstrip("=") + decoded = decode_character_id_with_provider(encoded_without_padding) + + assert decoded["character_id"] == "id" + assert decoded["custom_llm_provider"] == "openai" + assert decoded["model_id"] == "gpt-4o" + + +def test_video_create_character_target_model_names_returns_encoded_id(video_proxy_test_client): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.types.videos.utils import decode_character_id_with_provider + + captured_data = {} + + async def _mock_base_process(self, **kwargs): + captured_data.update(self.data) + return { + "id": "char_upstream_123", + "object": "character", + "created_at": 1712697600, + "name": "hero", + } + + with patch.object( + ProxyBaseLLMRequestProcessing, + "base_process_llm_request", + new=_mock_base_process, + ): + response = video_proxy_test_client.post( + "/v1/videos/characters", + headers={"Authorization": "Bearer sk-1234"}, + files={"video": ("character.mp4", b"fake-video", "video/mp4")}, + data={ + "name": "hero", + "target_model_names": "vertex-ai-sora-2", + "extra_body": json.dumps({"custom_llm_provider": "vertex_ai"}), + }, + ) + + assert response.status_code == 200, response.text + response_json = response.json() + decoded = decode_character_id_with_provider(response_json["id"]) + assert decoded["character_id"] == "char_upstream_123" + assert decoded["custom_llm_provider"] == "vertex_ai" + assert decoded["model_id"] == "vertex-ai-sora-2" + assert captured_data["model"] == "vertex-ai-sora-2" + assert captured_data["custom_llm_provider"] == "vertex_ai" + + +def test_video_get_character_accepts_encoded_character_id(video_proxy_test_client): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.types.videos.utils import ( + decode_character_id_with_provider, + encode_character_id_with_provider, + ) + + captured_data = {} + + async def _mock_base_process(self, **kwargs): + captured_data.update(self.data) + return { + "id": "char_upstream_123", + "object": "character", + "created_at": 1712697600, + "name": "hero", + } + + encoded_character_id = encode_character_id_with_provider( + character_id="char_upstream_123", + provider="vertex_ai", + model_id="veo-2.0-generate-001", + ) + mock_router = MagicMock() + mock_router.resolve_model_name_from_model_id.return_value = "vertex-ai-sora-2" + + with patch("litellm.proxy.proxy_server.llm_router", mock_router): + with patch.object( + ProxyBaseLLMRequestProcessing, + "base_process_llm_request", + new=_mock_base_process, + ): + response = video_proxy_test_client.get( + f"/v1/videos/characters/{encoded_character_id}", + headers={"Authorization": "Bearer sk-1234"}, + ) + + assert response.status_code == 200, response.text + assert captured_data["character_id"] == "char_upstream_123" + assert captured_data["custom_llm_provider"] == "vertex_ai" + assert captured_data["model"] == "vertex-ai-sora-2" + response_decoded = decode_character_id_with_provider(response.json()["id"]) + assert response_decoded["character_id"] == "char_upstream_123" + assert response_decoded["custom_llm_provider"] == "vertex_ai" + assert response_decoded["model_id"] == "veo-2.0-generate-001" + + +@pytest.mark.parametrize("endpoint", ["/v1/videos/edits", "/v1/videos/extensions"]) +def test_edit_and_extension_support_custom_provider_from_extra_body( + video_proxy_test_client, endpoint +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + + captured_data = {} + + async def _mock_base_process(self, **kwargs): + captured_data.update(self.data) + return { + "id": "video_resp_123", + "object": "video", + "status": "queued", + "created_at": 1712697600, + } + + payload = { + "prompt": "test", + "video": {"id": "video_raw_123"}, + "extra_body": {"custom_llm_provider": "vertex_ai"}, + } + if endpoint.endswith("extensions"): + payload["seconds"] = "4" + + with patch.object( + ProxyBaseLLMRequestProcessing, + "base_process_llm_request", + new=_mock_base_process, + ): + response = video_proxy_test_client.post( + endpoint, + headers={"Authorization": "Bearer sk-1234"}, + json=payload, + ) + + assert response.status_code == 200, response.text + assert captured_data["custom_llm_provider"] == "vertex_ai" + + +@pytest.mark.parametrize("endpoint", ["/v1/videos/edits", "/v1/videos/extensions"]) +def test_edit_and_extension_route_with_encoded_video_ids( + video_proxy_test_client, endpoint +): + from litellm.proxy.common_request_processing import ProxyBaseLLMRequestProcessing + from litellm.types.videos.utils import encode_video_id_with_provider + + captured_data = {} + + async def _mock_base_process(self, **kwargs): + captured_data.update(self.data) + return { + "id": "video_resp_123", + "object": "video", + "status": "queued", + "created_at": 1712697600, + } + + encoded_video_id = encode_video_id_with_provider( + video_id="video_raw_123", + provider="vertex_ai", + model_id="veo-2.0-generate-001", + ) + payload = {"prompt": "test", "video": {"id": encoded_video_id}} + if endpoint.endswith("extensions"): + payload["seconds"] = "4" + + mock_router = MagicMock() + mock_router.resolve_model_name_from_model_id.return_value = "vertex-ai-sora-2" + + with patch("litellm.proxy.proxy_server.llm_router", mock_router): + with patch.object( + ProxyBaseLLMRequestProcessing, + "base_process_llm_request", + new=_mock_base_process, + ): + response = video_proxy_test_client.post( + endpoint, + headers={"Authorization": "Bearer sk-1234"}, + json=payload, + ) + + assert response.status_code == 200, response.text + assert captured_data["video_id"] == encoded_video_id + assert captured_data["custom_llm_provider"] == "vertex_ai" + assert captured_data["model"] == "vertex-ai-sora-2" From 430f3ac4292b565373e8c2eb27fd91623cd0a668 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 16 Mar 2026 17:57:14 +0530 Subject: [PATCH 14/23] Add new videos docs --- .../docs/providers/openai/videos.md | 75 ++++++++++++++++++ docs/my-website/docs/videos.md | 76 +++++++++++++++++++ 2 files changed, 151 insertions(+) diff --git a/docs/my-website/docs/providers/openai/videos.md b/docs/my-website/docs/providers/openai/videos.md index 202c79c2446..b67800092a4 100644 --- a/docs/my-website/docs/providers/openai/videos.md +++ b/docs/my-website/docs/providers/openai/videos.md @@ -135,6 +135,81 @@ curl --location --request POST 'http://localhost:4000/v1/videos/video_id/remix' }' ``` +### Character, Edit, and Extension Routes + +OpenAI video routes supported by LiteLLM proxy: + +- `POST /v1/videos/characters` +- `GET /v1/videos/characters/{character_id}` +- `POST /v1/videos/edits` +- `POST /v1/videos/extensions` + +#### `target_model_names` support on character creation + +`POST /v1/videos/characters` supports `target_model_names` for model-based routing (same behavior as video create). + +```bash +curl --location 'http://localhost:4000/v1/videos/characters' \ +--header 'Authorization: Bearer sk-1234' \ +-F 'name=hero' \ +-F 'target_model_names=gpt-4' \ +-F 'video=@/path/to/character.mp4' +``` + +When `target_model_names` is used, LiteLLM returns an encoded character ID: + +```json +{ + "id": "character_...", + "object": "character", + "created_at": 1712697600, + "name": "hero" +} +``` + +Use that encoded ID directly on get: + +```bash +curl --location 'http://localhost:4000/v1/videos/characters/character_...' \ +--header 'Authorization: Bearer sk-1234' +``` + +#### Encoded and non-encoded video IDs for edit/extension + +Both routes accept either plain or encoded `video.id`: + +- `POST /v1/videos/edits` +- `POST /v1/videos/extensions` + +```bash +curl --location 'http://localhost:4000/v1/videos/edits' \ +--header 'Authorization: Bearer sk-1234' \ +--header 'Content-Type: application/json' \ +--data '{ + "prompt": "Make this brighter", + "video": { "id": "video_..." } +}' +``` + +```bash +curl --location 'http://localhost:4000/v1/videos/extensions' \ +--header 'Authorization: Bearer sk-1234' \ +--header 'Content-Type: application/json' \ +--data '{ + "prompt": "Continue this scene", + "seconds": "4", + "video": { "id": "video_..." } +}' +``` + +#### `custom_llm_provider` input sources + +For these routes, `custom_llm_provider` may be supplied via: + +- header: `custom-llm-provider` +- query: `?custom_llm_provider=...` +- body: `custom_llm_provider` (and `extra_body.custom_llm_provider` where supported) + Test OpenAI video generation request ```bash diff --git a/docs/my-website/docs/videos.md b/docs/my-website/docs/videos.md index 0c284aa3c42..846e551435a 100644 --- a/docs/my-website/docs/videos.md +++ b/docs/my-website/docs/videos.md @@ -290,6 +290,82 @@ curl --location 'http://localhost:4000/v1/videos' \ --header 'custom-llm-provider: azure' ``` +### Character, Edit, and Extension Endpoints + +LiteLLM proxy also supports these OpenAI-compatible video routes: + +- `POST /v1/videos/characters` +- `GET /v1/videos/characters/{character_id}` +- `POST /v1/videos/edits` +- `POST /v1/videos/extensions` + +#### Routing Behavior (`target_model_names`, encoded IDs, and provider overrides) + +- `POST /v1/videos/characters` supports `target_model_names` like `POST /v1/videos`. +- When `target_model_names` is provided on character creation, LiteLLM encodes the returned `character_id` with routing metadata. +- `GET /v1/videos/characters/{character_id}` accepts encoded character IDs directly. LiteLLM decodes the ID internally and routes with the correct model/provider metadata. +- `POST /v1/videos/edits` and `POST /v1/videos/extensions` support both: + - plain `video.id` + - encoded `video.id` values returned by LiteLLM +- `custom_llm_provider` can be supplied using the same patterns as other proxy endpoints: + - header: `custom-llm-provider` + - query: `?custom_llm_provider=...` + - body: `custom_llm_provider` (or `extra_body.custom_llm_provider` where applicable) + +#### Character create with `target_model_names` + +```bash +curl --location 'http://localhost:4000/v1/videos/characters' \ +--header 'Authorization: Bearer sk-1234' \ +-F 'name=hero' \ +-F 'target_model_names=gpt-4' \ +-F 'video=@/path/to/character.mp4' +``` + +Example response (encoded `id`): + +```json +{ + "id": "character_...", + "object": "character", + "created_at": 1712697600, + "name": "hero" +} +``` + +#### Get character using encoded `character_id` + +```bash +curl --location 'http://localhost:4000/v1/videos/characters/character_...' \ +--header 'Authorization: Bearer sk-1234' +``` + +#### Video edit with encoded `video.id` + +```bash +curl --location 'http://localhost:4000/v1/videos/edits' \ +--header 'Authorization: Bearer sk-1234' \ +--header 'Content-Type: application/json' \ +--data '{ + "prompt": "Make this brighter", + "video": { "id": "video_..." } +}' +``` + +#### Video extension with provider override from `extra_body` + +```bash +curl --location 'http://localhost:4000/v1/videos/extensions' \ +--header 'Authorization: Bearer sk-1234' \ +--header 'Content-Type: application/json' \ +--data '{ + "prompt": "Continue this scene", + "seconds": "4", + "video": { "id": "video_..." }, + "extra_body": { "custom_llm_provider": "openai" } +}' +``` + Test Azure video generation request ```bash From 1ccf67dd936e3f87c07ca82cba12bf37fd3a031b Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 16 Mar 2026 19:17:06 +0530 Subject: [PATCH 15/23] fix(greptile-review): address backward compatibility and code quality issues - Remove duplicate DecodedCharacterId TypedDict from litellm/types/videos/main.py - Remove dead LITELLM_MANAGED_VIDEO_CHARACTER_COMPLETE_STR constant from litellm/types/utils.py - Add FastAPI Form validation for name field in video_create_character endpoint Made-with: Cursor --- litellm/proxy/video_endpoints/endpoints.py | 3 ++- litellm/types/utils.py | 3 --- litellm/types/videos/main.py | 8 -------- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/litellm/proxy/video_endpoints/endpoints.py b/litellm/proxy/video_endpoints/endpoints.py index 9e8784df0aa..8d1c8059dca 100644 --- a/litellm/proxy/video_endpoints/endpoints.py +++ b/litellm/proxy/video_endpoints/endpoints.py @@ -3,7 +3,7 @@ from typing import Any, Dict, Optional import orjson -from fastapi import APIRouter, Depends, File, Request, Response, UploadFile +from fastapi import APIRouter, Depends, File, Form, Request, Response, UploadFile from fastapi.responses import ORJSONResponse from litellm.proxy._types import * @@ -530,6 +530,7 @@ async def video_create_character( request: Request, fastapi_response: Response, video: UploadFile = File(...), + name: str = Form(...), user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth), ): """ diff --git a/litellm/types/utils.py b/litellm/types/utils.py index f20958f3f84..38425c7ac4a 100644 --- a/litellm/types/utils.py +++ b/litellm/types/utils.py @@ -3493,9 +3493,6 @@ class SpecialEnums(Enum): LITELLM_MANAGED_VIDEO_COMPLETE_STR = ( "litellm:custom_llm_provider:{};model_id:{};video_id:{}" ) - LITELLM_MANAGED_VIDEO_CHARACTER_COMPLETE_STR = ( - "litellm:custom_llm_provider:{};model_id:{};character_id:{}" - ) class ServiceTier(Enum): diff --git a/litellm/types/videos/main.py b/litellm/types/videos/main.py index 2ee44944bcc..ec0277c789a 100644 --- a/litellm/types/videos/main.py +++ b/litellm/types/videos/main.py @@ -106,14 +106,6 @@ class DecodedVideoId(TypedDict, total=False): video_id: str -class DecodedCharacterId(TypedDict, total=False): - """Structure representing a decoded character ID""" - - custom_llm_provider: Optional[str] - model_id: Optional[str] - character_id: str - - class CharacterObject(BaseModel): """Represents a character created from a video.""" From ddf62e0651d493a6bde3798027dc8feb6eb4ad3e Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 16 Mar 2026 19:20:03 +0530 Subject: [PATCH 16/23] fix(critical): add HTTP error checks before parsing response bodies in video handlers Add response.raise_for_status() before transform_*_response() calls in all eight video character/edit/extension handler methods (sync and async): - video_create_character_handler / async_video_create_character_handler - video_get_character_handler / async_video_get_character_handler - video_edit_handler / async_video_edit_handler - video_extension_handler / async_video_extension_handler Without these checks, httpx does not raise on 4xx/5xx responses, so provider errors (e.g., 401 Unauthorized) pass directly to Pydantic model constructors, causing ValidationError instead of meaningful HTTP errors. The raise_for_status() ensures the exception handler receives proper HTTPStatusError for translation into actionable messages. Made-with: Cursor --- litellm/llms/custom_httpx/llm_http_handler.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/litellm/llms/custom_httpx/llm_http_handler.py b/litellm/llms/custom_httpx/llm_http_handler.py index ab95001a2e9..204fa4d0cca 100644 --- a/litellm/llms/custom_httpx/llm_http_handler.py +++ b/litellm/llms/custom_httpx/llm_http_handler.py @@ -6187,6 +6187,7 @@ def video_create_character_handler( files=files_list, timeout=timeout, ) + response.raise_for_status() return video_provider_config.transform_video_create_character_response( raw_response=response, logging_obj=logging_obj, @@ -6254,6 +6255,7 @@ async def async_video_create_character_handler( files=files_list, timeout=timeout, ) + response.raise_for_status() return video_provider_config.transform_video_create_character_response( raw_response=response, logging_obj=logging_obj, @@ -6327,6 +6329,7 @@ def video_get_character_handler( headers=headers, params=params ) + response.raise_for_status() return video_provider_config.transform_video_get_character_response( raw_response=response, logging_obj=logging_obj, @@ -6387,6 +6390,7 @@ async def async_video_get_character_handler( headers=headers, params=params ) + response.raise_for_status() return video_provider_config.transform_video_get_character_response( raw_response=response, logging_obj=logging_obj, @@ -6472,6 +6476,7 @@ def video_edit_handler( json=data, timeout=timeout, ) + response.raise_for_status() return video_provider_config.transform_video_edit_response( raw_response=response, logging_obj=logging_obj, @@ -6543,6 +6548,7 @@ async def async_video_edit_handler( json=data, timeout=timeout, ) + response.raise_for_status() return video_provider_config.transform_video_edit_response( raw_response=response, logging_obj=logging_obj, @@ -6632,6 +6638,7 @@ def video_extension_handler( json=data, timeout=timeout, ) + response.raise_for_status() return video_provider_config.transform_video_extension_response( raw_response=response, logging_obj=logging_obj, @@ -6705,6 +6712,7 @@ async def async_video_extension_handler( json=data, timeout=timeout, ) + response.raise_for_status() return video_provider_config.transform_video_extension_response( raw_response=response, logging_obj=logging_obj, From 2ec4ce178c1b21eecf78a6a39a08377d928ca6aa Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 16 Mar 2026 19:21:18 +0530 Subject: [PATCH 17/23] fix(routing): include avideo_create_character and avideo_get_character in router-first routing Add avideo_create_character and avideo_get_character to the list of video endpoints that use router-first routing when a model is provided (either from decoded IDs or target_model_names). Previously only avideo_edit and avideo_extension were in the router-first block. This ensures both character endpoints benefit from multi-deployment load balancing and model resolution, making them consistent with the other video operations. This allows: - avideo_create_character: Router picks among multiple deployments when target_model_names is set - avideo_get_character: Router assists with multi-model environments for consistency Made-with: Cursor --- litellm/proxy/route_llm_request.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/route_llm_request.py b/litellm/proxy/route_llm_request.py index a8169ca6d08..e5fc9fe76a4 100644 --- a/litellm/proxy/route_llm_request.py +++ b/litellm/proxy/route_llm_request.py @@ -461,10 +461,13 @@ async def route_request( # noqa: PLR0915 - Complex routing function, refactorin "avideo_status", "avideo_content", "avideo_remix", + "avideo_create_character", + "avideo_get_character", "avideo_edit", "avideo_extension", ]: - # Video endpoints: If model is provided (e.g., from decoded video_id), try router first + # Video endpoints: If model is provided (e.g., from decoded video_id or target_model_names), + # try router first to allow for multi-deployment load balancing try: return getattr(llm_router, f"{route_type}")(**data) except Exception: From 48e0f5952015e82e4356e7c82de89ee6c6962d01 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 16 Mar 2026 19:24:19 +0530 Subject: [PATCH 18/23] docs: add concise blog post on reusable video characters - Clear examples for SDK and proxy usage - Feature highlights: router support, encoding, error handling - Best practices for character uploads and prompting - Available from LiteLLM v1.83.0+ - Troubleshooting guide for common issues Made-with: Cursor --- .../blog/video_characters_litellm/index.md | 150 ++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 docs/my-website/blog/video_characters_litellm/index.md diff --git a/docs/my-website/blog/video_characters_litellm/index.md b/docs/my-website/blog/video_characters_litellm/index.md new file mode 100644 index 00000000000..b6e3fe61f1c --- /dev/null +++ b/docs/my-website/blog/video_characters_litellm/index.md @@ -0,0 +1,150 @@ +--- +slug: video_characters_api +title: "Reusable Video Characters with LiteLLM" +date: 2026-03-16T10:00:00 +authors: + - name: Sameer Kankute + title: SWE @ LiteLLM + url: https://www.linkedin.com/in/sameer-kankute/ + image_url: https://pbs.twimg.com/profile_images/2001352686994907136/ONgNuSk5_400x400.jpg + - name: Krrish Dholakia + title: "CEO, LiteLLM" + url: https://www.linkedin.com/in/krish-d/ + image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg +description: "LiteLLM now supports creating, retrieving, and managing reusable video characters across multiple video generations." +tags: [videos, characters, proxy, routing] +hide_table_of_contents: false +--- + +Upload a video character once, reference it across unlimited generations. LiteLLM now handles character management with full router support. + +## What's New + +Four new endpoints for video character operations: +- **Create character** - Upload a video to create a reusable asset +- **Get character** - Retrieve character metadata +- **Edit video** - Modify generated videos +- **Extend video** - Continue clips with character consistency + +**Available from:** LiteLLM v1.83.0+ + +## Quick Example + +```python +import litellm + +# Create character from video +character = litellm.avideo_create_character( + name="Luna", + video=open("luna.mp4", "rb"), + custom_llm_provider="openai", + model="sora-2" +) +print(f"Character: {character.id}") + +# Use in generation +video = litellm.avideo( + model="sora-2", + prompt="Luna dances through a magical forest.", + characters=[{"id": character.id}], + seconds="8" +) + +# Get character info +fetched = litellm.avideo_get_character( + character_id=character.id, + custom_llm_provider="openai" +) + +# Edit with character preserved +edited = litellm.avideo_edit( + video_id=video.id, + prompt="Add warm golden lighting" +) + +# Extend sequence +extended = litellm.avideo_extension( + video_id=video.id, + prompt="Luna waves goodbye", + seconds="5" +) +``` + +## Via Proxy + +```bash +# Create character +curl -X POST "http://localhost:4000/v1/videos/characters" \ + -H "Authorization: Bearer sk-litellm-key" \ + -F "video=@luna.mp4" \ + -F "name=Luna" + +# Get character +curl -X GET "http://localhost:4000/v1/videos/characters/char_xyz123" \ + -H "Authorization: Bearer sk-litellm-key" +``` + +## Key Features + +✅ **Full Router Support** - Load balance across multiple model deployments +✅ **Character Encoding** - Automatic provider/model tracking in character IDs +✅ **Error Handling** - Proper HTTP status checks before response parsing +✅ **Backward Compatible** - External providers receive NotImplementedError, not instantiation errors +✅ **Multi-Deployment** - Router picks optimal deployment when target_model_names is set + +## Best Practices + +**Character uploads:** +- 2-4 seconds optimal +- Match target resolution (16:9, 9:16, or 1:1) +- 720p-1080p +- Clear character isolation + +**Prompting:** +``` +✅ "Luna the fox dances through a cosmic forest, stars trailing her movement" +❌ "A character that looks like Luna" +``` + +Always mention character name verbatim in prompt. + +## Implementation Notes + +All four handler methods now include: +- `response.raise_for_status()` - Proper error detection before model parsing +- Router-first dispatch - Consistent with avideo_edit/extension +- Async support - Full async/await pattern + +## What's Inside + +- 8 handler methods (sync + async pairs) +- Character transformation classes +- SDK functions + Router wiring +- Full test coverage +- Comprehensive error handling + +## Common Issues + +**Character doesn't appear?** +- Include character ID in `characters` array +- Use character name in prompt (exact match) +- Ensure character occupies meaningful screen space + +**Distorted character?** +- Character video aspect ratio must match target resolution +- Upload again with matching dimensions + +**Want to edit with character?** +- Use avideo_edit (currently no character support in extensions) +- Edit preserves original composition + +## Next Steps + +- Try the [examples](https://docs.litellm.ai/video_characters) +- Check out [character best practices](https://docs.litellm.ai/docs/video_characters#best-practices) +- Deploy the proxy and start routing + +**Resources:** +- [Docs](https://docs.litellm.ai/docs/video_characters) +- [SDK Reference](https://github.com/BerriAI/litellm) +- [Support](https://github.com/BerriAI/litellm/issues) From c1179b835dedeb6c9344c8d47501db241b4ecbf4 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 16 Mar 2026 19:27:15 +0530 Subject: [PATCH 19/23] docs: add edit/extension curl examples and managed ID explanation - Add curl examples for avideo_edit and avideo_extension APIs - Explain how LiteLLM encodes/decodes managed character IDs - Show metadata included in character IDs (provider, model_id) - Detail transparent router-first routing benefits Made-with: Cursor --- .../blog/video_characters_litellm/index.md | 100 +++++++----------- 1 file changed, 40 insertions(+), 60 deletions(-) diff --git a/docs/my-website/blog/video_characters_litellm/index.md b/docs/my-website/blog/video_characters_litellm/index.md index b6e3fe61f1c..57d6bba893e 100644 --- a/docs/my-website/blog/video_characters_litellm/index.md +++ b/docs/my-website/blog/video_characters_litellm/index.md @@ -16,7 +16,7 @@ tags: [videos, characters, proxy, routing] hide_table_of_contents: false --- -Upload a video character once, reference it across unlimited generations. LiteLLM now handles character management with full router support. +LiteLLM now supoports videos character, edit and extension apis. ## What's New @@ -80,71 +80,51 @@ curl -X POST "http://localhost:4000/v1/videos/characters" \ -F "name=Luna" # Get character -curl -X GET "http://localhost:4000/v1/videos/characters/char_xyz123" \ +curl -X GET "http://localhost:4000/v1/videos/characters/char_abc123def456" \ -H "Authorization: Bearer sk-litellm-key" -``` - -## Key Features -✅ **Full Router Support** - Load balance across multiple model deployments -✅ **Character Encoding** - Automatic provider/model tracking in character IDs -✅ **Error Handling** - Proper HTTP status checks before response parsing -✅ **Backward Compatible** - External providers receive NotImplementedError, not instantiation errors -✅ **Multi-Deployment** - Router picks optimal deployment when target_model_names is set +# Edit video +curl -X POST "http://localhost:4000/v1/videos/edits" \ + -H "Authorization: Bearer sk-litellm-key" \ + -H "Content-Type: application/json" \ + -d '{ + "video": {"id": "video_xyz789"}, + "prompt": "Add warm golden lighting and enhance colors" + }' + +# Extend video +curl -X POST "http://localhost:4000/v1/videos/extensions" \ + -H "Authorization: Bearer sk-litellm-key" \ + -H "Content-Type: application/json" \ + -d '{ + "video": {"id": "video_xyz789"}, + "prompt": "Luna waves goodbye and walks into the sunset", + "seconds": "5" + }' +``` -## Best Practices +## Managed Character IDs -**Character uploads:** -- 2-4 seconds optimal -- Match target resolution (16:9, 9:16, or 1:1) -- 720p-1080p -- Clear character isolation +LiteLLM automatically encodes provider and model metadata into character IDs: -**Prompting:** +**What happens:** ``` -✅ "Luna the fox dances through a cosmic forest, stars trailing her movement" -❌ "A character that looks like Luna" +Upload character "Luna" with model "sora-2" on OpenAI + ↓ +LiteLLM creates: char_abc123def456 (contains provider + model_id) + ↓ +When you reference it later, LiteLLM decodes automatically + ↓ +Router knows exactly which deployment to use ``` -Always mention character name verbatim in prompt. - -## Implementation Notes - -All four handler methods now include: -- `response.raise_for_status()` - Proper error detection before model parsing -- Router-first dispatch - Consistent with avideo_edit/extension -- Async support - Full async/await pattern - -## What's Inside - -- 8 handler methods (sync + async pairs) -- Character transformation classes -- SDK functions + Router wiring -- Full test coverage -- Comprehensive error handling - -## Common Issues - -**Character doesn't appear?** -- Include character ID in `characters` array -- Use character name in prompt (exact match) -- Ensure character occupies meaningful screen space - -**Distorted character?** -- Character video aspect ratio must match target resolution -- Upload again with matching dimensions - -**Want to edit with character?** -- Use avideo_edit (currently no character support in extensions) -- Edit preserves original composition - -## Next Steps - -- Try the [examples](https://docs.litellm.ai/video_characters) -- Check out [character best practices](https://docs.litellm.ai/docs/video_characters#best-practices) -- Deploy the proxy and start routing +**Behind the scenes:** +- Character ID format: `character_` +- Metadata includes: provider, model_id, original_character_id +- Transparent to you - just use the ID, LiteLLM handles routing -**Resources:** -- [Docs](https://docs.litellm.ai/docs/video_characters) -- [SDK Reference](https://github.com/BerriAI/litellm) -- [Support](https://github.com/BerriAI/litellm/issues) +**Benefits:** +- Multi-deployment load balancing +- Automatic model resolution +- Encoded IDs work across proxy restarts +- Router picks optimal deployment From 32842a52bc0ef06da1e8b230d8e7dfc15c67e9ce Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 16 Mar 2026 19:33:23 +0530 Subject: [PATCH 20/23] Fix docs --- docs/my-website/blog/video_characters_litellm/index.md | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/docs/my-website/blog/video_characters_litellm/index.md b/docs/my-website/blog/video_characters_litellm/index.md index 57d6bba893e..6d6c0c33ba6 100644 --- a/docs/my-website/blog/video_characters_litellm/index.md +++ b/docs/my-website/blog/video_characters_litellm/index.md @@ -1,6 +1,6 @@ --- slug: video_characters_api -title: "Reusable Video Characters with LiteLLM" +title: "New Video Characters, Edit and Extension API support" date: 2026-03-16T10:00:00 authors: - name: Sameer Kankute @@ -121,10 +121,4 @@ Router knows exactly which deployment to use **Behind the scenes:** - Character ID format: `character_` - Metadata includes: provider, model_id, original_character_id -- Transparent to you - just use the ID, LiteLLM handles routing - -**Benefits:** -- Multi-deployment load balancing -- Automatic model resolution -- Encoded IDs work across proxy restarts -- Router picks optimal deployment +- Transparent to you - just use the ID, LiteLLM handles routing \ No newline at end of file From 1255382fb7f57bacfa678c15a4ec7fd88e0caec9 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 16 Mar 2026 19:39:22 +0530 Subject: [PATCH 21/23] Fix docs --- docs/my-website/blog/video_characters_litellm/index.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/my-website/blog/video_characters_litellm/index.md b/docs/my-website/blog/video_characters_litellm/index.md index 6d6c0c33ba6..263a17d7191 100644 --- a/docs/my-website/blog/video_characters_litellm/index.md +++ b/docs/my-website/blog/video_characters_litellm/index.md @@ -11,6 +11,10 @@ authors: title: "CEO, LiteLLM" url: https://www.linkedin.com/in/krish-d/ image_url: https://pbs.twimg.com/profile_images/1298587542745358340/DZv3Oj-h_400x400.jpg + - name: Ishaan Jaff + title: "CTO, LiteLLM" + url: https://www.linkedin.com/in/reffajnaahsi/ + image_url: https://pbs.twimg.com/profile_images/1613813310264340481/lz54oEiB_400x400.jpg description: "LiteLLM now supports creating, retrieving, and managing reusable video characters across multiple video generations." tags: [videos, characters, proxy, routing] hide_table_of_contents: false From ee24abe86e4305ac11c1f0a8525d0d25c6d4ccfe Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 16 Mar 2026 19:45:57 +0530 Subject: [PATCH 22/23] fix(test): skip new video character endpoints in Azure SDK initialization test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add avideo_create_character, avideo_get_character, avideo_edit, and avideo_extension to the skip condition since Azure video calls don't use initialize_azure_sdk_client. Tests now properly skip with expected behavior instead of failing: - test_ensure_initialize_azure_sdk_client_always_used[avideo_create_character] ✓ - test_ensure_initialize_azure_sdk_client_always_used[avideo_get_character] ✓ - test_ensure_initialize_azure_sdk_client_always_used[avideo_edit] ✓ - test_ensure_initialize_azure_sdk_client_always_used[avideo_extension] ✓ Made-with: Cursor --- tests/test_litellm/llms/azure/test_azure_common_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test_litellm/llms/azure/test_azure_common_utils.py b/tests/test_litellm/llms/azure/test_azure_common_utils.py index dabbd72e49c..d689c676580 100644 --- a/tests/test_litellm/llms/azure/test_azure_common_utils.py +++ b/tests/test_litellm/llms/azure/test_azure_common_utils.py @@ -564,6 +564,10 @@ async def test_ensure_initialize_azure_sdk_client_always_used(call_type): call_type == CallTypes.avideo_content or call_type == CallTypes.avideo_list or call_type == CallTypes.avideo_remix + or call_type == CallTypes.avideo_create_character + or call_type == CallTypes.avideo_get_character + or call_type == CallTypes.avideo_edit + or call_type == CallTypes.avideo_extension ): # Skip video call types as they don't use Azure SDK client initialization pytest.skip(f"Skipping {call_type.value} because Azure video calls don't use initialize_azure_sdk_client") From 1a6eb016bfde2cb2db67057279866e01c16ca159 Mon Sep 17 00:00:00 2001 From: Sameer Kankute Date: Mon, 16 Mar 2026 19:48:28 +0530 Subject: [PATCH 23/23] fix(critical): remove @abstractmethod from video character/edit/extension methods Convert all 8 new video methods from @abstractmethod to concrete implementations that raise NotImplementedError. This prevents breaking external third-party BaseVideoConfig subclasses at import time. Methods affected: - transform_video_create_character_request/response - transform_video_get_character_request/response - transform_video_edit_request/response - transform_video_extension_request/response External integrators can now upgrade without instantiation errors; NotImplementedError is only raised when operations are actually called on unsupported providers. This restores backward compatibility with the project's policy. Made-with: Cursor --- .../llms/base_llm/videos/transformation.py | 40 +++++++++++-------- 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/litellm/llms/base_llm/videos/transformation.py b/litellm/llms/base_llm/videos/transformation.py index 6fb6f69a9e8..a2892e20601 100644 --- a/litellm/llms/base_llm/videos/transformation.py +++ b/litellm/llms/base_llm/videos/transformation.py @@ -268,7 +268,6 @@ def transform_video_status_retrieve_response( ) -> VideoObject: pass - @abstractmethod def transform_video_create_character_request( self, name: str, @@ -283,17 +282,19 @@ def transform_video_create_character_request( Returns: Tuple[str, list]: (url, files_list) for the multipart POST request """ - pass + raise NotImplementedError( + "video create character is not supported for this provider" + ) - @abstractmethod def transform_video_create_character_response( self, raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, ) -> CharacterObject: - pass + raise NotImplementedError( + "video create character is not supported for this provider" + ) - @abstractmethod def transform_video_get_character_request( self, character_id: str, @@ -307,17 +308,19 @@ def transform_video_get_character_request( Returns: Tuple[str, Dict]: (url, params) for the GET request """ - pass + raise NotImplementedError( + "video get character is not supported for this provider" + ) - @abstractmethod def transform_video_get_character_response( self, raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, ) -> CharacterObject: - pass + raise NotImplementedError( + "video get character is not supported for this provider" + ) - @abstractmethod def transform_video_edit_request( self, prompt: str, @@ -333,18 +336,20 @@ def transform_video_edit_request( Returns: Tuple[str, Dict]: (url, data) for the POST request """ - pass + raise NotImplementedError( + "video edit is not supported for this provider" + ) - @abstractmethod def transform_video_edit_response( self, raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str] = None, ) -> VideoObject: - pass + raise NotImplementedError( + "video edit is not supported for this provider" + ) - @abstractmethod def transform_video_extension_request( self, prompt: str, @@ -361,16 +366,19 @@ def transform_video_extension_request( Returns: Tuple[str, Dict]: (url, data) for the POST request """ - pass + raise NotImplementedError( + "video extension is not supported for this provider" + ) - @abstractmethod def transform_video_extension_response( self, raw_response: httpx.Response, logging_obj: LiteLLMLoggingObj, custom_llm_provider: Optional[str] = None, ) -> VideoObject: - pass + raise NotImplementedError( + "video extension is not supported for this provider" + ) def get_error_class( self, error_message: str, status_code: int, headers: Union[dict, httpx.Headers]