Skip to content
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
512e730
fix(azure_ai): add passthrough config so router-model relays reach th…
mateo-berri Sep 5, 2026
4c9537f
chore: merge origin/litellm_internal_staging into litellm_lit_7022_az…
mateo-berri Sep 5, 2026
cf275cf
chore: merge origin/litellm_internal_staging into litellm_lit_7022_az…
mateo-berri Sep 5, 2026
a276690
fix(router): keep a model's own provider prefix for generic SDK calls
mateo-berri Sep 5, 2026
cd25eb9
fix(azure_ai): cost streaming relays and return upstream errors from …
mateo-berri Sep 5, 2026
9ea9aa2
fix(azure_ai): count prompt tokens for streaming relays that carry no…
mateo-berri Sep 5, 2026
bbbdccb
fix(azure_ai): count relayed image prompt tokens without fetching the…
mateo-berri Sep 5, 2026
3f69584
fix(router): rewrite the passthrough model group as a whole path segment
mateo-berri Sep 5, 2026
df6fb9e
fix(azure_ai): relay from the Foundry root and log non-chat relays fo…
mateo-berri Sep 5, 2026
2f981d1
refactor(passthrough): inject the streaming prompt-token counter inst…
mateo-berri Sep 5, 2026
6ed7269
Merge remote-tracking branch 'origin/litellm_internal_staging' into l…
mateo-berri Sep 5, 2026
4706ace
fix(passthrough): charge remote high-detail images at the high-detail…
mateo-berri Sep 5, 2026
614b151
chore: merge origin/litellm_internal_staging into litellm_lit_7022_az…
mateo-berri Sep 7, 2026
5c23d29
fix(azure_ai): cost OCR relays per page so non-chat relays debit budgets
mateo-berri Sep 7, 2026
22d7616
Merge remote-tracking branch 'origin/litellm_internal_staging' into l…
mateo-berri Sep 8, 2026
6f99917
fix(router): rewrite multi-segment model groups as whole passthrough …
mateo-berri Sep 8, 2026
c698dde
fix(azure): let the caller's api-version win over the deployment's on…
mateo-berri Sep 8, 2026
17b7003
fix(azure_ai): cost embeddings, responses, images, and rerank relays …
mateo-berri Sep 8, 2026
12d9413
refactor(passthrough): share the relay shape table and price FLUX 2 p…
mateo-berri Sep 8, 2026
02b18b4
fix(passthrough): match image URL schemes case-insensitively when cou…
mateo-berri Sep 8, 2026
1de369a
fix(passthrough): bound azure relays to the key's model group and rej…
mateo-berri Sep 8, 2026
52e9cfe
fix(azure): price streamed Responses relays and keep a full-URL api_b…
mateo-berri Sep 8, 2026
9666a21
Merge remote-tracking branch 'origin/litellm_internal_staging' into l…
mateo-berri Sep 8, 2026
13db79d
fix(azure_ai): price streamed Responses relays from their terminal event
mateo-berri Sep 8, 2026
db59e99
fix(azure): let the caller's api-version override a full-URL api_base…
mateo-berri Sep 8, 2026
5e056a2
refactor(azure): move the passthrough deployment-segment helpers unde…
mateo-berri Sep 8, 2026
a892e67
fix(azure): match deployment segments case-insensitively and keep rel…
mateo-berri Sep 9, 2026
4fa34ac
Merge remote-tracking branch 'origin/litellm_internal_staging' into HEAD
mateo-berri Sep 9, 2026
5ea447e
fix(azure): keep the model group segment exact and casefold only depl…
mateo-berri Sep 9, 2026
13745e8
chore: merge litellm_internal_staging into litellm_lit_7022_azure_ai_…
mateo-berri Sep 9, 2026
cc0c608
chore: merge litellm_internal_staging into litellm_lit_7022_azure_ai_…
mateo-berri Sep 9, 2026
a25ecca
fix(azure_ai): drop the api_base path segments a relay already repeats
mateo-berri Sep 10, 2026
8627576
fix(azure_ai): keep a deployment named like the first native path seg…
mateo-berri Sep 10, 2026
a668195
Merge branch 'litellm_internal_staging' into litellm_lit_7022_azure_a…
mateo-berri Sep 10, 2026
966a58d
Merge branch 'litellm_internal_staging' into litellm_lit_7022_azure_a…
mateo-berri Sep 10, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions basedpyright-code-budget.json
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@
"limit": 5570
},
"reportMissingTypeArgument": {
"limit": 15281
"limit": 15277
},
"reportMissingTypeStubs": {
"limit": 40
Expand Down Expand Up @@ -105,13 +105,13 @@
"limit": 109
},
"reportUnknownMemberType": {
"limit": 38283
"limit": 38271
},
"reportUnknownParameterType": {
"limit": 19584
"limit": 19580
},
"reportUnknownVariableType": {
"limit": 29829
"limit": 29821
},
"reportUnnecessaryCast": {
"limit": 110
Expand Down
4 changes: 2 additions & 2 deletions litellm/litellm_core_utils/streaming_chunk_builder_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,7 @@ def apply_grounding_request_counts(


class ChunkProcessor:
def __init__(self, chunks: list, messages: list | None = None):
def __init__(self, chunks: list, messages: Sequence | None = None):
self.chunks = self._sort_chunks(chunks)
self.messages = messages
self.first_chunk = chunks[0]
Expand Down Expand Up @@ -992,7 +992,7 @@ def calculate_usage(
chunks: Sequence["_UsageBearingChunk | ModelResponse"],
model: str,
completion_output: str,
messages: list | None = None,
messages: Sequence | None = None,
reasoning_tokens: int | None = None,
) -> Usage:
"""
Expand Down
42 changes: 41 additions & 1 deletion litellm/llms/azure/passthrough/transformation.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,9 @@
from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Final, Optional

import httpx
from httpx import Response
from pydantic import BaseModel, ValidationError

from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.llms.azure.common_utils import BaseAzureLLM
Expand All @@ -16,6 +18,22 @@
from litellm.types.utils import CostResponseTypes


class RelayedChatRequest(BaseModel):
messages: Sequence[Mapping[str, object]] | None = None


class RelayedCallDetails(BaseModel):
request_data: RelayedChatRequest | None = None


def _relayed_messages(litellm_logging_obj: Logging) -> Sequence[Mapping[str, object]] | None:
try:
details: Final = RelayedCallDetails.model_validate(litellm_logging_obj.model_call_details)
except ValidationError:
return None
return details.request_data.messages if details.request_data else None


class AzurePassthroughConfig(BasePassthroughConfig):
def is_streaming_request(self, endpoint: str, request_data: dict) -> bool:
return "stream" in request_data
Expand Down Expand Up @@ -43,7 +61,7 @@ def get_complete_url(
api_base=base_target_url,
litellm_params=litellm_params,
route=endpoint,
default_api_version=litellm_params.get("api_version"),
default_api_version=request_query_params.get("api-version") if request_query_params else None,
)
return (
httpx.URL(complete_url),
Expand Down Expand Up @@ -116,3 +134,25 @@ def logging_non_streaming_response(
)

return litellm_model_response

def handle_logging_collected_chunks(
self,
all_chunks: Sequence[str],
litellm_logging_obj: Logging,
model: str,
custom_llm_provider: str,
endpoint: str,
) -> Optional["CostResponseTypes"]:
from litellm.proxy.pass_through_endpoints.llm_provider_handlers.openai_passthrough_logging_handler import (
OpenAIPassthroughLoggingHandler,
)

if "chat/completions" not in endpoint:
return None

return OpenAIPassthroughLoggingHandler()._build_complete_streaming_response( # pyright: ignore[reportPrivateUsage] # the only OpenAI SSE-to-ModelResponse assembler; reimplementing it would fork the parser
all_chunks=all_chunks,
litellm_logging_obj=litellm_logging_obj,
model=model,
messages=_relayed_messages(litellm_logging_obj),
Comment thread
veria-ai[bot] marked this conversation as resolved.
Resolved
)
12 changes: 5 additions & 7 deletions litellm/llms/azure_ai/chat/transformation.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
import enum
import re
from typing import TYPE_CHECKING, Final, cast
from urllib.parse import urlparse

import httpx
from httpx import Response
Expand All @@ -15,7 +14,10 @@
filter_value_from_dict,
)
from litellm.llms.azure.common_utils import BaseAzureLLM
from litellm.llms.azure_ai.common_utils import is_foundry_model_inference_base
from litellm.llms.azure_ai.common_utils import (
api_key_header_for_base,
is_foundry_model_inference_base,
)
from litellm.llms.base_llm.chat.transformation import LiteLLMLoggingObj
from litellm.llms.openai.common_utils import drop_params_from_unprocessable_entity_error
from litellm.llms.openai.openai import OpenAIConfig
Expand Down Expand Up @@ -99,11 +101,7 @@ def _should_use_api_key_header(self, api_base: str) -> bool:
"""
Returns True if the request should use `api-key` header for authentication.
"""
parsed_url: Final = urlparse(api_base)
host: Final = parsed_url.hostname
if host and (host.endswith(".services.ai.azure.com") or host.endswith(".openai.azure.com")):
return True
return False
return api_key_header_for_base(api_base) == "api-key"

def get_complete_url(
self,
Expand Down
7 changes: 7 additions & 0 deletions litellm/llms/azure_ai/common_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,13 @@ def is_foundry_model_inference_base(api_base: str) -> bool:
return "/openai/deployments" not in parsed.path


def api_key_header_for_base(api_base: str | None) -> AzureAIApiKeyHeader:
host: Final = urlparse(api_base).hostname if api_base else None
if host and (host.endswith(".services.ai.azure.com") or host.endswith(".openai.azure.com")):
return "api-key"
return "Authorization"


def get_azure_ai_entra_token(litellm_params: Mapping[str, object] | None = None) -> str | None:
"""
Resolve an Entra ID / OAuth access token for an Azure AI Foundry deployment.
Expand Down
124 changes: 124 additions & 0 deletions litellm/llms/azure_ai/passthrough/transformation.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,124 @@
from __future__ import annotations

from collections.abc import Mapping, Sequence
from typing import TYPE_CHECKING, Final

from pydantic import BaseModel, ConfigDict, ValidationError

from litellm.llms.azure_ai.common_utils import (
AzureFoundryModelInfo,
api_key_header_for_base,
get_azure_ai_auth_headers,
)
from litellm.llms.base_llm.passthrough.transformation import BasePassthroughConfig
from litellm.types.llms.openai import AllMessageValues

if TYPE_CHECKING:
from httpx import URL, Response

from litellm.litellm_core_utils.litellm_logging import Logging
from litellm.types.utils import CostResponseTypes


def strip_leading_model_segment(endpoint: str, model_names: tuple[str, ...]) -> str:
path: Final = endpoint.lstrip("/")
for model_name in model_names:
if not model_name:
continue
if path == model_name:
return ""
if path.startswith(f"{model_name}/"):
return path[len(model_name) + 1 :]
return path


class PassthroughMetadata(BaseModel):
model_config = ConfigDict(extra="ignore")

model_group: str = ""


def model_group_from(litellm_params: Mapping[str, object]) -> str:
try:
return PassthroughMetadata.model_validate(litellm_params.get("litellm_metadata")).model_group
except ValidationError:
return ""


class AzureAIPassthroughConfig(AzureFoundryModelInfo, BasePassthroughConfig):
def is_streaming_request(self, endpoint: str, request_data: Mapping[str, object]) -> bool:
return request_data.get("stream") is True

def get_complete_url(
self,
api_base: str | None,
api_key: str | None,
model: str,
endpoint: str,
request_query_params: Mapping[str, object] | None,
litellm_params: Mapping[str, object],
) -> tuple[URL, str]:
base_target_url: Final = self.get_api_base(api_base)
if base_target_url is None:
raise ValueError("Azure AI api base not found: set `api_base` on the deployment or AZURE_AI_API_BASE")

native_endpoint: Final = strip_leading_model_segment(endpoint, (model, model_group_from(litellm_params)))
return (
self.format_url(native_endpoint, base_target_url, request_query_params),
base_target_url,
)

def validate_environment(
self,
headers: Mapping[str, str],
model: str,
messages: Sequence[AllMessageValues],
optional_params: Mapping[str, object],
litellm_params: Mapping[str, object],
api_key: str | None = None,
api_base: str | None = None,
) -> dict[str, str]: # mutable-ok: base class contract returns dict for httpx
auth_headers: Final = get_azure_ai_auth_headers(
api_key=api_key,
litellm_params=litellm_params,
api_key_header=api_key_header_for_base(api_base),
)
return {**headers, **auth_headers} # mutable-ok: base class contract returns dict for httpx

def logging_non_streaming_response(
self,
model: str,
custom_llm_provider: str,
httpx_response: Response,
request_data: Mapping[str, object],
logging_obj: Logging,
endpoint: str,
) -> CostResponseTypes | None:
from litellm.llms.azure.passthrough.transformation import AzurePassthroughConfig

return AzurePassthroughConfig().logging_non_streaming_response( # pyright: ignore[reportUnknownMemberType] # the Azure config still types request_data as a bare dict
model=model,
custom_llm_provider=custom_llm_provider,
httpx_response=httpx_response,
request_data=dict(request_data), # mutable-ok: AzurePassthroughConfig wants a dict
logging_obj=logging_obj,
endpoint=endpoint,
)

def handle_logging_collected_chunks(
self,
all_chunks: Sequence[str],
litellm_logging_obj: Logging,
model: str,
custom_llm_provider: str,
endpoint: str,
) -> CostResponseTypes | None:
from litellm.llms.azure.passthrough.transformation import AzurePassthroughConfig

return AzurePassthroughConfig().handle_logging_collected_chunks(
all_chunks=all_chunks,
litellm_logging_obj=litellm_logging_obj,
model=model,
custom_llm_provider=custom_llm_provider,
endpoint=endpoint,
)
3 changes: 2 additions & 1 deletion litellm/llms/base_llm/passthrough/transformation.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
from abc import abstractmethod
from collections.abc import Mapping
from typing import TYPE_CHECKING, Final, Optional, Union

from ..base_utils import BaseLLMModelInfo
Expand All @@ -23,7 +24,7 @@ def format_url(
self,
endpoint: str,
base_target_url: str,
request_query_params: dict | None,
request_query_params: Mapping[str, object] | None,
) -> "URL":
"""
Helper function to add query params to the url
Expand Down
4 changes: 2 additions & 2 deletions litellm/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -8553,7 +8553,7 @@ def config_completion(**kwargs):
)


def stream_chunk_builder_text_completion(chunks: list, messages: list | None = None) -> TextCompletionResponse:
def stream_chunk_builder_text_completion(chunks: list, messages: Sequence | None = None) -> TextCompletionResponse:
id: Final = chunks[0]["id"]
object: Final = chunks[0]["object"]
created: Final = chunks[0]["created"]
Expand Down Expand Up @@ -8670,7 +8670,7 @@ def _stamp_streaming_usage_cost(usage: Usage, response: ModelResponse, logging_o

def stream_chunk_builder(
chunks: list,
messages: list | None = None,
messages: Sequence | None = None,
start_time=None,
end_time=None,
logging_obj: Optional["Logging"] = None,
Expand Down
4 changes: 2 additions & 2 deletions litellm/proxy/common_utils/http_parsing_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ def _is_form_content_type(content_type: str) -> bool:
return _normalize_media_type(content_type) in _FORM_CONTENT_TYPES


def _is_json_content_type(content_type: str) -> bool:
def is_json_content_type(content_type: str) -> bool:
"""True iff the body should be parsed as JSON."""
return _normalize_media_type(content_type) == "application/json"

Expand Down Expand Up @@ -406,7 +406,7 @@ async def get_request_body(request: Request) -> dict[str, Any]:
"""
if request.method == "POST":
content_type: Final = request.headers.get("content-type", "")
if _is_json_content_type(content_type):
if is_json_content_type(content_type):
return await _read_request_body(request)
elif _is_form_content_type(content_type):
return await get_form_data(request)
Expand Down
Loading
Loading