Skip to content
Closed
Show file tree
Hide file tree
Changes from 16 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
07f32a2
fix(transformation): merge consecutive system and developer messages …
sowmy86 Sep 2, 2026
12699bd
fix(base_utils): keep later cache_control and billing metadata bounda…
sowmy86 Sep 2, 2026
eecea12
perf(base_utils): fold consecutive system messages in a single linear…
sowmy86 Sep 3, 2026
25186b1
fix(base_utils): hoist later developer messages into the leading syst…
mateo-berri Sep 5, 2026
356fae0
fix(openai): hoist later developer messages into one leading system m…
mateo-berri Sep 5, 2026
9b859f0
refactor(base_utils): fold each run of system messages in one pass
mateo-berri Sep 5, 2026
f500578
fix(base_utils): keep a closing developer message in place and hoist …
mateo-berri Sep 5, 2026
9820d2f
fix(base_utils): keep a closing developer message in place only after…
mateo-berri Sep 5, 2026
1d26ee5
Merge remote-tracking branch 'origin/litellm_internal_staging' into l…
mateo-berri Sep 5, 2026
20b3f1e
refactor(openai): reuse is_openai_backed_api_base for the hosted endp…
mateo-berri Sep 5, 2026
69fd884
test(base_utils): cover an empty system message inside a block-conten…
sowmy86 Sep 5, 2026
ea840eb
fix(openai): hoist developer messages for litellm_proxy and harden th…
mateo-berri Sep 5, 2026
4e3ea74
fix(litellm_proxy): keep the AllMessageValues import the transform si…
mateo-berri Sep 5, 2026
d5d70da
Merge branch 'litellm_internal_staging' of https://github.com/BerriAI…
mateo-berri Sep 8, 2026
22da54b
fix(fireworks_ai): hoist developer items after the first turn on the …
mateo-berri Sep 8, 2026
8b49a64
Merge branch 'litellm_internal_staging' of https://github.com/BerriAI…
mateo-berri Sep 10, 2026
08c1a1c
fix(transformation): carry the developer-message hoist onto main 9633…
mateo-berri Oct 20, 2026
e91fda4
chore: merge the pre-merge branch history (8b49a6438c) into the rebui…
mateo-berri Oct 20, 2026
88923bc
fix(transformation): join a lone plain text block as text when hoisti…
mateo-berri Oct 20, 2026
5c9607d
fix(transformation): hoist later developer messages only for models t…
mateo-berri Oct 20, 2026
08eac7b
fix(responses): fold later developer items into instructions on hoste…
mateo-berri Oct 20, 2026
dbc4d11
fix(llms): place developer messages only for models declared system-f…
mateo-berri Oct 20, 2026
0b99586
Merge remote-tracking branch 'origin/main' into litellm_lit_7019_hois…
mateo-berri Oct 20, 2026
a6a9910
fix(proxy): keep a closing developer message's cache mark when it is …
mateo-berri Oct 20, 2026
7c4ef7c
Merge remote-tracking branch 'origin/main' into litellm_lit_7019_hois…
mateo-berri Oct 28, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions litellm/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -472,6 +472,7 @@
BEDROCK_MIN_THINKING_BUDGET_TOKENS: Final = int(os.getenv("BEDROCK_MIN_THINKING_BUDGET_TOKENS", 1024))
# Anthropic's Messages API rejects thinking.budget_tokens < 1024.
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS: Final = 1024
ANTHROPIC_BILLING_METADATA_PREFIX: Final = "x-anthropic-billing-header:"
REPLICATE_POLLING_DELAY_SECONDS: Final = float(os.getenv("REPLICATE_POLLING_DELAY_SECONDS", 0.5))
DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS: Final = int(os.getenv("DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS", 4096))
DEFAULT_OCI_CHAT_MAX_TOKENS: Final = 4096
Expand Down
5 changes: 3 additions & 2 deletions litellm/llms/anthropic/chat/transformation.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@

import litellm
from litellm.constants import (
ANTHROPIC_BILLING_METADATA_PREFIX,
ANTHROPIC_MIN_THINKING_BUDGET_TOKENS,
ANTHROPIC_WEB_SEARCH_TOOL_MAX_USES,
DEFAULT_ANTHROPIC_CHAT_MAX_TOKENS,
Expand Down Expand Up @@ -1691,7 +1692,7 @@ def translate_system_message(self, messages: list[AllMessageValues]) -> list[Ant
if not system_message_block["content"]:
continue
if self.should_strip_billing_metadata() and system_message_block["content"].startswith(
"x-anthropic-billing-header:"
ANTHROPIC_BILLING_METADATA_PREFIX
):
continue
anthropic_system_message_content = AnthropicSystemMessageContent(
Expand All @@ -1711,7 +1712,7 @@ def translate_system_message(self, messages: list[AllMessageValues]) -> list[Ant
self.should_strip_billing_metadata()
and _content.get("type") == "text"
and text_value
and text_value.startswith("x-anthropic-billing-header:")
and text_value.startswith(ANTHROPIC_BILLING_METADATA_PREFIX)
):
continue
anthropic_system_message_content = AnthropicSystemMessageContent(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

import httpx

from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX
from litellm.exceptions import AuthenticationError
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.litellm_core_utils.litellm_logging import verbose_logger
Expand Down Expand Up @@ -125,7 +126,7 @@ def _filter_billing_headers_from_system(system_param):
"""
if isinstance(system_param, str):
# If it's a string and starts with billing header, filter it out
if system_param.startswith("x-anthropic-billing-header:"):
if system_param.startswith(ANTHROPIC_BILLING_METADATA_PREFIX):
return None
return system_param
elif isinstance(system_param, list):
Expand All @@ -136,7 +137,7 @@ def _filter_billing_headers_from_system(system_param):
text = content_block.get("text", "")
content_type = content_block.get("type", "")
# Skip text blocks that start with billing header
if content_type == "text" and text.startswith("x-anthropic-billing-header:"):
if content_type == "text" and text.startswith(ANTHROPIC_BILLING_METADATA_PREFIX):
continue
filtered_list.append(content_block)
else:
Expand Down
17 changes: 16 additions & 1 deletion litellm/llms/azure/chat/gpt_transformation.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from collections.abc import Mapping
from collections.abc import Mapping, Sequence
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final

Expand All @@ -13,6 +13,7 @@
from litellm.litellm_core_utils.prompt_templates.factory import (
convert_to_azure_openai_messages,
)
from litellm.llms.base_llm.base_utils import map_developer_role_to_system_role
from litellm.llms.base_llm.chat.transformation import BaseLLMException
from litellm.llms.openai.chat.gpt_5_transformation import GPT_REASONING_SERIES_MARKERS
from litellm.types.llms.azure import (
Expand Down Expand Up @@ -94,6 +95,20 @@ def __init__(
if key != "self" and value is not None:
setattr(self.__class__, key, value)

def translate_developer_role_to_system_role(
self,
messages: Sequence[AllMessageValues],
*,
custom_llm_provider: str | None,
api_base: str | None,
) -> Sequence[AllMessageValues]:
"""
Azure OpenAI accepts `system` messages at any position, so a developer message
only changes role. Declared here so `AzureOpenAIGPT5Config` resolves to it
ahead of the OpenAI-compatible hoist it also inherits.
"""
return map_developer_role_to_system_role(messages=messages)

@classmethod
def get_config(cls):
return super().get_config()
Expand Down
184 changes: 169 additions & 15 deletions litellm/llms/base_llm/base_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,23 @@
import copy
import json
from abc import ABC, abstractmethod
from typing import Any, Final
from collections.abc import Iterator, Sequence
from itertools import groupby
from typing import Any, Final, TypeAlias

from openai.lib import _parsing, _pydantic
from pydantic import BaseModel
from pydantic import BaseModel, TypeAdapter, ValidationError
from typing_extensions import TypeIs # noqa: TID251 # TypeIs lands in typing only on 3.13

from litellm._logging import verbose_logger
from litellm.types.llms.openai import AllMessageValues, ChatCompletionToolCallChunk
from litellm.constants import ANTHROPIC_BILLING_METADATA_PREFIX
from litellm.types.llms.openai import (
AllMessageValues,
ChatCompletionCachedContent,
ChatCompletionSystemMessage,
ChatCompletionTextObject,
ChatCompletionToolCallChunk,
)
from litellm.types.utils import Message, ProviderSpecificModelInfo, TokenCountResponse


Expand Down Expand Up @@ -204,19 +214,163 @@ def type_to_response_format_param(
}


SystemMessageContent: TypeAlias = str | list[object]

_system_content_adapter: Final = TypeAdapter[SystemMessageContent | None](SystemMessageContent | None)
_content_block_adapter: Final = TypeAdapter[dict[str, object]](dict[str, object])


def _system_content(message: ChatCompletionSystemMessage) -> SystemMessageContent:
content: Final = _system_content_adapter.validate_python(message.get("content"))
return "" if content is None else content


def _with_message_cache_control(
blocks: list[object], cache_control: ChatCompletionCachedContent | None
) -> list[object]: # mutable-ok: block lists are the wire format
if not blocks or cache_control is None:
return blocks
try:
last: Final = _content_block_adapter.validate_python(blocks[-1])
except ValidationError:
return blocks
if "cache_control" in last:
return blocks
return [*blocks[:-1], {**last, "cache_control": cache_control}] # mutable-ok: block lists are the wire format


def _as_system_message(message: AllMessageValues) -> AllMessageValues:
if message["role"] != "developer":
return message
verbose_logger.debug("Translating developer role to system role for non-OpenAI providers.")
translated: Final[ChatCompletionSystemMessage] = {**message, "role": "system"}
return translated


def map_developer_role_to_system_role(
messages: list[AllMessageValues],
) -> list[AllMessageValues]:
messages: Sequence[AllMessageValues],
) -> Sequence[AllMessageValues]:
"""
Translate `developer` role to `system` role for non-OpenAI providers.
Translate `developer` role to `system` role, keeping every message where the client put it.
"""
new_messages: Final[list[AllMessageValues]] = []
for m in messages:
if m["role"] == "developer":
verbose_logger.debug(
"Translating developer role to system role for non-OpenAI providers."
) # ensure user knows what's happening with their input.
new_messages.append({"role": "system", "content": m["content"]})
return tuple(_as_system_message(message) for message in messages)


def _text_blocks(message: ChatCompletionSystemMessage) -> list[object]: # mutable-ok: block lists are the wire format
content: Final = _system_content(message)
cache_control: Final = message.get("cache_control")
if not isinstance(content, str):
return _with_message_cache_control(content, cache_control)
if not content:
return [] # mutable-ok: block lists are the wire format
if cache_control:
cached_block: Final[ChatCompletionTextObject] = {
"type": "text",
"text": content,
"cache_control": cache_control,
}
return [cached_block] # mutable-ok: block lists are the wire format
text_block: Final[ChatCompletionTextObject] = {"type": "text", "text": content}
return [text_block] # mutable-ok: block lists are the wire format


def _plain_text(message: ChatCompletionSystemMessage) -> str | None:
content: Final = _system_content(message)
if (
not isinstance(content, str)
or message.get("cache_control")
or content.startswith(ANTHROPIC_BILLING_METADATA_PREFIX)
):
return None
return content


def _is_system_message(
message: AllMessageValues,
) -> TypeIs[ChatCompletionSystemMessage]: # guard-ok: the role literal picks the TypedDict member
return message["role"] == "system"


def _merged_system_run(run: Sequence[ChatCompletionSystemMessage]) -> ChatCompletionSystemMessage:
if len(run) == 1:
return run[0]
texts: Final = tuple(_plain_text(message) for message in run)
content: Final[SystemMessageContent] = (
"\n\n".join(text for text in texts if text)
if all(text is not None for text in texts)
else [
block for message in run for block in _text_blocks(message)
] # mutable-ok: block lists are the wire format
)
named: Final = next((message for message in reversed(run) if "name" in message), None)
if named is None:
merged: Final[ChatCompletionSystemMessage] = {"role": "system", "content": content}
return merged
with_name: Final[ChatCompletionSystemMessage] = {"role": "system", "content": content, "name": named["name"]}
return with_name


def _merged_system_runs(messages: Sequence[AllMessageValues]) -> Iterator[AllMessageValues]:
for is_system, run in groupby(messages, key=_is_system_message):
if is_system:
yield _merged_system_run(tuple(message for message in run if _is_system_message(message)))
else:
new_messages.append(m)
return new_messages
yield from run


_INSTRUCTION_ROLES: Final = frozenset(("system", "developer"))


def _leading_instruction_block_length(roles: Sequence[str | None]) -> int:
return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles))


def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int:
last_conversation_index: Final = next(
(index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES),
None,
)
if last_conversation_index is None or roles[last_conversation_index] != "assistant":
return len(roles)
return last_conversation_index + 1


def hoisted_instruction_indices(roles: Sequence[str | None]) -> tuple[int, ...]:
"""
Indices of the leading instruction block plus every developer item after it, except
a developer block that closes the conversation right after an assistant turn, which
stays in place so the request does not end on the assistant's turn. Items without a
role (tool calls and their outputs) count as conversation.
"""
leading_length: Final = _leading_instruction_block_length(roles)
closing_start: Final = _closing_instruction_block_start(roles, leading_length)
moved: Final = tuple(index for index in range(leading_length, closing_start) if roles[index] == "developer")
if moved:
verbose_logger.debug("Hoisting %d developer message(s) into the leading system block.", len(moved))
return (*range(leading_length), *moved)


def _hoisted_developer_item_order(roles: Sequence[str | None]) -> tuple[int, ...]:
hoisted: Final = hoisted_instruction_indices(roles)
return (*hoisted, *(index for index in range(len(roles)) if index not in hoisted))


def _move_later_developer_messages_up(messages: Sequence[AllMessageValues]) -> tuple[AllMessageValues, ...]:
order: Final = _hoisted_developer_item_order(tuple(message["role"] for message in messages))
return tuple(messages[index] for index in order)


def hoist_developer_messages_into_leading_system_message(
messages: Sequence[AllMessageValues],
) -> Sequence[AllMessageValues]:
"""
Translate `developer` role to `system` role for OpenAI-compatible backends whose
chat template allows a single system message and only at the start: developer
messages that arrive after the first user turn move into the leading system
block, except a developer message that closes the conversation right after an
assistant turn, which stays in place so the request does not end on the
assistant's turn. Each run of consecutive system messages is then folded into
one message in a single pass.
"""
translated: Final = tuple(map(_as_system_message, _move_later_developer_messages_up(messages)))
return tuple(_merged_system_runs(translated))
13 changes: 9 additions & 4 deletions litellm/llms/base_llm/chat/transformation.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import types
from abc import ABC, abstractmethod
from collections.abc import AsyncIterator, Iterator
from collections.abc import AsyncIterator, Iterator, Sequence
from typing import TYPE_CHECKING, Any, Final, Union

import httpx
Expand Down Expand Up @@ -149,12 +149,17 @@ def _add_tools_to_optional_params(self, optional_params: dict, tools: list) -> d

def translate_developer_role_to_system_role(
self,
messages: list[AllMessageValues],
) -> list[AllMessageValues]:
messages: Sequence[AllMessageValues],
*,
custom_llm_provider: str | None,
api_base: str | None,
) -> Sequence[AllMessageValues]:
"""
Translate `developer` role to `system` role for non-OpenAI providers.

Overriden by OpenAI/Azure
Overridden by OpenAI, which keeps developer messages in place on its own
endpoint and hoists them into one leading system message for OpenAI-compatible
backends, and by the o-series and Azure configs.
"""
return map_developer_role_to_system_role(messages=messages)
Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved

Expand Down
30 changes: 3 additions & 27 deletions litellm/llms/fireworks_ai/responses/transformation.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,12 @@
from collections.abc import Mapping, Sequence
from collections.abc import Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Final
from urllib.parse import unquote

import httpx
from openai.types.responses import EasyInputMessageParam, ResponseInputContentParam, ResponseInputItemParam

from litellm.llms.base_llm.base_utils import hoisted_instruction_indices
from litellm.llms.fireworks_ai.common_utils import (
resolve_fireworks_api_key,
resolve_fireworks_resource_name,
Expand All @@ -31,9 +32,6 @@ def _session_params(litellm_params: GenericLiteLLMParams) -> Mapping[str, object
)


_INSTRUCTION_ROLES: Final = frozenset({"system", "developer"})


def _role(item: ResponseInputItemParam) -> str | None:
match item:
case {"role": str(role)}:
Expand Down Expand Up @@ -73,28 +71,6 @@ def _text_only_content(item: ResponseInputItemParam) -> str | None:
return None


def _leading_instruction_block_length(roles: Sequence[str | None]) -> int:
return next((index for index, role in enumerate(roles) if role not in _INSTRUCTION_ROLES), len(roles))


def _closing_instruction_block_start(roles: Sequence[str | None], leading_length: int) -> int:
last_conversation_index: Final = next(
(index for index in range(len(roles) - 1, leading_length - 1, -1) if roles[index] not in _INSTRUCTION_ROLES),
None,
)
if last_conversation_index is None or roles[last_conversation_index] != "assistant":
return len(roles)
return last_conversation_index + 1


def _hoisted_indices(roles: Sequence[str | None]) -> tuple[int, ...]:
leading_length: Final = _leading_instruction_block_length(roles)
closing_start: Final = _closing_instruction_block_start(roles, leading_length)
return tuple(
index for index, role in enumerate(roles[:closing_start]) if index < leading_length or role == "developer"
)


def _with_instruction_items_folded(
input: str | ResponseInputParam, instructions: str | None
) -> tuple[str | None, str | ResponseInputParam]:
Expand All @@ -104,7 +80,7 @@ def _with_instruction_items_folded(
folded: Final = MappingProxyType(
{
index: text
for index in _hoisted_indices(tuple(map(_role, items)))
for index in hoisted_instruction_indices(tuple(map(_role, items)))
if (text := _text_only_content(items[index])) is not None
}
)
Expand Down
Loading
Loading