Skip to content
Merged
Show file tree
Hide file tree
Changes from 24 commits
Commits
Show all changes
26 commits
Select commit Hold shift + click to select a range
038e6a5
feat(anthropic): placement policy for mid-conversation system messages
oneKn8 Aug 24, 2026
2d56720
fix(anthropic): keep mid-conversation system out of the chat completi…
oneKn8 Aug 24, 2026
2780999
refactor(anthropic): single-source the converted system note
oneKn8 Aug 24, 2026
4e31132
test(e2e): prove the prompt cache survives a mid-conversation system …
oneKn8 Aug 24, 2026
5da7890
test(anthropic): cover the cache_control rebuild shapes and type the …
oneKn8 Aug 24, 2026
91a5c18
refactor(anthropic): read the mid-conversation flag through a public …
oneKn8 Aug 25, 2026
1dbc575
chore(typing): declare the mid-conversation type aliases with TypeAlias
oneKn8 Aug 26, 2026
d8c2086
fix(anthropic): let add_code_execution_tool take the pass-through mes…
oneKn8 Aug 27, 2026
184dba0
fix(bedrock): keep mid-conversation system messages in place on conve…
ege-arhan Sep 14, 2026
ff7c668
fix: ruff format + multi tool_result order + regression test
ege-arhan Sep 14, 2026
0cabad9
fix: satisfy type-discipline gate + update osv ignore for mlflow PYSE…
ege-arhan Sep 14, 2026
2d4f6d0
fix(bedrock): restore role narrowing in hoisted system loop for based…
ege-arhan Sep 14, 2026
4eb70cc
test(bedrock): cover mid-conversation system conversion branches
ege-arhan Sep 22, 2026
50d59e3
Place type-discipline suppressions on the lines the gate scans
ege-arhan Sep 22, 2026
26e0d60
Narrow hoisted loop to system role so basedpyright sees the right Typ…
ege-arhan Sep 22, 2026
26bc94f
fix(anthropic): place mid-conversation system runs by their neighbour…
mateo-berri Sep 23, 2026
8fefe41
refactor(bedrock): share the converted system note with the anthropic…
mateo-berri Sep 23, 2026
b5185a2
test: pin the replayed request prefix across preserved-thinking turns
mateo-berri Sep 23, 2026
38a3929
fix(anthropic): move mid-conversation system placement into prompt_te…
mateo-berri Sep 23, 2026
152fdb6
fix(anthropic): anchor system messages only on user turns that reach …
mateo-berri Sep 23, 2026
54ec578
Merge remote-tracking branch 'origin/main' into litellm_anthropic_thi…
mateo-berri Sep 23, 2026
98a2746
fix(bedrock): type the converse system-message helpers over the messa…
mateo-berri Sep 23, 2026
eb92eee
fix(anthropic): read replayed pydantic messages in the Converse helpe…
mateo-berri Sep 25, 2026
5e96d52
chore: merge main into litellm_anthropic_thinking_prefix_stable and s…
mateo-berri Sep 25, 2026
bf3ba0d
fix(anthropic): treat bridged encrypted reasoning as a vanishing assi…
mateo-berri Sep 25, 2026
e05e9dc
fix(anthropic): let an inline thinking part hide separate thinking_bl…
mateo-berri Sep 25, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions litellm/litellm_core_utils/prompt_templates/factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
import xml.etree.ElementTree as ET
from collections.abc import Iterator, Mapping, Sequence
from enum import Enum
from typing import Any, Final, TypedDict, cast, overload
from typing import Any, Final, TypeAlias, TypedDict, cast, overload

from jinja2.sandbox import ImmutableSandboxedEnvironment

Expand All @@ -17,6 +17,7 @@
from litellm import verbose_logger
from litellm._uuid import uuid
from litellm.constants import REDACTED_BY_LITELLM
from litellm.litellm_core_utils.prompt_templates.mid_conversation_system import anthropic_system_messages
from litellm.litellm_core_utils.url_utils import async_safe_get, safe_get
from litellm.llms.custom_httpx.http_handler import HTTPHandler, get_async_httpx_client
from litellm.types.files import get_file_extension_from_mime_type
Expand Down Expand Up @@ -2352,14 +2353,19 @@ def _drop_unsignable_thinking_blocks(
return [block for block in thinking_blocks if not _is_unsignable_thinking_block(block)]


_AnthropicMessageList: TypeAlias = list[AllAnthropicPassThroughMessageValues]


def anthropic_messages_pt(
messages: list[AllMessageValues],
model: str,
llm_provider: str,
) -> list[AnthropicMessagesUserMessageParam | AnthopicMessagesAssistantMessageParam]:
) -> _AnthropicMessageList:
"""
format messages for anthropic
1. Anthropic supports roles like "user" and "assistant" (system prompt sent separately)
1. Anthropic supports roles like "user" and "assistant" (system prompt sent separately).
Models flagged ``supports_mid_conversation_system`` also accept "system" inside
messages after a user turn; the caller decides placement, this keeps such messages.
2. The first message always needs to be of role "user"
3. Each message must alternate between "user" and "assistant" (this is not addressed as now by litellm)
4. final assistant content cannot end with trailing whitespace (anthropic raises an error otherwise)
Expand All @@ -2384,7 +2390,7 @@ def anthropic_messages_pt(
# add role=tool support to allow function call result/error submission
user_message_types: Final = {"user", "tool", "function"}
# reformat messages to ensure user/assistant are alternating, if there's either 2 consecutive 'user' messages or 2 consecutive 'assistant' message, merge them.
new_messages: Final[list[AnthropicMessagesUserMessageParam | AnthopicMessagesAssistantMessageParam]] = []
new_messages: Final[_AnthropicMessageList] = [] # mutable-ok: accumulator behind the mutable return contract

if len(messages) == 0:
if not litellm.modify_params:
Expand Down Expand Up @@ -2777,6 +2783,11 @@ def anthropic_messages_pt(
if assistant_content:
new_messages.append({"role": "assistant", "content": assistant_content})

## MID-CONVERSATION SYSTEM MESSAGES (placement is the caller's job) ##
while msg_i < len(messages) and messages[msg_i]["role"] == "system":
new_messages.extend(anthropic_system_messages(messages[msg_i]))
msg_i += 1

if msg_i == init_msg_i: # prevent infinite loops
raise litellm.BadRequestError(
message=BAD_MESSAGE_ERROR_STR + f"passed in {messages[msg_i]}",
Expand Down
Loading
Loading