Skip to content
Merged
Show file tree
Hide file tree
Changes from 41 commits
Commits
Show all changes
116 commits
Select commit Hold shift + click to select a range
b38b867
feat(batches): emit per-line JSONL batch records to callbacks
shivamrawat1 Sep 17, 2026
0ea8ce7
fix(batches): only emit line items for final batches
shivamrawat1 Sep 17, 2026
7fa8c92
chore(deps): bump soupsieve to 2.9.2 for GHSA-gjv8-xp57-g29c and GHSA…
shivamrawat1 Sep 17, 2026
c7edcce
style(batches): satisfy ruff format for line item logging
shivamrawat1 Sep 17, 2026
7f06708
chore(ui): regenerate schema.d.ts for store_batch_line_items_in_callb…
shivamrawat1 Sep 17, 2026
8908023
test(batches): cover line-item edge shapes and general settings propa…
shivamrawat1 Sep 17, 2026
8aca0c3
fix(batches): keep deployment credentials out of callback-visible params
shivamrawat1 Sep 17, 2026
ff9d30b
refactor(batches): declare credential transport attribute on Logging
shivamrawat1 Sep 17, 2026
feea6f2
chore: merge origin/main into litellm_batch_jsonl_line_item_callbacks
shivamrawat1 Sep 18, 2026
775d851
chore: merge origin/main into litellm_batch_jsonl_line_item_callbacks
yucheng-berri Sep 23, 2026
8484b2f
test(batches): integration contract for per-line batch callback events
yucheng-berri Sep 23, 2026
b4cb884
fix(proxy): resolve store_batch_line_items_in_callbacks through the s…
yucheng-berri Sep 23, 2026
61d5ab0
test(batches): assert per-line events beside the aggregate and create…
yucheng-berri Sep 23, 2026
bc53e23
fix(proxy): drop resurrected use_legacy_interactions_schema shim from…
yucheng-berri Sep 23, 2026
4b6efef
test(batches): assert line item events carry no credentials
yucheng-berri Sep 23, 2026
ed23e2e
fix(proxy): reset batch line-item flag when the setting is absent
yucheng-berri Sep 23, 2026
096ba84
fix(batches): map anthropic batch lines through the anthropic message…
yucheng-berri Sep 23, 2026
a8fafaf
style: tag the anthropic result cast with a cast-ok reason
yucheng-berri Sep 23, 2026
c746e57
style: shorten cast-ok reason so ruff format passes
yucheng-berri Sep 23, 2026
45b7509
fix(batches): skip line-item callbacks for providers without a line r…
yucheng-berri Sep 23, 2026
bef2f04
fix(batches): reconstruct bedrock and mistral batch lines through the…
yucheng-berri Sep 23, 2026
6cbad5c
style: wrap widened provider literals so ruff format passes
yucheng-berri Sep 23, 2026
a760991
fix(batches): pair bedrock lines by recordId and type them from the r…
yucheng-berri Sep 23, 2026
271b6c6
chore: merge main into litellm_batch_jsonl_line_item_callbacks
yucheng-berri Sep 23, 2026
c0b69ae
fix(batches): keep line item events out of proxy limiters and strip c…
yucheng-berri Sep 23, 2026
67d4471
fix(batches): keep cast-ok tags on the flagged cast lines for the dis…
yucheng-berri Sep 23, 2026
ee8dcfc
fix(batches): skip failed line item events in parallel request limite…
yucheng-berri Sep 23, 2026
a2fd2a8
test(batches): tolerate lifecycle spend rows when asserting no line i…
yucheng-berri Sep 23, 2026
fe11251
chore: merge main into litellm_batch_jsonl_line_item_callbacks
yucheng-berri Sep 24, 2026
05d6977
fix(batches): place mutable-ok tags on the lines the discipline gate …
yucheng-berri Sep 24, 2026
f5670e0
fix(batches): skip router budget accounting for batch line item events
yucheng-berri Sep 24, 2026
0ca85b4
chore: merge main into litellm_batch_jsonl_line_item_callbacks
yucheng-berri Sep 24, 2026
dd9d4fd
chore: merge main into litellm_batch_jsonl_line_item_callbacks
yucheng-berri Sep 24, 2026
4c86f0b
fix(batches): keep anthropic line errors and skip native vertex batch…
yucheng-berri Sep 25, 2026
5777f7e
test(batches): restore test-quality tags on line item tests
yucheng-berri Sep 25, 2026
87d82b9
chore: merge main into litellm_batch_jsonl_line_item_callbacks
yucheng-berri Sep 25, 2026
f4a6e12
Merge remote-tracking branch 'origin/main' into litellm_batch_jsonl_l…
yucheng-berri Sep 26, 2026
f3ba65d
test: move batch line-item unit test to tests/unit after merging main
yucheng-berri Sep 26, 2026
f65b5b1
fix(batches): emit batch line-item callbacks once per batch
yucheng-berri Sep 27, 2026
cffa640
fix(batches): release line-item claim when fan-out emitted nothing
yucheng-berri Sep 27, 2026
a7c9ded
fix(batches): make the line-item claim release ownership-safe
yucheng-berri Sep 29, 2026
f69d5c5
fix(batches): keep bodyless failed batch lines reaching callbacks
yucheng-berri Sep 30, 2026
5f35c25
fix(logging): seed batch line hidden params in get_hidden_params defa…
yucheng-berri Sep 30, 2026
a891065
merge: main into litellm_batch_jsonl_line_item_callbacks
yucheng-berri Sep 30, 2026
ff77170
perf(batches): reuse fetched batch result files for line-item logging
yucheng-berri Sep 30, 2026
1388dbc
fix(batches): release line-item claim when nothing was emitted
yucheng-berri Sep 30, 2026
ee21adb
Merge remote-tracking branch 'origin/main' into litellm_batch_jsonl_l…
yucheng-berri Oct 1, 2026
4d96efc
Merge remote-tracking branch 'origin/main' into litellm_batch_jsonl_l…
yucheng-berri Oct 1, 2026
64c514c
fix(batches): type batch line-item callback sites to satisfy basedpyr…
yucheng-berri Oct 1, 2026
aebde2a
style: wrap batch hidden-params cast for ruff format
yucheng-berri Oct 1, 2026
06fb342
Merge remote-tracking branch 'origin/main' into litellm_batch_jsonl_l…
yucheng-berri Oct 1, 2026
eac1ed0
fix(batches): reason suppressions for batch line-item lint deltas
yucheng-berri Oct 1, 2026
e1b9449
Merge remote-tracking branch 'origin/main' into litellm_batch_jsonl_l…
yucheng-berri Oct 1, 2026
60a47e2
chore: drop stale mutable-ok suppressions after LIT002 removal
yucheng-berri Oct 1, 2026
a7f1978
fix(batches): skip batch line-item events in ClickHouse spend sink an…
yucheng-berri Oct 4, 2026
9ca41c9
Merge origin/main into litellm_batch_jsonl_line_item_callbacks
yucheng-berri Oct 4, 2026
9c71dae
fix(clickhouse): skip failed batch line items in the spend sink too
yucheng-berri Oct 5, 2026
8ec50a2
fix(batches): declare batch_line_items service family on claim calls
yucheng-berri Oct 5, 2026
a35d4ee
fix(integrations): keep batch line items out of the built-in metering…
yucheng-berri Oct 5, 2026
c9f67ff
style: format metering-sinks test
yucheng-berri Oct 5, 2026
1b02cb6
fix(otel): skip synthetic line-item durations in v2 metrics; isolate …
yucheng-berri Oct 5, 2026
6348a7d
Merge remote-tracking branch 'origin/main' into litellm_batch_jsonl_l…
yucheng-berri Oct 6, 2026
b139924
chore: merge main into litellm_batch_jsonl_line_item_callbacks
yucheng-berri Oct 6, 2026
db33862
chore: merge main into litellm_batch_jsonl_line_item_callbacks
yucheng-berri Oct 6, 2026
15fa86c
chore: merge main into litellm_batch_jsonl_line_item_callbacks
yucheng-berri Oct 7, 2026
1b63787
chore: finish main merge conflict resolution for batch line items
yucheng-berri Oct 7, 2026
235019d
style: sort imports in litellm_logging after main merge
yucheng-berri Oct 7, 2026
6b8fb29
chore: merge origin/main into litellm_batch_jsonl_line_item_callbacks
yucheng-berri Oct 7, 2026
0632fdd
Merge remote-tracking branch 'origin/main' into fix41691
yucheng-berri Oct 8, 2026
b57e4f0
test(batches): use the public hook class names main now exports
yucheng-berri Oct 8, 2026
c653e2e
fix(batches): only the batch writer can mark a callback event as a li…
yucheng-berri Oct 8, 2026
e00471b
chore(batches): keep the lint gates within main's ceilings after the …
yucheng-berri Oct 8, 2026
317764c
fix(anthropic): read the message id off the validated message
yucheng-berri Oct 8, 2026
85fc449
fix(batches): an unreadable error file no longer drops the output lin…
yucheng-berri Oct 8, 2026
21cd892
test(batches): pin billing for batch_parent_id on every endpoint and …
yucheng-berri Oct 8, 2026
7cf3024
test(batches): bill every request of a concurrent burst mixing forged…
yucheng-berri Oct 8, 2026
4bd5762
revert: keep the line-item fan-out all-or-nothing on an unreadable er…
yucheng-berri Oct 8, 2026
4cc9f56
test(batches): pin the all-or-nothing retry and warm the burst model …
yucheng-berri Oct 8, 2026
586b022
Merge remote-tracking branch 'origin/main' into fix41691
yucheng-berri Oct 8, 2026
f26f86b
Merge remote-tracking branch 'origin/main' into fix41691
yucheng-berri Oct 9, 2026
07d543c
fix(batches): run the line fan-out outside the aggregate event's dead…
yucheng-berri Oct 9, 2026
6e826b4
fix(batches): let running line fan-outs finish before proxy shutdown …
yucheng-berri Oct 10, 2026
fd7a099
Merge remote-tracking branch 'origin/main' into fix41691
yucheng-berri Oct 12, 2026
9ed3211
fix(batches): keep batch line fields off callback kwargs and skip fan…
yucheng-berri Oct 12, 2026
8a46101
fix(batches): log batch lines under the retrieve's callback settings …
yucheng-berri Oct 12, 2026
7ae8fec
test(batches): line events follow the key's message logging opt-out
yucheng-berri Oct 12, 2026
903c593
test(batches): line events follow team logging settings, keep the lin…
yucheng-berri Oct 12, 2026
2b30a09
fix(batches): keep in-memory line claims for 10,000 batches and pin m…
yucheng-berri Oct 12, 2026
0865123
Keep batch line unit tests in-process and drop repeated per-sink comm…
yucheng-berri Oct 12, 2026
56b60e6
Share one batch provider alias across batch_utils and line item logging
yucheng-berri Oct 12, 2026
0470177
Report Bedrock line status codes, skip unknown Bedrock lines without …
yucheng-berri Oct 12, 2026
8f2c935
Read a swallowed Redis claim failure as unavailable, price unpriceabl…
yucheng-berri Oct 12, 2026
d62c5f2
Call handle_completed_batch when line items are off, keep a cache tes…
yucheng-berri Oct 12, 2026
f6dc422
Hand the cost poller's credential snapshot to line item file reads, s…
yucheng-berri Oct 12, 2026
6125f5f
Log batch lines through the sync callbacks a direct async call runs, …
yucheng-berri Oct 12, 2026
cbdbbb5
fix(batches): log failed Bedrock embedding lines as embeddings with t…
yucheng-berri Oct 13, 2026
5460242
fix(batches): keep a Bedrock line's system prompt in its logged messages
yucheng-berri Oct 13, 2026
f010f96
Merge remote-tracking branch 'origin/main' into fix41691
yucheng-berri Oct 13, 2026
4c120da
feat(batches): store per-line batch requests and responses in S3/GCS …
yucheng-berri Oct 13, 2026
7ab9a3f
fix(batches): store batch line items after the batch is billed and re…
yucheng-berri Oct 13, 2026
5cad620
test(batches): cover team-level and deleted-key redaction of batch li…
yucheng-berri Oct 13, 2026
aa76a2e
fix(batches): store batch line items in the background so a hung buck…
yucheng-berri Oct 13, 2026
15f3d73
fix(batches): store an unbilled batch's line items only once it is re…
yucheng-berri Oct 13, 2026
21a7109
fix(batches): price and write batch line items off the event loop
yucheng-berri Oct 13, 2026
082a859
fix(batches): pair no request for a shared custom_id and wait for an …
yucheng-berri Oct 13, 2026
459234f
fix(batches): store batch line items only once the poll has billed ev…
yucheng-berri Oct 13, 2026
400a6f9
fix(batches): hash line-item objects off the event loop and keep line…
yucheng-berri Oct 13, 2026
ee61a60
fix(batches): keep request and response for batches created without a…
yucheng-berri Oct 13, 2026
85169fc
fix(batches): pair line items with the right request on every deploym…
yucheng-berri Oct 13, 2026
2875f03
fix(batches): let OAuth2 batches follow team and proxy message loggin…
yucheng-berri Oct 13, 2026
4ba1eaa
Merge branch 'main' into litellm_batch_jsonl_line_item_callbacks
yucheng-berri Oct 14, 2026
0ee34c9
fix(batches): hash the creator credential and write batch line items …
yucheng-berri Oct 14, 2026
a02fc23
chore: merge main into litellm_batch_jsonl_line_item_callbacks
yucheng-berri Oct 14, 2026
bc925d4
docs(batches): say async_log_batch_line_items is called once per part
yucheng-berri Oct 15, 2026
315d705
fix(batches): log an error when a batch's deployment is gone before i…
yucheng-berri Oct 15, 2026
c75d257
Merge origin/main into litellm_batch_jsonl_line_item_callbacks
yucheng-berri Oct 15, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -897,6 +897,7 @@ async def _track_completed_batch_cost(
optional_params={},
custom_llm_provider=str(llm_provider) if llm_provider else None,
)
logging_obj._litellm_internal_model_credentials = MappingProxyType(dict(credentials)) # pyright: ignore[reportPrivateUsage] # trusted credentials transport, consumed by batch_line_item_logging

if not await self._claim_job_for_costing(job):
verbose_proxy_logger.info(
Expand Down
1 change: 1 addition & 0 deletions litellm/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -343,6 +343,7 @@ def _dev_env_hot_reload_enabled() -> bool:
)
openai_system_messages_first: bool = False
disable_vertex_batch_output_transformation: bool = False
store_batch_line_items_in_callbacks: bool = False
extra_spend_tag_headers: Optional[List[str]] = None
in_memory_llm_clients_cache: "LLMClientCache"
safe_memory_mode: bool = False
Expand Down
499 changes: 499 additions & 0 deletions litellm/batches/batch_line_item_logging.py

Large diffs are not rendered by default.

14 changes: 9 additions & 5 deletions litellm/batches/batch_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ def batch_cost_is_final(batch: Batch) -> bool:

async def calculate_batch_cost_and_usage(
file_content_dictionary: list[dict],
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"],
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
Comment thread
yucheng-berri marked this conversation as resolved.
Resolved
model_name: str | None = None,
model_info: ModelInfo | None = None,
) -> BatchCostUsageResult:
Expand All @@ -97,7 +97,7 @@ async def calculate_batch_cost_and_usage(

async def _handle_completed_batch(
batch: Batch,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"],
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
model_name: str | None = None,
litellm_params: dict | None = None,
model_info: ModelInfo | None = None,
Expand Down Expand Up @@ -434,7 +434,9 @@ def _provider_output_file_id(output_file_id: str) -> str:

async def _fetch_batch_managed_file_content(
file_id: str,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"] = "openai",
custom_llm_provider: Literal[
"openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"
] = "openai",
litellm_params: dict | None = None,
) -> bytes:
"""
Expand Down Expand Up @@ -464,7 +466,9 @@ async def _fetch_batch_managed_file_content(

async def _fetch_batch_output_file_content(
batch: Batch,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"] = "openai",
custom_llm_provider: Literal[
"openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"
] = "openai",
litellm_params: dict | None = None,
) -> bytes:
"""
Expand All @@ -486,7 +490,7 @@ async def _fetch_batch_output_file_content(

async def count_error_file_failed_requests(
Comment thread
yucheng-berri marked this conversation as resolved.
Resolved
batch: Batch,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "mistral"],
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "hosted_vllm", "anthropic", "bedrock", "mistral"],
litellm_params: dict | None,
) -> int:
"""Count failed requests reported only in the batch's separate error file.
Expand Down
13 changes: 12 additions & 1 deletion litellm/litellm_core_utils/core_helpers.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import re
from collections.abc import Collection, Iterable, Mapping
from types import MappingProxyType
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol
from typing import TYPE_CHECKING, Any, Final, Literal, Protocol, cast

import httpx
from pydantic import TypeAdapter, ValidationError
Expand Down Expand Up @@ -767,6 +767,17 @@ def set_response_cost_in_hidden_params(response: _CarriesHiddenParams, cost: flo
hidden_params["additional_headers"] = merged


def is_batch_line_item_event(kwargs: object) -> bool:
if not isinstance(kwargs, Mapping):
return False
typed_kwargs: Final = cast(Mapping[str, object], kwargs) # cast-ok: isinstance leaves Mapping unparameterized
litellm_params: Final = typed_kwargs.get("litellm_params")
if not isinstance(litellm_params, Mapping):
return False
typed_params: Final = cast(Mapping[str, object], litellm_params) # cast-ok: same narrowing limit
return bool(typed_params.get("batch_parent_id"))


_HIDDEN_PARAMS_ADAPTER: Final = TypeAdapter(Mapping[str, object])
_PROVIDER_HEADERS_ADAPTER: Final = TypeAdapter(Mapping[str, str])

Expand Down
34 changes: 20 additions & 14 deletions litellm/litellm_core_utils/litellm_logging.py
Original file line number Diff line number Diff line change
Expand Up @@ -642,6 +642,7 @@ def __init__(
self.streaming_chunks: list[Any] = [] # for generating complete stream response
self.sync_streaming_chunks: list[Any] = [] # for generating complete stream response
self.log_raw_request_response = log_raw_request_response
self._litellm_internal_model_credentials: Mapping[str, object] | None = None
self.raw_request_only = raw_request_only

# Initialize dynamic callbacks
Expand Down Expand Up @@ -3261,6 +3262,18 @@ async def _async_success_handler_body(
cost_for_built_in_tools_cost_usd_dollar=0.0,
)

if litellm.store_batch_line_items_in_callbacks and (has_explicit_batch_data or should_compute_batch_data):
from litellm.batches.batch_line_item_logging import log_batch_line_items

await log_batch_line_items(
batch=result,
custom_llm_provider=self.custom_llm_provider,
parent=self,
model_name=self.get_deployment_model_for_cost(),
litellm_params=self.litellm_params,
model_info=self.get_router_deployment_model_info(),
)

self.truncated_messages_for_logging = await truncate_base64_in_messages_async(
StandardLoggingPayloadSetup.append_system_prompt_messages(
kwargs=self.model_call_details, messages=self.model_call_details.get("messages")
Expand Down Expand Up @@ -4210,19 +4223,10 @@ def _handle_anthropic_messages_response_logging(self, result: Any) -> ModelRespo
verbose_logger.warning("LiteLLM: the anthropic_messages stream assembled no response, logging an empty one")
return litellm.ModelResponse(model=self.model)
else:
from litellm.types.llms.anthropic import AnthropicResponse

pydantic_result: Final = AnthropicResponse.model_validate(result)
import httpx
from litellm.llms.anthropic.chat.transformation import anthropic_message_to_model_response

result = litellm.AnthropicConfig().transform_parsed_response(
completion_response=pydantic_result.model_dump(),
raw_response=httpx.Response(
status_code=200,
headers={},
),
model_response=litellm.ModelResponse(id=provider_response_id),
json_mode=None,
result = anthropic_message_to_model_response(
cast(Mapping[str, object], result), # cast-ok: handler result is typed Any upstream
speed=self.optional_params.get("speed") if self.optional_params else None,
)
return result
Expand Down Expand Up @@ -6348,8 +6352,10 @@ def _extract_response_obj_and_hidden_params(
response_obj = {}

if original_exception is not None and hidden_params is None:
response_headers: Final = _get_response_headers(original_exception)
if response_headers is not None:
exception_hidden_params: Final = getattr(original_exception, "_hidden_params", None)
if isinstance(exception_hidden_params, dict) and exception_hidden_params:
hidden_params = dict(exception_hidden_params) # mutable-ok: hidden_params downstream expects a plain dict
elif (response_headers := _get_response_headers(original_exception)) is not None:
hidden_params = dict(
StandardLoggingHiddenParams(
additional_headers=StandardLoggingPayloadSetup.get_additional_headers(dict(response_headers)),
Expand Down
16 changes: 16 additions & 0 deletions litellm/llms/anthropic/chat/transformation.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
AnthropicMessagesToolChoice,
AnthropicOutputSchema,
AnthropicOutputTokensDetails,
AnthropicResponse,
AnthropicSystemMessageContent,
AnthropicThinkingParam,
AnthropicWebSearchTool,
Expand Down Expand Up @@ -2808,3 +2809,18 @@ def _valid_user_id(user_id: str) -> bool:
return False

return True


def anthropic_message_to_model_response(result: Mapping[str, object], speed: str | None) -> ModelResponse:
pydantic_result: Final = AnthropicResponse.model_validate(result)
result_id: Final = result.get("id")
return AnthropicConfig().transform_parsed_response(
completion_response=pydantic_result.model_dump(),
raw_response=httpx.Response(
status_code=200,
headers={}, # mutable-ok: httpx.Response wants a plain dict of headers
),
model_response=ModelResponse(id=result_id if isinstance(result_id, str) and result_id else None),
json_mode=None,
speed=speed,
)
11 changes: 10 additions & 1 deletion litellm/llms/base_llm/batches/transformation.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import types
from abc import ABC, abstractmethod
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any

import httpx
Expand All @@ -9,7 +10,7 @@
AllMessageValues,
CreateBatchRequest,
)
from litellm.types.utils import LiteLLMBatch, LlmProviders
from litellm.types.utils import EmbeddingResponse, LiteLLMBatch, LlmProviders, ModelResponse

if TYPE_CHECKING:
from litellm.litellm_core_utils.litellm_logging import Logging as _LiteLLMLoggingObj
Expand Down Expand Up @@ -39,6 +40,14 @@ def __init__(self):
def custom_llm_provider(self) -> LlmProviders:
"""Return the LLM provider type for this configuration."""

def transform_batch_output_line(
self, model_output: Mapping[str, object], model: str
) -> ModelResponse | EmbeddingResponse | None:
"""Reconstruct one provider batch output line into a litellm response, or
None when the line is OpenAI-shaped and the caller can use the generic
reconstruction."""
return None

@classmethod
def get_config(cls):
"""Get configuration dictionary for this class."""
Expand Down
50 changes: 49 additions & 1 deletion litellm/llms/bedrock/batches/transformation.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@
AllMessageValues,
CreateBatchRequest,
)
from litellm.types.utils import LiteLLMBatch, LlmProviders, Usage
from litellm.types.utils import EmbeddingResponse, LiteLLMBatch, LlmProviders, ModelResponse, Usage

from ..base_aws_llm import BaseAWSLLM
from ..common_utils import (
Expand Down Expand Up @@ -96,6 +96,49 @@ def titan_embedding_usage_from_batch_output(model_output: Mapping[str, object])
)


def bedrock_batch_line_to_response(
model_output: Mapping[str, object], model: str
) -> ModelResponse | EmbeddingResponse | None:
"""Reconstruct a Bedrock batch output line (the ``modelOutput`` object) into
the litellm response type its shape implies, or None when the shape is
unrecognized."""
if "embedding" in model_output:
embedding: Final = model_output.get("embedding")
return EmbeddingResponse(
model=model,
data=[ # mutable-ok: EmbeddingResponse takes a plain data list
{ # mutable-ok: plain row dict for EmbeddingResponse.data
"object": "embedding",
"index": 0,
"embedding": embedding
if isinstance(embedding, list)
else [], # mutable-ok: empty fallback for the row
}
],
usage=titan_embedding_usage_from_batch_output(model_output),
)
if "output" in model_output:
from ..chat.converse_transformation import AmazonConverseConfig

return AmazonConverseConfig()._transform_response( # pyright: ignore[reportPrivateUsage] # same reconstruction the converse chat path performs on the live response
model=model,
response=Response(200, json=dict(model_output)), # mutable-ok: httpx.Response json= wants a plain dict
model_response=ModelResponse(),
stream=False,
logging_obj=None,
optional_params={}, # mutable-ok: the converse transform signature takes a plain dict
api_key=None,
data="",
messages=[], # mutable-ok: the converse transform signature takes a plain list
encoding=None,
)
if "content" in model_output:
from litellm.llms.anthropic.chat.transformation import anthropic_message_to_model_response

return anthropic_message_to_model_response(model_output, None)
return None


class BedrockBatchesConfig(BaseAWSLLM, BaseBatchesConfig):
"""
Config for Bedrock Batches - handles batch job creation and management for Bedrock
Expand All @@ -109,6 +152,11 @@ def __init__(self):
def custom_llm_provider(self) -> LlmProviders:
return LlmProviders.BEDROCK

def transform_batch_output_line(
self, model_output: Mapping[str, object], model: str
) -> ModelResponse | EmbeddingResponse | None:
return bedrock_batch_line_to_response(model_output, model)

@classmethod
def _get_bare_model_name_from_s3_key(cls, object_key: str) -> str | None:
if not object_key.startswith(BEDROCK_MANAGED_S3_BATCH_PREFIX):
Expand Down
4 changes: 4 additions & 0 deletions litellm/proxy/_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -2943,6 +2943,10 @@ class ConfigGeneralSettings(LiteLLMPydanticObjectBase):
None,
description="If True, stores request messages and responses in spend logs. Default is False.",
)
store_batch_line_items_in_callbacks: bool | None = Field(
None,
description="If True, a completed batch logged via aretrieve_batch also emits one callback event per JSONL line item (request paired with its response or error). The aggregate batch callback is unchanged. Default is False.",
)
disable_auto_add_proxy_admin_to_teams: bool | None = Field(
None,
description="By default, the user calling /team/new is automatically added to the new team as a team admin. If True, proxy admins are no longer auto-added; members explicitly listed in members_with_roles are unaffected. Default is False.",
Expand Down
3 changes: 3 additions & 0 deletions litellm/proxy/hooks/dynamic_rate_limiter_v3.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import is_batch_line_item_event
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.proxy_rate_limit_error import (
ProxyRateLimitError,
Expand Down Expand Up @@ -693,6 +694,8 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti
- model_saturation_check: Model-wide token tracking
- priority_model: Priority-specific token tracking
"""
if is_batch_line_item_event(kwargs):
return
from litellm.litellm_core_utils.core_helpers import (
_get_parent_otel_span_from_kwargs,
)
Expand Down
3 changes: 3 additions & 0 deletions litellm/proxy/hooks/max_budget_per_session_limiter.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
from litellm.caching.redis_cache import log_redis_failure
from litellm.exceptions import RateLimitType
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import is_batch_line_item_event
from litellm.proxy._types import UserAPIKeyAuth
from litellm.proxy.common_utils.proxy_rate_limit_error import ProxyRateLimitError
from litellm.proxy.hooks.rate_limiter_utils import resolve_llm_provider_for_rate_limit
Expand Down Expand Up @@ -131,6 +132,8 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti
"""
After a successful LLM call, increment the session spend by the response cost.
"""
if is_batch_line_item_event(kwargs): # pyright: ignore[reportUnknownArgumentType] # hook kwargs arrive untyped from the logging dispatcher
return
try:
litellm_params: Final = kwargs.get("litellm_params") or {}
metadata: Final = litellm_params.get("metadata") or {}
Expand Down
3 changes: 3 additions & 0 deletions litellm/proxy/hooks/model_max_budget_limiter.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from litellm._logging import verbose_proxy_logger
from litellm.caching.caching import DualCache
from litellm.integrations.custom_logger import Span
from litellm.litellm_core_utils.core_helpers import is_batch_line_item_event
from litellm.litellm_core_utils.duration_parser import duration_in_seconds
from litellm.llms.bedrock.common_utils import get_bedrock_base_model
from litellm.proxy._types import Litellm_EntityType, UserAPIKeyAuth
Expand Down Expand Up @@ -490,6 +491,8 @@ async def async_log_success_event(self, kwargs, response_obj, start_time, end_ti

Example: key=sk-1234567890, model=gpt-4o, max_budget=100, time_period=1d
"""
if is_batch_line_item_event(kwargs):
return
verbose_proxy_logger.debug("in RouterBudgetLimiting.async_log_success_event")
standard_logging_payload: Final[StandardLoggingPayload | None] = kwargs.get("standard_logging_object", None)
if standard_logging_payload is None:
Expand Down
9 changes: 8 additions & 1 deletion litellm/proxy/hooks/parallel_request_limiter.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,10 @@
from litellm._logging import verbose_proxy_logger
from litellm.exceptions import RateLimitType
from litellm.integrations.custom_logger import CustomLogger
from litellm.litellm_core_utils.core_helpers import _get_parent_otel_span_from_kwargs
from litellm.litellm_core_utils.core_helpers import (
_get_parent_otel_span_from_kwargs,
is_batch_line_item_event,
)
from litellm.proxy._types import CommonProxyErrors, CurrentItemRateLimit, UserAPIKeyAuth
from litellm.proxy.auth.auth_utils import (
get_key_model_rpm_limit,
Expand Down Expand Up @@ -490,6 +493,8 @@ async def async_pre_call_hook(
)

Comment thread
greptile-apps[bot] marked this conversation as resolved.
Resolved
async def async_log_success_event(self, kwargs, response_obj: object, start_time, end_time):
if is_batch_line_item_event(kwargs):
return
from litellm.proxy.common_utils.callback_utils import (
get_model_group_from_litellm_kwargs,
)
Expand Down Expand Up @@ -695,6 +700,8 @@ async def async_log_success_event(self, kwargs, response_obj: object, start_time
self.print_verbose(e)

async def async_log_failure_event(self, kwargs, response_obj, start_time, end_time):
if is_batch_line_item_event(kwargs):
return
try:
self.print_verbose("Inside Max Parallel Request Failure Hook")
litellm_parent_otel_span: Final[Span | None] = _get_parent_otel_span_from_kwargs(kwargs=kwargs)
Expand Down
Loading
Loading