Skip to content

feat(xai): add native xAI batches and files support #42812

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mateo-berri merged 18 commits into from
Sep 25, 2026
Merged

feat(xai): add native xAI batches and files support #42812

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
9c89e31
feat(xai): add native xAI batches and files support
Sep 23, 2026
59d750e
Merge remote-tracking branch 'origin/main' into litellm_xai_batches
Sep 23, 2026
d3ccf6e
fix(xai): tighten batch handler typing and avoid Final redeclaration …
Sep 23, 2026
383fbc5
refactor(xai): walk batch result pages iteratively
Sep 23, 2026
de144a2
fix(xai): stop paging on empty pagination token and honor litellm.xai…
Sep 23, 2026
beff309
fix(xai): import NotRequired and TypedDict from typing_extensions for…
Sep 23, 2026
1dcd712
feat(batches): accept image and video endpoints on batch create
mubashir1osmani Sep 23, 2026
c7cd26e
test(xai): lock batch endpoint, auth, and result contracts
mubashir1osmani Sep 23, 2026
2fcb819
fix(xai): price grok batch usage at xAI's 20 percent batch discount
mateo-berri Sep 25, 2026
769be6f
fix(xai): map not-found file reads to 404, bill batch reasoning token…
mateo-berri Sep 25, 2026
76067e0
Merge remote-tracking branch 'origin/main' into litellm_xai_batches
mateo-berri Sep 25, 2026
d7ea3a6
Merge remote-tracking branch 'origin/main' into litellm_xai_batches
Sep 25, 2026
c49f03b
refactor(xai): drop routine prose and move tests under tests/unit
mateo-berri Sep 25, 2026
8c983d3
Merge remote-tracking branch 'origin/litellm_xai_batches' into litell…
mateo-berri Sep 25, 2026
cd35f82
Merge remote-tracking branch 'origin/main' into litellm_xai_batches
mateo-berri Sep 25, 2026
ea97f57
fix(health): hand the resolved provider to list_batches in batch-mode…
mateo-berri Sep 25, 2026
c29a1ff
test(xai): make tests/unit/llms/xai/batches a package
mateo-berri Sep 25, 2026
dc0e29c
fix(xai): walk every page of the files list by pagination_token
mateo-berri Sep 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions litellm/batches/batch_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -706,6 +706,10 @@ def _get_batch_job_usage_from_response_body(
if ResponseAPILoggingUtils._is_response_api_usage(_usage_dict):
return ResponseAPILoggingUtils._transform_response_api_usage_to_chat_usage(_usage_dict)
usage: Final[Usage] = Usage(**_usage_dict)
if custom_llm_provider == "xai":
from litellm.llms.xai.chat.transformation import XAIChatConfig

XAIChatConfig.fold_reasoning_tokens_into_completion(usage)
Comment thread
greptile-apps[bot] marked this conversation as resolved.
return usage


Expand Down
77 changes: 68 additions & 9 deletions litellm/batches/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
from litellm.llms.custom_httpx.llm_http_handler import BaseLLMHTTPHandler
from litellm.llms.openai.openai import OpenAIBatchesAPI
from litellm.llms.vertex_ai.batches.handler import VertexAIBatchPrediction
from litellm.llms.xai.batches.handler import XAIBatchesHandler
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
CancelBatchRequest,
Expand Down Expand Up @@ -59,6 +60,7 @@
azure_batches_instance: Final = AzureBatchesAPI()
vertex_ai_batches_instance: Final = VertexAIBatchPrediction(gcs_bucket_name="")
anthropic_batches_instance: Final = AnthropicBatchesHandler()
xai_batches_instance: Final = XAIBatchesHandler()
base_llm_http_handler = BaseLLMHTTPHandler()
#################################################

Expand Down Expand Up @@ -105,10 +107,22 @@ def _resolve_timeout(
@client
async def acreate_batch(
completion_window: Literal["24h"],
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses", "/v1/ocr"],
endpoint: Literal[
"/v1/chat/completions",
"/v1/embeddings",
"/v1/completions",
"/v1/responses",
"/v1/ocr",
"/v1/images/generations",
"/v1/images/edits",
"/v1/videos/generations",
"/v1/videos",
"/v1/videos/edits",
"/v1/videos/extensions",
],
input_file_id: str,
custom_llm_provider: Literal[
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "mistral"
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "mistral", "xai"
] = "openai",
metadata: dict[str, str] | None = None,
extra_headers: dict[str, str] | None = None,
Expand Down Expand Up @@ -157,10 +171,22 @@ async def acreate_batch(
@client
def create_batch(
completion_window: Literal["24h"],
endpoint: Literal["/v1/chat/completions", "/v1/embeddings", "/v1/completions", "/v1/responses", "/v1/ocr"],
endpoint: Literal[
"/v1/chat/completions",
"/v1/embeddings",
"/v1/completions",
"/v1/responses",
"/v1/ocr",
"/v1/images/generations",
"/v1/images/edits",
"/v1/videos/generations",
"/v1/videos",
"/v1/videos/edits",
"/v1/videos/extensions",
],
input_file_id: str,
custom_llm_provider: Literal[
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "mistral"
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "mistral", "xai"
] = "openai",
metadata: dict[str, str] | None = None,
extra_headers: dict[str, str] | None = None,
Expand Down Expand Up @@ -243,6 +269,14 @@ def create_batch(
model=model,
)
return response
if custom_llm_provider == LlmProviders.XAI.value:
return xai_batches_instance.create_batch(
_is_async=_is_async,
create_batch_data=_create_batch_request,
api_base=optional_params.api_base,
api_key=optional_params.api_key,
timeout=timeout,
)
api_base: str | None = None
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
Expand Down Expand Up @@ -345,7 +379,7 @@ def create_batch(
async def aretrieve_batch(
batch_id: str,
custom_llm_provider: Literal[
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral"
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral", "xai"
] = "openai",
metadata: dict[str, str] | None = None,
extra_headers: dict[str, str] | None = None,
Expand Down Expand Up @@ -393,10 +427,18 @@ def _handle_retrieve_batch_providers_without_provider_config(
_retrieve_batch_request: RetrieveBatchRequest,
_is_async: bool,
custom_llm_provider: Literal[
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral"
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral", "xai"
] = "openai",
logging_obj: LiteLLMLoggingObj | None = None,
):
if custom_llm_provider == LlmProviders.XAI.value:
return xai_batches_instance.retrieve_batch(
_is_async=_is_async,
batch_id=batch_id,
api_base=optional_params.api_base,
api_key=optional_params.api_key,
timeout=timeout,
)
api_base: str | None = None
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
Expand Down Expand Up @@ -518,7 +560,7 @@ def _handle_retrieve_batch_providers_without_provider_config(
def retrieve_batch(
batch_id: str,
custom_llm_provider: Literal[
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral"
"openai", "azure", "vertex_ai", "bedrock", "hosted_vllm", "litellm_proxy", "anthropic", "mistral", "xai"
] = "openai",
metadata: dict[str, str] | None = None,
extra_headers: dict[str, str] | None = None,
Expand Down Expand Up @@ -741,6 +783,15 @@ def list_batches(
timeout = 600.0

_is_async: Final = kwargs.pop("alist_batches", False) is True
if custom_llm_provider == LlmProviders.XAI.value:
return xai_batches_instance.list_batches(
_is_async=_is_async,
api_base=optional_params.api_base,
api_key=optional_params.api_key,
timeout=timeout,
after=after,
limit=limit,
)
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
# for deepinfra/perplexity/anyscale/groq we check in get_llm_provider and pass in the api base from there
api_base = (
Expand Down Expand Up @@ -837,7 +888,7 @@ def list_batches(
async def acancel_batch(
batch_id: str,
model: str | None = None,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "litellm_proxy"] = "openai",
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "litellm_proxy", "xai"] = "openai",
metadata: dict[str, str] | None = None,
extra_headers: dict[str, str] | None = None,
extra_body: dict[str, str] | None = None,
Expand Down Expand Up @@ -883,7 +934,7 @@ async def acancel_batch(
def cancel_batch(
batch_id: str,
model: str | None = None,
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "litellm_proxy"] | str = "openai",
custom_llm_provider: Literal["openai", "azure", "vertex_ai", "bedrock", "litellm_proxy", "xai"] | str = "openai",
metadata: dict[str, str] | None = None,
extra_headers: dict[str, str] | None = None,
extra_body: dict[str, str] | None = None,
Expand Down Expand Up @@ -933,6 +984,14 @@ def cancel_batch(
)

_is_async: Final = kwargs.pop("acancel_batch", False) is True
if custom_llm_provider == LlmProviders.XAI.value:
return xai_batches_instance.cancel_batch(
_is_async=_is_async,
batch_id=batch_id,
api_base=optional_params.api_base,
api_key=optional_params.api_key,
timeout=timeout,
)
api_base: str | None = None
if custom_llm_provider in OPENAI_COMPATIBLE_BATCH_AND_FILES_PROVIDERS:
api_base = (
Expand Down
21 changes: 18 additions & 3 deletions litellm/files/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,12 +28,15 @@
"manus",
"anthropic",
"mistral",
"xai",
]
FileRetrieveProvider = Literal[
"openai", "azure", "gemini", "vertex_ai", "hosted_vllm", "litellm_proxy", "manus", "anthropic", "mistral"
"openai", "azure", "gemini", "vertex_ai", "hosted_vllm", "litellm_proxy", "manus", "anthropic", "mistral", "xai"
]
FileDeleteProvider = Literal["openai", "azure", "gemini", "bedrock", "litellm_proxy", "manus", "anthropic", "mistral"]
FileListProvider = Literal["openai", "azure", "litellm_proxy", "manus", "anthropic", "mistral"]
FileDeleteProvider = Literal[
"openai", "azure", "gemini", "bedrock", "litellm_proxy", "manus", "anthropic", "mistral", "xai"
]
FileListProvider = Literal["openai", "azure", "litellm_proxy", "manus", "anthropic", "mistral", "xai"]
import litellm
from litellm import get_secret_str
from litellm.files.streaming import FileContentStreamingResponse
Expand All @@ -49,6 +52,8 @@
from litellm.llms.openai.common_utils import get_openai_credentials
from litellm.llms.openai.openai import FileDeleted, FileObject, OpenAIFilesAPI
from litellm.llms.vertex_ai.files.handler import VertexAIFilesHandler
from litellm.llms.xai.batches.handler import XAIBatchesHandler
from litellm.llms.xai.batches.transformation import is_xai_batch_results_id
from litellm.types.llms.openai import (
CreateFileRequest,
FileContentRequest,
Expand Down Expand Up @@ -103,6 +108,7 @@ def _file_content_logging_obj(kwargs: dict[str, object], _is_async: bool) -> Lit
azure_files_instance: Final = AzureOpenAIFilesAPI()
vertex_ai_files_instance: Final = VertexAIFilesHandler()
bedrock_files_instance: Final = BedrockFilesHandler()
xai_batch_results_instance: Final = XAIBatchesHandler()
#################################################


Expand Down Expand Up @@ -920,6 +926,15 @@ def file_content(
client=client,
)

if custom_llm_provider == LlmProviders.XAI.value and is_xai_batch_results_id(file_id):
return xai_batch_results_instance.batch_results_content(
_is_async=_is_async,
batch_id=file_id,
api_base=optional_params.api_base,
api_key=optional_params.api_key,
timeout=timeout,
)

# Check if provider has a custom files config (e.g., Anthropic, Manus)
provider_config: Final = ProviderConfigManager.get_provider_files_config(
model="",
Expand Down
12 changes: 5 additions & 7 deletions litellm/litellm_core_utils/health_check_helpers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,10 +114,9 @@ async def _batch_health_check(
"""
Health check for batch mode.

Calls list_batches for providers that support it (openai, hosted_vllm, azure,
vertex_ai). For all other providers (e.g. bedrock) the batch API surface doesn't
include list_batches, so we fall back to acompletion to verify connectivity and
credential validity instead.
Calls list_batches for providers that support it. For all other providers (e.g. bedrock)
the batch API surface doesn't include list_batches, so we fall back to acompletion to
verify connectivity and credential validity instead.
"""
import litellm

Expand All @@ -132,10 +131,9 @@ async def _batch_health_check(
litellm_params={"api_base": api_base} if api_base else None,
)

if custom_llm_provider in LIST_BATCHES_SUPPORTED_PROVIDERS:
return await litellm.alist_batches(**filtered_model_params)
else:
if custom_llm_provider not in LIST_BATCHES_SUPPORTED_PROVIDERS:
return await litellm.acompletion(**model_params)
return await litellm.alist_batches(**{**filtered_model_params, "custom_llm_provider": custom_llm_provider})

@staticmethod
async def _image_edit_health_check(edit_request: Callable[[], Awaitable["ImageResponse"]]) -> "ImageResponse":
Expand Down
3 changes: 3 additions & 0 deletions litellm/litellm_core_utils/litellm_logging.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -380,9 +380,12 @@ class RawRequestCaptured(Exception):
"output_cost_per_token",
"input_cost_per_token_batches",
"output_cost_per_token_batches",
"input_cost_per_token_above_200k_tokens_batches",
"input_cost_per_token_above_272k_tokens_batches",
"output_cost_per_token_above_200k_tokens_batches",
"output_cost_per_token_above_272k_tokens_batches",
"cache_read_input_token_cost_batches",
"cache_read_input_token_cost_above_200k_tokens_batches",
"cache_read_input_token_cost_above_272k_tokens_batches",
"cache_creation_input_token_cost_batches",
"cache_creation_input_token_cost_above_272k_tokens_batches",
Expand Down
Empty file added litellm/llms/xai/batches/__init__.py
View file
Open in desktop
Empty file.
Loading
Loading