Skip to content
Merged
Show file tree
Hide file tree
Changes from 44 commits
Commits
Show all changes
50 commits
Select commit Hold shift + click to select a range
aee740c
Cato Networks guardrail, based on Aim (#26597)
trexinc May 28, 2026
dbba9f1
Support Lemonade runtime context metadata (#28135)
neubig May 28, 2026
4469d6d
fix(cato): address guardrail review feedback
Sameerlite May 28, 2026
8747c17
fix(vertex_ai): route google/gemma-*-maas through partner-models Open…
icep87 May 28, 2026
a982643
fix(cato): guardrail all completion choices on output
Sameerlite May 28, 2026
9b0a7c5
Fix review
Sameerlite May 28, 2026
f4d247d
fix(cato_networks): harden output anonymize handling and restructure …
Sameerlite May 28, 2026
82d24e3
Fix mypy
Sameerlite May 29, 2026
bb73ed6
fix(cato): guard missing policy_drill_down and all_redacted_messages …
mateo-berri May 30, 2026
f885865
fix(cato): avoid KeyError bypassing block action on missing analysis_…
mateo-berri May 30, 2026
21856c8
fix(cato): preserve non-text message fields during anonymize
mateo-berri May 30, 2026
31f2fff
fix(cato): preserve trailing messages when fewer redacted messages re…
mateo-berri May 30, 2026
6677812
Merge remote-tracking branch 'origin/litellm_internal_staging' into p…
mateo-berri May 30, 2026
17675d1
fix(cato,model-info): preserve stream block signal on sender teardown…
mateo-berri May 30, 2026
212d899
fix(cato): surface mid-stream forwarding errors instead of blocking o…
mateo-berri May 30, 2026
f9b3415
fix(cato): drop spoofable end_user_id from guardrail user identity
mateo-berri May 30, 2026
fad053d
fix(cato): harden output anonymize path against missing content key
mateo-berri May 30, 2026
5fa9e8d
fix(cato): fall back to original message when redacted content key is…
mateo-berri May 30, 2026
451672b
refactor(model-info): drop unused api_key from cached model-info helper
mateo-berri May 30, 2026
3adc1df
fix(cato): preserve None content on tool-call-only choices in output …
mateo-berri May 30, 2026
e67add9
fix(ollama): respect static-model guard in OllamaConfig.get_model_info
mateo-berri May 30, 2026
1b9e3d3
fix(lemonade,ollama): treat empty api_key as unset to avoid leaking s…
mateo-berri May 30, 2026
f1180b2
fix(cato): inspect and redact Responses-API input, not just messages
mateo-berri May 30, 2026
0ab80b0
perf(utils): keep api_key out of get_model_info lru_cache key
mateo-berri May 30, 2026
1273d35
fix(cato): propagate ssl_verify to streaming WebSocket connection
mateo-berri May 30, 2026
1915626
refactor(utils): rename shadowing local in _get_model_info_helper
mateo-berri May 30, 2026
9551ed5
fix(cato): flatten multimodal chat content before inspection
mateo-berri May 30, 2026
8b45257
test(lemonade): clear get_model_info cache around api_base test
mateo-berri May 30, 2026
d9c5d4c
fix(cato): inspect and redact Responses-API input even when messages …
mateo-berri May 30, 2026
34c758e
Merge branch 'litellm_internal_staging' into litellm_oss_staging_28_0…
mateo-berri May 30, 2026
5f300d0
test(log_db_metrics): assert table_name event_metadata contract
mateo-berri May 30, 2026
5caba25
fix(cato): inspect and redact completion prompt and Responses-API ins…
mateo-berri May 30, 2026
d0bee40
Merge remote-tracking branch 'origin/litellm_internal_staging' into l…
mateo-berri May 30, 2026
7bd62af
fix(cato): serialize non-str/bytes websocket chunks before forwarding
mateo-berri May 30, 2026
f6b190d
fix(cato): inspect tool descriptions and tool-call arguments
mateo-berri May 30, 2026
71d57ad
fix(cato): map redacted output by assistant index; restore get_model_…
mateo-berri May 30, 2026
0879db6
fix(cato): block output even when detection_message is null/empty
mateo-berri May 30, 2026
9c11f0c
fix(cato): inspect and redact nested tool param and legacy function d…
mateo-berri May 31, 2026
53104d7
fix(cato): traverse schema descriptions iteratively to satisfy recurs…
mateo-berri May 31, 2026
02a6458
fix(cato): inspect and redact response_format JSON schema descriptions
mateo-berri May 31, 2026
cfe4830
fix(cato): skip output rewrite when Cato returns no redaction
mateo-berri May 31, 2026
eeb1116
refactor(utils): resolve explicit api_key model info without the cache
mateo-berri May 31, 2026
75d108c
fix(cato): inspect and redact non-description schema string values
mateo-berri May 31, 2026
040d316
fix(model-info): surface swallowed dynamic model-info errors
mateo-berri May 31, 2026
03970bd
fix(cato): inspect and redact Responses API output in post-call hook
mateo-berri May 31, 2026
808bb92
Merge remote-tracking branch 'origin/litellm_internal_staging' into l…
mateo-berri May 31, 2026
1ac9885
Merge remote-tracking branch 'origin/litellm_internal_staging' into l…
mateo-berri Jun 1, 2026
e65c15a
chore: reset _experimental/out folder
mateo-berri Jun 1, 2026
9e8353b
chore(ui): remove orphaned prebuilt dashboard chunk files
mateo-berri Jun 2, 2026
97c986b
fix(guardrails,ollama): forward ssl_verify to Cato init and raise_for…
mateo-berri Jun 2, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions litellm/constants.py
Original file line number Diff line number Diff line change
Expand Up @@ -868,6 +868,7 @@
_openai_like_providers: List = [
"predibase",
"databricks",
"lemonade",
"watsonx",
] # private helper. similar to openai but require some custom auth / endpoint handling, so can't use the openai sdk
# well supported replicate llms
Expand Down
5 changes: 4 additions & 1 deletion litellm/litellm_core_utils/exception_mapping_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ def is_error_str_context_window_exceeded(error_str: str) -> bool:
"is longer than the model's context length",
"input tokens exceed the configured limit",
"`inputs` tokens + `max_new_tokens` must be",
"exceeds the available context size", # llama.cpp/Lemonade
"exceeds the maximum number of tokens allowed", # Gemini
]
for substring in known_exception_substrings:
Expand Down Expand Up @@ -891,12 +892,14 @@ def exception_type( # type: ignore # noqa: PLR0915
response=getattr(original_exception, "response", None),
litellm_debug_info=extra_information,
)
elif "model's maximum context limit" in error_str:
elif ExceptionCheckers.is_error_str_context_window_exceeded(error_str):
exception_mapping_worked = True
raise ContextWindowExceededError(
message=f"{custom_llm_provider.capitalize()}Exception: Context Window Error - {error_str}",
model=model,
llm_provider=custom_llm_provider,
response=getattr(original_exception, "response", None),
litellm_debug_info=extra_information,
)
elif "token_quota_reached" in error_str:
exception_mapping_worked = True
Expand Down
123 changes: 120 additions & 3 deletions litellm/llms/lemonade/chat/transformation.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@
"""

from typing import Any, List, Optional, Tuple, Union
from urllib.parse import quote

import httpx

import litellm
from litellm._logging import verbose_logger
from litellm.litellm_core_utils.litellm_logging import Logging as LiteLLMLoggingObj
from litellm.secret_managers.main import get_secret_str
from litellm.types.llms.openai import (
Expand All @@ -18,6 +20,8 @@


class LemonadeChatConfig(OpenAILikeChatConfig):
_DEFAULT_API_KEY = "lemonade"

repeat_penalty: Optional[float] = None
functions: Optional[list] = None
logit_bias: Optional[dict] = None
Expand Down Expand Up @@ -68,7 +72,7 @@ def get_models(self, api_key: Optional[str] = None, api_base: Optional[str] = No
This method queries the Lemonade /models endpoint to retrieve the list of available models.

Args:
api_key: Optional API key (Lemonade doesn't require authentication)
api_key: Optional API key for authenticated Lemonade servers
api_base: Optional API base URL (defaults to LEMONADE_API_BASE env var or http://localhost:8000)

Returns:
Expand All @@ -87,6 +91,7 @@ def get_models(self, api_key: Optional[str] = None, api_base: Optional[str] = No
try:
response = litellm.module_level_client.get(
url=f"{api_base}/models",
headers=self._get_auth_headers(api_key),
)
except Exception as e:
raise ValueError(
Expand All @@ -101,19 +106,131 @@ def get_models(self, api_key: Optional[str] = None, api_base: Optional[str] = No
model_list = response.json().get("data", [])
return ["lemonade/" + model["id"] for model in model_list]

@staticmethod
def _get_positive_int(value: Any) -> Optional[int]:
if isinstance(value, bool):
return None
if isinstance(value, int) and value > 0:
return value
if isinstance(value, str):
try:
parsed = int(value)
except ValueError:
return None
if parsed > 0:
return parsed
return None

@staticmethod
def _get_provider_specific_entry(model_info: dict) -> dict:
provider_specific_entry = model_info.get("provider_specific_entry")
if not isinstance(provider_specific_entry, dict):
provider_specific_entry = {}
else:
provider_specific_entry = provider_specific_entry.copy()

for key in ("recipe_options", "context_window", "max_context_window"):
if key in model_info:
provider_specific_entry[key] = model_info[key]

return provider_specific_entry

def _get_context_window(self, model_info: dict) -> Optional[int]:
provider_specific_entry = self._get_provider_specific_entry(model_info)
recipe_options = provider_specific_entry.get("recipe_options")
if not isinstance(recipe_options, dict):
recipe_options = {}

for value in (
recipe_options.get("ctx_size"),
model_info.get("max_input_tokens"),
provider_specific_entry.get("context_window"),
provider_specific_entry.get("max_context_window"),
):
parsed = self._get_positive_int(value)
if parsed is not None:
return parsed
return None

def _get_default_model_info(self, model: str) -> dict:
return {
"key": "lemonade/" + model,
"litellm_provider": "lemonade",
"mode": "chat",
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
"max_tokens": None,
"max_input_tokens": None,
"max_output_tokens": None,
}

def get_model_info(
self,
model: str,
api_base: Optional[str] = None,
api_key: Optional[str] = None,
) -> Any:
if model.startswith("lemonade/"):
model = model.split("/", 1)[1]

api_base, api_key = self._get_openai_compatible_provider_info(
api_base=api_base, api_key=api_key
)
encoded_model = quote(model, safe="")

try:
response = litellm.module_level_client.get(
url=f"{api_base}/models/{encoded_model}",
headers=self._get_auth_headers(api_key),
)
response.raise_for_status()
model_info = response.json()
except Exception:
verbose_logger.debug("LemonadeError: Could not get model info.")
return self._get_default_model_info(model)

max_input_tokens = self._get_context_window(model_info)
max_output_tokens = self._get_positive_int(model_info.get("max_output_tokens"))
max_tokens = self._get_positive_int(model_info.get("max_tokens"))
provider_specific_entry = self._get_provider_specific_entry(model_info)

model_info_response = self._get_default_model_info(model)
model_info_response.update(
{
"max_tokens": max_tokens or max_output_tokens,
"max_input_tokens": max_input_tokens,
"max_output_tokens": max_output_tokens,
}
)
if provider_specific_entry:
model_info_response["provider_specific_entry"] = provider_specific_entry
return model_info_response

def _get_openai_compatible_provider_info(
self, api_base: Optional[str], api_key: Optional[str]
) -> Tuple[Optional[str], Optional[str]]:
# lemonade is openai compatible, we just need to set this to custom_openai and have the api_base be lemonade's endpoint
passed_api_base = api_base
api_base = (
api_base
or get_secret_str("LEMONADE_API_BASE")
or "http://localhost:8000/api/v1"
) # type: ignore
# Lemonade doesn't check the key
key = "lemonade"
key = self._DEFAULT_API_KEY
if passed_api_base is None or api_key:
key = (
api_key
or litellm.lemonade_key
or get_secret_str("LEMONADE_API_KEY")
or self._DEFAULT_API_KEY
)
return api_base, key

def _get_auth_headers(self, api_key: Optional[str]) -> dict:
if api_key is None or api_key == self._DEFAULT_API_KEY:
return {}
return {"Authorization": f"Bearer {api_key}"}

def transform_response(
self,
model: str,
Expand Down
123 changes: 119 additions & 4 deletions litellm/llms/ollama/common_utils.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from typing import List, Optional, Union
from typing import Any, List, Optional, Union

import httpx

Expand Down Expand Up @@ -65,7 +65,8 @@ def get_api_key(api_key=None) -> Optional[str]:
from litellm.secret_managers.main import get_secret_str

return (
os.environ.get("OLLAMA_API_KEY")
api_key
or os.environ.get("OLLAMA_API_KEY")
or litellm.api_key
or litellm.openai_key
or get_secret_str("OLLAMA_API_KEY")
Expand All @@ -78,13 +79,31 @@ def get_api_base(api_base: Optional[str] = None) -> str:
# env var OLLAMA_API_BASE or default
return api_base or get_secret_str("OLLAMA_API_BASE") or "http://localhost:11434"

@classmethod
def get_server_api_base(cls, api_base: Optional[str] = None) -> str:
api_base = cls.get_api_base(api_base).rstrip("/")
for suffix in (
"/api/generate",
"/api/chat",
"/api/embed",
"/api/embeddings",
"/api/show",
"/api/tags",
):
if api_base.endswith(suffix):
return api_base[: -len(suffix)]
return api_base

def get_models(self, api_key=None, api_base: Optional[str] = None) -> List[str]:
"""
List all models available on the Ollama server via /api/tags endpoint.
"""

base = self.get_api_base(api_base)
api_key = self.get_api_key()
passed_api_base = api_base
base = self.get_server_api_base(api_base)
api_key = (
self.get_api_key(api_key) if passed_api_base is None or api_key else None
)
headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}

names: set[str] = set()
Expand Down Expand Up @@ -126,6 +145,102 @@ def get_models(self, api_key=None, api_base: Optional[str] = None) -> List[str]:
result = sorted(names)
return result

@staticmethod
def _strip_ollama_model_prefix(model: str) -> str:
if model.startswith("ollama/") or model.startswith("ollama_chat/"):
return model.split("/", 1)[1]
return model

@staticmethod
def _is_static_ollama_model(model: str) -> bool:
from litellm import model_cost

stripped_model = OllamaModelInfo._strip_ollama_model_prefix(model)
potential_model_names = {
model,
stripped_model,
"ollama/" + stripped_model,
"ollama_chat/" + stripped_model,
}
model_cost_keys = {key.lower() for key in model_cost}
return any(name.lower() in model_cost_keys for name in potential_model_names)

@staticmethod
def _supports_function_calling(ollama_model_info: dict) -> bool:
_template: str = str(ollama_model_info.get("template", "") or "")
return "tools" in _template.lower()

@staticmethod
def _get_max_tokens(ollama_model_info: dict) -> Optional[int]:
_model_info: dict = ollama_model_info.get("model_info", {})

for key, value in _model_info.items():
if "context_length" in key:
return value
return None

def get_runtime_model_info(
self,
model: str,
api_base: Optional[str] = None,
api_key: Optional[str] = None,
) -> dict[str, Any]:
from litellm import module_level_client

model = self._strip_ollama_model_prefix(model)
passed_api_base = api_base
api_base = self.get_server_api_base(api_base)
api_key = (
self.get_api_key(api_key) if passed_api_base is None or api_key else None
)
headers = {"Authorization": f"Bearer {api_key}"} if api_key else {}

try:
response = module_level_client.post(
url=f"{api_base}/api/show",
json={"name": model},
headers=headers,
)
except Exception:
verbose_logger.debug("OllamaError: Could not get model info.")
return {
"key": model,
"litellm_provider": "ollama",
"mode": "chat",
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
"max_tokens": None,
"max_input_tokens": None,
"max_output_tokens": None,
}

model_info = response.json()
max_tokens = self._get_max_tokens(model_info)

return {
"key": model,
"litellm_provider": "ollama",
"mode": "chat",
"supports_function_calling": self._supports_function_calling(model_info),
"input_cost_per_token": 0.0,
"output_cost_per_token": 0.0,
"max_tokens": max_tokens,
"max_input_tokens": max_tokens,
"max_output_tokens": max_tokens,
}

def get_model_info(
self,
model: str,
api_base: Optional[str] = None,
api_key: Optional[str] = None,
) -> Optional[dict[str, Any]]:
if self._is_static_ollama_model(model):
return None
return self.get_runtime_model_info(
model=model, api_base=api_base, api_key=api_key
)

def validate_environment(
self,
headers: dict,
Expand Down
Loading
Loading