feat(proxy): proactive model deprecation alerts and /model/deprecations endpoint - #26900
Merged
mateo-berri merged 17 commits intoAug 18, 2026
Merged
Conversation
|
|
Contributor
Author
|
bugbot run |
![cursor[bot]](https://avatars.githubusercontent.com/in/1210556?s=60&v=4){kind=small}
Contributor
There was a problem hiding this comment.
Cursor Bugbot has reviewed your changes and found 1 potential issue.
Autofix Details
Bugbot Autofix prepared a fix for the issue found in the latest run.
- ✅ Fixed:
datetimesubclass ofdatecauses arithmetic TypeError- Added an explicit
isinstance(raw_value, datetime)check before thedatecheck in_parse_deprecation_dateto convert datetime values via.date()so downstreamdep_date - todayarithmetic stays consistent.
- Added an explicit
Preview (dd3282fdbb)
diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py
--- a/litellm/integrations/SlackAlerting/slack_alerting.py
+++ b/litellm/integrations/SlackAlerting/slack_alerting.py
@@ -1150,6 +1150,81 @@
async def model_removed_alert(self, model_name: str):
pass
+ async def send_model_deprecation_alert(
+ self, llm_router: Optional[Any] = None
+ ) -> bool:
+ """Aggregate deprecation metadata for the configured models and alert.
+
+ Returns ``True`` when an alert payload was dispatched, ``False``
+ otherwise. The ``send_alert`` helper itself is responsible for honoring
+ the user's webhook configuration; this method only owns producing the
+ message and choosing whether to send it.
+ """
+ if (
+ self.alerting is None
+ or AlertType.model_deprecation_warnings not in self.alert_types
+ ):
+ return False
+
+ from litellm.proxy.common_utils.model_deprecation import (
+ collect_model_deprecations,
+ format_deprecation_alert_message,
+ )
+
+ try:
+ snapshot = collect_model_deprecations(llm_router=llm_router)
+ except Exception as e:
+ verbose_proxy_logger.exception(
+ "Error collecting model deprecation snapshot: %s", e
+ )
+ return False
+
+ message = format_deprecation_alert_message(snapshot)
+ if message is None:
+ return False
+
+ level: Literal["Low", "Medium", "High"] = (
+ "High" if snapshot.deprecated else "Medium"
+ )
+
+ await self.send_alert(
+ message=message,
+ level=level,
+ alert_type=AlertType.model_deprecation_warnings,
+ alerting_metadata={
+ "deprecated_count": len(snapshot.deprecated),
+ "imminent_count": len(snapshot.imminent),
+ "upcoming_count": len(snapshot.upcoming),
+ },
+ )
+ return True
+
+ async def _run_scheduled_deprecation_check(self, llm_router: Optional[Any] = None):
+ """Periodic background task that emits a model deprecation alert.
+
+ Runs immediately on startup (so operators see the current state in
+ Slack) and then sleeps ``DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS``
+ between runs. Exits silently if the alert type is not enabled.
+ """
+ from litellm.types.proxy.model_deprecation import (
+ DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS,
+ )
+
+ if (
+ self.alerting is None
+ or AlertType.model_deprecation_warnings not in self.alert_types
+ ):
+ return
+
+ while True:
+ try:
+ await self.send_model_deprecation_alert(llm_router=llm_router)
+ except Exception as e:
+ verbose_proxy_logger.exception(
+ "Error in model deprecation alert loop: %s", e
+ )
+ await asyncio.sleep(DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS)
+
async def send_webhook_alert(self, webhook_event: WebhookEvent) -> bool:
"""
Sends structured alert to webhook, if set.
diff --git a/litellm/proxy/common_utils/model_deprecation.py b/litellm/proxy/common_utils/model_deprecation.py
new file mode 100644
--- /dev/null
+++ b/litellm/proxy/common_utils/model_deprecation.py
@@ -1,0 +1,249 @@
+"""Helpers for surfacing model deprecation/sunset information.
+
+This module reads ``deprecation_date`` metadata that is bundled in
+``model_prices_and_context_window.json`` (exposed at runtime via
+``litellm.model_cost``) and classifies the proxy's configured models into
+``upcoming``, ``imminent`` and ``deprecated`` buckets. It is the single
+source of truth used by both the ``/model/deprecations`` endpoint and the
+proactive Slack alert.
+
+Resolution order for a deployment's deprecation date:
+
+1. ``model_info.deprecation_date`` – an explicit override on the deployment.
+2. ``model_info.base_model`` looked up in ``litellm.model_cost``.
+3. The ``litellm_params.model`` string looked up in ``litellm.model_cost``.
+
+Models without any deprecation metadata are skipped silently (most models
+are not deprecated, and we don't want to pollute the response).
+"""
+
+from __future__ import annotations
+
+from datetime import date, datetime, timezone
+from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple
+
+import litellm
+from litellm._logging import verbose_logger
+from litellm.types.proxy.model_deprecation import (
+ DEFAULT_DEPRECATION_WARN_DAYS,
+ ModelDeprecationInfo,
+ ModelDeprecationResponse,
+)
+
+if TYPE_CHECKING:
+ from litellm.router import Router as _Router
+
+ Router = _Router
+else:
+ Router = Any
+
+
+def _parse_deprecation_date(raw_value: Any) -> Optional[date]:
+ """Parse a ``deprecation_date`` string in YYYY-MM-DD form.
+
+ Returns ``None`` for missing, malformed, or sentinel placeholder values
+ (the JSON map ships a documentation sentinel of the form ``"date when..."``).
+ """
+ if raw_value is None:
+ return None
+ if isinstance(raw_value, datetime):
+ return raw_value.date()
+ if isinstance(raw_value, date):
+ return raw_value
+ if not isinstance(raw_value, str):
+ return None
+ try:
+ return datetime.strptime(raw_value.strip(), "%Y-%m-%d").date()
+ except ValueError:
+ return None
+
+
+def _lookup_deprecation_date_from_cost_map(
+ model_key: Optional[str],
+) -> Tuple[Optional[date], Optional[str]]:
+ """Look up a deprecation date in ``litellm.model_cost`` for ``model_key``.
+
+ Returns a tuple of (deprecation_date, litellm_provider).
+ """
+ if not model_key:
+ return None, None
+ entry = litellm.model_cost.get(model_key)
+ if not isinstance(entry, dict):
+ return None, None
+ return (
+ _parse_deprecation_date(entry.get("deprecation_date")),
+ entry.get("litellm_provider"),
+ )
+
+
+def _resolve_deployment_deprecation(
+ deployment: Dict[str, Any],
+) -> Tuple[Optional[date], Optional[str], Optional[str]]:
+ """Resolve a deployment's deprecation metadata.
+
+ Returns a tuple of (deprecation_date, litellm_model, litellm_provider).
+ """
+ model_info = deployment.get("model_info") or {}
+ explicit = _parse_deprecation_date(model_info.get("deprecation_date"))
+ if explicit is not None:
+ litellm_params = deployment.get("litellm_params") or {}
+ return (
+ explicit,
+ litellm_params.get("model"),
+ model_info.get("litellm_provider"),
+ )
+
+ base_model = model_info.get("base_model")
+ dep_date, provider = _lookup_deprecation_date_from_cost_map(base_model)
+ if dep_date is not None:
+ return dep_date, base_model, provider
+
+ litellm_params = deployment.get("litellm_params") or {}
+ raw_model = litellm_params.get("model")
+ dep_date, provider = _lookup_deprecation_date_from_cost_map(raw_model)
+ if dep_date is not None:
+ return dep_date, raw_model, provider
+
+ if isinstance(raw_model, str) and "/" in raw_model:
+ # Try the un-prefixed lookup (e.g. "openai/gpt-4o" → "gpt-4o").
+ bare = raw_model.split("/", 1)[1]
+ dep_date, provider = _lookup_deprecation_date_from_cost_map(bare)
+ if dep_date is not None:
+ return dep_date, bare, provider
+
+ return None, raw_model, model_info.get("litellm_provider")
+
+
+def _classify(days_until: int, warn_within_days: int) -> str:
+ if days_until < 0:
+ return "deprecated"
+ if days_until <= warn_within_days:
+ return "imminent"
+ return "upcoming"
+
+
+def _model_dump_compat(deployment: Any) -> Dict[str, Any]:
+ """Return a plain dict for both pydantic models and dicts."""
+ if isinstance(deployment, dict):
+ return deployment
+ if hasattr(deployment, "model_dump"):
+ return deployment.model_dump(exclude_none=True)
+ if hasattr(deployment, "dict"):
+ return deployment.dict()
+ return dict(deployment)
+
+
+def collect_model_deprecations(
+ llm_router: Optional[Router],
+ warn_within_days: int = DEFAULT_DEPRECATION_WARN_DAYS,
+ today: Optional[date] = None,
+) -> ModelDeprecationResponse:
+ """Aggregate deprecation info for all deployments configured on the router.
+
+ De-duplicates by ``(model_name, deprecation_date)`` so multi-deployment
+ model groups (load-balanced across regions) only surface once per
+ deprecation date.
+ """
+ snapshot_time = datetime.now(timezone.utc)
+ today = today or snapshot_time.date()
+
+ response = ModelDeprecationResponse(
+ warn_within_days=warn_within_days,
+ checked_at=snapshot_time,
+ )
+
+ if llm_router is None:
+ return response
+
+ seen: set = set()
+ deployments = llm_router.get_model_list() or []
+ for deployment in deployments:
+ deployment_dict = _model_dump_compat(deployment)
+ model_name = deployment_dict.get("model_name")
+ if not model_name:
+ continue
+
+ dep_date, litellm_model, provider = _resolve_deployment_deprecation(
+ deployment_dict
+ )
+ if dep_date is None:
+ continue
+
+ dedup_key = (model_name, dep_date.isoformat())
+ if dedup_key in seen:
+ continue
+ seen.add(dedup_key)
+
+ days_until = (dep_date - today).days
+ status = _classify(days_until, warn_within_days)
+
+ info = ModelDeprecationInfo(
+ model_name=model_name,
+ litellm_model=litellm_model,
+ deprecation_date=dep_date,
+ days_until_deprecation=days_until,
+ status=status,
+ litellm_provider=provider,
+ )
+
+ if status == "deprecated":
+ response.deprecated.append(info)
+ elif status == "imminent":
+ response.imminent.append(info)
+ else:
+ response.upcoming.append(info)
+
+ response.deprecated.sort(key=lambda m: m.deprecation_date)
+ response.imminent.sort(key=lambda m: m.deprecation_date)
+ response.upcoming.sort(key=lambda m: m.deprecation_date)
+
+ verbose_logger.debug(
+ "model_deprecation: deprecated=%d imminent=%d upcoming=%d",
+ len(response.deprecated),
+ len(response.imminent),
+ len(response.upcoming),
+ )
+
+ return response
+
+
+def format_deprecation_alert_message(
+ snapshot: ModelDeprecationResponse,
+) -> Optional[str]:
+ """Format a Slack-friendly alert message for the warning buckets.
+
+ Only ``deprecated`` and ``imminent`` models are included; ``upcoming``
+ models are intentionally omitted to avoid alert fatigue. Returns
+ ``None`` when there is nothing to alert on.
+ """
+ if not snapshot.deprecated and not snapshot.imminent:
+ return None
+
+ lines: List[str] = ["*⚠️ Model Deprecation Warning*"]
+
+ def _format_entry(info: ModelDeprecationInfo) -> str:
+ suffix = (
+ f"already deprecated {abs(info.days_until_deprecation)}d ago"
+ if info.days_until_deprecation < 0
+ else f"in {info.days_until_deprecation}d"
+ )
+ return (
+ f"• `{info.model_name}` "
+ f"(provider: {info.litellm_provider or 'unknown'}, "
+ f"deprecates {info.deprecation_date.isoformat()} – {suffix})"
+ )
+
+ if snapshot.deprecated:
+ lines.append("\n*Already deprecated:*")
+ lines.extend(_format_entry(i) for i in snapshot.deprecated)
+
+ if snapshot.imminent:
+ lines.append(f"\n*Deprecating within {snapshot.warn_within_days} days:*")
+ lines.extend(_format_entry(i) for i in snapshot.imminent)
+
+ lines.append(
+ "\nPlan migrations to a supported model. See "
+ "https://docs.litellm.ai/docs/proxy/model_management for guidance."
+ )
+
+ return "\n".join(lines)
diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py
--- a/litellm/proxy/proxy_server.py
+++ b/litellm/proxy/proxy_server.py
@@ -313,6 +313,7 @@
get_config_file_contents_from_gcs,
get_file_contents_from_s3,
)
+from litellm.proxy.common_utils.model_deprecation import collect_model_deprecations
from litellm.proxy.common_utils.openai_endpoint_utils import (
remove_sensitive_info_from_deployment,
)
@@ -552,6 +553,10 @@
from litellm.types.proxy.management_endpoints.model_management_endpoints import (
ModelGroupInfoProxy,
)
+from litellm.types.proxy.model_deprecation import (
+ DEFAULT_DEPRECATION_WARN_DAYS,
+ ModelDeprecationResponse,
+)
from litellm.types.proxy.management_endpoints.ui_sso import (
DefaultTeamSSOParams,
LiteLLM_UpperboundKeyGenerateParams,
@@ -11215,6 +11220,52 @@
return {"data": all_models}
+@router.get(
+ "/model/deprecations",
+ tags=["model management"],
+ dependencies=[Depends(user_api_key_auth)],
+ response_model=ModelDeprecationResponse,
+)
+@router.get(
+ "/v1/model/deprecations",
+ tags=["model management"],
+ dependencies=[Depends(user_api_key_auth)],
+ response_model=ModelDeprecationResponse,
+)
+async def model_deprecations(
+ user_api_key_dict: UserAPIKeyAuth = Depends(user_api_key_auth),
+ warn_within_days: int = DEFAULT_DEPRECATION_WARN_DAYS,
+) -> ModelDeprecationResponse:
+ """List models with known deprecation/sunset dates, bucketed by urgency.
+
+ Reads `deprecation_date` metadata from `model_prices_and_context_window.json`
+ (and any per-deployment `model_info.deprecation_date` overrides) for the
+ models configured on this proxy.
+
+ Parameters:
+ warn_within_days: Window (in days) used to bucket "imminent" models.
+ Defaults to `LITELLM_MODEL_DEPRECATION_WARN_DAYS` env var (or 30).
+
+ Returns:
+ A payload with three lists of `ModelDeprecationInfo` entries:
+
+ - `deprecated`: deprecation date is in the past — these requests may
+ fail at any time.
+ - `imminent`: deprecation date is within `warn_within_days` from today.
+ - `upcoming`: deprecation date is further out.
+
+ Example:
+ ```shell
+ curl -X GET 'http://localhost:4000/model/deprecations' \\
+ -H 'Authorization: Bearer sk-1234'
+ ```
+ """
+ global llm_router
+ return collect_model_deprecations(
+ llm_router=llm_router, warn_within_days=warn_within_days
+ )
+
+
def _get_model_group_info(
llm_router: Router, all_models_str: List[str], model_group: Optional[str]
) -> List[ModelGroupInfoProxy]:
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -379,6 +379,7 @@
# Guard flags to prevent duplicate background tasks
self.daily_report_started: bool = False
self.hanging_requests_check_started: bool = False
+ self.deprecation_check_started: bool = False
def startup_event(
self,
@@ -421,6 +422,19 @@
) # RUN HANGING REQUEST CHECK (if user wants to alert on hanging requests)
self.hanging_requests_check_started = True
+ if (
+ self.slack_alerting_instance is not None
+ and AlertType.model_deprecation_warnings
+ in self.slack_alerting_instance.alert_types
+ and not self.deprecation_check_started
+ ):
+ asyncio.create_task(
+ self.slack_alerting_instance._run_scheduled_deprecation_check(
+ llm_router=llm_router
+ )
+ ) # RUN MODEL DEPRECATION ALERT LOOP (if scheduled)
+ self.deprecation_check_started = True
+
def update_values(
self,
alerting: Optional[List] = None,
diff --git a/litellm/types/integrations/slack_alerting.py b/litellm/types/integrations/slack_alerting.py
--- a/litellm/types/integrations/slack_alerting.py
+++ b/litellm/types/integrations/slack_alerting.py
@@ -1,7 +1,7 @@
import os
from datetime import datetime as dt
from enum import Enum
-from typing import Any, Dict, List, Literal, Optional, Set, Union
+from typing import List, Optional, Set, Union
from pydantic import BaseModel, Field
from typing_extensions import TypedDict
@@ -146,6 +146,7 @@
# Deployment alerts
cooldown_deployment = "cooldown_deployment"
new_model_added = "new_model_added"
+ model_deprecation_warnings = "model_deprecation_warnings"
# Outage alerts
outage_alerts = "outage_alerts"
@@ -186,6 +187,7 @@
# Deployment alerts
AlertType.cooldown_deployment,
AlertType.new_model_added,
+ AlertType.model_deprecation_warnings,
# Outage alerts
AlertType.outage_alerts,
AlertType.region_outage_alerts,
diff --git a/litellm/types/proxy/model_deprecation.py b/litellm/types/proxy/model_deprecation.py
new file mode 100644
--- /dev/null
+++ b/litellm/types/proxy/model_deprecation.py
@@ -1,0 +1,89 @@
+"""Type definitions for model deprecation tracking and proactive alerts.
+
+The proxy reads deprecation/sunset metadata from
+``litellm.model_cost`` (sourced from ``model_prices_and_context_window.json``)
+and surfaces it through the ``/model/deprecations`` endpoint and Slack
+alerting. These types describe the response payload and the alert payload.
+"""
+
+from __future__ import annotations
+
+from datetime import date, datetime
+from typing import List, Optional
+
+from pydantic import BaseModel, Field
+
+
+DEFAULT_DEPRECATION_WARN_DAYS = 30
+"""Default warning window (in days) for the ``imminent`` bucket.
+
+Matches the typical migration window most LLM providers offer between
+deprecation announcement and removal. Callers of ``/model/deprecations``
+can override this per-request via the ``?warn_within_days=N`` query
+parameter without restarting the proxy.
+"""
+
+DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS = 24 * 60 * 60
+"""How often the periodic background check runs. Once per day."""
+
+
+DeprecationStatusLiteral = str
+"""One of ``"upcoming"``, ``"imminent"``, ``"deprecated"``.
+
+* ``upcoming`` – deprecation is scheduled but more than the warn window away.
+* ``imminent`` – deprecation date is within ``warn_within_days`` from today.
+* ``deprecated`` – deprecation date has already passed.
+"""
+
+
+class ModelDeprecationInfo(BaseModel):
+ """Per-model deprecation metadata returned by ``/model/deprecations``."""
+
+ model_name: str = Field(
+ description="The public name of the model on the proxy (model_group)."
+ )
+ litellm_model: Optional[str] = Field(
+ default=None,
+ description="The underlying litellm model string the deprecation date is sourced from.",
+ )
+ deprecation_date: date = Field(
+ description="The date (UTC) when the model becomes deprecated."
+ )
+ days_until_deprecation: int = Field(
+ description=(
+ "Days remaining until the deprecation date. Negative if the model "
+ "is already deprecated."
+ ),
+ )
+ status: DeprecationStatusLiteral = Field(
+ description="One of 'upcoming', 'imminent', or 'deprecated'.",
+ )
+ litellm_provider: Optional[str] = Field(
+ default=None, description="The provider this model belongs to."
+ )
+
+
+class ModelDeprecationResponse(BaseModel):
+ """Response payload for ``GET /model/deprecations``."""
+
+ deprecated: List[ModelDeprecationInfo] = Field(
+ default_factory=list,
+ description="Models whose deprecation date has already passed.",
+ )
+ imminent: List[ModelDeprecationInfo] = Field(
+ default_factory=list,
+ description=(
+ "Models whose deprecation date is within ``warn_within_days`` from "
+ "today and require immediate migration planning."
+ ),
+ )
+ upcoming: List[ModelDeprecationInfo] = Field(
+ default_factory=list,
+ description="Models with a future deprecation date outside the warn window.",
+ )
+ warn_within_days: int = Field(
+ description="The window (in days) used to bucket 'imminent' models."
+ )
+ checked_at: datetime = Field(
+ description="UTC timestamp when the deprecation snapshot was generated."
+ )
diff --git a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py
new file mode 100644
--- /dev/null
+++ b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py
@@ -1,0 +1,100 @@
+"""Tests for the Slack alerting model deprecation hook."""
+
+import os
+import sys
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+sys.path.insert(0, os.path.abspath("../../../.."))
+
+import litellm
+from litellm.integrations.SlackAlerting.slack_alerting import SlackAlerting
+from litellm.proxy._types import AlertType
+
+
+def _make_router(deployments):
+ router = MagicMock()
+ router.get_model_list.return_value = deployments
+ return router
+
+
+@pytest.mark.asyncio
+async def test_should_skip_when_alert_type_disabled():
+ alerting = SlackAlerting(
+ alerting=["slack"],
+ alert_types=[AlertType.llm_exceptions],
+ )
+ sent = await alerting.send_model_deprecation_alert(llm_router=MagicMock())
+ assert sent is False
+
+
+@pytest.mark.asyncio
+async def test_should_skip_when_no_alerting_configured():
+ alerting = SlackAlerting(
+ alerting=None,
+ alert_types=[AlertType.model_deprecation_warnings],
+ )
+ sent = await alerting.send_model_deprecation_alert(llm_router=MagicMock())
+ assert sent is False
+
+
+@pytest.mark.asyncio
+async def test_should_skip_when_no_deprecations_found(monkeypatch):
+ monkeypatch.setattr(litellm, "model_cost", {})
+ alerting = SlackAlerting(
+ alerting=["slack"],
+ alert_types=[AlertType.model_deprecation_warnings],
+ )
+ router = _make_router(
+ [
+ {
+ "model_name": "fresh",
+ "litellm_params": {"model": "openai/gpt-4o"},
+ "model_info": {"id": "x"},
+ }
+ ]
+ )
+ sent = await alerting.send_model_deprecation_alert(llm_router=router)
+ assert sent is False
+
+
+@pytest.mark.asyncio
+async def test_should_dispatch_high_severity_when_deprecated(monkeypatch):
+ monkeypatch.setattr(
+ litellm,
+ "model_cost",
+ {
+ "dead-model": {
+ "deprecation_date": "2020-01-01",
+ "litellm_provider": "openai",
+ }
+ },
+ )
+ alerting = SlackAlerting(
+ alerting=["slack"],
+ alert_types=[AlertType.model_deprecation_warnings],
+ )
+ router = _make_router(
+ [
+ {
+ "model_name": "dead-alias",
+ "litellm_params": {"model": "dead-model"},
+ "model_info": {"id": "1"},
+ }
+ ]
+ )
+
+ with patch.object(
+ alerting, "send_alert", new_callable=AsyncMock
+ ) as mock_send_alert:
+ sent = await alerting.send_model_deprecation_alert(llm_router=router)
+
+ assert sent is True
+ mock_send_alert.assert_awaited_once()
+ call_kwargs = mock_send_alert.await_args.kwargs
+ assert call_kwargs["alert_type"] == AlertType.model_deprecation_warnings
+ assert call_kwargs["level"] == "High"
+ assert call_kwargs["alerting_metadata"]["deprecated_count"] == 1
+ assert call_kwargs["alerting_metadata"]["imminent_count"] == 0
+ assert "dead-alias" in call_kwargs["message"]
diff --git a/tests/test_litellm/proxy/common_utils/test_model_deprecation.py b/tests/test_litellm/proxy/common_utils/test_model_deprecation.py
new file mode 100644
--- /dev/null
+++ b/tests/test_litellm/proxy/common_utils/test_model_deprecation.py
@@ -1,0 +1,280 @@
+"""Tests for the model deprecation helper module.
+
+These tests focus on the helper itself — not on the proxy endpoint or
+Slack integration — so they can run without the full proxy stack.
+"""
+
+import os
+import sys
+from datetime import date
+from unittest.mock import MagicMock
+
+
+sys.path.insert(0, os.path.abspath("../../../.."))
+
+import litellm
+from litellm.proxy.common_utils.model_deprecation import (
+ _classify,
+ _parse_deprecation_date,
+ collect_model_deprecations,
+ format_deprecation_alert_message,
+)
+
+
+def _make_router(deployments):
+ router = MagicMock()
+ router.get_model_list.return_value = deployments
+ return router
+
+
+class TestParseDeprecationDate:
+ def test_should_parse_iso_string(self):
+ assert _parse_deprecation_date("2026-12-31") == date(2026, 12, 31)
+
+ def test_should_pass_through_date_object(self):
+ d = date(2026, 1, 1)
+ assert _parse_deprecation_date(d) == d
+
+ def test_should_return_none_for_documentation_sentinel(self):
+ # The JSON map ships a sentinel string under the "sample_spec" key.
+ assert (
+ _parse_deprecation_date(
+ "date when the model becomes deprecated in the format YYYY-MM-DD"
+ )
+ is None
+ )
+
+ def test_should_return_none_for_none(self):
+ assert _parse_deprecation_date(None) is None
+
+ def test_should_return_none_for_unsupported_type(self):
+ assert _parse_deprecation_date(12345) is None
+
+
+class TestClassify:
+ def test_should_classify_past_dates_as_deprecated(self):
+ assert _classify(-1, warn_within_days=30) == "deprecated"
+ assert _classify(-365, warn_within_days=30) == "deprecated"
+
+ def test_should_classify_inside_window_as_imminent(self):
+ assert _classify(0, warn_within_days=30) == "imminent"
+ assert _classify(15, warn_within_days=30) == "imminent"
+ assert _classify(30, warn_within_days=30) == "imminent"
+
+ def test_should_classify_outside_window_as_upcoming(self):
+ assert _classify(31, warn_within_days=30) == "upcoming"
+ assert _classify(365, warn_within_days=30) == "upcoming"
+
+
+class TestCollectModelDeprecations:
+ def test_should_return_empty_response_when_router_is_none(self):
+ snapshot = collect_model_deprecations(llm_router=None)
+ assert snapshot.deprecated == []
+ assert snapshot.imminent == []
+ assert snapshot.upcoming == []
+
+ def test_should_skip_models_without_deprecation_metadata(self, monkeypatch):
+ monkeypatch.setattr(litellm, "model_cost", {})
+ router = _make_router(
+ [
+ {
+ "model_name": "gpt-4o",
+ "litellm_params": {"model": "openai/gpt-4o"},
+ "model_info": {"id": "abc"},
+ }
+ ]
+ )
+ snapshot = collect_model_deprecations(llm_router=router)
+ assert snapshot.deprecated == []
+ assert snapshot.imminent == []
+ assert snapshot.upcoming == []
+
+ def test_should_classify_into_three_buckets(self, monkeypatch):
+ today = date(2026, 6, 1)
+ monkeypatch.setattr(
+ litellm,
+ "model_cost",
+ {
+ "deprecated-model": {
+ "deprecation_date": "2026-01-01",
+ "litellm_provider": "openai",
+ },
+ "imminent-model": {
+ "deprecation_date": "2026-06-15",
+ "litellm_provider": "openai",
+ },
+ "upcoming-model": {
+ "deprecation_date": "2027-01-01",
+ "litellm_provider": "openai",
+ },
+ },
+ )
+ router = _make_router(
+ [
+ {
+ "model_name": "deprecated-alias",
+ "litellm_params": {"model": "openai/deprecated-model"},
... diff truncated: showing 800 of 964 linesYou can send follow-ups to the cloud agent here.
Contributor
Author
|
bugbot run |
Contributor
There was a problem hiding this comment.
✅ Bugbot reviewed your changes and found no new issues!
Comment @cursor review or bugbot run to trigger another review on this PR
Reviewed by Cursor Bugbot for commit dd3282f. Configure here.
4 tasks
Contributor
|
This pull request has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. |
This was referenced Aug 6, 2026
Closed
Closed
…ns endpoint Surfaces deprecation_date metadata that is already shipped in model_prices_and_context_window.json so operators get lead time to migrate before a provider sunsets a model. - New helper litellm.proxy.common_utils.model_deprecation classifies the router's configured models into deprecated / imminent / upcoming buckets. Resolution order: explicit model_info.deprecation_date > model_info.base_model > litellm_params.model. - New GET /model/deprecations (and /v1/model/deprecations) endpoint returns a ModelDeprecationResponse, gated by user_api_key_auth. - New AlertType.model_deprecation_warnings (in DEFAULT_ALERT_TYPES) plus SlackAlerting.send_model_deprecation_alert dispatches a Slack message for deprecated/imminent models. Severity is High when any model is already past its date, Medium when only imminent. - ProxyLogging.startup_event schedules a daily background task (_run_scheduled_deprecation_check) when the alert type is enabled. The interval is configurable via LITELLM_MODEL_DEPRECATION_CHECK_INTERVAL and the warn window via LITELLM_MODEL_DEPRECATION_WARN_DAYS. - Tests: 16 unit tests for the helper plus 4 for the Slack hook in tests/test_litellm/. Co-authored-by: Mateo Wang <mateo-berri@users.noreply.github.com>
…tion The proxy documentation lives in BerriAI/litellm-docs and any new env key flagged by os.getenv() must be added there before the test_env_keys.py CI check passes. Rather than fork the docs repo for two niche tunables, hard-code the defaults: - DEFAULT_DEPRECATION_WARN_DAYS = 30 (already overridable per-request via ?warn_within_days=N on /model/deprecations). - DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS = 24h. Both can still be raised as env-var follow-ups together with their docs update if operators ask for it. Co-authored-by: Mateo Wang <mateo-berri@users.noreply.github.com>
Drops Any-typed router plumbing, immutable bucketing, generated dashboard API types, and adds endpoint plus resolution-fallback tests Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
devin-ai-integration
Bot
force-pushed
the
litellm_model-deprecation-alerts-55bc
branch
from
August 10, 2026 22:58
dd3282f to
8f1aea5
Compare
Codecov Report❌ Patch coverage is 📢 Thoughts on this report? Let us know! |
Contributor
Greptile SummaryThis PR exposes configured model sunset information through authenticated management endpoints and adds coordinated Slack warnings for deprecated or imminent models.
Confidence Score: 5/5The PR appears safe to merge. No blocking failure remains.
|
| Filename | Overview |
|---|---|
| litellm/integrations/SlackAlerting/slack_alerting.py | Adds deprecation alert formatting, daily coordination, current-Router lookup, and a polling loop that observes runtime configuration changes. |
| litellm/proxy/common_utils/model_deprecation.py | Resolves registry or deployment override dates, deduplicates deployments, and produces deprecated, imminent, and upcoming buckets. |
| litellm/proxy/proxy_server.py | Adds authenticated /model/deprecations route aliases with configurable warning windows. |
| litellm/proxy/utils.py | Schedules the deprecation loop during startup and when alerting is enabled through runtime configuration. |
| tests/test_litellm/proxy/utils/proxy_logging/test_lifecycle.py | Covers startup and runtime scheduling behavior for the deprecation alert loop. |
Reviews (13): Last reviewed commit: "fix(alerting): back off a day after a de..." | Re-trigger Greptile
![greptile-apps[bot]](https://avatars.githubusercontent.com/in/867647?s=60&v=4){kind=small}
Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
The daily loop no longer captures the startup Router or bails when the alert type is off at startup, so config reloads take effect Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Contributor
|
@greptileai re-review: the daily deprecation check now re-reads the router and alert types each pass, with a regression test |
…gured Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
4 tasks
Contributor
Author
Contributor
4 tasks
…itellm_model-deprecation-alerts-55bc
4 tasks
… a dead None check Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Contributor
|
Pushed a type-gate fix: the loop entrypoint is public now and a dead None check is gone. bugbot run |
Closed
4 tasks
Contributor
Author
|
bugbot run |
Contributor
There was a problem hiding this comment.
Cursor Bugbot has reviewed your changes using default effort and found 1 potential issue.
Autofix Details
Bugbot Autofix prepared a fix for the issue found in the latest run.
- ✅ Fixed: Duplicate multi-pod Slack alerts
- Gated the daily model deprecation alert with PodLockManager (new SLACK_MODEL_DEPRECATION_ALERT_LOCK_ID, TTL = check interval, allow_reentrant=False) plumbed through from ProxyLogging, mirroring _run_scheduled_daily_report so only one pod posts per window.
Or push these changes by commenting:
@cursor push 967ae93adf
Preview (967ae93adf)
diff --git a/litellm/constants.py b/litellm/constants.py
--- a/litellm/constants.py
+++ b/litellm/constants.py
@@ -1485,6 +1485,7 @@
MONTHLY_SPEND_REPORT_JOB_ID: Final = "monthly_spend_report_job"
PROMETHEUS_FALLBACK_STATS_JOB_ID: Final = "prometheus_fallback_stats_job"
SLACK_DAILY_REPORT_LOCK_ID: Final = "slack_daily_report"
+SLACK_MODEL_DEPRECATION_ALERT_LOCK_ID: Final = "slack_model_deprecation_alert"
SPEND_LOG_RUN_LOOPS: Final = int(os.getenv("SPEND_LOG_RUN_LOOPS", 500))
SPEND_LOG_CLEANUP_BATCH_SIZE: Final = int(os.getenv("SPEND_LOG_CLEANUP_BATCH_SIZE", 1000))
SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES = int(os.getenv("SPEND_LOG_CLEANUP_MAX_CONSECUTIVE_BATCH_FAILURES", 3))
diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py
--- a/litellm/integrations/SlackAlerting/slack_alerting.py
+++ b/litellm/integrations/SlackAlerting/slack_alerting.py
@@ -18,7 +18,11 @@
import litellm.types
from litellm._logging import verbose_logger, verbose_proxy_logger
from litellm.caching.caching import DualCache
-from litellm.constants import HOURS_IN_A_DAY, SLACK_DAILY_REPORT_LOCK_ID
+from litellm.constants import (
+ HOURS_IN_A_DAY,
+ SLACK_DAILY_REPORT_LOCK_ID,
+ SLACK_MODEL_DEPRECATION_ALERT_LOCK_ID,
+)
from litellm.integrations.custom_batch_logger import CustomBatchLogger
from litellm.integrations.SlackAlerting.budget_alert_types import get_budget_alert_type
from litellm.integrations.SlackAlerting.hanging_request_check import (
@@ -1088,15 +1092,30 @@
return True
async def run_scheduled_deprecation_check(
- self, get_llm_router: Callable[[], Router | None] = _proxy_llm_router
+ self,
+ get_llm_router: Callable[[], Router | None] = _proxy_llm_router,
+ pod_lock_manager: "PodLockManager | None" = None,
) -> None:
- """Alert once the router is loaded and the alert is on, then daily, re-reading both each pass"""
+ """Alert once the router is loaded and the alert is on, then daily, re-reading both each pass
+
+ `pod_lock_manager` dedupes the alert across a multi-pod fleet: only the pod that acquires
+ the daily lock sends, so the Slack webhook receives one message per interval, not N.
+ """
while True:
if (llm_router := get_llm_router()) is None or not self._deprecation_alerts_enabled():
await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS)
continue
try:
- await self.send_model_deprecation_alert(llm_router=llm_router)
+ if (
+ pod_lock_manager is None
+ or await pod_lock_manager.acquire_lock(
+ cronjob_id=SLACK_MODEL_DEPRECATION_ALERT_LOCK_ID,
+ ttl=DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS,
+ allow_reentrant=False,
+ )
+ is not False
+ ):
+ await self.send_model_deprecation_alert(llm_router=llm_router)
except Exception as e: # noqa: BLE001 # a failed alert must not kill the daily loop
verbose_proxy_logger.exception("Error in model deprecation alert loop: %s", e)
await asyncio.sleep(DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS)
diff --git a/litellm/proxy/utils.py b/litellm/proxy/utils.py
--- a/litellm/proxy/utils.py
+++ b/litellm/proxy/utils.py
@@ -503,7 +503,11 @@
except RuntimeError:
return
- asyncio.create_task(self.slack_alerting_instance.run_scheduled_deprecation_check())
+ asyncio.create_task(
+ self.slack_alerting_instance.run_scheduled_deprecation_check(
+ pod_lock_manager=self.db_spend_update_writer.pod_lock_manager,
+ )
+ )
self.deprecation_check_started = True
def update_values(
diff --git a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py
--- a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py
+++ b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py
@@ -202,3 +202,136 @@
]
mock_send_alert.assert_awaited_once()
assert "dead-alias" in mock_send_alert.await_args.kwargs["message"]
+
+
+@pytest.mark.asyncio
+async def test_should_skip_alert_when_pod_lock_is_held_by_another_pod(monkeypatch):
+ """Multi-pod fleets must send exactly one Slack alert per interval, not one per replica"""
+ from litellm.constants import SLACK_MODEL_DEPRECATION_ALERT_LOCK_ID
+
+ monkeypatch.setattr(
+ litellm,
+ "model_cost",
+ {"dead-model": {"deprecation_date": "2020-01-01", "litellm_provider": "openai"}},
+ )
+ alerting = SlackAlerting(
+ alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings]
+ )
+ router = _make_router(
+ [
+ {
+ "model_name": "dead-alias",
+ "litellm_params": {"model": "dead-model"},
+ "model_info": {"id": "1"},
+ }
+ ]
+ )
+
+ pod_lock_manager = MagicMock()
+ pod_lock_manager.acquire_lock = AsyncMock(return_value=False)
+
+ async def stop_after_one_pass(_seconds):
+ raise asyncio.CancelledError
+
+ with (
+ patch.object(alerting, "send_alert", new_callable=AsyncMock) as mock_send_alert,
+ patch(
+ "litellm.integrations.SlackAlerting.slack_alerting.asyncio.sleep",
+ side_effect=stop_after_one_pass,
+ ),
+ pytest.raises(asyncio.CancelledError),
+ ):
+ await alerting.run_scheduled_deprecation_check(
+ get_llm_router=lambda: router,
+ pod_lock_manager=pod_lock_manager,
+ )
+
+ mock_send_alert.assert_not_awaited()
+ pod_lock_manager.acquire_lock.assert_awaited_once_with(
+ cronjob_id=SLACK_MODEL_DEPRECATION_ALERT_LOCK_ID,
+ ttl=DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS,
+ allow_reentrant=False,
+ )
+
+
+@pytest.mark.asyncio
+async def test_should_send_alert_when_pod_lock_is_acquired(monkeypatch):
+ """The pod that wins the daily lock is the one that actually posts to Slack"""
+ monkeypatch.setattr(
+ litellm,
+ "model_cost",
+ {"dead-model": {"deprecation_date": "2020-01-01", "litellm_provider": "openai"}},
+ )
+ alerting = SlackAlerting(
+ alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings]
+ )
+ router = _make_router(
+ [
+ {
+ "model_name": "dead-alias",
+ "litellm_params": {"model": "dead-model"},
+ "model_info": {"id": "1"},
+ }
+ ]
+ )
+
+ pod_lock_manager = MagicMock()
+ pod_lock_manager.acquire_lock = AsyncMock(return_value=True)
+
+ async def stop_after_one_pass(_seconds):
+ raise asyncio.CancelledError
+
+ with (
+ patch.object(alerting, "send_alert", new_callable=AsyncMock) as mock_send_alert,
+ patch(
+ "litellm.integrations.SlackAlerting.slack_alerting.asyncio.sleep",
+ side_effect=stop_after_one_pass,
+ ),
+ pytest.raises(asyncio.CancelledError),
+ ):
+ await alerting.run_scheduled_deprecation_check(
+ get_llm_router=lambda: router,
+ pod_lock_manager=pod_lock_manager,
+ )
+
+ mock_send_alert.assert_awaited_once()
+
+
+@pytest.mark.asyncio
+async def test_should_send_alert_when_no_pod_lock_manager_is_configured(monkeypatch):
+ """Deployments without Redis have no lock manager, so the loop must still send"""
+ monkeypatch.setattr(
+ litellm,
+ "model_cost",
+ {"dead-model": {"deprecation_date": "2020-01-01", "litellm_provider": "openai"}},
+ )
+ alerting = SlackAlerting(
+ alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings]
+ )
+ router = _make_router(
+ [
+ {
+ "model_name": "dead-alias",
+ "litellm_params": {"model": "dead-model"},
+ "model_info": {"id": "1"},
+ }
+ ]
+ )
+
+ async def stop_after_one_pass(_seconds):
+ raise asyncio.CancelledError
+
+ with (
+ patch.object(alerting, "send_alert", new_callable=AsyncMock) as mock_send_alert,
+ patch(
+ "litellm.integrations.SlackAlerting.slack_alerting.asyncio.sleep",
+ side_effect=stop_after_one_pass,
+ ),
+ pytest.raises(asyncio.CancelledError),
+ ):
+ await alerting.run_scheduled_deprecation_check(
+ get_llm_router=lambda: router,
+ pod_lock_manager=None,
+ )
+
+ mock_send_alert.assert_awaited_once()
diff --git a/tests/test_litellm/proxy/utils/proxy_logging/test_lifecycle.py b/tests/test_litellm/proxy/utils/proxy_logging/test_lifecycle.py
--- a/tests/test_litellm/proxy/utils/proxy_logging/test_lifecycle.py
+++ b/tests/test_litellm/proxy/utils/proxy_logging/test_lifecycle.py
@@ -142,7 +142,9 @@
proxy_logging.startup_event(llm_router=None, redis_usage_cache=None)
assert proxy_logging.deprecation_check_started is True
- proxy_logging.slack_alerting_instance.run_scheduled_deprecation_check.assert_called_once_with()
+ proxy_logging.slack_alerting_instance.run_scheduled_deprecation_check.assert_called_once_with(
+ pod_lock_manager=proxy_logging.db_spend_update_writer.pod_lock_manager,
+ )
@pytest.mark.asyncio
@@ -159,7 +161,9 @@
proxy_logging.update_values(alerting=["slack"])
assert proxy_logging.deprecation_check_started is True
- proxy_logging.slack_alerting_instance.run_scheduled_deprecation_check.assert_called_once_with()
+ proxy_logging.slack_alerting_instance.run_scheduled_deprecation_check.assert_called_once_with(
+ pod_lock_manager=proxy_logging.db_spend_update_writer.pod_lock_manager,
+ )
def test_startup_event_propagates_init_callbacks_failure_raises(proxy_logging):You can send follow-ups to the cloud agent here.
… alert per day Co-Authored-By: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com>
Contributor
|
Verified live: two pods on shared redis send one deprecation alert (lock TTL 86400); without redis each pod alerts, endpoint buckets unchanged. |
Contributor
Author
Contributor
Author
|
bugbot run |
Contributor
There was a problem hiding this comment.
Cursor Bugbot has reviewed your changes using default effort and found 2 potential issues.
Autofix Details
Bugbot Autofix prepared fixes for both issues found in the latest run.
- ✅ Fixed: Failed lock claim sleeps full day
- The scheduled loop now sleeps a short DEPRECATION_LOCK_RETRY_SECONDS (5 min) whenever send_model_deprecation_alert returns False (including a transient Redis-induced lock-claim failure), and only sleeps the full 24h when an alert was actually sent.
- ✅ Fixed: Empty check burns daily lock window
- send_model_deprecation_alert now claims the fleet lock only after collect_model_deprecations/format_deprecation_alert_message produce a non-empty message, so an empty pass never burns the 24h window.
Or push these changes by commenting:
@cursor push 0c4ff7bff5
Preview (0c4ff7bff5)
diff --git a/litellm/integrations/SlackAlerting/slack_alerting.py b/litellm/integrations/SlackAlerting/slack_alerting.py
--- a/litellm/integrations/SlackAlerting/slack_alerting.py
+++ b/litellm/integrations/SlackAlerting/slack_alerting.py
@@ -53,6 +53,7 @@
from litellm.types.proxy.model_deprecation import (
DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS,
DEPRECATION_IDLE_POLL_SECONDS,
+ DEPRECATION_LOCK_RETRY_SECONDS,
)
from ..email_templates.templates import *
@@ -1062,8 +1063,16 @@
def _deprecation_alerts_enabled(self) -> bool:
return self.alerting is not None and AlertType.model_deprecation_warnings in self.alert_types
- async def send_model_deprecation_alert(self, llm_router: Router | None = None) -> bool:
- """Alert on the router's deprecated and imminent models, True when one was sent"""
+ async def send_model_deprecation_alert(
+ self,
+ llm_router: Router | None = None,
+ pod_lock_manager: "PodLockManager | None" = None,
+ ) -> bool:
+ """Alert on the router's deprecated and imminent models, True when one was sent
+
+ The daily fleet lock is claimed only after we confirm there is content to alert on,
+ so an empty pass does not burn the 24h window for every other replica.
+ """
if not self._deprecation_alerts_enabled():
return False
@@ -1077,6 +1086,9 @@
if message is None:
return False
+ if not await self._claimed_deprecation_alert_window(pod_lock_manager):
+ return False
+
level: Final[Literal["Low", "Medium", "High"]] = "High" if snapshot.deprecated else "Medium"
await self.send_alert(
@@ -1103,22 +1115,40 @@
)
) is not False
+ async def _send_deprecation_alert_swallowing_errors(
+ self,
+ llm_router: Router,
+ pod_lock_manager: "PodLockManager | None",
+ ) -> bool:
+ """A failed pass must not kill the daily loop, so exceptions here fall through as 'not sent'"""
+ try:
+ return await self.send_model_deprecation_alert(
+ llm_router=llm_router, pod_lock_manager=pod_lock_manager
+ )
+ except Exception as e: # noqa: BLE001 # a failed alert must not kill the daily loop
+ verbose_proxy_logger.exception("Error in model deprecation alert loop: %s", e)
+ return False
+
async def run_scheduled_deprecation_check(
self,
get_llm_router: Callable[[], Router | None] = _proxy_llm_router,
pod_lock_manager: "PodLockManager | None" = None,
) -> None:
- """Alert once the router is loaded and the alert is on, then daily, re-reading both each pass"""
+ """Alert once the router is loaded and the alert is on, then daily, re-reading both each pass
+
+ A failed lock claim (transient Redis error or another pod holds the day) and an empty check
+ retry on a short cadence, so a boot-time Redis blip does not silence the fleet for 24h.
+ """
while True:
if (llm_router := get_llm_router()) is None or not self._deprecation_alerts_enabled():
await asyncio.sleep(DEPRECATION_IDLE_POLL_SECONDS)
continue
- try:
- if await self._claimed_deprecation_alert_window(pod_lock_manager):
- await self.send_model_deprecation_alert(llm_router=llm_router)
- except Exception as e: # noqa: BLE001 # a failed alert must not kill the daily loop
- verbose_proxy_logger.exception("Error in model deprecation alert loop: %s", e)
- await asyncio.sleep(DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS)
+ alert_sent: Final = await self._send_deprecation_alert_swallowing_errors(
+ llm_router=llm_router, pod_lock_manager=pod_lock_manager
+ )
+ await asyncio.sleep(
+ DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS if alert_sent else DEPRECATION_LOCK_RETRY_SECONDS
+ )
async def send_webhook_alert(self, webhook_event: WebhookEvent) -> bool:
"""
diff --git a/litellm/types/proxy/model_deprecation.py b/litellm/types/proxy/model_deprecation.py
--- a/litellm/types/proxy/model_deprecation.py
+++ b/litellm/types/proxy/model_deprecation.py
@@ -11,6 +11,8 @@
DEPRECATION_IDLE_POLL_SECONDS: Final = 30
+DEPRECATION_LOCK_RETRY_SECONDS: Final = 5 * 60
+
DeprecationStatus = Literal["upcoming", "imminent", "deprecated"]
diff --git a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py
--- a/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py
+++ b/tests/test_litellm/integrations/SlackAlerting/test_model_deprecation_alert.py
@@ -17,6 +17,7 @@
from litellm.types.proxy.model_deprecation import (
DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS,
DEPRECATION_IDLE_POLL_SECONDS,
+ DEPRECATION_LOCK_RETRY_SECONDS,
)
@@ -253,3 +254,88 @@
"ttl": DEFAULT_DEPRECATION_CHECK_INTERVAL_SECONDS,
"allow_reentrant": False,
}
+
+
+@pytest.mark.asyncio
+async def test_should_retry_soon_when_lock_claim_fails(monkeypatch):
+ """A transient Redis error surfaces as acquire_lock=False, and must not silence the fleet 24h"""
+ monkeypatch.setattr(
+ litellm,
+ "model_cost",
+ {"dead-model": {"deprecation_date": "2020-01-01", "litellm_provider": "openai"}},
+ )
+ alerting = SlackAlerting(
+ alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings]
+ )
+ router = _make_router(
+ [
+ {
+ "model_name": "dead-alias",
+ "litellm_params": {"model": "dead-model"},
+ "model_info": {"id": "1"},
+ }
+ ]
+ )
+ pod_lock_manager = MagicMock()
+ pod_lock_manager.acquire_lock = AsyncMock(return_value=False)
+ slept: list[float] = []
+
+ async def stop_after_first_sleep(seconds):
+ slept.append(seconds)
+ raise asyncio.CancelledError
+
+ with (
+ patch.object(alerting, "send_alert", new_callable=AsyncMock) as mock_send_alert,
+ patch(
+ "litellm.integrations.SlackAlerting.slack_alerting.asyncio.sleep",
+ side_effect=stop_after_first_sleep,
+ ),
+ pytest.raises(asyncio.CancelledError),
+ ):
+ await alerting.run_scheduled_deprecation_check(
+ get_llm_router=lambda: router, pod_lock_manager=pod_lock_manager
+ )
+
+ assert slept == [DEPRECATION_LOCK_RETRY_SECONDS]
+ mock_send_alert.assert_not_awaited()
+
+
+@pytest.mark.asyncio
+async def test_should_not_claim_lock_when_no_deprecations(monkeypatch):
+ """An empty pass must not burn the 24h fleet lock, so mid-day sunsets can still alert"""
+ monkeypatch.setattr(litellm, "model_cost", {})
+ alerting = SlackAlerting(
+ alerting=["slack"], alert_types=[AlertType.model_deprecation_warnings]
+ )
+ router = _make_router(
+ [
+ {
+ "model_name": "fresh",
+ "litellm_params": {"model": "openai/gpt-4o"},
+ "model_info": {"id": "x"},
+ }
+ ]
+ )
+ pod_lock_manager = MagicMock()
+ pod_lock_manager.acquire_lock = AsyncMock(return_value=True)
+ slept: list[float] = []
+
+ async def stop_after_first_sleep(seconds):
+ slept.append(seconds)
+ raise asyncio.CancelledError
+
+ with (
+ patch.object(alerting, "send_alert", new_callable=AsyncMock) as mock_send_alert,
+ patch(
+ "litellm.integrations.SlackAlerting.slack_alerting.asyncio.sleep",
+ side_effect=stop_after_first_sleep,
+ ),
+ pytest.raises(asyncio.CancelledError),
+ ):
+ await alerting.run_scheduled_deprecation_check(
+ get_llm_router=lambda: router, pod_lock_manager=pod_lock_manager
+ )
+
+ assert slept == [DEPRECATION_LOCK_RETRY_SECONDS]
+ pod_lock_manager.acquire_lock.assert_not_awaited()
+ mock_send_alert.assert_not_awaited()You can send follow-ups to the cloud agent here.
4 tasks
… failed claims next poll An empty pass no longer holds the daily lock, a False lock claim (held or redis error) is retried on the next 30 second poll instead of sleeping a day, and a sent alert is stamped in the shared cache for a day so sibling pods and restarts stay quiet
Contributor
Author
…el the alert in the UI A pass that raises (a missing Slack webhook, say) now waits the daily interval instead of logging the same exception every 30 seconds, and the Admin UI alerting settings list the new alert type so it can be toggled like the others
Contributor
Author
Contributor
Author
|
bugbot run |
Contributor
There was a problem hiding this comment.
✅ Bugbot reviewed your changes and found no new issues!
Comment @cursor review or bugbot run to trigger another review on this PR
Reviewed by Cursor Bugbot for commit 7017df5. Configure here.
mateo-berri
enabled auto-merge
August 18, 2026 01:15
yucheng-berri
approved these changes
Aug 18, 2026
This was referenced Aug 18, 2026
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
4 participants
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.
TLDR
Problem this solves:
How it solves it:
GET /model/deprecationsbuckets configured models by urgencymodel_info.deprecation_dateoverrides the registryUser Flow
Before: a proxy admin running a fleet of deployments has no way to see which of them a provider is about to sunset
model_infothemselves, so they keep routing traffic to a dying modelAfter: the same admin gets the sunset dates up front, and Slack nags them daily while there is still time to migrate
deprecated,imminent, andupcoming, each entry naming the model as they configured it, the date, days remaining, and the providerimminentdeprecation_dateundermodel_infoon that deployment and it shows up in the same listsRelevant issues
Linear ticket
Resolves LIT-2701
Pre-Submission checklist
Please complete all items before asking a LiteLLM maintainer to review your PR
@greptileaito re-request a review after pushing changes)Delays in PR merge?
If you're seeing a delay in your PR being merged, ping the LiteLLM Team on Slack (#pr-review).
Screenshots / Proof of Fix
Every case below was run against a live proxy at the named commit, both halves loading the same config with five deployments:
claude-opus-4-1(registry date already passed),gemini/imagen-4.0-generate-001(registry date passed the day of the run),openai/sora-2(37 days out), an Azure deployment whosebase_modelcarries no date, andinternal-legacywith a hand setmodel_info.deprecation_date. The After proxy also picks up three Azure deployments stored in the dev database, all dated 2028, which is whyupcominglists more than the config. The Slack alert cases pointSLACK_WEBHOOK_URLat a tiny local HTTP sink (python3 sink.py 41735 sink.log) that logs every POST with a timestamp, and the redis cases share one localredis-server --port 41738throughlitellm_settings.cache_paramsBefore (909a2e6)
GET /model/deprecations
curl -s -o /dev/null -w "%{http_code}\n" "http://localhost:41739/model/deprecations" -H "Authorization: Bearer sk-1234"404, the route does not existRe-bucket with warn_within_days=90, no restart
curl -s -o /dev/null -w "%{http_code}\n" "http://localhost:41739/v1/model/deprecations?warn_within_days=90" -H "Authorization: Bearer sk-1234"404Auth enforced like the other model management routes
curl -s -o /dev/null -w "%{http_code}\n" "http://localhost:41739/model/deprecations"404, no route to protectBoot-time Slack alert reaches the webhook
SLACK_WEBHOOK_URL=http://127.0.0.1:41735/webhook litellm --config qa_deprecation_config.yaml --port 41739 &, then wait for/health/livelinesssleep 45; cat sink.logprints nothing: no deprecation alert exists, so a fleet running two already deprecated models hears nothingMulti replica fleet sends one alert
sink.logstays emptyRestart within a day stays quiet
sink.logstays emptyA pod with nothing to report leaves the daily lock free
sink.logstays emptyNo webhook configured logs once a day
alerting: ["slack"]and noSLACK_WEBHOOK_URL(port 41739), wait for/health/liveliness(18:07:06), thensleep 75grep -c "Error in model deprecation alert loop" pod.logprints0andgrep -c deprecation pod.logprints0: nothing checks sunset dates, so nothing complains eitherAfter (7017df5)
GET /model/deprecations
curl -s -X GET "http://localhost:41733/model/deprecations" -H "Authorization: Bearer sk-1234"internal-legacytaking its date frommodel_info.deprecation_dateon the deployment because the registry has none foropenai/gpt-4o, and the dateless Azure config deployment correctly absent{ "deprecated": [ { "model_name": "opus-4-1", "litellm_model": "claude-opus-4-1", "deprecation_date": "2026-08-05", "days_until_deprecation": -13, "status": "deprecated", "litellm_provider": "anthropic" }, { "model_name": "imagen-4", "litellm_model": "gemini/imagen-4.0-generate-001", "deprecation_date": "2026-08-17", "days_until_deprecation": -1, "status": "deprecated", "litellm_provider": "gemini" } ], "imminent": [ { "model_name": "internal-legacy", "litellm_model": "openai/gpt-4o", "deprecation_date": "2026-08-20", "days_until_deprecation": 2, "status": "imminent", "litellm_provider": null } ], "upcoming": [ { "model_name": "sora-2", "litellm_model": "openai/sora-2", "deprecation_date": "2026-09-24", "days_until_deprecation": 37, "status": "upcoming", "litellm_provider": "openai" }, { "model_name": "gpt-5-6-luna-azure-openai", "litellm_model": "azure/gpt-5.6-luna", "deprecation_date": "2028-01-11", "days_until_deprecation": 511, "status": "upcoming", "litellm_provider": "azure" }, { "model_name": "gpt-5-6-sol-azure-openai", "litellm_model": "azure/gpt-5.6-sol", "deprecation_date": "2028-01-11", "days_until_deprecation": 511, "status": "upcoming", "litellm_provider": "azure" }, { "model_name": "gpt-5-6-terra-azure-openai", "litellm_model": "azure/gpt-5.6-terra", "deprecation_date": "2028-01-11", "days_until_deprecation": 511, "status": "upcoming", "litellm_provider": "azure" } ], "warn_within_days": 30, "checked_at": "2026-08-18T01:06:07.407468Z" }Re-bucket with warn_within_days=90, no restart
curl -s "http://localhost:41733/v1/model/deprecations?warn_within_days=90" -H "Authorization: Bearer sk-1234"sora-2moves fromupcomingintoimminent(model names only, for brevity){"deprecated": ["opus-4-1", "imagen-4"], "imminent": ["internal-legacy", "sora-2"], "upcoming": ["gpt-5-6-luna-azure-openai", "gpt-5-6-sol-azure-openai", "gpt-5-6-terra-azure-openai"], "warn_within_days": 90, "checked_at": "2026-08-18T01:06:07.465843Z"}Auth enforced like the other model management routes
curl -s -o /dev/null -w "%{http_code}\n" "http://localhost:41733/model/deprecations"401Boot-time Slack alert reaches the webhook
SLACK_WEBHOOK_URL=http://127.0.0.1:41735/webhook litellm --config qa_deprecation_config.yaml --port 41733 &until curl -s -o /dev/null http://127.0.0.1:41733/health/liveliness; do sleep 2; done; date +%H:%M:%Sprints18:05:59cat sink.logshows the alert 2 seconds after the proxy reported healthyMulti replica fleet sends one alert
redis-cli -p 41738 flushall; : > sink.log, then boot two pods onqa_deprecation_config_redis.yaml(ports 41736 and 41737) and wait for both/health/liveliness;dateprints17:58:20sleep 10; cut -c1-120 sink.logshows exactly one alertredis-cli -p 41738 --scanplusttlshows the daily lock and the shared sent stamp, both a day longRestart within a day stays quiet
pkill -f "port 41737", boot it again on the same redis, wait for/health/liveliness(17:58:48), thensleep 45cut -c1-120 sink.logstill shows only the17:58:24deprecation alert (plus an unrelatedspend_reportsalert): the restarted pod sees the sent stamp and never asks redis for the lockA pod with nothing to report leaves the daily lock free
redis-cli -p 41738 flushall; : > sink.log, boot a pod onqa_empty_config_redis.yaml(one undatedopenai/gpt-4o, port 41737), wait for/health/liveliness(17:59:58), thensleep 40so at least two passes ranredis-cli -p 41738 --scan | grep -ci "cronjob_lock\|deprecation"prints0, andsink.logis empty: an empty pass claims nothingqa_deprecation_config_redis.yaml(port 41736) against the same redis;/health/livelinessat18:00:54sleep 10; cut -c1-160 sink.logshows the alert seconds later instead of a day later, and redis now holds the lock and stampNo webhook configured logs once a day
alerting: ["slack"]and noSLACK_WEBHOOK_URL(port 41733), wait for/health/liveliness(18:01:25), thensleep 75so three polls would have rungrep -c "Error in model deprecation alert loop" pod.logprints1, and the loop backs off a full day instead of logging every 30 secondsType
🆕 New Feature
Caveats (if any)
base_modelormodel_info.deprecation_dateto pin itFinal Attestation
Link to Devin session: https://app.devin.ai/sessions/680cda1b74c34159899de6c6ae064ef6
Requested by: @mateo-berri
Note
Cursor Bugbot is generating a summary for commit 4e7e2f5. Configure here.
Link to Devin session: https://app.devin.ai/sessions/9ac267a1b87043189b066e7a44373a8e