From b2086221ca35661aa9fdafa08f8710875a477382 Mon Sep 17 00:00:00 2001 From: rain <1504569896@qq.com> Date: Mon, 21 Sep 2026 10:11:49 +0000 Subject: [PATCH 1/2] fix(proxy): keep externally-managed deployments across config reconcile The 30s reconcile in _delete_deployment evicts every router deployment absent from both the DB and config.yaml. A deployment injected at runtime (by a plugin, callback or set_model_list/upsert_deployment) meets exactly that condition, so with store_model_in_db on it is silently deleted within seconds. Let a deployment declare an external owner via model_info.managed_by: such a deployment is not an orphan of the DB+config reconcile, so the cleanup loop skips it (with a log line) instead of evicting it. Deployments without an owner and unknown to DB/config are still removed as before. Adds regression coverage: an externally-managed deployment survives the reconcile when config.yaml lists other models, while an owner-less orphan is still evicted. --- litellm/proxy/proxy_server.py | 11 +++ .../test_delete_deployment_managed_by.py | 98 +++++++++++++++++++ .../test_update_llm_router_resilience.py | 4 + 3 files changed, 113 insertions(+) create mode 100644 tests/test_litellm/proxy/test_delete_deployment_managed_by.py diff --git a/litellm/proxy/proxy_server.py b/litellm/proxy/proxy_server.py index 7634237a59f5..e985559437ac 100644 --- a/litellm/proxy/proxy_server.py +++ b/litellm/proxy/proxy_server.py @@ -6877,6 +6877,17 @@ async def _delete_deployment(self, db_models: list) -> frozenset[str] | None: for model_id in router_model_ids: if model_id not in combined_id_list: + deployment: Final = llm_router.get_deployment(model_id=model_id) + if deployment is not None and getattr(deployment.model_info, "managed_by", None): + # A deployment with a declared external owner (e.g. injected at runtime by a + # plugin/callback) is not an orphan of the DB+config reconcile - skip it. + verbose_proxy_logger.info( + "Skipping deployment %s (id=%s): declares external owner %r, not an orphan.", + deployment.model_name, + model_id, + getattr(deployment.model_info, "managed_by", None), + ) + continue llm_router.delete_deployment(id=model_id) return frozenset(combined_id_list) diff --git a/tests/test_litellm/proxy/test_delete_deployment_managed_by.py b/tests/test_litellm/proxy/test_delete_deployment_managed_by.py new file mode 100644 index 000000000000..95de6ef3785e --- /dev/null +++ b/tests/test_litellm/proxy/test_delete_deployment_managed_by.py @@ -0,0 +1,98 @@ +""" +Regression tests for runtime-injected deployments being silently deleted by the +config reconcile (_delete_deployment) when store_model_in_db is enabled. + +A deployment injected into the Router at runtime (via set_model_list / +upsert_deployment by a callback or plugin) is present in the router but absent +from both the DB and config.yaml, so the clean-up loop treats it as an orphan +and evicts it on the next 30s tick - silently. + +The fix: a deployment whose model_info declares an external owner (`managed_by`) +is not an orphan. The reconcile skips it and logs the skip. +""" + +import pytest +from unittest.mock import AsyncMock, MagicMock, patch + +from litellm.proxy.proxy_server import ProxyConfig + + +def _make_config_deployment(model_name: str, model_id: str) -> dict: + """A config.yaml-style deployment entry.""" + return { + "model_name": model_name, + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"}, + "model_info": {"id": model_id}, + } + + +@pytest.fixture +def runtime_router(): + """A Router holding one config-sourced deployment and one runtime-injected + deployment that declares an external owner via model_info.managed_by.""" + from litellm.router import Router + + router = Router( + model_list=[ + { + "model_name": "plugin/model", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"}, + "model_info": {"id": "plugin-1", "managed_by": "my-plugin"}, + }, + _make_config_deployment("gpt-4o", "config-1"), + ] + ) + return router + + +class TestDeleteDeploymentKeepsExternallyManagedModels: + @pytest.mark.asyncio + async def test_runtime_injected_deployment_survives_reconcile(self, runtime_router): + """Deployments declaring model_info.managed_by must not be evicted when + config.yaml exists and lists only other models.""" + cfg = ProxyConfig() + + async def fake_get_config(config_file_path=None): + return {"model_list": [_make_config_deployment("gpt-4o", "config-1")]} + + cfg.get_config = fake_get_config + + router = runtime_router + with patch("litellm.proxy.proxy_server.llm_router", router): + await cfg._delete_deployment(db_models=[]) + + model_ids = sorted(router.get_model_ids()) + assert "plugin-1" in model_ids, "runtime-injected deployment was evicted" + assert "config-1" in model_ids + assert len(model_ids) == 2 + + @pytest.mark.asyncio + async def test_orphan_without_owner_is_still_evicted(self): + """A router deployment that is in neither config nor DB and declares no + external owner must still be deleted - the keep rule must not become a + blanket stay-deleted rule.""" + from litellm.router import Router + + router = Router( + model_list=[ + { + "model_name": "legacy/model", + "litellm_params": {"model": "openai/gpt-4o", "api_key": "sk-fake"}, + "model_info": {"id": "orphan-1"}, + }, + _make_config_deployment("gpt-4o", "config-1"), + ] + ) + cfg = ProxyConfig() + + async def fake_get_config(config_file_path=None): + return {"model_list": [_make_config_deployment("gpt-4o", "config-1")]} + + cfg.get_config = fake_get_config + + with patch("litellm.proxy.proxy_server.llm_router", router): + await cfg._delete_deployment(db_models=[]) + + model_ids = sorted(router.get_model_ids()) + assert "orphan-1" not in model_ids, "orphan without owner should be evicted" + assert model_ids == ["config-1"] \ No newline at end of file diff --git a/tests/test_litellm/proxy/test_update_llm_router_resilience.py b/tests/test_litellm/proxy/test_update_llm_router_resilience.py index d6ebfde10919..f8c0224ba162 100644 --- a/tests/test_litellm/proxy/test_update_llm_router_resilience.py +++ b/tests/test_litellm/proxy/test_update_llm_router_resilience.py @@ -153,6 +153,10 @@ async def test_normal_delete_still_works(self): mock_router = MagicMock() # Router has a model ID that's not in DB or config -> should be deleted mock_router.get_model_ids.return_value = ["db-id-1", "stale-id"] + # get_deployment returns None when the id is unknown to the router, which must + # NOT shield it from eviction - only a deployment declaring model_info.managed_by + # is treated as externally owned and spared. + mock_router.get_deployment.return_value = None mock_router.delete_deployment.return_value = True mock_router.generate_model_id = MagicMock(return_value="config-id-1") From da7973e8e9a9bdd992107ea1a6091a273a4cb59f Mon Sep 17 00:00:00 2001 From: rain <1504569896@qq.com> Date: Mon, 21 Sep 2026 11:10:44 +0000 Subject: [PATCH 2/2] test: mock get_deployment=None in reconcile eviction tests _delete_deployment now skips deployments declaring an external owner via model_info.managed_by. These pre-existing tests use MagicMock routers whose get_deployment returns an auto-created (truthy) mock, which the new skip branch misread as a managed deployment, suppressing expected evictions. Pin get_deployment.return_value to None so the ids are treated as unknown to the router - the correct condition under which they must still be evicted. --- tests/test_litellm/proxy/test_proxy_server.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_litellm/proxy/test_proxy_server.py b/tests/test_litellm/proxy/test_proxy_server.py index f71f9c20f3b4..ac7b8ba2c820 100644 --- a/tests/test_litellm/proxy/test_proxy_server.py +++ b/tests/test_litellm/proxy/test_proxy_server.py @@ -3006,6 +3006,9 @@ def mock_delete_deployment(id): return True # Simulate successful deletion mock_llm_router.delete_deployment = MagicMock(side_effect=mock_delete_deployment) + # None means these ids are unknown to the router, so they must NOT be shielded + # from eviction (only a deployment declaring model_info.managed_by is spared). + mock_llm_router.get_deployment = MagicMock(return_value=None) async def mock_get_config(config_file_path): return { @@ -11234,6 +11237,8 @@ async def test_delete_deployment_evicts_model_when_all_db_models_deleted(self): proxy_config = ProxyConfig() mock_router = MagicMock() mock_router.get_model_ids.return_value = ["model-id-to-evict"] + # None: the id is unknown to the router, so it must not be spared from eviction + mock_router.get_deployment.return_value = None mock_router.delete_deployment.return_value = MagicMock() with patch("litellm.proxy.proxy_server.llm_router", mock_router):