feat(price-sync): drop friendliai entries the catalog no longer returns

.github/scripts/auto_update_price_and_context_window_file.py

@@ -97,7 +97,7 @@ def _modality_flags(input_mods: list) -> dict:
     }
 
 
-def transform_friendli_data(data: list, local_data: dict) -> dict:
+def transform_friendli_data(data: list, local_data: dict) -> dict[str, dict[str, object]]:
     transformed: dict[str, dict] = {}
     if not data:
         return transformed
@@ -165,7 +165,33 @@ def transform_friendli_data(data: list, local_data: dict) -> dict:
     return transformed
 
 # Synchronize local data with remote data
-def sync_local_data_with_remote(local_data, remote_data, replace_keys=frozenset()):
+def _friendli_catalog_keys(data: list | None) -> frozenset[str]:
+    if not data:
+        return frozenset()
+    return frozenset(f"{FRIENDLI_PROVIDER}/{model['id']}" for model in data if "id" in model)
+
+
+def _stale_friendli_keys(
+    local_keys: frozenset[str],
+    friendli_keys_in_remote: frozenset[str],
+) -> frozenset[str]:
+    """friendliai/* keys absent from the transformed Friendli catalog are dead:
+    Friendli delists deprecated serverless models, so a registry entry the
+    catalog no longer returns bills stale prices against a model that fails
+    at the provider. Legacy pre-sync rows (no `source`) are included since
+    scoping is by key prefix, not by provenance"""
+    prefix = f"{FRIENDLI_PROVIDER}/"
+    return frozenset(
+        key for key in local_keys if key.startswith(prefix) and key not in friendli_keys_in_remote
+    )
+
+
+def sync_local_data_with_remote(
+    local_data: dict[str, dict[str, object]],
+    remote_data: dict[str, dict[str, object]],
+    replace_keys: frozenset[str] = frozenset(),
+    catalog_dropped_keys: frozenset[str] = frozenset(),
+) -> None:
     # Update existing keys in local_data with values from remote_data
     # (replace_keys entries are swapped wholesale so a field the remote catalog
     #  dropped, e.g. cache pricing, cannot survive as a stale value)
@@ -179,6 +205,9 @@ def sync_local_data_with_remote(local_data, remote_data, replace_keys=frozenset(
     for key in (set(remote_data) - set(local_data)):
         local_data[key] = remote_data[key]
 
+    for key in catalog_dropped_keys:
+        local_data.pop(key, None)
+
 # Write data to the json file
 def write_to_file(file_path, data):
     try:
@@ -302,14 +331,32 @@ def main():
     vercel_data = transform_vercel_ai_gateway_data(vercel_data)
 
     friendli_data = asyncio.run(fetch_data(FRIENDLI_API_URL))
+    friendli_catalog_keys = _friendli_catalog_keys(friendli_data)
     friendli_data = transform_friendli_data(friendli_data, local_data)
-    
+
     # Combine both datasets
     all_remote_data = {**openrouter_data, **vercel_data, **friendli_data}
 
+    # Deletion only runs when Friendli returns a non-empty catalog. Catalog rows
+    # with invalid pricing still prove the model is live, so deletion compares
+    # against raw catalog ids instead of transformed priced rows
+    catalog_dropped_keys: frozenset[str] = (
+        frozenset()
+        if not friendli_catalog_keys
+        else _stale_friendli_keys(
+            local_keys=frozenset(local_data or {}),
+            friendli_keys_in_remote=friendli_catalog_keys,
+        )
+    )
+
     # If both local and openrouter data are available, synchronize and save
     if local_data and all_remote_data:
-        sync_local_data_with_remote(local_data, all_remote_data, replace_keys=frozenset(friendli_data))
+        sync_local_data_with_remote(
+            local_data,
+            all_remote_data,
+            replace_keys=frozenset(friendli_data),
+            catalog_dropped_keys=catalog_dropped_keys,
+        )
         write_to_file(local_file_path, local_data)
     else:
         print("Failed to fetch model data from either local file or URL.")

tests/test_litellm/test_auto_update_price_and_context_window_file.py

@@ -140,6 +140,27 @@ def test_transform_skips_rows_without_valid_token_prices_so_priced_local_entries
     assert local["friendliai/zai-org/GLM-Test"]["output_cost_per_token"] == 5e-07
 
 
+def test_sync_keeps_live_friendli_entry_when_catalog_row_has_invalid_pricing(sync_module):
+    local = {
+        "friendliai/zai-org/GLM-Test": {
+            "litellm_provider": "friendliai",
+            "input_cost_per_token": 1.5e-07,
+            "output_cost_per_token": 5e-07,
+        }
+    }
+    raw_catalog = [_reasoning_model(pricing={})]
+    remote = sync_module.transform_friendli_data(raw_catalog, local)
+    dropped = sync_module._stale_friendli_keys(
+        local_keys=frozenset(local),
+        friendli_keys_in_remote=sync_module._friendli_catalog_keys(raw_catalog),
+    )
+    sync_module.sync_local_data_with_remote(
+        local, remote, replace_keys=frozenset(remote), catalog_dropped_keys=dropped
+    )
+    assert "friendliai/zai-org/GLM-Test" in local
+    assert local["friendliai/zai-org/GLM-Test"]["input_cost_per_token"] == 1.5e-07
+
+
 def test_transform_keeps_zero_priced_rows(sync_module):
     free_model = _reasoning_model(pricing={"input": "0", "output": "0"})
     entry = sync_module.transform_friendli_data([free_model], {})["friendliai/zai-org/GLM-Test"]
@@ -220,3 +241,79 @@ def test_transform_inherits_allowlisted_keys_from_base_model_entry(sync_module):
     assert entry["supports_pdf_input"] is True
     assert entry["supports_assistant_prefill"] is True
     assert entry["input_cost_per_token"] == 1.5e-07
+
+
+def test_sync_drops_friendli_entries_absent_from_catalog(sync_module):
+    local = {
+        "friendliai/dead-model": {"litellm_provider": "friendliai", "input_cost_per_token": 1e-06},
+        "friendliai/zai-org/GLM-Test": {"litellm_provider": "friendliai"},
+        "openrouter/still-listed": {"input_cost_per_token": 1e-06},
+    }
+    remote = sync_module.transform_friendli_data([_reasoning_model()], local)
+    dropped = sync_module._stale_friendli_keys(
+        local_keys=frozenset(local), friendli_keys_in_remote=frozenset(remote)
+    )
+    sync_module.sync_local_data_with_remote(
+        local, remote, replace_keys=frozenset(remote), catalog_dropped_keys=dropped
+    )
+    assert "friendliai/dead-model" not in local
+    assert "friendliai/zai-org/GLM-Test" in local
+    assert "openrouter/still-listed" in local
+
+
+def test_stale_keys_scoped_to_friendli_prefix(sync_module):
+    local_keys = frozenset({"friendliai/gone", "openrouter/also-absent", "zhipuai/glm-test"})
+    remote_keys = frozenset({"friendliai/zai-org/GLM-Test"})
+    dropped = sync_module._stale_friendli_keys(local_keys, remote_keys)
+    assert dropped == frozenset({"friendliai/gone"})
+
+
+def test_sync_keeps_all_entries_when_friendli_fetch_fails_or_is_empty(sync_module):
+    local = {
+        "friendliai/zai-org/GLM-Test": {"litellm_provider": "friendliai"},
+        "openrouter/kept": {"input_cost_per_token": 1e-06},
+    }
+    for failed_fetch in (None, []):
+        remote = sync_module.transform_friendli_data(failed_fetch, local)
+        catalog_keys = sync_module._friendli_catalog_keys(failed_fetch)
+        dropped = (
+            frozenset()
+            if not catalog_keys
+            else sync_module._stale_friendli_keys(
+                local_keys=frozenset(local), friendli_keys_in_remote=catalog_keys
+            )
+        )
+        sync_module.sync_local_data_with_remote(
+            local, remote, replace_keys=frozenset(remote), catalog_dropped_keys=dropped
+        )
+        assert "friendliai/zai-org/GLM-Test" in local
+        assert "openrouter/kept" in local
+
+
+def test_sync_deletes_nothing_when_friendli_fetch_fails_but_other_catalogs_succeed(sync_module):
+    # merged remote is non-empty (openrouter rows), friendli catalog unreachable:
+    # absence from the friendli catalog proves nothing, so its entries must survive
+    local = {
+        "friendliai/zai-org/GLM-Test": {"litellm_provider": "friendliai"},
+        "openrouter/live-model": {"input_cost_per_token": 1e-06},
+    }
+    remote = {"openrouter/live-model": {"input_cost_per_token": 2e-06}}
+    sync_module.sync_local_data_with_remote(
+        local, remote, replace_keys=frozenset(), catalog_dropped_keys=frozenset()
+    )
+    assert "friendliai/zai-org/GLM-Test" in local
+    assert "openrouter/live-model" in local
+
+
+def test_sync_drops_legacy_friendli_entries_without_source_field(sync_module):
+    local = {
+        "friendliai/legacy-model": {"litellm_provider": "friendliai", "max_tokens": 8192},
+    }
+    remote = sync_module.transform_friendli_data([_reasoning_model()], local)
+    dropped = sync_module._stale_friendli_keys(
+        local_keys=frozenset(local), friendli_keys_in_remote=frozenset(remote)
+    )
+    sync_module.sync_local_data_with_remote(
+        local, remote, replace_keys=frozenset(remote), catalog_dropped_keys=dropped
+    )
+    assert "friendliai/legacy-model" not in local