diff --git a/.github/scripts/auto_update_price_and_context_window_file.py b/.github/scripts/auto_update_price_and_context_window_file.py index 4c9a05bd4f04..a50015730ed5 100644 --- a/.github/scripts/auto_update_price_and_context_window_file.py +++ b/.github/scripts/auto_update_price_and_context_window_file.py @@ -97,7 +97,7 @@ def _modality_flags(input_mods: list) -> dict: } -def transform_friendli_data(data: list, local_data: dict) -> dict: +def transform_friendli_data(data: list, local_data: dict) -> dict[str, dict[str, object]]: transformed: dict[str, dict] = {} if not data: return transformed @@ -165,7 +165,33 @@ def transform_friendli_data(data: list, local_data: dict) -> dict: return transformed # Synchronize local data with remote data -def sync_local_data_with_remote(local_data, remote_data, replace_keys=frozenset()): +def _friendli_catalog_keys(data: list | None) -> frozenset[str]: + if not data: + return frozenset() + return frozenset(f"{FRIENDLI_PROVIDER}/{model['id']}" for model in data if "id" in model) + + +def _stale_friendli_keys( + local_keys: frozenset[str], + friendli_keys_in_remote: frozenset[str], +) -> frozenset[str]: + """friendliai/* keys absent from the transformed Friendli catalog are dead: + Friendli delists deprecated serverless models, so a registry entry the + catalog no longer returns bills stale prices against a model that fails + at the provider. Legacy pre-sync rows (no `source`) are included since + scoping is by key prefix, not by provenance""" + prefix = f"{FRIENDLI_PROVIDER}/" + return frozenset( + key for key in local_keys if key.startswith(prefix) and key not in friendli_keys_in_remote + ) + + +def sync_local_data_with_remote( + local_data: dict[str, dict[str, object]], + remote_data: dict[str, dict[str, object]], + replace_keys: frozenset[str] = frozenset(), + catalog_dropped_keys: frozenset[str] = frozenset(), +) -> None: # Update existing keys in local_data with values from remote_data # (replace_keys entries are swapped wholesale so a field the remote catalog # dropped, e.g. cache pricing, cannot survive as a stale value) @@ -179,6 +205,9 @@ def sync_local_data_with_remote(local_data, remote_data, replace_keys=frozenset( for key in (set(remote_data) - set(local_data)): local_data[key] = remote_data[key] + for key in catalog_dropped_keys: + local_data.pop(key, None) + # Write data to the json file def write_to_file(file_path, data): try: @@ -302,14 +331,32 @@ def main(): vercel_data = transform_vercel_ai_gateway_data(vercel_data) friendli_data = asyncio.run(fetch_data(FRIENDLI_API_URL)) + friendli_catalog_keys = _friendli_catalog_keys(friendli_data) friendli_data = transform_friendli_data(friendli_data, local_data) - + # Combine both datasets all_remote_data = {**openrouter_data, **vercel_data, **friendli_data} + # Deletion only runs when Friendli returns a non-empty catalog. Catalog rows + # with invalid pricing still prove the model is live, so deletion compares + # against raw catalog ids instead of transformed priced rows + catalog_dropped_keys: frozenset[str] = ( + frozenset() + if not friendli_catalog_keys + else _stale_friendli_keys( + local_keys=frozenset(local_data or {}), + friendli_keys_in_remote=friendli_catalog_keys, + ) + ) + # If both local and openrouter data are available, synchronize and save if local_data and all_remote_data: - sync_local_data_with_remote(local_data, all_remote_data, replace_keys=frozenset(friendli_data)) + sync_local_data_with_remote( + local_data, + all_remote_data, + replace_keys=frozenset(friendli_data), + catalog_dropped_keys=catalog_dropped_keys, + ) write_to_file(local_file_path, local_data) else: print("Failed to fetch model data from either local file or URL.") diff --git a/tests/test_litellm/test_auto_update_price_and_context_window_file.py b/tests/test_litellm/test_auto_update_price_and_context_window_file.py index 435747e9a09e..a5cf2f80c681 100644 --- a/tests/test_litellm/test_auto_update_price_and_context_window_file.py +++ b/tests/test_litellm/test_auto_update_price_and_context_window_file.py @@ -140,6 +140,27 @@ def test_transform_skips_rows_without_valid_token_prices_so_priced_local_entries assert local["friendliai/zai-org/GLM-Test"]["output_cost_per_token"] == 5e-07 +def test_sync_keeps_live_friendli_entry_when_catalog_row_has_invalid_pricing(sync_module): + local = { + "friendliai/zai-org/GLM-Test": { + "litellm_provider": "friendliai", + "input_cost_per_token": 1.5e-07, + "output_cost_per_token": 5e-07, + } + } + raw_catalog = [_reasoning_model(pricing={})] + remote = sync_module.transform_friendli_data(raw_catalog, local) + dropped = sync_module._stale_friendli_keys( + local_keys=frozenset(local), + friendli_keys_in_remote=sync_module._friendli_catalog_keys(raw_catalog), + ) + sync_module.sync_local_data_with_remote( + local, remote, replace_keys=frozenset(remote), catalog_dropped_keys=dropped + ) + assert "friendliai/zai-org/GLM-Test" in local + assert local["friendliai/zai-org/GLM-Test"]["input_cost_per_token"] == 1.5e-07 + + def test_transform_keeps_zero_priced_rows(sync_module): free_model = _reasoning_model(pricing={"input": "0", "output": "0"}) entry = sync_module.transform_friendli_data([free_model], {})["friendliai/zai-org/GLM-Test"] @@ -220,3 +241,79 @@ def test_transform_inherits_allowlisted_keys_from_base_model_entry(sync_module): assert entry["supports_pdf_input"] is True assert entry["supports_assistant_prefill"] is True assert entry["input_cost_per_token"] == 1.5e-07 + + +def test_sync_drops_friendli_entries_absent_from_catalog(sync_module): + local = { + "friendliai/dead-model": {"litellm_provider": "friendliai", "input_cost_per_token": 1e-06}, + "friendliai/zai-org/GLM-Test": {"litellm_provider": "friendliai"}, + "openrouter/still-listed": {"input_cost_per_token": 1e-06}, + } + remote = sync_module.transform_friendli_data([_reasoning_model()], local) + dropped = sync_module._stale_friendli_keys( + local_keys=frozenset(local), friendli_keys_in_remote=frozenset(remote) + ) + sync_module.sync_local_data_with_remote( + local, remote, replace_keys=frozenset(remote), catalog_dropped_keys=dropped + ) + assert "friendliai/dead-model" not in local + assert "friendliai/zai-org/GLM-Test" in local + assert "openrouter/still-listed" in local + + +def test_stale_keys_scoped_to_friendli_prefix(sync_module): + local_keys = frozenset({"friendliai/gone", "openrouter/also-absent", "zhipuai/glm-test"}) + remote_keys = frozenset({"friendliai/zai-org/GLM-Test"}) + dropped = sync_module._stale_friendli_keys(local_keys, remote_keys) + assert dropped == frozenset({"friendliai/gone"}) + + +def test_sync_keeps_all_entries_when_friendli_fetch_fails_or_is_empty(sync_module): + local = { + "friendliai/zai-org/GLM-Test": {"litellm_provider": "friendliai"}, + "openrouter/kept": {"input_cost_per_token": 1e-06}, + } + for failed_fetch in (None, []): + remote = sync_module.transform_friendli_data(failed_fetch, local) + catalog_keys = sync_module._friendli_catalog_keys(failed_fetch) + dropped = ( + frozenset() + if not catalog_keys + else sync_module._stale_friendli_keys( + local_keys=frozenset(local), friendli_keys_in_remote=catalog_keys + ) + ) + sync_module.sync_local_data_with_remote( + local, remote, replace_keys=frozenset(remote), catalog_dropped_keys=dropped + ) + assert "friendliai/zai-org/GLM-Test" in local + assert "openrouter/kept" in local + + +def test_sync_deletes_nothing_when_friendli_fetch_fails_but_other_catalogs_succeed(sync_module): + # merged remote is non-empty (openrouter rows), friendli catalog unreachable: + # absence from the friendli catalog proves nothing, so its entries must survive + local = { + "friendliai/zai-org/GLM-Test": {"litellm_provider": "friendliai"}, + "openrouter/live-model": {"input_cost_per_token": 1e-06}, + } + remote = {"openrouter/live-model": {"input_cost_per_token": 2e-06}} + sync_module.sync_local_data_with_remote( + local, remote, replace_keys=frozenset(), catalog_dropped_keys=frozenset() + ) + assert "friendliai/zai-org/GLM-Test" in local + assert "openrouter/live-model" in local + + +def test_sync_drops_legacy_friendli_entries_without_source_field(sync_module): + local = { + "friendliai/legacy-model": {"litellm_provider": "friendliai", "max_tokens": 8192}, + } + remote = sync_module.transform_friendli_data([_reasoning_model()], local) + dropped = sync_module._stale_friendli_keys( + local_keys=frozenset(local), friendli_keys_in_remote=frozenset(remote) + ) + sync_module.sync_local_data_with_remote( + local, remote, replace_keys=frozenset(remote), catalog_dropped_keys=dropped + ) + assert "friendliai/legacy-model" not in local