From f04c4999c8820d8a659152d2817b80da4c6b2787 Mon Sep 17 00:00:00 2001 From: Tim Stranske Date: Mon, 20 Jul 2026 05:28:25 -0500 Subject: [PATCH] fix: preserve slot allowlist fail-closed behavior --- templates/consumer-repo/tools/llm_registry.py | 14 +++++++++----- tests/tools/test_langchain_client.py | 4 ++-- tests/tools/test_llm_registry_selection.py | 12 ++++-------- tools/llm_registry.py | 14 +++++++++----- 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/templates/consumer-repo/tools/llm_registry.py b/templates/consumer-repo/tools/llm_registry.py index 321bddb66..41f1c9ed0 100644 --- a/templates/consumer-repo/tools/llm_registry.py +++ b/templates/consumer-repo/tools/llm_registry.py @@ -133,7 +133,6 @@ def load_model_registry() -> list[ModelRegistryEntry]: model=model, blocked=bool(raw_entry.get("blocked", False)), lifecycle=str(raw_entry.get("lifecycle", "unknown")).strip().lower(), - quality={}, ) ) return entries @@ -432,10 +431,15 @@ def resolve_slots( env_slot_prefix: str = "LANGCHAIN_SLOT", ) -> list[SlotDefinition]: slots = load_slot_config(github_default_model=github_default_model) - # Preserve an explicit runtime override as an emergency bootstrap when the - # registry file is unavailable. Empty models are never invoked directly; - # langchain_client skips them when the override cannot serve that provider. - if not slots and not os.environ.get(ENV_SLOT_CONFIG) and os.environ.get(env_model_name): + # Preserve an explicit runtime override only when no default slot allowlist + # is available. A present slot config that resolves to no usable slots fails + # closed rather than broadening execution to the environment model. + if ( + not slots + and not os.environ.get(ENV_SLOT_CONFIG) + and not _slot_path().exists() + and os.environ.get(env_model_name) + ): slots = [ SlotDefinition(name=f"slot{index}", provider=provider, model="") for index, provider in enumerate( diff --git a/tests/tools/test_langchain_client.py b/tests/tools/test_langchain_client.py index 5f4e64f08..55f8c92b9 100644 --- a/tests/tools/test_langchain_client.py +++ b/tests/tools/test_langchain_client.py @@ -335,7 +335,7 @@ def test_load_model_registry_ignores_malformed_nested_values( entries = llm_registry.load_model_registry() assert len(entries) == 1 - assert entries[0].quality == {} + assert entries[0].quality is None def test_load_model_registry_rejects_non_list_models( @@ -372,7 +372,7 @@ def test_load_model_registry_ignores_v1_quality_scores( [entry] = llm_registry.load_model_registry() - assert entry.quality == {} + assert entry.quality is None def test_load_slot_config_ignores_tier_slot_when_registry_invalid( diff --git a/tests/tools/test_llm_registry_selection.py b/tests/tools/test_llm_registry_selection.py index f3e557577..0db693279 100644 --- a/tests/tools/test_llm_registry_selection.py +++ b/tests/tools/test_llm_registry_selection.py @@ -68,14 +68,14 @@ def test_profile_selection_uses_explicit_decision_not_model_position( assert registry.select_model_for_profile(provider="openai") == "model-balanced" -def test_loaded_registry_entries_keep_empty_compatibility_quality( +def test_loaded_registry_entries_leave_compatibility_quality_unset( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: registry_path = tmp_path / "registry.json" _write_registry(registry_path) monkeypatch.setenv(registry.ENV_MODEL_REGISTRY_CONFIG, str(registry_path)) - assert registry.load_model_registry()[0].quality == {} + assert registry.load_model_registry()[0].quality is None def test_new_catalog_model_does_not_auto_promote( @@ -207,7 +207,7 @@ def test_missing_explicit_slot_config_fails_closed( assert registry.configured_model_for_provider("openai") == "" -def test_unusable_bundled_slot_config_bootstraps_explicit_env_model( +def test_unusable_bundled_slot_config_fails_closed_despite_env_model( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: registry_path = tmp_path / "registry.json" @@ -219,11 +219,7 @@ def test_unusable_bundled_slot_config_bootstraps_explicit_env_model( monkeypatch.setattr(registry, "DEFAULT_SLOT_CONFIG_PATH", slots_path) monkeypatch.setenv("LANGCHAIN_MODEL", "emergency-model") - assert [slot.provider for slot in registry.resolve_slots()] == [ - "openai", - "anthropic", - "github-models", - ] + assert registry.resolve_slots() == [] def test_invalid_registry_with_explicit_profile_slot_fails_closed( diff --git a/tools/llm_registry.py b/tools/llm_registry.py index 321bddb66..41f1c9ed0 100644 --- a/tools/llm_registry.py +++ b/tools/llm_registry.py @@ -133,7 +133,6 @@ def load_model_registry() -> list[ModelRegistryEntry]: model=model, blocked=bool(raw_entry.get("blocked", False)), lifecycle=str(raw_entry.get("lifecycle", "unknown")).strip().lower(), - quality={}, ) ) return entries @@ -432,10 +431,15 @@ def resolve_slots( env_slot_prefix: str = "LANGCHAIN_SLOT", ) -> list[SlotDefinition]: slots = load_slot_config(github_default_model=github_default_model) - # Preserve an explicit runtime override as an emergency bootstrap when the - # registry file is unavailable. Empty models are never invoked directly; - # langchain_client skips them when the override cannot serve that provider. - if not slots and not os.environ.get(ENV_SLOT_CONFIG) and os.environ.get(env_model_name): + # Preserve an explicit runtime override only when no default slot allowlist + # is available. A present slot config that resolves to no usable slots fails + # closed rather than broadening execution to the environment model. + if ( + not slots + and not os.environ.get(ENV_SLOT_CONFIG) + and not _slot_path().exists() + and os.environ.get(env_model_name) + ): slots = [ SlotDefinition(name=f"slot{index}", provider=provider, model="") for index, provider in enumerate(