From c748ed7012e1ebdff04ef86139290ed63fb99477 Mon Sep 17 00:00:00 2001 From: Tim Stranske Date: Sun, 21 Jun 2026 22:38:50 -0500 Subject: [PATCH] fix: sync llm registry helper to consumers --- .github/sync-manifest.yml | 3 + .../consumer-repo/tools/langchain_client.py | 228 +++++++++------ templates/consumer-repo/tools/llm_registry.py | 276 ++++++++++++++++++ tests/tools/test_langchain_client.py | 12 +- .../workflows/test_sync_manifest_delivery.py | 9 + tools/langchain_client.py | 148 +++++----- tools/llm_registry.py | 276 ++++++++++++++++++ 7 files changed, 783 insertions(+), 169 deletions(-) create mode 100644 templates/consumer-repo/tools/llm_registry.py create mode 100644 tools/llm_registry.py diff --git a/.github/sync-manifest.yml b/.github/sync-manifest.yml index 6080f7647..b695eea79 100644 --- a/.github/sync-manifest.yml +++ b/.github/sync-manifest.yml @@ -672,6 +672,9 @@ scripts: - source: tools/langchain_client.py description: "LangChain client builder - multi-provider client with slot-based fallback and configuration" + - source: tools/llm_registry.py + description: "LLM model registry helper - shared slot/model selection and blocked-model enforcement" + - source: tools/embedding_provider.py description: "Embedding provider registry used by synced semantic matching helpers" diff --git a/templates/consumer-repo/tools/langchain_client.py b/templates/consumer-repo/tools/langchain_client.py index 8197183f0..dd4217d74 100644 --- a/templates/consumer-repo/tools/langchain_client.py +++ b/templates/consumer-repo/tools/langchain_client.py @@ -8,13 +8,28 @@ from __future__ import annotations import contextlib -import json import logging import os from dataclasses import dataclass -from pathlib import Path +from tools import llm_registry as _llm_registry from tools.llm_provider import DEFAULT_MODEL, GITHUB_MODELS_BASE_URL +from tools.llm_registry import ( + PROVIDER_ANTHROPIC, + PROVIDER_GITHUB, + PROVIDER_OPENAI, + ModelRegistryEntry, + SlotDefinition, + apply_slot_env_overrides, + default_slots, + is_model_blocked, + load_model_registry, + load_slot_config, + normalize_provider, + registry_entry_for, + resolve_slots, + select_model_for_tier, +) logger = logging.getLogger(__name__) @@ -22,15 +37,12 @@ ENV_MODEL = "LANGCHAIN_MODEL" ENV_TIMEOUT = "LANGCHAIN_TIMEOUT" ENV_MAX_RETRIES = "LANGCHAIN_MAX_RETRIES" -ENV_SLOT_CONFIG = "LANGCHAIN_SLOT_CONFIG" +ENV_SLOT_CONFIG = _llm_registry.ENV_SLOT_CONFIG +ENV_MODEL_REGISTRY_CONFIG = _llm_registry.ENV_MODEL_REGISTRY_CONFIG ENV_SLOT_PREFIX = "LANGCHAIN_SLOT" ENV_ANTHROPIC_KEY = "CLAUDE_API_STRANSKE" - -PROVIDER_OPENAI = "openai" -PROVIDER_ANTHROPIC = "anthropic" -PROVIDER_GITHUB = "github-models" - -DEFAULT_SLOT_CONFIG_PATH = Path(__file__).resolve().parent.parent / "config" / "llm_slots.json" +DEFAULT_SLOT_CONFIG_PATH = _llm_registry.DEFAULT_SLOT_CONFIG_PATH +DEFAULT_MODEL_REGISTRY_CONFIG_PATH = _llm_registry.DEFAULT_MODEL_REGISTRY_CONFIG_PATH def _env_int(name: str, default: int) -> int: @@ -59,24 +71,8 @@ def provider_label(self) -> str: return f"{self.provider}/{self.model}" -@dataclass(frozen=True) -class SlotDefinition: - name: str - provider: str - model: str - - def _normalize_provider(value: str | None) -> str | None: - if not value: - return None - normalized = value.strip().lower() - if normalized in {"github", "github_models", "github-models"}: - return PROVIDER_GITHUB - if normalized in {"anthropic", "claude"}: - return PROVIDER_ANTHROPIC - if normalized in {"openai"}: - return PROVIDER_OPENAI - return None + return normalize_provider(value) def _resolve_provider(provider: str | None, *, force_openai: bool) -> tuple[str | None, bool]: @@ -93,69 +89,81 @@ def _resolve_model(model: str | None) -> str: return model or env_model or DEFAULT_MODEL +def _load_model_registry() -> list[ModelRegistryEntry]: + return load_model_registry() + + +def _registry_entry_for( + provider: str, model: str, registry: list[ModelRegistryEntry] | None = None +) -> ModelRegistryEntry | None: + return registry_entry_for(provider, model, registry=registry) + + +def _is_model_blocked( + provider: str, model: str, registry: list[ModelRegistryEntry] | None = None +) -> bool: + return is_model_blocked(provider, model, registry=registry) + + +def _select_model_for_tier( + *, + provider: str, + tier: str, + registry: list[ModelRegistryEntry] | None = None, +) -> str | None: + return select_model_for_tier(provider=provider, tier=tier, registry=registry) + + def _default_slots() -> list[SlotDefinition]: - return [ - SlotDefinition(name="slot1", provider=PROVIDER_OPENAI, model="gpt-5.4"), - SlotDefinition(name="slot2", provider=PROVIDER_ANTHROPIC, model="claude-sonnet-4-6"), - SlotDefinition(name="slot3", provider=PROVIDER_GITHUB, model=DEFAULT_MODEL), - ] + return default_slots(github_default_model=DEFAULT_MODEL) def _load_slot_config() -> list[SlotDefinition]: - config_path = os.environ.get(ENV_SLOT_CONFIG) - path = Path(config_path) if config_path else DEFAULT_SLOT_CONFIG_PATH - if not path.is_file(): - return _default_slots() - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return _default_slots() - - slots: list[SlotDefinition] = [] - for idx, entry in enumerate(payload.get("slots", []), start=1): - provider = _normalize_provider(str(entry.get("provider", ""))) - model = str(entry.get("model", "")).strip() - if not provider or not model: - continue - name = str(entry.get("name") or f"slot{idx}").strip() or f"slot{idx}" - slots.append(SlotDefinition(name=name, provider=provider, model=model)) - - return slots or _default_slots() + return load_slot_config(github_default_model=DEFAULT_MODEL) def _apply_slot_env_overrides(slots: list[SlotDefinition]) -> list[SlotDefinition]: - updated: list[SlotDefinition] = [] - for idx, slot in enumerate(slots, start=1): - provider_key = f"{ENV_SLOT_PREFIX}{idx}_PROVIDER" - model_key = f"{ENV_SLOT_PREFIX}{idx}_MODEL" - provider_override = _normalize_provider(os.environ.get(provider_key)) - model_override = os.environ.get(model_key) - if idx == 1: - model_override = model_override or os.environ.get(ENV_MODEL) - updated.append( - SlotDefinition( - name=slot.name, - provider=provider_override or slot.provider, - model=(model_override or slot.model).strip(), - ) - ) - return updated + return apply_slot_env_overrides( + slots, + env_model_name=ENV_MODEL, + env_slot_prefix=ENV_SLOT_PREFIX, + ) def _resolve_slots() -> list[SlotDefinition]: - return _apply_slot_env_overrides(_load_slot_config()) + return resolve_slots( + github_default_model=DEFAULT_MODEL, + env_model_name=ENV_MODEL, + env_slot_prefix=ENV_SLOT_PREFIX, + ) + + +def _is_reasoning_model(model: str) -> bool: + """Return True if the model is an OpenAI reasoning model that rejects temperature. + + Supported naming pattern: `o` + digits, with optional suffixes. Examples: `o1`, + `o1-preview`, `o1-preview-2024-09-12`, `o3`, `o3-mini`, `o3-pro`, `o4-mini`, + `o4-mini-deep-research`. Non-matching examples: `o`, `o-1`, `openai-o1`, `oasis-1`. + """ + name = model.lower().strip() + # o-series reasoning models use an `o` prefix followed by digits with optional + # hyphen-separated suffixes: o1, o1-preview, o1-preview-2024-09-12, o3, o3-mini, + # o3-pro, o4-mini, o4-mini-deep-research. + return bool(__import__("re").fullmatch(r"o[0-9]+(?:-[a-z0-9]+)*", name)) def _build_openai_client( chat_openai: type, *, model: str, token: str, timeout: int, max_retries: int ) -> object: - return chat_openai( - model=model, - api_key=token, - temperature=0.1, - timeout=timeout, - max_retries=max_retries, - ) + kwargs: dict = { + "model": model, + "api_key": token, + "timeout": timeout, + "max_retries": max_retries, + } + if not _is_reasoning_model(model): + kwargs["temperature"] = 0.1 + return chat_openai(**kwargs) def _build_anthropic_client( @@ -173,14 +181,16 @@ def _build_anthropic_client( def _build_github_client( chat_openai: type, *, model: str, token: str, timeout: int, max_retries: int ) -> object: - return chat_openai( - model=model, - base_url=GITHUB_MODELS_BASE_URL, - api_key=token, - temperature=0.1, - timeout=timeout, - max_retries=max_retries, - ) + kwargs: dict = { + "model": model, + "base_url": GITHUB_MODELS_BASE_URL, + "api_key": token, + "timeout": timeout, + "max_retries": max_retries, + } + if not _is_reasoning_model(model): + kwargs["temperature"] = 0.1 + return chat_openai(**kwargs) def build_chat_client( @@ -197,9 +207,11 @@ def build_chat_client( return None try: - from langchain_anthropic import ChatAnthropic as ChatAnthropicClass + from langchain_anthropic import ChatAnthropic except ImportError: - ChatAnthropicClass = None + chat_anthropic_cls = None + else: + chat_anthropic_cls = ChatAnthropic github_token = os.environ.get("GITHUB_TOKEN") openai_token = os.environ.get("OPENAI_API_KEY") @@ -214,6 +226,9 @@ def build_chat_client( selected_provider, provider_explicit = _resolve_provider(provider, force_openai=force_openai) if provider_explicit and selected_provider is None: return None + if selected_provider and _is_model_blocked(selected_provider, selected_model): + logger.warning("Refusing blocked LLM model: %s/%s", selected_provider, selected_model) + return None if selected_provider == PROVIDER_GITHUB: if not github_token: @@ -246,11 +261,11 @@ def build_chat_client( return None if selected_provider == PROVIDER_ANTHROPIC: - if not anthropic_token or not ChatAnthropicClass: + if not anthropic_token or not chat_anthropic_cls: return None try: client = _build_anthropic_client( - ChatAnthropicClass, + chat_anthropic_cls, model=selected_model, token=anthropic_token, timeout=selected_timeout, @@ -263,6 +278,13 @@ def build_chat_client( # Auto-select: slot order (OpenAI -> Claude -> GitHub Models by default). slots = _resolve_slots() model_override = model or os.environ.get(ENV_MODEL) + if model_override: + override_provider = selected_provider or (slots[0].provider if slots else "") + if override_provider and _is_model_blocked(override_provider, model_override): + logger.warning( + "Refusing blocked LLM model override: %s/%s", override_provider, model_override + ) + return None used_override = False for slot in slots: slot_model = model_override if model_override and not used_override else slot.model @@ -277,10 +299,10 @@ def build_chat_client( ) used_override = True return ClientInfo(client=client, provider=PROVIDER_OPENAI, model=slot_model) - if slot.provider == PROVIDER_ANTHROPIC and anthropic_token and ChatAnthropicClass: + if slot.provider == PROVIDER_ANTHROPIC and anthropic_token and chat_anthropic_cls: with contextlib.suppress(Exception): client = _build_anthropic_client( - ChatAnthropicClass, + chat_anthropic_cls, model=slot_model, token=anthropic_token, timeout=selected_timeout, @@ -317,9 +339,11 @@ def build_chat_clients( return [] try: - from langchain_anthropic import ChatAnthropic as ChatAnthropicClass + from langchain_anthropic import ChatAnthropic except ImportError: - ChatAnthropicClass = None + chat_anthropic_cls = None + else: + chat_anthropic_cls = ChatAnthropic github_token = os.environ.get("GITHUB_TOKEN") openai_token = os.environ.get("OPENAI_API_KEY") @@ -336,6 +360,15 @@ def build_chat_clients( selected_provider, provider_explicit = _resolve_provider(provider, force_openai=False) if provider_explicit and selected_provider is None: return [] + registry = _load_model_registry() + if selected_provider: + blocked_models = [candidate for candidate in (first_model, second_model) if candidate] + if any( + _is_model_blocked(selected_provider, candidate, registry=registry) + for candidate in blocked_models + ): + logger.warning("Refusing blocked LLM model for provider %s", selected_provider) + return [] clients: list[ClientInfo] = [] @@ -400,12 +433,12 @@ def build_chat_clients( model=second_model, ) ) - elif selected_provider == PROVIDER_ANTHROPIC and anthropic_token and ChatAnthropicClass: + elif selected_provider == PROVIDER_ANTHROPIC and anthropic_token and chat_anthropic_cls: with contextlib.suppress(Exception): clients.append( ClientInfo( client=_build_anthropic_client( - ChatAnthropicClass, + chat_anthropic_cls, model=first_model, token=anthropic_token, timeout=selected_timeout, @@ -420,7 +453,7 @@ def build_chat_clients( clients.append( ClientInfo( client=_build_anthropic_client( - ChatAnthropicClass, + chat_anthropic_cls, model=second_model, token=anthropic_token, timeout=selected_timeout, @@ -439,7 +472,7 @@ def build_chat_clients( if any( ( slot.provider == PROVIDER_OPENAI and openai_token, - slot.provider == PROVIDER_ANTHROPIC and anthropic_token and ChatAnthropicClass, + slot.provider == PROVIDER_ANTHROPIC and anthropic_token and chat_anthropic_cls, slot.provider == PROVIDER_GITHUB and github_token, ) ): @@ -453,6 +486,9 @@ def build_chat_clients( for idx, slot in enumerate(candidate_slots): slot_model = model_overrides[idx] if idx < len(model_overrides) else None slot_model = slot_model or slot.model + if _is_model_blocked(slot.provider, slot_model, registry=registry): + logger.warning("Skipping blocked LLM model override: %s/%s", slot.provider, slot_model) + continue if slot.provider == PROVIDER_OPENAI and openai_token: with contextlib.suppress(Exception): clients.append( @@ -468,12 +504,12 @@ def build_chat_clients( model=slot_model, ) ) - if slot.provider == PROVIDER_ANTHROPIC and anthropic_token and ChatAnthropicClass: + if slot.provider == PROVIDER_ANTHROPIC and anthropic_token and chat_anthropic_cls: with contextlib.suppress(Exception): clients.append( ClientInfo( client=_build_anthropic_client( - ChatAnthropicClass, + chat_anthropic_cls, model=slot_model, token=anthropic_token, timeout=selected_timeout, diff --git a/templates/consumer-repo/tools/llm_registry.py b/templates/consumer-repo/tools/llm_registry.py new file mode 100644 index 000000000..05226b09e --- /dev/null +++ b/templates/consumer-repo/tools/llm_registry.py @@ -0,0 +1,276 @@ +"""Shared LLM slot and model-registry resolution helpers.""" + +from __future__ import annotations + +import json +import logging +import os +from dataclasses import dataclass +from pathlib import Path + +logger = logging.getLogger(__name__) + +ENV_MODEL_REGISTRY_CONFIG = "LANGCHAIN_MODEL_REGISTRY_CONFIG" +ENV_SLOT_CONFIG = "LANGCHAIN_SLOT_CONFIG" + +PROVIDER_OPENAI = "openai" +PROVIDER_ANTHROPIC = "anthropic" +PROVIDER_GITHUB = "github-models" + +DEFAULT_SLOT_CONFIG_PATH = Path(__file__).resolve().parent.parent / "config" / "llm_slots.json" +DEFAULT_MODEL_REGISTRY_CONFIG_PATH = ( + Path(__file__).resolve().parent.parent / "config" / "model_registry.json" +) + + +@dataclass(frozen=True) +class ModelRegistryEntry: + provider: str + model: str + blocked: bool + quality: dict[str, float] + + +@dataclass(frozen=True) +class SlotDefinition: + name: str + provider: str + model: str + + +def normalize_provider(value: str | None) -> str | None: + if not value: + return None + normalized = value.strip().lower() + if normalized in {"github", "github_models", "github-models"}: + return PROVIDER_GITHUB + if normalized in {"anthropic", "claude"}: + return PROVIDER_ANTHROPIC + if normalized == PROVIDER_OPENAI: + return PROVIDER_OPENAI + return None + + +def _slot_entries(payload: dict[str, object], path: Path) -> list[dict[str, object]]: + raw_slots = payload.get("slots", []) + if not isinstance(raw_slots, list): + logger.warning("Invalid slot config format in %s; expected slots list", path) + return [] + slots: list[dict[str, object]] = [] + for raw_slot in raw_slots: + if isinstance(raw_slot, dict): + slots.append(raw_slot) + else: + logger.warning("Ignoring invalid slot entry in %s; expected object", path) + return slots + + +def load_model_registry() -> list[ModelRegistryEntry]: + config_path = os.environ.get(ENV_MODEL_REGISTRY_CONFIG) + path = Path(config_path) if config_path else DEFAULT_MODEL_REGISTRY_CONFIG_PATH + if not path.is_file(): + return [] + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + logger.warning("Could not read model registry %s; continuing without registry", path) + return [] + if not isinstance(payload, dict): + logger.warning("Invalid model registry format in %s; expected object", path) + return [] + + entries: list[ModelRegistryEntry] = [] + for raw_entry in payload.get("models", []): + if not isinstance(raw_entry, dict): + logger.warning("Ignoring invalid model registry entry in %s; expected object", path) + continue + provider = normalize_provider(str(raw_entry.get("provider", ""))) + model = str(raw_entry.get("model_id", "")).strip() + if not provider or not model: + continue + quality_payload = raw_entry.get("quality", {}) + quality = { + str(tier).upper(): float(score) + for tier, score in quality_payload.items() + if isinstance(score, int | float) + } + entries.append( + ModelRegistryEntry( + provider=provider, + model=model, + blocked=bool(raw_entry.get("blocked", False)), + quality=quality, + ) + ) + return entries + + +def registry_entry_for( + provider: str, model: str, registry: list[ModelRegistryEntry] | None = None +) -> ModelRegistryEntry | None: + entries = registry if registry is not None else load_model_registry() + normalized_provider = normalize_provider(provider) + normalized_model = model.strip() + for entry in entries: + if entry.provider == normalized_provider and entry.model == normalized_model: + return entry + return None + + +def is_model_blocked( + provider: str, model: str, registry: list[ModelRegistryEntry] | None = None +) -> bool: + entry = registry_entry_for(provider, model, registry=registry) + return bool(entry and entry.blocked) + + +def select_model_for_tier( + *, + provider: str, + tier: str, + registry: list[ModelRegistryEntry] | None = None, +) -> str | None: + entries = registry if registry is not None else load_model_registry() + normalized_provider = normalize_provider(provider) + normalized_tier = tier.strip().upper() + candidates = [ + entry + for entry in entries + if entry.provider == normalized_provider + and not entry.blocked + and normalized_tier in entry.quality + ] + if not candidates: + return None + selected = max(candidates, key=lambda entry: entry.quality[normalized_tier]) + return selected.model + + +def configured_model_for_provider( + provider: str, + *, + fallback: str, + tier: str = "T3", + registry: list[ModelRegistryEntry] | None = None, +) -> str: + normalized_provider = normalize_provider(provider) + entries = registry if registry is not None else load_model_registry() + + config_path = os.environ.get(ENV_SLOT_CONFIG) + path = Path(config_path) if config_path else DEFAULT_SLOT_CONFIG_PATH + if path.is_file(): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + payload = {} + if not isinstance(payload, dict): + logger.warning("Invalid slot config format in %s; expected object", path) + payload = {} + for slot in _slot_entries(payload, path): + slot_provider = normalize_provider(str(slot.get("provider", ""))) + if slot_provider != normalized_provider: + continue + model = str(slot.get("model", "")).strip() + slot_tier = str(slot.get("quality_tier") or slot.get("tier") or tier).strip() + if not model and slot_tier: + model = ( + select_model_for_tier( + provider=slot_provider or "", + tier=slot_tier, + registry=entries, + ) + or "" + ) + if model and not is_model_blocked(slot_provider or "", model, registry=entries): + return model + + selected = select_model_for_tier(provider=provider, tier=tier, registry=entries) + if selected: + return selected + if not is_model_blocked(provider, fallback, registry=entries): + return fallback + return "" + + +def default_slots(*, github_default_model: str) -> list[SlotDefinition]: + return [ + SlotDefinition(name="slot1", provider=PROVIDER_OPENAI, model="gpt-5.4"), + SlotDefinition(name="slot2", provider=PROVIDER_ANTHROPIC, model="claude-sonnet-4-6"), + SlotDefinition(name="slot3", provider=PROVIDER_GITHUB, model=github_default_model), + ] + + +def load_slot_config(*, github_default_model: str) -> list[SlotDefinition]: + config_path = os.environ.get(ENV_SLOT_CONFIG) + path = Path(config_path) if config_path else DEFAULT_SLOT_CONFIG_PATH + fallback_slots = default_slots(github_default_model=github_default_model) + if not path.is_file(): + return fallback_slots + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return fallback_slots + if not isinstance(payload, dict): + logger.warning("Invalid slot config format in %s; expected object", path) + return fallback_slots + + registry = load_model_registry() + slots: list[SlotDefinition] = [] + for idx, entry in enumerate(_slot_entries(payload, path), start=1): + provider = normalize_provider(str(entry.get("provider", ""))) + model = str(entry.get("model", "")).strip() + tier = str(entry.get("quality_tier") or entry.get("tier") or "").strip() + if provider and not model and tier: + model = select_model_for_tier(provider=provider, tier=tier, registry=registry) or "" + if not provider or not model: + continue + if is_model_blocked(provider, model, registry=registry): + logger.warning("Skipping blocked LLM model in slot config: %s/%s", provider, model) + continue + name = str(entry.get("name") or f"slot{idx}").strip() or f"slot{idx}" + slots.append(SlotDefinition(name=name, provider=provider, model=model)) + + return slots or fallback_slots + + +def apply_slot_env_overrides( + slots: list[SlotDefinition], + *, + env_model_name: str = "LANGCHAIN_MODEL", + env_slot_prefix: str = "LANGCHAIN_SLOT", +) -> list[SlotDefinition]: + registry = load_model_registry() + updated: list[SlotDefinition] = [] + for idx, slot in enumerate(slots, start=1): + provider_key = f"{env_slot_prefix}{idx}_PROVIDER" + model_key = f"{env_slot_prefix}{idx}_MODEL" + provider_override = normalize_provider(os.environ.get(provider_key)) + model_override = os.environ.get(model_key) + if idx == 1: + model_override = model_override or os.environ.get(env_model_name) + provider = provider_override or slot.provider + model = (model_override or slot.model).strip() + if is_model_blocked(provider, model, registry=registry): + logger.warning("Skipping blocked LLM slot override: %s/%s", provider, model) + continue + updated.append( + SlotDefinition( + name=slot.name, + provider=provider, + model=model, + ) + ) + return updated + + +def resolve_slots( + *, + github_default_model: str, + env_model_name: str = "LANGCHAIN_MODEL", + env_slot_prefix: str = "LANGCHAIN_SLOT", +) -> list[SlotDefinition]: + return apply_slot_env_overrides( + load_slot_config(github_default_model=github_default_model), + env_model_name=env_model_name, + env_slot_prefix=env_slot_prefix, + ) diff --git a/tests/tools/test_langchain_client.py b/tests/tools/test_langchain_client.py index 37bed668b..f312f9eab 100644 --- a/tests/tools/test_langchain_client.py +++ b/tests/tools/test_langchain_client.py @@ -199,13 +199,13 @@ def test_build_chat_clients_env_provider_override(monkeypatch: pytest.MonkeyPatc monkeypatch.setenv("OPENAI_API_KEY", "oa-token") monkeypatch.setenv(langchain_client.ENV_PROVIDER, "openai") - clients = langchain_client.build_chat_clients(model1="gpt-4o-mini", model2="gpt-4o") + clients = langchain_client.build_chat_clients(model1="gpt-4.1-mini", model2="gpt-4o") assert [client.provider for client in clients] == [ langchain_client.PROVIDER_OPENAI, langchain_client.PROVIDER_OPENAI, ] - assert [client.model for client in clients] == ["gpt-4o-mini", "gpt-4o"] + assert [client.model for client in clients] == ["gpt-4.1-mini", "gpt-4o"] assert all(isinstance(client.client, FakeChatOpenAI) for client in clients) @@ -233,13 +233,13 @@ def test_build_chat_clients_env_model_override(monkeypatch: pytest.MonkeyPatch) FakeChatOpenAI = _install_fake_langchain_openai(monkeypatch) monkeypatch.setenv("GITHUB_TOKEN", "gh-token") monkeypatch.setenv("OPENAI_API_KEY", "oa-token") - monkeypatch.setenv(langchain_client.ENV_MODEL, "gpt-4o-mini") + monkeypatch.setenv(langchain_client.ENV_MODEL, "gpt-4.1-mini") monkeypatch.delenv(langchain_client.ENV_PROVIDER, raising=False) clients = langchain_client.build_chat_clients() assert [client.model for client in clients] == [ - "gpt-4o-mini", + "gpt-4.1-mini", langchain_client.DEFAULT_MODEL, ] assert isinstance(clients[0].client, FakeChatOpenAI) @@ -253,12 +253,12 @@ def test_build_chat_clients_env_model_with_provider_override( monkeypatch.delenv("GITHUB_TOKEN", raising=False) monkeypatch.setenv("OPENAI_API_KEY", "oa-token") monkeypatch.setenv(langchain_client.ENV_PROVIDER, "openai") - monkeypatch.setenv(langchain_client.ENV_MODEL, "gpt-4o-mini") + monkeypatch.setenv(langchain_client.ENV_MODEL, "gpt-4.1-mini") clients = langchain_client.build_chat_clients() assert [client.provider for client in clients] == [langchain_client.PROVIDER_OPENAI] - assert [client.model for client in clients] == ["gpt-4o-mini"] + assert [client.model for client in clients] == ["gpt-4.1-mini"] assert all(isinstance(client.client, FakeChatOpenAI) for client in clients) diff --git a/tests/workflows/test_sync_manifest_delivery.py b/tests/workflows/test_sync_manifest_delivery.py index 96a4ff870..1e64aaa40 100644 --- a/tests/workflows/test_sync_manifest_delivery.py +++ b/tests/workflows/test_sync_manifest_delivery.py @@ -145,6 +145,15 @@ def test_copy_dependent_langchain_scripts_stay_copy_synced() -> None: ) +def test_langchain_client_registry_dependency_stays_copy_synced() -> None: + """The copy-synced LangChain client imports tools.llm_registry in consumers.""" + manifest = _load_manifest() + copy_sources = _sources_in_sections(manifest, COPY_SYNCED_SECTIONS) + + assert "tools/langchain_client.py" in copy_sources + assert "tools/llm_registry.py" in copy_sources + + def test_all_langchain_entries_have_a_delivery_channel() -> None: """Every scripts/langchain/* entry must declare its delivery channel.""" manifest = _load_manifest() diff --git a/tools/langchain_client.py b/tools/langchain_client.py index 1f1617db7..dd4217d74 100644 --- a/tools/langchain_client.py +++ b/tools/langchain_client.py @@ -8,13 +8,28 @@ from __future__ import annotations import contextlib -import json import logging import os from dataclasses import dataclass -from pathlib import Path +from tools import llm_registry as _llm_registry from tools.llm_provider import DEFAULT_MODEL, GITHUB_MODELS_BASE_URL +from tools.llm_registry import ( + PROVIDER_ANTHROPIC, + PROVIDER_GITHUB, + PROVIDER_OPENAI, + ModelRegistryEntry, + SlotDefinition, + apply_slot_env_overrides, + default_slots, + is_model_blocked, + load_model_registry, + load_slot_config, + normalize_provider, + registry_entry_for, + resolve_slots, + select_model_for_tier, +) logger = logging.getLogger(__name__) @@ -22,15 +37,12 @@ ENV_MODEL = "LANGCHAIN_MODEL" ENV_TIMEOUT = "LANGCHAIN_TIMEOUT" ENV_MAX_RETRIES = "LANGCHAIN_MAX_RETRIES" -ENV_SLOT_CONFIG = "LANGCHAIN_SLOT_CONFIG" +ENV_SLOT_CONFIG = _llm_registry.ENV_SLOT_CONFIG +ENV_MODEL_REGISTRY_CONFIG = _llm_registry.ENV_MODEL_REGISTRY_CONFIG ENV_SLOT_PREFIX = "LANGCHAIN_SLOT" ENV_ANTHROPIC_KEY = "CLAUDE_API_STRANSKE" - -PROVIDER_OPENAI = "openai" -PROVIDER_ANTHROPIC = "anthropic" -PROVIDER_GITHUB = "github-models" - -DEFAULT_SLOT_CONFIG_PATH = Path(__file__).resolve().parent.parent / "config" / "llm_slots.json" +DEFAULT_SLOT_CONFIG_PATH = _llm_registry.DEFAULT_SLOT_CONFIG_PATH +DEFAULT_MODEL_REGISTRY_CONFIG_PATH = _llm_registry.DEFAULT_MODEL_REGISTRY_CONFIG_PATH def _env_int(name: str, default: int) -> int: @@ -59,24 +71,8 @@ def provider_label(self) -> str: return f"{self.provider}/{self.model}" -@dataclass(frozen=True) -class SlotDefinition: - name: str - provider: str - model: str - - def _normalize_provider(value: str | None) -> str | None: - if not value: - return None - normalized = value.strip().lower() - if normalized in {"github", "github_models", "github-models"}: - return PROVIDER_GITHUB - if normalized in {"anthropic", "claude"}: - return PROVIDER_ANTHROPIC - if normalized in {"openai"}: - return PROVIDER_OPENAI - return None + return normalize_provider(value) def _resolve_provider(provider: str | None, *, force_openai: bool) -> tuple[str | None, bool]: @@ -93,57 +89,53 @@ def _resolve_model(model: str | None) -> str: return model or env_model or DEFAULT_MODEL +def _load_model_registry() -> list[ModelRegistryEntry]: + return load_model_registry() + + +def _registry_entry_for( + provider: str, model: str, registry: list[ModelRegistryEntry] | None = None +) -> ModelRegistryEntry | None: + return registry_entry_for(provider, model, registry=registry) + + +def _is_model_blocked( + provider: str, model: str, registry: list[ModelRegistryEntry] | None = None +) -> bool: + return is_model_blocked(provider, model, registry=registry) + + +def _select_model_for_tier( + *, + provider: str, + tier: str, + registry: list[ModelRegistryEntry] | None = None, +) -> str | None: + return select_model_for_tier(provider=provider, tier=tier, registry=registry) + + def _default_slots() -> list[SlotDefinition]: - return [ - SlotDefinition(name="slot1", provider=PROVIDER_OPENAI, model="gpt-5.4"), - SlotDefinition(name="slot2", provider=PROVIDER_ANTHROPIC, model="claude-sonnet-4-6"), - SlotDefinition(name="slot3", provider=PROVIDER_GITHUB, model=DEFAULT_MODEL), - ] + return default_slots(github_default_model=DEFAULT_MODEL) def _load_slot_config() -> list[SlotDefinition]: - config_path = os.environ.get(ENV_SLOT_CONFIG) - path = Path(config_path) if config_path else DEFAULT_SLOT_CONFIG_PATH - if not path.is_file(): - return _default_slots() - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError): - return _default_slots() - - slots: list[SlotDefinition] = [] - for idx, entry in enumerate(payload.get("slots", []), start=1): - provider = _normalize_provider(str(entry.get("provider", ""))) - model = str(entry.get("model", "")).strip() - if not provider or not model: - continue - name = str(entry.get("name") or f"slot{idx}").strip() or f"slot{idx}" - slots.append(SlotDefinition(name=name, provider=provider, model=model)) - - return slots or _default_slots() + return load_slot_config(github_default_model=DEFAULT_MODEL) def _apply_slot_env_overrides(slots: list[SlotDefinition]) -> list[SlotDefinition]: - updated: list[SlotDefinition] = [] - for idx, slot in enumerate(slots, start=1): - provider_key = f"{ENV_SLOT_PREFIX}{idx}_PROVIDER" - model_key = f"{ENV_SLOT_PREFIX}{idx}_MODEL" - provider_override = _normalize_provider(os.environ.get(provider_key)) - model_override = os.environ.get(model_key) - if idx == 1: - model_override = model_override or os.environ.get(ENV_MODEL) - updated.append( - SlotDefinition( - name=slot.name, - provider=provider_override or slot.provider, - model=(model_override or slot.model).strip(), - ) - ) - return updated + return apply_slot_env_overrides( + slots, + env_model_name=ENV_MODEL, + env_slot_prefix=ENV_SLOT_PREFIX, + ) def _resolve_slots() -> list[SlotDefinition]: - return _apply_slot_env_overrides(_load_slot_config()) + return resolve_slots( + github_default_model=DEFAULT_MODEL, + env_model_name=ENV_MODEL, + env_slot_prefix=ENV_SLOT_PREFIX, + ) def _is_reasoning_model(model: str) -> bool: @@ -234,6 +226,9 @@ def build_chat_client( selected_provider, provider_explicit = _resolve_provider(provider, force_openai=force_openai) if provider_explicit and selected_provider is None: return None + if selected_provider and _is_model_blocked(selected_provider, selected_model): + logger.warning("Refusing blocked LLM model: %s/%s", selected_provider, selected_model) + return None if selected_provider == PROVIDER_GITHUB: if not github_token: @@ -283,6 +278,13 @@ def build_chat_client( # Auto-select: slot order (OpenAI -> Claude -> GitHub Models by default). slots = _resolve_slots() model_override = model or os.environ.get(ENV_MODEL) + if model_override: + override_provider = selected_provider or (slots[0].provider if slots else "") + if override_provider and _is_model_blocked(override_provider, model_override): + logger.warning( + "Refusing blocked LLM model override: %s/%s", override_provider, model_override + ) + return None used_override = False for slot in slots: slot_model = model_override if model_override and not used_override else slot.model @@ -358,6 +360,15 @@ def build_chat_clients( selected_provider, provider_explicit = _resolve_provider(provider, force_openai=False) if provider_explicit and selected_provider is None: return [] + registry = _load_model_registry() + if selected_provider: + blocked_models = [candidate for candidate in (first_model, second_model) if candidate] + if any( + _is_model_blocked(selected_provider, candidate, registry=registry) + for candidate in blocked_models + ): + logger.warning("Refusing blocked LLM model for provider %s", selected_provider) + return [] clients: list[ClientInfo] = [] @@ -475,6 +486,9 @@ def build_chat_clients( for idx, slot in enumerate(candidate_slots): slot_model = model_overrides[idx] if idx < len(model_overrides) else None slot_model = slot_model or slot.model + if _is_model_blocked(slot.provider, slot_model, registry=registry): + logger.warning("Skipping blocked LLM model override: %s/%s", slot.provider, slot_model) + continue if slot.provider == PROVIDER_OPENAI and openai_token: with contextlib.suppress(Exception): clients.append( diff --git a/tools/llm_registry.py b/tools/llm_registry.py new file mode 100644 index 000000000..05226b09e --- /dev/null +++ b/tools/llm_registry.py @@ -0,0 +1,276 @@ +"""Shared LLM slot and model-registry resolution helpers.""" + +from __future__ import annotations + +import json +import logging +import os +from dataclasses import dataclass +from pathlib import Path + +logger = logging.getLogger(__name__) + +ENV_MODEL_REGISTRY_CONFIG = "LANGCHAIN_MODEL_REGISTRY_CONFIG" +ENV_SLOT_CONFIG = "LANGCHAIN_SLOT_CONFIG" + +PROVIDER_OPENAI = "openai" +PROVIDER_ANTHROPIC = "anthropic" +PROVIDER_GITHUB = "github-models" + +DEFAULT_SLOT_CONFIG_PATH = Path(__file__).resolve().parent.parent / "config" / "llm_slots.json" +DEFAULT_MODEL_REGISTRY_CONFIG_PATH = ( + Path(__file__).resolve().parent.parent / "config" / "model_registry.json" +) + + +@dataclass(frozen=True) +class ModelRegistryEntry: + provider: str + model: str + blocked: bool + quality: dict[str, float] + + +@dataclass(frozen=True) +class SlotDefinition: + name: str + provider: str + model: str + + +def normalize_provider(value: str | None) -> str | None: + if not value: + return None + normalized = value.strip().lower() + if normalized in {"github", "github_models", "github-models"}: + return PROVIDER_GITHUB + if normalized in {"anthropic", "claude"}: + return PROVIDER_ANTHROPIC + if normalized == PROVIDER_OPENAI: + return PROVIDER_OPENAI + return None + + +def _slot_entries(payload: dict[str, object], path: Path) -> list[dict[str, object]]: + raw_slots = payload.get("slots", []) + if not isinstance(raw_slots, list): + logger.warning("Invalid slot config format in %s; expected slots list", path) + return [] + slots: list[dict[str, object]] = [] + for raw_slot in raw_slots: + if isinstance(raw_slot, dict): + slots.append(raw_slot) + else: + logger.warning("Ignoring invalid slot entry in %s; expected object", path) + return slots + + +def load_model_registry() -> list[ModelRegistryEntry]: + config_path = os.environ.get(ENV_MODEL_REGISTRY_CONFIG) + path = Path(config_path) if config_path else DEFAULT_MODEL_REGISTRY_CONFIG_PATH + if not path.is_file(): + return [] + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + logger.warning("Could not read model registry %s; continuing without registry", path) + return [] + if not isinstance(payload, dict): + logger.warning("Invalid model registry format in %s; expected object", path) + return [] + + entries: list[ModelRegistryEntry] = [] + for raw_entry in payload.get("models", []): + if not isinstance(raw_entry, dict): + logger.warning("Ignoring invalid model registry entry in %s; expected object", path) + continue + provider = normalize_provider(str(raw_entry.get("provider", ""))) + model = str(raw_entry.get("model_id", "")).strip() + if not provider or not model: + continue + quality_payload = raw_entry.get("quality", {}) + quality = { + str(tier).upper(): float(score) + for tier, score in quality_payload.items() + if isinstance(score, int | float) + } + entries.append( + ModelRegistryEntry( + provider=provider, + model=model, + blocked=bool(raw_entry.get("blocked", False)), + quality=quality, + ) + ) + return entries + + +def registry_entry_for( + provider: str, model: str, registry: list[ModelRegistryEntry] | None = None +) -> ModelRegistryEntry | None: + entries = registry if registry is not None else load_model_registry() + normalized_provider = normalize_provider(provider) + normalized_model = model.strip() + for entry in entries: + if entry.provider == normalized_provider and entry.model == normalized_model: + return entry + return None + + +def is_model_blocked( + provider: str, model: str, registry: list[ModelRegistryEntry] | None = None +) -> bool: + entry = registry_entry_for(provider, model, registry=registry) + return bool(entry and entry.blocked) + + +def select_model_for_tier( + *, + provider: str, + tier: str, + registry: list[ModelRegistryEntry] | None = None, +) -> str | None: + entries = registry if registry is not None else load_model_registry() + normalized_provider = normalize_provider(provider) + normalized_tier = tier.strip().upper() + candidates = [ + entry + for entry in entries + if entry.provider == normalized_provider + and not entry.blocked + and normalized_tier in entry.quality + ] + if not candidates: + return None + selected = max(candidates, key=lambda entry: entry.quality[normalized_tier]) + return selected.model + + +def configured_model_for_provider( + provider: str, + *, + fallback: str, + tier: str = "T3", + registry: list[ModelRegistryEntry] | None = None, +) -> str: + normalized_provider = normalize_provider(provider) + entries = registry if registry is not None else load_model_registry() + + config_path = os.environ.get(ENV_SLOT_CONFIG) + path = Path(config_path) if config_path else DEFAULT_SLOT_CONFIG_PATH + if path.is_file(): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + payload = {} + if not isinstance(payload, dict): + logger.warning("Invalid slot config format in %s; expected object", path) + payload = {} + for slot in _slot_entries(payload, path): + slot_provider = normalize_provider(str(slot.get("provider", ""))) + if slot_provider != normalized_provider: + continue + model = str(slot.get("model", "")).strip() + slot_tier = str(slot.get("quality_tier") or slot.get("tier") or tier).strip() + if not model and slot_tier: + model = ( + select_model_for_tier( + provider=slot_provider or "", + tier=slot_tier, + registry=entries, + ) + or "" + ) + if model and not is_model_blocked(slot_provider or "", model, registry=entries): + return model + + selected = select_model_for_tier(provider=provider, tier=tier, registry=entries) + if selected: + return selected + if not is_model_blocked(provider, fallback, registry=entries): + return fallback + return "" + + +def default_slots(*, github_default_model: str) -> list[SlotDefinition]: + return [ + SlotDefinition(name="slot1", provider=PROVIDER_OPENAI, model="gpt-5.4"), + SlotDefinition(name="slot2", provider=PROVIDER_ANTHROPIC, model="claude-sonnet-4-6"), + SlotDefinition(name="slot3", provider=PROVIDER_GITHUB, model=github_default_model), + ] + + +def load_slot_config(*, github_default_model: str) -> list[SlotDefinition]: + config_path = os.environ.get(ENV_SLOT_CONFIG) + path = Path(config_path) if config_path else DEFAULT_SLOT_CONFIG_PATH + fallback_slots = default_slots(github_default_model=github_default_model) + if not path.is_file(): + return fallback_slots + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return fallback_slots + if not isinstance(payload, dict): + logger.warning("Invalid slot config format in %s; expected object", path) + return fallback_slots + + registry = load_model_registry() + slots: list[SlotDefinition] = [] + for idx, entry in enumerate(_slot_entries(payload, path), start=1): + provider = normalize_provider(str(entry.get("provider", ""))) + model = str(entry.get("model", "")).strip() + tier = str(entry.get("quality_tier") or entry.get("tier") or "").strip() + if provider and not model and tier: + model = select_model_for_tier(provider=provider, tier=tier, registry=registry) or "" + if not provider or not model: + continue + if is_model_blocked(provider, model, registry=registry): + logger.warning("Skipping blocked LLM model in slot config: %s/%s", provider, model) + continue + name = str(entry.get("name") or f"slot{idx}").strip() or f"slot{idx}" + slots.append(SlotDefinition(name=name, provider=provider, model=model)) + + return slots or fallback_slots + + +def apply_slot_env_overrides( + slots: list[SlotDefinition], + *, + env_model_name: str = "LANGCHAIN_MODEL", + env_slot_prefix: str = "LANGCHAIN_SLOT", +) -> list[SlotDefinition]: + registry = load_model_registry() + updated: list[SlotDefinition] = [] + for idx, slot in enumerate(slots, start=1): + provider_key = f"{env_slot_prefix}{idx}_PROVIDER" + model_key = f"{env_slot_prefix}{idx}_MODEL" + provider_override = normalize_provider(os.environ.get(provider_key)) + model_override = os.environ.get(model_key) + if idx == 1: + model_override = model_override or os.environ.get(env_model_name) + provider = provider_override or slot.provider + model = (model_override or slot.model).strip() + if is_model_blocked(provider, model, registry=registry): + logger.warning("Skipping blocked LLM slot override: %s/%s", provider, model) + continue + updated.append( + SlotDefinition( + name=slot.name, + provider=provider, + model=model, + ) + ) + return updated + + +def resolve_slots( + *, + github_default_model: str, + env_model_name: str = "LANGCHAIN_MODEL", + env_slot_prefix: str = "LANGCHAIN_SLOT", +) -> list[SlotDefinition]: + return apply_slot_env_overrides( + load_slot_config(github_default_model=github_default_model), + env_model_name=env_model_name, + env_slot_prefix=env_slot_prefix, + )