From b3f30274b743f8f75d901c4042e4a103bf51031a Mon Sep 17 00:00:00 2001 From: Tim Stranske Date: Sat, 20 Jun 2026 18:27:02 -0500 Subject: [PATCH 1/3] Wire provider models through LLM registry config --- tests/tools/test_langchain_client_config.py | 66 +++++++- tools/llm_provider.py | 31 +++- tools/llm_registry.py | 159 ++++++++++++++++++++ 3 files changed, 249 insertions(+), 7 deletions(-) create mode 100644 tools/llm_registry.py diff --git a/tests/tools/test_langchain_client_config.py b/tests/tools/test_langchain_client_config.py index 0af075eb4..17d3a4403 100644 --- a/tests/tools/test_langchain_client_config.py +++ b/tests/tools/test_langchain_client_config.py @@ -6,7 +6,7 @@ import pytest -from tools import langchain_client +from tools import langchain_client, llm_provider class FakeChatOpenAI: @@ -140,3 +140,67 @@ def test_build_chat_client_refuses_explicit_blocked_model(tmp_path, monkeypatch) assert client is None assert FakeChatOpenAI.calls == [] + + +def test_llm_provider_uses_configured_slot_model(tmp_path, monkeypatch) -> None: + registry_path = _write_json( + tmp_path / "model_registry.json", + { + "models": [ + { + "model_id": "gpt-configured", + "provider": "openai", + "quality": {"T3": 0.91}, + } + ] + }, + ) + slots_path = _write_json( + tmp_path / "llm_slots.json", + {"slots": [{"name": "primary", "provider": "openai", "model": "gpt-configured"}]}, + ) + fake_openai_module = types.SimpleNamespace(ChatOpenAI=FakeChatOpenAI) + monkeypatch.setitem(sys.modules, "langchain_openai", fake_openai_module) + monkeypatch.setenv(langchain_client.ENV_MODEL_REGISTRY_CONFIG, registry_path) + monkeypatch.setenv(langchain_client.ENV_SLOT_CONFIG, slots_path) + monkeypatch.setenv("OPENAI_API_KEY", "test-token") + + client = llm_provider.OpenAIProvider()._get_client() + + assert client is not None + assert FakeChatOpenAI.calls[-1]["model"] == "gpt-configured" + + +def test_llm_provider_skips_blocked_slot_model(tmp_path, monkeypatch) -> None: + registry_path = _write_json( + tmp_path / "model_registry.json", + { + "models": [ + { + "model_id": "gpt-blocked", + "provider": "openai", + "blocked": True, + "quality": {"T3": 0.99}, + }, + { + "model_id": "gpt-safe", + "provider": "openai", + "quality": {"T3": 0.80}, + }, + ] + }, + ) + slots_path = _write_json( + tmp_path / "llm_slots.json", + {"slots": [{"name": "primary", "provider": "openai", "model": "gpt-blocked"}]}, + ) + fake_openai_module = types.SimpleNamespace(ChatOpenAI=FakeChatOpenAI) + monkeypatch.setitem(sys.modules, "langchain_openai", fake_openai_module) + monkeypatch.setenv(langchain_client.ENV_MODEL_REGISTRY_CONFIG, registry_path) + monkeypatch.setenv(langchain_client.ENV_SLOT_CONFIG, slots_path) + monkeypatch.setenv("OPENAI_API_KEY", "test-token") + + client = llm_provider.OpenAIProvider()._get_client() + + assert client is not None + assert FakeChatOpenAI.calls[-1]["model"] == "gpt-safe" diff --git a/tools/llm_provider.py b/tools/llm_provider.py index 7fb0603a2..ac3e10b72 100644 --- a/tools/llm_provider.py +++ b/tools/llm_provider.py @@ -29,6 +29,13 @@ from abc import ABC, abstractmethod from dataclasses import dataclass +from tools.llm_registry import ( + PROVIDER_ANTHROPIC, + PROVIDER_GITHUB, + PROVIDER_OPENAI, + configured_model_for_provider, +) + logger = logging.getLogger(__name__) # GitHub Models API endpoint (OpenAI-compatible) @@ -337,8 +344,11 @@ def _get_client(self): logger.warning("langchain_openai not installed") return None + model = configured_model_for_provider(PROVIDER_GITHUB, fallback="gpt-4.1") + if not model: + return None return ChatOpenAI( - model="gpt-4.1", # Battle-tested, reliable, available on GitHub Models + model=model, base_url=GITHUB_MODELS_BASE_URL, api_key=os.environ.get("GITHUB_TOKEN"), temperature=0.1, # Low temperature for consistent analysis @@ -550,7 +560,7 @@ def _parse_response( confidence=adjusted_confidence, reasoning=reasoning, provider_used=self.name, - model_name="gpt-4.1", # Actual model used by GitHubModelsProvider + model_name=configured_model_for_provider(PROVIDER_GITHUB, fallback="gpt-4.1"), raw_confidence=raw_confidence if adjusted_confidence != raw_confidence else None, confidence_adjusted=adjusted_confidence != raw_confidence, quality_warnings=warnings if warnings else None, @@ -565,7 +575,7 @@ def _parse_response( confidence=0.0, reasoning=f"Failed to parse response: {e}", provider_used=self.name, - model_name="gpt-4.1", # Actual model used by GitHubModelsProvider + model_name=configured_model_for_provider(PROVIDER_GITHUB, fallback="gpt-4.1"), ) @@ -590,8 +600,11 @@ def _get_client(self): logger.warning("langchain_openai not installed") return None + model = configured_model_for_provider(PROVIDER_OPENAI, fallback="gpt-5.1-codex") + if not model: + return None return ChatOpenAI( - model="gpt-5.1-codex", # Purpose-built for analyzing Codex coding sessions + model=model, api_key=os.environ.get("OPENAI_API_KEY"), temperature=0.1, ) @@ -626,7 +639,7 @@ def analyze_completion( confidence=result.confidence, reasoning=result.reasoning, provider_used=self.name, - model_name="gpt-5.1-codex", # Actual model used by OpenAIProvider + model_name=configured_model_for_provider(PROVIDER_OPENAI, fallback="gpt-5.1-codex"), raw_confidence=result.raw_confidence, confidence_adjusted=result.confidence_adjusted, quality_warnings=result.quality_warnings, @@ -656,8 +669,14 @@ def _get_client(self): logger.warning("langchain_anthropic not installed") return None + model = configured_model_for_provider( + PROVIDER_ANTHROPIC, + fallback="claude-sonnet-4-5-20250929", + ) + if not model: + return None return ChatAnthropic( - model="claude-sonnet-4-5-20250929", + model=model, anthropic_api_key=os.environ.get(ANTHROPIC_API_KEY_ENV), temperature=0.1, ) diff --git a/tools/llm_registry.py b/tools/llm_registry.py new file mode 100644 index 000000000..2eb94011d --- /dev/null +++ b/tools/llm_registry.py @@ -0,0 +1,159 @@ +"""Shared LLM slot and model-registry resolution helpers.""" + +from __future__ import annotations + +import json +import logging +import os +from dataclasses import dataclass +from pathlib import Path + +logger = logging.getLogger(__name__) + +ENV_MODEL_REGISTRY_CONFIG = "LANGCHAIN_MODEL_REGISTRY_CONFIG" +ENV_SLOT_CONFIG = "LANGCHAIN_SLOT_CONFIG" + +PROVIDER_OPENAI = "openai" +PROVIDER_ANTHROPIC = "anthropic" +PROVIDER_GITHUB = "github-models" + +DEFAULT_SLOT_CONFIG_PATH = Path(__file__).resolve().parent.parent / "config" / "llm_slots.json" +DEFAULT_MODEL_REGISTRY_CONFIG_PATH = ( + Path(__file__).resolve().parent.parent / "config" / "model_registry.json" +) + + +@dataclass(frozen=True) +class ModelRegistryEntry: + provider: str + model: str + blocked: bool + quality: dict[str, float] + + +def normalize_provider(value: str | None) -> str | None: + if not value: + return None + normalized = value.strip().lower() + if normalized in {"github", "github_models", "github-models"}: + return PROVIDER_GITHUB + if normalized in {"anthropic", "claude"}: + return PROVIDER_ANTHROPIC + if normalized == PROVIDER_OPENAI: + return PROVIDER_OPENAI + return None + + +def load_model_registry() -> list[ModelRegistryEntry]: + config_path = os.environ.get(ENV_MODEL_REGISTRY_CONFIG) + path = Path(config_path) if config_path else DEFAULT_MODEL_REGISTRY_CONFIG_PATH + if not path.is_file(): + return [] + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + logger.warning("Could not read model registry %s; continuing without registry", path) + return [] + + entries: list[ModelRegistryEntry] = [] + for raw_entry in payload.get("models", []): + provider = normalize_provider(str(raw_entry.get("provider", ""))) + model = str(raw_entry.get("model_id", "")).strip() + if not provider or not model: + continue + quality_payload = raw_entry.get("quality", {}) + quality = { + str(tier).upper(): float(score) + for tier, score in quality_payload.items() + if isinstance(score, int | float) + } + entries.append( + ModelRegistryEntry( + provider=provider, + model=model, + blocked=bool(raw_entry.get("blocked", False)), + quality=quality, + ) + ) + return entries + + +def registry_entry_for( + provider: str, model: str, registry: list[ModelRegistryEntry] | None = None +) -> ModelRegistryEntry | None: + entries = registry if registry is not None else load_model_registry() + normalized_provider = normalize_provider(provider) + normalized_model = model.strip() + for entry in entries: + if entry.provider == normalized_provider and entry.model == normalized_model: + return entry + return None + + +def is_model_blocked( + provider: str, model: str, registry: list[ModelRegistryEntry] | None = None +) -> bool: + entry = registry_entry_for(provider, model, registry=registry) + return bool(entry and entry.blocked) + + +def select_model_for_tier( + *, + provider: str, + tier: str, + registry: list[ModelRegistryEntry] | None = None, +) -> str | None: + entries = registry if registry is not None else load_model_registry() + normalized_provider = normalize_provider(provider) + normalized_tier = tier.strip().upper() + candidates = [ + entry + for entry in entries + if entry.provider == normalized_provider + and not entry.blocked + and normalized_tier in entry.quality + ] + if not candidates: + return None + selected = max(candidates, key=lambda entry: entry.quality[normalized_tier]) + return selected.model + + +def configured_model_for_provider( + provider: str, + *, + fallback: str, + tier: str = "T3", + registry: list[ModelRegistryEntry] | None = None, +) -> str: + normalized_provider = normalize_provider(provider) + entries = registry if registry is not None else load_model_registry() + + config_path = os.environ.get(ENV_SLOT_CONFIG) + path = Path(config_path) if config_path else DEFAULT_SLOT_CONFIG_PATH + if path.is_file(): + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + payload = {} + for slot in payload.get("slots", []): + slot_provider = normalize_provider(str(slot.get("provider", ""))) + if slot_provider != normalized_provider: + continue + model = str(slot.get("model", "")).strip() + slot_tier = str(slot.get("quality_tier") or slot.get("tier") or tier).strip() + if not model and slot_tier: + model = select_model_for_tier( + provider=slot_provider or "", + tier=slot_tier, + registry=entries, + ) or "" + if model and not is_model_blocked(slot_provider or "", model, registry=entries): + return model + + selected = select_model_for_tier(provider=provider, tier=tier, registry=entries) + if selected: + return selected + if not is_model_blocked(provider, fallback, registry=entries): + return fallback + return "" From c9c5def136f78fd3d04e9a65de2346be8f45152e Mon Sep 17 00:00:00 2001 From: Tim Stranske Date: Sat, 20 Jun 2026 19:25:01 -0500 Subject: [PATCH 2/3] Report configured Anthropic model in completion analysis --- tests/tools/test_langchain_client_config.py | 61 +++++++++++++++++++++ tools/llm_provider.py | 5 +- 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/tests/tools/test_langchain_client_config.py b/tests/tools/test_langchain_client_config.py index 17d3a4403..ecc7e309b 100644 --- a/tests/tools/test_langchain_client_config.py +++ b/tests/tools/test_langchain_client_config.py @@ -204,3 +204,64 @@ def test_llm_provider_skips_blocked_slot_model(tmp_path, monkeypatch) -> None: assert client is not None assert FakeChatOpenAI.calls[-1]["model"] == "gpt-safe" + + +def test_anthropic_completion_reports_configured_model(tmp_path, monkeypatch) -> None: + registry_path = _write_json( + tmp_path / "model_registry.json", + { + "models": [ + { + "model_id": "claude-configured", + "provider": "anthropic", + "quality": {"T3": 0.91}, + } + ] + }, + ) + slots_path = _write_json( + tmp_path / "llm_slots.json", + { + "slots": [ + { + "name": "primary", + "provider": "anthropic", + "model": "claude-configured", + } + ] + }, + ) + + class FakeAnthropicClient: + def invoke(self, prompt, **kwargs): + return types.SimpleNamespace(content="ok") + + monkeypatch.setenv(langchain_client.ENV_MODEL_REGISTRY_CONFIG, registry_path) + monkeypatch.setenv(langchain_client.ENV_SLOT_CONFIG, slots_path) + monkeypatch.setattr( + llm_provider.AnthropicProvider, + "_get_client", + lambda self: FakeAnthropicClient(), + ) + monkeypatch.setattr( + llm_provider.GitHubModelsProvider, + "_build_analysis_prompt", + lambda self, session_output, tasks, context: "prompt", + ) + monkeypatch.setattr( + llm_provider.GitHubModelsProvider, + "_parse_response", + lambda self, content, tasks, quality_context=None: llm_provider.CompletionAnalysis( + completed_tasks=[], + in_progress_tasks=[], + blocked_tasks=[], + confidence=0.9, + reasoning="ok", + provider_used="github-models", + model_name="stale-hardcoded", + ), + ) + + result = llm_provider.AnthropicProvider().analyze_completion("output", ["task"]) + + assert result.model_name == "claude-configured" diff --git a/tools/llm_provider.py b/tools/llm_provider.py index ac3e10b72..4b1a0cad3 100644 --- a/tools/llm_provider.py +++ b/tools/llm_provider.py @@ -721,7 +721,10 @@ def analyze_completion( confidence=result.confidence, reasoning=result.reasoning, provider_used=self.name, - model_name="claude-sonnet-4-5-20250929", + model_name=configured_model_for_provider( + PROVIDER_ANTHROPIC, + fallback="claude-sonnet-4-5-20250929", + ), raw_confidence=result.raw_confidence, confidence_adjusted=result.confidence_adjusted, quality_warnings=result.quality_warnings, From 5aaa810b1b9ffb6813a4463e939c320fa75acb59 Mon Sep 17 00:00:00 2001 From: Tim Stranske Date: Sat, 20 Jun 2026 20:27:21 -0500 Subject: [PATCH 3/3] Handle malformed LLM config payloads --- tests/tools/test_langchain_client_config.py | 46 +++++++++++++++++++++ tools/langchain_client.py | 6 +++ tools/llm_registry.py | 6 +++ 3 files changed, 58 insertions(+) diff --git a/tests/tools/test_langchain_client_config.py b/tests/tools/test_langchain_client_config.py index ecc7e309b..bd1bb9e6f 100644 --- a/tests/tools/test_langchain_client_config.py +++ b/tests/tools/test_langchain_client_config.py @@ -39,6 +39,52 @@ def _write_json(path, payload: dict[str, object]) -> str: return str(path) +def test_slot_config_ignores_non_object_payload(tmp_path, monkeypatch) -> None: + registry_path = _write_json( + tmp_path / "model_registry.json", + { + "models": [ + { + "model_id": "gpt-safe", + "provider": "openai", + "quality": {"T3": 0.80}, + } + ] + }, + ) + slots_path = tmp_path / "llm_slots.json" + slots_path.write_text(json.dumps(["not", "an", "object"]), encoding="utf-8") + monkeypatch.setenv(langchain_client.ENV_MODEL_REGISTRY_CONFIG, registry_path) + monkeypatch.setenv(langchain_client.ENV_SLOT_CONFIG, str(slots_path)) + + slots = langchain_client._resolve_slots() + + assert [(slot.name, slot.provider, slot.model) for slot in slots] == [ + ("slot1", langchain_client.PROVIDER_OPENAI, "gpt-5.4"), + ("slot2", langchain_client.PROVIDER_ANTHROPIC, "claude-sonnet-4-6"), + ("slot3", langchain_client.PROVIDER_GITHUB, langchain_client.DEFAULT_MODEL), + ] + + +def test_model_registry_ignores_non_object_payload(tmp_path, monkeypatch) -> None: + registry_path = tmp_path / "model_registry.json" + registry_path.write_text(json.dumps(["not", "an", "object"]), encoding="utf-8") + slots_path = _write_json( + tmp_path / "llm_slots.json", + {"slots": [{"name": "primary", "provider": "openai", "quality_tier": "T3"}]}, + ) + monkeypatch.setenv(langchain_client.ENV_MODEL_REGISTRY_CONFIG, str(registry_path)) + monkeypatch.setenv(langchain_client.ENV_SLOT_CONFIG, slots_path) + + slots = langchain_client._resolve_slots() + + assert [(slot.name, slot.provider, slot.model) for slot in slots] == [ + ("slot1", langchain_client.PROVIDER_OPENAI, "gpt-5.4"), + ("slot2", langchain_client.PROVIDER_ANTHROPIC, "claude-sonnet-4-6"), + ("slot3", langchain_client.PROVIDER_GITHUB, langchain_client.DEFAULT_MODEL), + ] + + def test_slot_config_skips_model_registry_blocked_models(tmp_path, monkeypatch) -> None: registry_path = _write_json( tmp_path / "model_registry.json", diff --git a/tools/langchain_client.py b/tools/langchain_client.py index 48d6f4c99..96f622b65 100644 --- a/tools/langchain_client.py +++ b/tools/langchain_client.py @@ -115,6 +115,9 @@ def _load_model_registry() -> list[ModelRegistryEntry]: except (OSError, json.JSONDecodeError): logger.warning("Could not read model registry %s; continuing without registry", path) return [] + if not isinstance(payload, dict): + logger.warning("Invalid model registry format in %s; expected object", path) + return [] entries: list[ModelRegistryEntry] = [] for raw_entry in payload.get("models", []): @@ -197,6 +200,9 @@ def _load_slot_config() -> list[SlotDefinition]: payload = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): return _default_slots() + if not isinstance(payload, dict): + logger.warning("Invalid slot config format in %s; expected object", path) + return _default_slots() registry = _load_model_registry() slots: list[SlotDefinition] = [] diff --git a/tools/llm_registry.py b/tools/llm_registry.py index 2eb94011d..6816ca155 100644 --- a/tools/llm_registry.py +++ b/tools/llm_registry.py @@ -54,6 +54,9 @@ def load_model_registry() -> list[ModelRegistryEntry]: except (OSError, json.JSONDecodeError): logger.warning("Could not read model registry %s; continuing without registry", path) return [] + if not isinstance(payload, dict): + logger.warning("Invalid model registry format in %s; expected object", path) + return [] entries: list[ModelRegistryEntry] = [] for raw_entry in payload.get("models", []): @@ -136,6 +139,9 @@ def configured_model_for_provider( payload = json.loads(path.read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): payload = {} + if not isinstance(payload, dict): + logger.warning("Invalid slot config format in %s; expected object", path) + payload = {} for slot in payload.get("slots", []): slot_provider = normalize_provider(str(slot.get("provider", ""))) if slot_provider != normalized_provider: