From 9e3ebfd89e8cdfd55637f3f137d5dafe6dabd19b Mon Sep 17 00:00:00 2001 From: Tim Stranske Date: Wed, 15 Jul 2026 15:30:03 -0500 Subject: [PATCH 1/7] fix: preserve consumer slot compatibility --- scripts/api_client.py | 2 +- templates/consumer-repo/scripts/api_client.py | 2 +- .../consumer-repo/tools/ci_failure_triage.py | 25 +++++++++---------- templates/consumer-repo/tools/llm_registry.py | 15 ++++++----- tests/scripts/test_api_client.py | 2 +- tests/tools/test_llm_registry_selection.py | 17 ++++++------- tools/ci_failure_triage.py | 25 +++++++++---------- tools/evaluate_model_benchmark.py | 2 +- tools/llm_registry.py | 15 ++++++----- 9 files changed, 54 insertions(+), 51 deletions(-) diff --git a/scripts/api_client.py b/scripts/api_client.py index fbc03729c4..3a4eebd5ff 100644 --- a/scripts/api_client.py +++ b/scripts/api_client.py @@ -199,7 +199,7 @@ def fetch_pull_request_diff( url, token, payload=None, - accept="application/vnd.github.diff", + accept="application/vnd.github.v3.diff", **_retry_kwargs(retry_attempts, retry_backoff), ) return response.text diff --git a/templates/consumer-repo/scripts/api_client.py b/templates/consumer-repo/scripts/api_client.py index fbc03729c4..3a4eebd5ff 100644 --- a/templates/consumer-repo/scripts/api_client.py +++ b/templates/consumer-repo/scripts/api_client.py @@ -199,7 +199,7 @@ def fetch_pull_request_diff( url, token, payload=None, - accept="application/vnd.github.diff", + accept="application/vnd.github.v3.diff", **_retry_kwargs(retry_attempts, retry_backoff), ) return response.text diff --git a/templates/consumer-repo/tools/ci_failure_triage.py b/templates/consumer-repo/tools/ci_failure_triage.py index 1498917830..2e9729807c 100644 --- a/templates/consumer-repo/tools/ci_failure_triage.py +++ b/templates/consumer-repo/tools/ci_failure_triage.py @@ -297,20 +297,19 @@ def _get_llm_client() -> tuple[_LLMClient, str] | None: if github_token: model = configured_model_for_provider("github-models") - if not model: - return None - return ( - cast( - _LLMClient, - ChatOpenAI( - model=model, - base_url=GITHUB_MODELS_BASE_URL, - api_key=cast(Any, github_token), - temperature=0.1, + if model: + return ( + cast( + _LLMClient, + ChatOpenAI( + model=model, + base_url=GITHUB_MODELS_BASE_URL, + api_key=cast(Any, github_token), + temperature=0.1, + ), ), - ), - "github-models", - ) + "github-models", + ) if openai_token is None: return None model = configured_model_for_provider("openai") diff --git a/templates/consumer-repo/tools/llm_registry.py b/templates/consumer-repo/tools/llm_registry.py index b3c253be3a..79616f7bce 100644 --- a/templates/consumer-repo/tools/llm_registry.py +++ b/templates/consumer-repo/tools/llm_registry.py @@ -332,7 +332,12 @@ def load_slot_config(*, github_default_model: str = "") -> list[SlotDefinition]: select_model_for_profile(provider=provider, profile=profile, registry=registry) or "" ) - if provider and explicit_model and explicit_model != model: + explicit_entry = registry_entry_for(provider, explicit_model, registry=registry) + if provider and explicit_model and not configured_profile and explicit_entry and not ( + explicit_entry.blocked or explicit_entry.lifecycle != "current" + ): + model = explicit_model + elif provider and explicit_model and explicit_model != model: logger.warning( "Ignoring slot model pin %s/%s; reviewed %s selection is %s", provider, @@ -357,11 +362,9 @@ def load_slot_config(*, github_default_model: str = "") -> list[SlotDefinition]: continue name = str(entry.get("name") or f"slot{idx}").strip() or f"slot{idx}" slots.append(SlotDefinition(name=name, provider=provider, model=model)) - # A syntactically valid config can still yield no usable entries (for - # example, every profile was retired or every model was blocked). That is - # equivalent to no usable slot configuration; retain the reviewed defaults - # instead of silently disabling every provider. - return slots or fallback_slots + # A present slot file is an allowlist. If every configured slot is + # unusable, fail closed instead of broadening execution to default providers. + return slots if slot_entries else fallback_slots def apply_slot_env_overrides( diff --git a/tests/scripts/test_api_client.py b/tests/scripts/test_api_client.py index b58100e3fc..3105f39f31 100644 --- a/tests/scripts/test_api_client.py +++ b/tests/scripts/test_api_client.py @@ -215,7 +215,7 @@ def fake_response(*args: object, **kwargs: object) -> object: assert api_client.fetch_pull_request("owner/repo", 7, "tok") == {"title": "PR"} assert api_client.fetch_pull_request_diff("owner/repo", 7, "tok") == response.text - assert calls == [{"payload": None, "accept": "application/vnd.github.diff"}] + assert calls == [{"payload": None, "accept": "application/vnd.github.v3.diff"}] def test_fetch_pull_request_rejects_nonobject_payload(monkeypatch: pytest.MonkeyPatch) -> None: diff --git a/tests/tools/test_llm_registry_selection.py b/tests/tools/test_llm_registry_selection.py index 66c177b5fd..05cbb98d44 100644 --- a/tests/tools/test_llm_registry_selection.py +++ b/tests/tools/test_llm_registry_selection.py @@ -102,18 +102,18 @@ def test_registry_decision_update_changes_slot_without_slot_edit( assert "model" not in json.loads(slots_path.read_text())["slots"][0] -def test_legacy_slot_pin_cannot_override_reviewed_selection( +def test_legacy_slot_pin_is_honored_when_current_and_unblocked( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: registry_path = tmp_path / "registry.json" slots_path = tmp_path / "slots.json" _write_registry(registry_path) - _write_slots(slots_path, model="model-frontier") + _write_slots(slots_path, model="model-frontier", profile="") monkeypatch.setenv(registry.ENV_MODEL_REGISTRY_CONFIG, str(registry_path)) monkeypatch.setenv(registry.ENV_SLOT_CONFIG, str(slots_path)) - assert registry.load_slot_config()[0].model == "model-balanced" - assert registry.configured_model_for_provider("openai") == "model-balanced" + assert registry.load_slot_config()[0].model == "model-frontier" + assert registry.configured_model_for_provider("openai") == "model-frontier" def test_unknown_explicit_profile_fails_closed( @@ -126,13 +126,12 @@ def test_unknown_explicit_profile_fails_closed( monkeypatch.setenv(registry.ENV_MODEL_REGISTRY_CONFIG, str(registry_path)) monkeypatch.setenv(registry.ENV_SLOT_CONFIG, str(slots_path)) - # An unresolved slot profile must not disable the provider-level reviewed - # selection used as the safe fallback. - assert registry.load_slot_config()[0].model == "model-balanced" + # A present slot config is an allowlist; an unresolved profile fails closed. + assert registry.load_slot_config() == [] assert registry.configured_model_for_provider("openai") == "model-balanced" -def test_all_unusable_slot_entries_fall_back_to_reviewed_defaults( +def test_all_unusable_slot_entries_fail_closed( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: registry_path = tmp_path / "registry.json" @@ -142,7 +141,7 @@ def test_all_unusable_slot_entries_fall_back_to_reviewed_defaults( monkeypatch.setenv(registry.ENV_MODEL_REGISTRY_CONFIG, str(registry_path)) monkeypatch.setenv(registry.ENV_SLOT_CONFIG, str(slots_path)) - assert registry.load_slot_config()[0].model == "model-balanced" + assert registry.load_slot_config() == [] def test_noncurrent_selected_model_fails_closed( diff --git a/tools/ci_failure_triage.py b/tools/ci_failure_triage.py index 1498917830..2e9729807c 100644 --- a/tools/ci_failure_triage.py +++ b/tools/ci_failure_triage.py @@ -297,20 +297,19 @@ def _get_llm_client() -> tuple[_LLMClient, str] | None: if github_token: model = configured_model_for_provider("github-models") - if not model: - return None - return ( - cast( - _LLMClient, - ChatOpenAI( - model=model, - base_url=GITHUB_MODELS_BASE_URL, - api_key=cast(Any, github_token), - temperature=0.1, + if model: + return ( + cast( + _LLMClient, + ChatOpenAI( + model=model, + base_url=GITHUB_MODELS_BASE_URL, + api_key=cast(Any, github_token), + temperature=0.1, + ), ), - ), - "github-models", - ) + "github-models", + ) if openai_token is None: return None model = configured_model_for_provider("openai") diff --git a/tools/evaluate_model_benchmark.py b/tools/evaluate_model_benchmark.py index 41eab9981b..a9a081eb75 100644 --- a/tools/evaluate_model_benchmark.py +++ b/tools/evaluate_model_benchmark.py @@ -153,7 +153,7 @@ def evaluate_benchmark(payload: dict[str, Any], policy: dict[str, Any]) -> dict[ noninferiority_margin = float(gates["paired_success_noninferiority_margin"]) for candidate in candidates: - model_id = str(candidate["model_id"]) + model_id = str(candidate["model_id"]).strip() metrics = metrics_by_model[model_id] gate_results = { "minimum_adjudicated_cases": metrics["sample_count"] >= minimum_cases, diff --git a/tools/llm_registry.py b/tools/llm_registry.py index b3c253be3a..79616f7bce 100644 --- a/tools/llm_registry.py +++ b/tools/llm_registry.py @@ -332,7 +332,12 @@ def load_slot_config(*, github_default_model: str = "") -> list[SlotDefinition]: select_model_for_profile(provider=provider, profile=profile, registry=registry) or "" ) - if provider and explicit_model and explicit_model != model: + explicit_entry = registry_entry_for(provider, explicit_model, registry=registry) + if provider and explicit_model and not configured_profile and explicit_entry and not ( + explicit_entry.blocked or explicit_entry.lifecycle != "current" + ): + model = explicit_model + elif provider and explicit_model and explicit_model != model: logger.warning( "Ignoring slot model pin %s/%s; reviewed %s selection is %s", provider, @@ -357,11 +362,9 @@ def load_slot_config(*, github_default_model: str = "") -> list[SlotDefinition]: continue name = str(entry.get("name") or f"slot{idx}").strip() or f"slot{idx}" slots.append(SlotDefinition(name=name, provider=provider, model=model)) - # A syntactically valid config can still yield no usable entries (for - # example, every profile was retired or every model was blocked). That is - # equivalent to no usable slot configuration; retain the reviewed defaults - # instead of silently disabling every provider. - return slots or fallback_slots + # A present slot file is an allowlist. If every configured slot is + # unusable, fail closed instead of broadening execution to default providers. + return slots if slot_entries else fallback_slots def apply_slot_env_overrides( From e6cedcc25c017dabfa58d4628e3b2004a37c6173 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 15 Jul 2026 20:45:47 +0000 Subject: [PATCH 2/7] chore(autofix): formatting/lint --- templates/consumer-repo/tools/llm_registry.py | 8 ++++++-- tools/llm_registry.py | 8 ++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/templates/consumer-repo/tools/llm_registry.py b/templates/consumer-repo/tools/llm_registry.py index 79616f7bce..0a34bb618f 100644 --- a/templates/consumer-repo/tools/llm_registry.py +++ b/templates/consumer-repo/tools/llm_registry.py @@ -333,8 +333,12 @@ def load_slot_config(*, github_default_model: str = "") -> list[SlotDefinition]: or "" ) explicit_entry = registry_entry_for(provider, explicit_model, registry=registry) - if provider and explicit_model and not configured_profile and explicit_entry and not ( - explicit_entry.blocked or explicit_entry.lifecycle != "current" + if ( + provider + and explicit_model + and not configured_profile + and explicit_entry + and not (explicit_entry.blocked or explicit_entry.lifecycle != "current") ): model = explicit_model elif provider and explicit_model and explicit_model != model: diff --git a/tools/llm_registry.py b/tools/llm_registry.py index 79616f7bce..0a34bb618f 100644 --- a/tools/llm_registry.py +++ b/tools/llm_registry.py @@ -333,8 +333,12 @@ def load_slot_config(*, github_default_model: str = "") -> list[SlotDefinition]: or "" ) explicit_entry = registry_entry_for(provider, explicit_model, registry=registry) - if provider and explicit_model and not configured_profile and explicit_entry and not ( - explicit_entry.blocked or explicit_entry.lifecycle != "current" + if ( + provider + and explicit_model + and not configured_profile + and explicit_entry + and not (explicit_entry.blocked or explicit_entry.lifecycle != "current") ): model = explicit_model elif provider and explicit_model and explicit_model != model: From 740a29e0efde7f10e2cb2a6c24b88891c062d16b Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 20:46:46 +0000 Subject: [PATCH 3/7] chore(codex-autofix): apply updates (PR #2781) --- langsmith-fleet-worker-attempt.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/langsmith-fleet-worker-attempt.json b/langsmith-fleet-worker-attempt.json index 0c18aa06c4..797e7b4087 100644 --- a/langsmith-fleet-worker-attempt.json +++ b/langsmith-fleet-worker-attempt.json @@ -1,13 +1,13 @@ { "agent": "codex", "cli_version": "0.125.0", - "emitted_at": "2026-07-15T14:46:51.200865Z", + "emitted_at": "2026-07-15T20:46:44.622969Z", "execution_profile": "codex-default", "fallback_models": [ "gpt-5.4" ], "operation_role": "worker", - "pr_number": "2779", + "pr_number": "2781", "requested_model": "gpt-5.5", "runner": "reusable-codex-run", "schema": "langsmith-fleet/v1", From c0c8b35dbaa1ab2b647b0f20f3cd275eeb38c775 Mon Sep 17 00:00:00 2001 From: Codex Automation Date: Wed, 15 Jul 2026 16:25:11 -0500 Subject: [PATCH 4/7] fix: preserve slot allowlist semantics --- templates/consumer-repo/tools/llm_registry.py | 8 +++++- tests/test_check_model_registry_freshness.py | 20 +++++++++++++++ tests/tools/test_ci_failure_triage.py | 25 ++++++++++++++++++- tests/tools/test_llm_registry_selection.py | 16 +++++++++++- tools/check_model_registry_freshness.py | 20 +++++++++++---- tools/llm_registry.py | 8 +++++- 6 files changed, 88 insertions(+), 9 deletions(-) diff --git a/templates/consumer-repo/tools/llm_registry.py b/templates/consumer-repo/tools/llm_registry.py index 0a34bb618f..c2e854b88e 100644 --- a/templates/consumer-repo/tools/llm_registry.py +++ b/templates/consumer-repo/tools/llm_registry.py @@ -278,6 +278,12 @@ def configured_model_for_provider( ): return slot.model + # A readable slot config is an execution allowlist. Do not broaden to a + # provider-level registry decision when that config has no usable slot for + # this provider (including an empty or invalid-only slots list). + if _load_object(_slot_path(), label="slot config") is not None: + return "" + selected = select_model_for_profile(provider=provider, profile=profile, registry=entries) if selected: return selected @@ -368,7 +374,7 @@ def load_slot_config(*, github_default_model: str = "") -> list[SlotDefinition]: slots.append(SlotDefinition(name=name, provider=provider, model=model)) # A present slot file is an allowlist. If every configured slot is # unusable, fail closed instead of broadening execution to default providers. - return slots if slot_entries else fallback_slots + return slots def apply_slot_env_overrides( diff --git a/tests/test_check_model_registry_freshness.py b/tests/test_check_model_registry_freshness.py index 4e436cd994..680d57bba2 100644 --- a/tests/test_check_model_registry_freshness.py +++ b/tests/test_check_model_registry_freshness.py @@ -164,6 +164,26 @@ def test_explicit_pin_without_profile_is_flagged_against_default_selection(): assert "selection_override" in _kinds(findings) +def test_current_unblocked_legacy_pin_is_compatible_with_default_selection(): + registry = _registry() + registry["models"].append( + { + "model_id": "model-frontier", + "provider": "openai", + "lifecycle": "current", + "source_ids": ["models-1"], + "pricing": {"as_of": "2026-07-10"}, + } + ) + findings = gate.evaluate( + registry, + {"slots": [{"name": "slot1", "provider": "openai", "model": "model-frontier"}]}, + today=TODAY, + policy=_policy(), + ) + assert "selection_override" not in _kinds(findings) + + def test_slot_requires_profile_selection(): findings = gate.evaluate( _registry(), diff --git a/tests/tools/test_ci_failure_triage.py b/tests/tools/test_ci_failure_triage.py index 871b36e53d..85abfcbee9 100644 --- a/tests/tools/test_ci_failure_triage.py +++ b/tests/tools/test_ci_failure_triage.py @@ -1,4 +1,7 @@ -from tools import ci_failure_triage +import sys +import types + +from tools import ci_failure_triage, llm_registry def test_extract_pytest_failures_parses_unique() -> None: @@ -32,3 +35,23 @@ def test_triage_report_includes_failed_tests() -> None: "tests/workflows/test_keepalive_workflow.py::test_keepalive_requires_dispatch_token" ] assert "Pytest failures: 1" in report.summary + + +def test_llm_triage_does_not_fall_back_outside_slot_allowlist(monkeypatch) -> None: + created: list[object] = [] + + class FakeChatOpenAI: + def __init__(self, **kwargs: object) -> None: + created.append(kwargs) + + monkeypatch.setitem( + sys.modules, "langchain_openai", types.SimpleNamespace(ChatOpenAI=FakeChatOpenAI) + ) + monkeypatch.setenv("GITHUB_TOKEN", "github-token") + monkeypatch.setenv("OPENAI_API_KEY", "openai-token") + monkeypatch.setattr( + llm_registry, "configured_model_for_provider", lambda _provider: "" + ) + + assert ci_failure_triage._get_llm_client() is None + assert created == [] diff --git a/tests/tools/test_llm_registry_selection.py b/tests/tools/test_llm_registry_selection.py index 05cbb98d44..27f641f61c 100644 --- a/tests/tools/test_llm_registry_selection.py +++ b/tests/tools/test_llm_registry_selection.py @@ -128,7 +128,7 @@ def test_unknown_explicit_profile_fails_closed( # A present slot config is an allowlist; an unresolved profile fails closed. assert registry.load_slot_config() == [] - assert registry.configured_model_for_provider("openai") == "model-balanced" + assert registry.configured_model_for_provider("openai") == "" def test_all_unusable_slot_entries_fail_closed( @@ -144,6 +144,20 @@ def test_all_unusable_slot_entries_fail_closed( assert registry.load_slot_config() == [] +def test_empty_slot_config_does_not_broaden_to_default_providers( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + registry_path = tmp_path / "registry.json" + slots_path = tmp_path / "slots.json" + _write_registry(registry_path) + slots_path.write_text(json.dumps({"slots": []}), encoding="utf-8") + monkeypatch.setenv(registry.ENV_MODEL_REGISTRY_CONFIG, str(registry_path)) + monkeypatch.setenv(registry.ENV_SLOT_CONFIG, str(slots_path)) + + assert registry.load_slot_config() == [] + assert registry.configured_model_for_provider("openai") == "" + + def test_noncurrent_selected_model_fails_closed( monkeypatch: pytest.MonkeyPatch, tmp_path: Path ) -> None: diff --git a/tools/check_model_registry_freshness.py b/tools/check_model_registry_freshness.py index ac12d45f81..a3c49ecaf4 100644 --- a/tools/check_model_registry_freshness.py +++ b/tools/check_model_registry_freshness.py @@ -318,12 +318,23 @@ def evaluate( findings.append(_finding("invalid_slot", f"slot {name!r} has no provider.")) continue if explicit_model: - # A slot without an explicit profile still resolves through the - # default reviewed profile at runtime. Compare it against that - # decision so an old model pin cannot silently bypass review. + model = model_by_key.get((provider, explicit_model)) + # Explicit-profile pins must match their reviewed decision. A + # legacy pin without a profile is also valid at runtime when the + # pin is a current, unblocked catalog model. effective_profile = profile or "verifier-balanced" selected = selection_by_key.get((effective_profile, provider)) - if selected and selected.get("model_id") != explicit_model: + legacy_pin_is_current = bool( + not profile + and model is not None + and not model.get("blocked") + and str(model.get("lifecycle", "")).strip().lower() == "current" + ) + if ( + selected + and selected.get("model_id") != explicit_model + and not legacy_pin_is_current + ): findings.append( _finding( "selection_override", @@ -331,7 +342,6 @@ def evaluate( f"selection {selected.get('model_id')} for {effective_profile}.", ) ) - model = model_by_key.get((provider, explicit_model)) if model is None: findings.append( _finding( diff --git a/tools/llm_registry.py b/tools/llm_registry.py index 0a34bb618f..c2e854b88e 100644 --- a/tools/llm_registry.py +++ b/tools/llm_registry.py @@ -278,6 +278,12 @@ def configured_model_for_provider( ): return slot.model + # A readable slot config is an execution allowlist. Do not broaden to a + # provider-level registry decision when that config has no usable slot for + # this provider (including an empty or invalid-only slots list). + if _load_object(_slot_path(), label="slot config") is not None: + return "" + selected = select_model_for_profile(provider=provider, profile=profile, registry=entries) if selected: return selected @@ -368,7 +374,7 @@ def load_slot_config(*, github_default_model: str = "") -> list[SlotDefinition]: slots.append(SlotDefinition(name=name, provider=provider, model=model)) # A present slot file is an allowlist. If every configured slot is # unusable, fail closed instead of broadening execution to default providers. - return slots if slot_entries else fallback_slots + return slots def apply_slot_env_overrides( From 90c642479f67aafb245aac2c8dc1f6148c818bc6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Wed, 15 Jul 2026 21:29:45 +0000 Subject: [PATCH 5/7] chore(autofix): formatting/lint --- tests/tools/test_ci_failure_triage.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/tools/test_ci_failure_triage.py b/tests/tools/test_ci_failure_triage.py index 85abfcbee9..7164339f69 100644 --- a/tests/tools/test_ci_failure_triage.py +++ b/tests/tools/test_ci_failure_triage.py @@ -49,9 +49,7 @@ def __init__(self, **kwargs: object) -> None: ) monkeypatch.setenv("GITHUB_TOKEN", "github-token") monkeypatch.setenv("OPENAI_API_KEY", "openai-token") - monkeypatch.setattr( - llm_registry, "configured_model_for_provider", lambda _provider: "" - ) + monkeypatch.setattr(llm_registry, "configured_model_for_provider", lambda _provider: "") assert ci_failure_triage._get_llm_client() is None assert created == [] From cc319aa62a7dc7096e7aece385f0f226cdacc61a Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 15 Jul 2026 21:30:41 +0000 Subject: [PATCH 6/7] chore(codex-autofix): apply updates (PR #2781) --- langsmith-fleet-worker-attempt.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/langsmith-fleet-worker-attempt.json b/langsmith-fleet-worker-attempt.json index 797e7b4087..35abb6a54b 100644 --- a/langsmith-fleet-worker-attempt.json +++ b/langsmith-fleet-worker-attempt.json @@ -1,7 +1,7 @@ { "agent": "codex", "cli_version": "0.125.0", - "emitted_at": "2026-07-15T20:46:44.622969Z", + "emitted_at": "2026-07-15T21:30:40.959289Z", "execution_profile": "codex-default", "fallback_models": [ "gpt-5.4" From 6634d1314e709f99cd7fd83468916358b617ad43 Mon Sep 17 00:00:00 2001 From: Codex Automation Date: Wed, 15 Jul 2026 17:24:11 -0500 Subject: [PATCH 7/7] test: cover CI triage provider fallback --- tests/tools/test_ci_failure_triage.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/tools/test_ci_failure_triage.py b/tests/tools/test_ci_failure_triage.py index 7164339f69..6721a03c5c 100644 --- a/tests/tools/test_ci_failure_triage.py +++ b/tests/tools/test_ci_failure_triage.py @@ -53,3 +53,30 @@ def __init__(self, **kwargs: object) -> None: assert ci_failure_triage._get_llm_client() is None assert created == [] + + +def test_llm_triage_falls_back_from_github_models_to_openai(monkeypatch) -> None: + created: list[dict[str, object]] = [] + + class FakeChatOpenAI: + def __init__(self, **kwargs: object) -> None: + created.append(kwargs) + + monkeypatch.setitem( + sys.modules, "langchain_openai", types.SimpleNamespace(ChatOpenAI=FakeChatOpenAI) + ) + monkeypatch.setenv("GITHUB_TOKEN", "github-token") + monkeypatch.setenv("OPENAI_API_KEY", "openai-token") + monkeypatch.setattr( + llm_registry, + "configured_model_for_provider", + lambda provider: "" if provider == "github-models" else "gpt-5-mini", + ) + + client_and_provider = ci_failure_triage._get_llm_client() + + assert client_and_provider is not None + client, provider = client_and_provider + assert provider == "openai" + assert isinstance(client, FakeChatOpenAI) + assert created == [{"model": "gpt-5-mini", "api_key": "openai-token", "temperature": 0.1}]