Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
161 changes: 161 additions & 0 deletions scripts/check_agents_md_freshness.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,161 @@
#!/usr/bin/env python3
"""Warn when the managed Orchestrator AGENTS.md section cites stale repo facts."""

from __future__ import annotations

import argparse
import json
import re
import shutil
import sys
from dataclasses import dataclass
from pathlib import Path

MANAGED_START = "<!-- BEGIN orch-playbook -->"
MANAGED_END = "<!-- END orch-playbook -->"
PATH_SUFFIXES = {
".cfg",
".ini",
".js",
".json",
".md",
".py",
".sh",
".toml",
".txt",
".yaml",
".yml",
}


@dataclass(frozen=True)
class Finding:
kind: str
value: str
message: str

def as_dict(self) -> dict[str, str]:
return {"kind": self.kind, "value": self.value, "message": self.message}


def managed_section(text: str) -> str | None:
start = text.find(MANAGED_START)
end = text.find(MANAGED_END, start + len(MANAGED_START)) if start >= 0 else -1
if start < 0 or end < 0 or end < start:
return None
return text[start : end + len(MANAGED_END)]


def _clean_ref(value: str) -> str:
value = value.strip().strip("\"'")
value = re.sub(r"[:#]L?\d+(?:-L?\d+)?$", "", value)
return value


def _looks_like_path(value: str) -> bool:
if value.startswith(("./", "../", ".github/", "docs/", "scripts/", "templates/", "tools/")):
return True
path = Path(value)
return "/" in value or path.suffix.lower() in PATH_SUFFIXES


def _path_exists(repo_root: Path, ref: str) -> bool:
return (repo_root / ref).exists()


def _command_exists(repo_root: Path, ref: str) -> bool:
parts = ref.split()
if not parts:
return True
command = parts[0]
if command.startswith(("./", "../")) or "/" in command:
return (repo_root / command).exists()
return shutil.which(command) is not None


def _check_command_ref(repo_root: Path, ref: str) -> list[Finding]:
findings: list[Finding] = []
if not _command_exists(repo_root, ref):
findings.append(Finding("command", ref, f"referenced command not found: {ref}"))
for arg in ref.split()[1:]:
arg = _clean_ref(arg)
if "=" in arg:
_, arg = arg.split("=", 1)
arg = _clean_ref(arg)
if _looks_like_path(arg) and not _path_exists(repo_root, arg):
findings.append(Finding("path", arg, f"referenced path not found: {arg}"))
return findings


def cited_refs(section: str) -> list[str]:
refs: list[str] = []
for raw in re.findall(r"`([^`]+)`", section):
value = _clean_ref(raw)
if not value or value.startswith(("http://", "https://")):
continue
refs.append(value)
return refs


def check_agents_md(repo_root: Path, agents_md: Path | None = None) -> list[Finding]:
agents_path = agents_md or repo_root / "AGENTS.md"
if not agents_path.exists():
return []
section = managed_section(agents_path.read_text(encoding="utf-8"))
if section is None:
return []

findings: list[Finding] = []
seen: set[tuple[str, str]] = set()
for ref in cited_refs(section):
if " " in ref:
for finding in _check_command_ref(repo_root, ref):
key = (finding.kind, finding.value)
if key not in seen:
findings.append(finding)
seen.add(key)
elif _looks_like_path(ref):
key = ("path", ref)
if key not in seen and not _path_exists(repo_root, ref):
findings.append(Finding("path", ref, f"referenced path not found: {ref}"))
seen.add(key)
return findings


def _emit_github_warnings(findings: list[Finding]) -> None:
for finding in findings:
message = finding.message.replace("%", "%25").replace("\n", "%0A").replace("\r", "%0D")
print(f"::warning title=AGENTS.md freshness::{message}")


def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--repo-root", type=Path, default=Path.cwd())
parser.add_argument("--agents-md", type=Path)
parser.add_argument("--github-annotations", action="store_true")
parser.add_argument("--json", action="store_true", dest="as_json")
parser.add_argument(
"--strict", action="store_true", help="Exit non-zero when findings are present."
)
args = parser.parse_args(argv)

repo_root = args.repo_root.resolve()
agents_md = args.agents_md.resolve() if args.agents_md else None
findings = check_agents_md(repo_root, agents_md)

if args.as_json:
print(json.dumps({"findings": [finding.as_dict() for finding in findings]}, indent=2))
elif findings:
for finding in findings:
print(finding.message)
else:
print("AGENTS.md managed section freshness check passed.")

if args.github_annotations and findings:
_emit_github_warnings(findings)

return 1 if args.strict and findings else 0


if __name__ == "__main__":
sys.exit(main())
146 changes: 67 additions & 79 deletions tools/langchain_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,41 +8,29 @@
from __future__ import annotations

import contextlib
import json
import logging
import os
from dataclasses import dataclass
from pathlib import Path

from tools import llm_registry as _llm_registry
from tools.llm_provider import DEFAULT_MODEL, GITHUB_MODELS_BASE_URL
from tools.llm_registry import (
PROVIDER_ANTHROPIC,
PROVIDER_GITHUB,
PROVIDER_OPENAI,
ModelRegistryEntry,
SlotDefinition,
apply_slot_env_overrides,
default_slots,
is_model_blocked,
load_model_registry,
load_slot_config,
normalize_provider,
registry_entry_for,
resolve_slots,
select_model_for_tier,
)

logger = logging.getLogger(__name__)

ENV_PROVIDER = "LANGCHAIN_PROVIDER"
ENV_MODEL = "LANGCHAIN_MODEL"
ENV_TIMEOUT = "LANGCHAIN_TIMEOUT"
ENV_MAX_RETRIES = "LANGCHAIN_MAX_RETRIES"
ENV_SLOT_CONFIG = _llm_registry.ENV_SLOT_CONFIG
ENV_MODEL_REGISTRY_CONFIG = _llm_registry.ENV_MODEL_REGISTRY_CONFIG
ENV_SLOT_CONFIG = "LANGCHAIN_SLOT_CONFIG"
ENV_SLOT_PREFIX = "LANGCHAIN_SLOT"
ENV_ANTHROPIC_KEY = "CLAUDE_API_STRANSKE"
DEFAULT_SLOT_CONFIG_PATH = _llm_registry.DEFAULT_SLOT_CONFIG_PATH
DEFAULT_MODEL_REGISTRY_CONFIG_PATH = _llm_registry.DEFAULT_MODEL_REGISTRY_CONFIG_PATH

PROVIDER_OPENAI = "openai"
PROVIDER_ANTHROPIC = "anthropic"
PROVIDER_GITHUB = "github-models"

DEFAULT_SLOT_CONFIG_PATH = Path(__file__).resolve().parent.parent / "config" / "llm_slots.json"


def _env_int(name: str, default: int) -> int:
Expand Down Expand Up @@ -71,8 +59,24 @@ def provider_label(self) -> str:
return f"{self.provider}/{self.model}"


@dataclass(frozen=True)
class SlotDefinition:
name: str
provider: str
model: str


def _normalize_provider(value: str | None) -> str | None:
return normalize_provider(value)
if not value:
return None
normalized = value.strip().lower()
if normalized in {"github", "github_models", "github-models"}:
return PROVIDER_GITHUB
if normalized in {"anthropic", "claude"}:
return PROVIDER_ANTHROPIC
if normalized in {"openai"}:
return PROVIDER_OPENAI
return None


def _resolve_provider(provider: str | None, *, force_openai: bool) -> tuple[str | None, bool]:
Expand All @@ -89,53 +93,57 @@ def _resolve_model(model: str | None) -> str:
return model or env_model or DEFAULT_MODEL


def _load_model_registry() -> list[ModelRegistryEntry]:
return load_model_registry()


def _registry_entry_for(
provider: str, model: str, registry: list[ModelRegistryEntry] | None = None
) -> ModelRegistryEntry | None:
return registry_entry_for(provider, model, registry=registry)


def _is_model_blocked(
provider: str, model: str, registry: list[ModelRegistryEntry] | None = None
) -> bool:
return is_model_blocked(provider, model, registry=registry)


def _select_model_for_tier(
*,
provider: str,
tier: str,
registry: list[ModelRegistryEntry] | None = None,
) -> str | None:
return select_model_for_tier(provider=provider, tier=tier, registry=registry)


def _default_slots() -> list[SlotDefinition]:
return default_slots(github_default_model=DEFAULT_MODEL)
return [
SlotDefinition(name="slot1", provider=PROVIDER_OPENAI, model="gpt-5.4"),
SlotDefinition(name="slot2", provider=PROVIDER_ANTHROPIC, model="claude-sonnet-4-6"),
SlotDefinition(name="slot3", provider=PROVIDER_GITHUB, model=DEFAULT_MODEL),
]


def _load_slot_config() -> list[SlotDefinition]:
return load_slot_config(github_default_model=DEFAULT_MODEL)
config_path = os.environ.get(ENV_SLOT_CONFIG)
path = Path(config_path) if config_path else DEFAULT_SLOT_CONFIG_PATH
if not path.is_file():
return _default_slots()
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return _default_slots()

slots: list[SlotDefinition] = []
for idx, entry in enumerate(payload.get("slots", []), start=1):
provider = _normalize_provider(str(entry.get("provider", "")))
model = str(entry.get("model", "")).strip()
if not provider or not model:
continue
name = str(entry.get("name") or f"slot{idx}").strip() or f"slot{idx}"
slots.append(SlotDefinition(name=name, provider=provider, model=model))

return slots or _default_slots()


def _apply_slot_env_overrides(slots: list[SlotDefinition]) -> list[SlotDefinition]:
return apply_slot_env_overrides(
slots,
env_model_name=ENV_MODEL,
env_slot_prefix=ENV_SLOT_PREFIX,
)
updated: list[SlotDefinition] = []
for idx, slot in enumerate(slots, start=1):
provider_key = f"{ENV_SLOT_PREFIX}{idx}_PROVIDER"
model_key = f"{ENV_SLOT_PREFIX}{idx}_MODEL"
provider_override = _normalize_provider(os.environ.get(provider_key))
model_override = os.environ.get(model_key)
if idx == 1:
model_override = model_override or os.environ.get(ENV_MODEL)
updated.append(
SlotDefinition(
name=slot.name,
provider=provider_override or slot.provider,
model=(model_override or slot.model).strip(),
)
)
return updated


def _resolve_slots() -> list[SlotDefinition]:
return resolve_slots(
github_default_model=DEFAULT_MODEL,
env_model_name=ENV_MODEL,
env_slot_prefix=ENV_SLOT_PREFIX,
)
return _apply_slot_env_overrides(_load_slot_config())


def _is_reasoning_model(model: str) -> bool:
Expand Down Expand Up @@ -226,9 +234,6 @@ def build_chat_client(
selected_provider, provider_explicit = _resolve_provider(provider, force_openai=force_openai)
if provider_explicit and selected_provider is None:
return None
if selected_provider and _is_model_blocked(selected_provider, selected_model):
logger.warning("Refusing blocked LLM model: %s/%s", selected_provider, selected_model)
return None

if selected_provider == PROVIDER_GITHUB:
if not github_token:
Expand Down Expand Up @@ -278,11 +283,6 @@ def build_chat_client(
# Auto-select: slot order (OpenAI -> Claude -> GitHub Models by default).
slots = _resolve_slots()
model_override = model or os.environ.get(ENV_MODEL)
if model_override:
override_provider = selected_provider or (slots[0].provider if slots else "")
if override_provider and _is_model_blocked(override_provider, model_override):
logger.warning("Refusing blocked LLM model override: %s/%s", override_provider, model_override)
return None
used_override = False
for slot in slots:
slot_model = model_override if model_override and not used_override else slot.model
Expand Down Expand Up @@ -358,15 +358,6 @@ def build_chat_clients(
selected_provider, provider_explicit = _resolve_provider(provider, force_openai=False)
if provider_explicit and selected_provider is None:
return []
registry = _load_model_registry()
if selected_provider:
blocked_models = [candidate for candidate in (first_model, second_model) if candidate]
if any(
_is_model_blocked(selected_provider, candidate, registry=registry)
for candidate in blocked_models
):
logger.warning("Refusing blocked LLM model for provider %s", selected_provider)
return []

clients: list[ClientInfo] = []

Expand Down Expand Up @@ -484,9 +475,6 @@ def build_chat_clients(
for idx, slot in enumerate(candidate_slots):
slot_model = model_overrides[idx] if idx < len(model_overrides) else None
slot_model = slot_model or slot.model
if _is_model_blocked(slot.provider, slot_model, registry=registry):
logger.warning("Skipping blocked LLM model override: %s/%s", slot.provider, slot_model)
continue
if slot.provider == PROVIDER_OPENAI and openai_token:
with contextlib.suppress(Exception):
clients.append(
Expand Down
Loading
Loading