From 60d3d057047b311b49e63521f6124b72e6711ab3 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 14 Jul 2026 11:10:41 -0700 Subject: [PATCH 01/12] feat(cli): add lite autoroute to QA complexity-based auto-routing against a real proxy Lets a customer try litellm's complexity_router against models they already have on their existing, unmodified production proxy, with no config.yaml edits and no new infra. lite autoroute configure discovers accessible models via /model_group/info and walks through tier assignment (plus optional LLM classifier / semantic matching / adaptive selection); every referenced model becomes its own litellm_proxy/ deployment forwarding back to the real proxy with the real key, so every actual call, routed completions, classifier calls, embedding calls, still lands on their real proxy. lite autoroute up launches that generated config as an ephemeral local proxy, patches ~/.claude/settings.json to point Claude Code at it, and streams routing decisions live; Ctrl-C/SIGTERM (or lite autoroute down after an unclean exit) restores everything. Also adds lite model-groups list (a thin CLI wrapper over the existing ModelGroupsManagementClient), and generalizes up.py's settings-backup/restore helpers to take explicit paths so this feature can reuse them instead of duplicating the logic. Depends on litellm_lite_up_down (#33231) for that generalization. --- litellm/proxy/client/cli/README.md | 55 +++++ .../client/cli/commands/autoroute/__init__.py | 0 .../client/cli/commands/autoroute/commands.py | 157 ++++++++++++ .../client/cli/commands/autoroute/config.py | 223 ++++++++++++++++++ .../client/cli/commands/autoroute/process.py | 158 +++++++++++++ .../client/cli/commands/autoroute/settings.py | 35 +++ .../client/cli/commands/autoroute/wizard.py | 106 +++++++++ .../proxy/client/cli/commands/model_groups.py | 54 +++++ litellm/proxy/client/cli/commands/up.py | 34 +-- litellm/proxy/client/cli/main.py | 6 + .../proxy/client/cli/autoroute/__init__.py | 0 .../client/cli/autoroute/test_commands.py | 175 ++++++++++++++ .../proxy/client/cli/autoroute/test_config.py | 167 +++++++++++++ .../client/cli/autoroute/test_process.py | 113 +++++++++ .../client/cli/autoroute/test_settings.py | 36 +++ .../proxy/client/cli/autoroute/test_wizard.py | 188 +++++++++++++++ .../client/cli/test_model_groups_commands.py | 104 ++++++++ 17 files changed, 1597 insertions(+), 14 deletions(-) create mode 100644 litellm/proxy/client/cli/commands/autoroute/__init__.py create mode 100644 litellm/proxy/client/cli/commands/autoroute/commands.py create mode 100644 litellm/proxy/client/cli/commands/autoroute/config.py create mode 100644 litellm/proxy/client/cli/commands/autoroute/process.py create mode 100644 litellm/proxy/client/cli/commands/autoroute/settings.py create mode 100644 litellm/proxy/client/cli/commands/autoroute/wizard.py create mode 100644 litellm/proxy/client/cli/commands/model_groups.py create mode 100644 tests/test_litellm/proxy/client/cli/autoroute/__init__.py create mode 100644 tests/test_litellm/proxy/client/cli/autoroute/test_commands.py create mode 100644 tests/test_litellm/proxy/client/cli/autoroute/test_config.py create mode 100644 tests/test_litellm/proxy/client/cli/autoroute/test_process.py create mode 100644 tests/test_litellm/proxy/client/cli/autoroute/test_settings.py create mode 100644 tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py create mode 100644 tests/test_litellm/proxy/client/cli/test_model_groups_commands.py diff --git a/litellm/proxy/client/cli/README.md b/litellm/proxy/client/cli/README.md index e8b024dc31d..ecd5f3d48e2 100644 --- a/litellm/proxy/client/cli/README.md +++ b/litellm/proxy/client/cli/README.md @@ -489,6 +489,61 @@ This is a one-time file patch and restore, not a live traffic interceptor. A Cla Cursor is not supported: it has no equivalent file-based config to hot-patch this way, since its model routing lives in its own app storage and is configured through its GUI. +### QA Complexity-Based Auto-Routing Against Your Real Proxy + +`lite autoroute` lets you try LiteLLM's complexity-based auto-routing -- picking a cheaper or more expensive model depending on how complex a prompt looks -- against models your key already has access to on your real, running proxy, without editing that proxy's `config.yaml` and without any real request ever bypassing it. It builds a second, throwaway proxy locally that forwards every request back to your real proxy, and points Claude Code at that local proxy for the duration of the session. + +#### List Your Accessible Model Groups + +```bash +lite model-groups list [--format table|json] +``` + +Lists the model groups your key can reach on the proxy, via `/model_group/info`, along with each group's mode (`chat`, `embedding`, etc.) and per-token pricing. This is also what `lite autoroute configure` uses internally to discover what it can offer you. + +#### Configure the Auto-Router + +```bash +lite autoroute configure +``` + +An interactive wizard. It runs the same model-group discovery as above, splits the results into chat-capable and embedding-capable pools, and asks you to assign a model from the chat pool to each of the four complexity tiers -- SIMPLE, MEDIUM, COMPLEX, REASONING. From there it optionally offers: classifying prompt complexity with an LLM (again picked from your discovered pool) instead of the free built-in heuristic scorer, semantic keyword matching for tier assignment (needs an embedding model from the pool), and adaptive (bandit-based) selection layered on top of tiering. + +The wizard writes the result to `~/.litellm/autorouter/config.yaml` with `0600` permissions, since the file embeds your real proxy API key. Every model referenced anywhere in that config -- tier targets, the classifier model, the embedding model -- becomes its own `litellm_proxy/` deployment whose `api_base` and `api_key` point back at your real proxy. That is the trick that keeps your real proxy's config untouched: every actual network call this generates, whether it is the routed completion, an LLM-classifier call, or an embedding call, forwards transparently through your real, already-running proxy with your real key. + +You must run `configure` at least once before `up`; running `up` first fails with a clear error telling you to configure first. + +#### Launch the Ephemeral Auto-Router Proxy + +```bash +lite autoroute up +``` + +Starts a local, throwaway litellm proxy on a random free port, running the config `configure` generated, with a freshly-minted random API key baked in for this session only (your real proxy key never leaves the generated config -- it only appears there, forwarding to your real proxy). It waits for the ephemeral proxy to report healthy, then patches `~/.claude/settings.json` the same way `lite up` does, except with a static `ANTHROPIC_AUTH_TOKEN` env var instead of an `apiKeyHelper`, since this key is short-lived and self-issued rather than something needing SSO refresh. Any `claude` session started afterward, from any terminal, routes through the ephemeral proxy. + +`lite autoroute up` runs in the foreground and streams the ephemeral proxy's own log file into your terminal, so you can watch its routing decisions -- which tier and model got picked for each request -- as you use Claude Code normally. Press Ctrl-C (or send SIGTERM) to stop it; this kills the child proxy process and restores your original Claude Code settings, in that order. + +#### Recover From an Unclean Shutdown + +```bash +lite autoroute down +``` + +If the `lite autoroute up` process dies uncleanly -- `kill -9`, a crash -- rather than being stopped with Ctrl-C, `down` is the manual recovery path: it kills any leftover ephemeral proxy process found via a recorded pid file and restores Claude Code's settings from whatever backup is on disk. + +#### Example + +```bash +lite autoroute configure +lite autoroute up +# use Claude Code as normal in another terminal; routing decisions stream live +lite autoroute down # only needed if `up` was killed uncleanly instead of Ctrl-C'd +``` + +#### Caveats + +Adaptive mode's learned state does not persist across `lite autoroute up` sessions -- there is no local database, so every session starts adaptive selection cold. A Claude Code session already running before `up` started, or still running when it stops, keeps whatever settings it loaded at its own startup; like `lite up`, this is a one-time file patch and restore, not a live traffic interceptor. Only Claude Code is supported, for the same reason as `lite up`: no other supported agent (for example Cursor) has an equivalent hot-patchable config file. + ## Environment Variables The CLI respects the following environment variables: diff --git a/litellm/proxy/client/cli/commands/autoroute/__init__.py b/litellm/proxy/client/cli/commands/autoroute/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/litellm/proxy/client/cli/commands/autoroute/commands.py b/litellm/proxy/client/cli/commands/autoroute/commands.py new file mode 100644 index 00000000000..aca2f395615 --- /dev/null +++ b/litellm/proxy/client/cli/commands/autoroute/commands.py @@ -0,0 +1,157 @@ +import atexit +import json +import secrets +import signal +import threading +from types import FrameType +from typing import Dict, Optional + +import click +import yaml +from pydantic import JsonValue, TypeAdapter + +from ..up import CLAUDE_SETTINGS_PATH +from ..up import BackupRecord as ClaudeBackupRecord +from ..up import load_json_or_empty, restore_claude_settings, write_backup +from .process import ( + AUTOROUTE_DIR, + CONFIG_PATH, + LOG_PATH, + PidRecord, + ProcessLaunchError, + allocate_free_port, + clear_pid_record, + is_running, + launch_proxy, + poll_liveliness, + read_pid_record, + stream_log, + terminate, + write_pid_record, +) +from .settings import merge_claude_settings_static_token +from .wizard import run_configure_wizard + +AUTOROUTE_BACKUP_PATH = AUTOROUTE_DIR / "claude_settings_backup.json" + +_GENERATED_CONFIG_ADAPTER = TypeAdapter(Dict[str, JsonValue]) + + +def _mint_and_embed_master_key() -> str: + """Generate a fresh key for this session and write it into the generated config.yaml. + + Must go under general_settings, not litellm_settings -- the proxy server only ever + reads general_settings.master_key (proxy_server.py:4530) to authenticate requests. A + key placed under litellm_settings is silently ignored, leaving the ephemeral proxy with + no real auth: any request reaches it regardless of the token Claude Code sends. + """ + master_key = secrets.token_urlsafe(32) + with open(CONFIG_PATH, "r") as f: + generated = _GENERATED_CONFIG_ADAPTER.validate_python(yaml.safe_load(f)) + general_settings = generated.get("general_settings") + updated_settings: Dict[str, JsonValue] = { + **(general_settings if isinstance(general_settings, dict) else {}), + "master_key": master_key, + } + updated: Dict[str, JsonValue] = {**generated, "general_settings": updated_settings} + with open(CONFIG_PATH, "w") as f: + yaml.safe_dump(updated, f, sort_keys=False) + CONFIG_PATH.chmod(0o600) + return master_key + + +@click.group(name="autoroute") +def autoroute_group() -> None: + """QA complexity-based auto-routing against models your key can already use""" + + +@autoroute_group.command("configure") +@click.pass_context +def configure(ctx: click.Context) -> None: + """Discover accessible models and generate an ephemeral auto-router config""" + run_configure_wizard(ctx) + + +@autoroute_group.command("up") +def up() -> None: + """Launch the ephemeral auto-router proxy and route Claude Code through it""" + if not CONFIG_PATH.exists(): + raise click.ClickException("No config found. Run `lite autoroute configure` first.") + + existing_pid = read_pid_record() + if existing_pid is not None and is_running(existing_pid.pid): + raise click.ClickException( + "An ephemeral proxy is already running (lite autoroute up looks already active). " + "Run `lite autoroute down` first." + ) + + master_key = _mint_and_embed_master_key() + port = allocate_free_port() + base_url = f"http://127.0.0.1:{port}" + process = launch_proxy(CONFIG_PATH, port, LOG_PATH) + write_pid_record(PidRecord(pid=process.pid, port=port, config_path=str(CONFIG_PATH), log_path=str(LOG_PATH))) + + try: + poll_liveliness(base_url, LOG_PATH, process) + except ProcessLaunchError as e: + clear_pid_record() + raise click.ClickException(str(e)) + + original_existed = CLAUDE_SETTINGS_PATH.exists() + original_settings = load_json_or_empty(CLAUDE_SETTINGS_PATH) + write_backup( + ClaudeBackupRecord(existed=original_existed, content=original_settings if original_existed else None), + AUTOROUTE_BACKUP_PATH, + ) + merged = merge_claude_settings_static_token(original_settings, base_url, master_key) + CLAUDE_SETTINGS_PATH.parent.mkdir(parents=True, exist_ok=True) + with open(CLAUDE_SETTINGS_PATH, "w") as f: + json.dump(merged, f, indent=2) + + click.echo(f"litellm: ephemeral auto-router proxy up at {base_url} (pid {process.pid})") + click.echo("Claude Code sessions started now will route through it. Press Ctrl-C to stop and restore.") + + stop_event = threading.Event() + restored = threading.Lock() + + def _teardown() -> None: + if not restored.acquire(blocking=False): + return + terminate(process.pid) + clear_pid_record() + restore_claude_settings(CLAUDE_SETTINGS_PATH, AUTOROUTE_BACKUP_PATH) + click.echo("\nStopped ephemeral proxy and restored Claude Code settings.") + + def _handle_signal(_signum: int, _frame: Optional[FrameType]) -> None: + stop_event.set() + + signal.signal(signal.SIGINT, _handle_signal) + signal.signal(signal.SIGTERM, _handle_signal) + atexit.register(_teardown) + + log_thread = threading.Thread(target=stream_log, args=(LOG_PATH, stop_event), daemon=True) + log_thread.start() + + stop_event.wait() + _teardown() + + +@autoroute_group.command("down") +def down() -> None: + """Restore Claude Code settings and stop a leftover ephemeral proxy, if any""" + record: Optional[PidRecord] = read_pid_record() + if record is not None and is_running(record.pid): + terminate(record.pid) + click.echo(f"Stopped leftover ephemeral proxy (pid {record.pid}).") + clear_pid_record() + + restored = restore_claude_settings(CLAUDE_SETTINGS_PATH, AUTOROUTE_BACKUP_PATH) + if restored is None: + click.echo("Nothing to restore.") + elif restored.existed: + click.echo(f"Restored {CLAUDE_SETTINGS_PATH} to its original contents.") + else: + click.echo(f"Removed {CLAUDE_SETTINGS_PATH} (it did not exist before `lite autoroute up`).") + + +__all__ = ["autoroute_group"] diff --git a/litellm/proxy/client/cli/commands/autoroute/config.py b/litellm/proxy/client/cli/commands/autoroute/config.py new file mode 100644 index 00000000000..85b5223524b --- /dev/null +++ b/litellm/proxy/client/cli/commands/autoroute/config.py @@ -0,0 +1,223 @@ +from typing import Dict, FrozenSet, List, Literal, Tuple, Union + +from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter + +TIER_NAMES: Tuple[str, ...] = ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING") + + +class ConfigGenerationError(Exception): + """Raised when an AutorouteConfig references a model the discovery step didn't find.""" + + +class DiscoveredModel(BaseModel): + model_config = ConfigDict(frozen=True) + + name: str + mode: str = "chat" + input_cost_per_token: float | None = None + output_cost_per_token: float | None = None + + +class _RawModelGroup(BaseModel): + model_config = ConfigDict(extra="ignore") + + model_group: str + # Optional: some real deployments return an explicit `"mode": null` for models that + # were registered without a mode (seen for embedding models like voyage-4-large). + # ModelGroupInfo's own "chat" default (litellm/types/router.py) only applies when the + # key is missing entirely, not when it's present as null, so this must tolerate None. + mode: str | None = "chat" + input_cost_per_token: float | None = None + output_cost_per_token: float | None = None + + +_RAW_MODEL_GROUPS_ADAPTER = TypeAdapter(List[_RawModelGroup]) + + +def parse_discovered_models(raw: List[JsonValue]) -> Tuple[DiscoveredModel, ...]: + """Validate a raw `/model_group/info` response into typed models.""" + parsed = _RAW_MODEL_GROUPS_ADAPTER.validate_python(raw) + return tuple( + DiscoveredModel( + name=group.model_group, + # A null mode means the server genuinely doesn't know what this model does; + # "unknown" (rather than guessing "chat") keeps it out of both chat_models() + # and embedding_models() instead of risking a wrong-mode deployment. + mode=group.mode or "unknown", + input_cost_per_token=group.input_cost_per_token, + output_cost_per_token=group.output_cost_per_token, + ) + for group in parsed + ) + + +def chat_models(models: Tuple[DiscoveredModel, ...]) -> Tuple[DiscoveredModel, ...]: + return tuple(m for m in models if m.mode == "chat") + + +def embedding_models(models: Tuple[DiscoveredModel, ...]) -> Tuple[DiscoveredModel, ...]: + return tuple(m for m in models if m.mode == "embedding") + + +class HeuristicClassifier(BaseModel): + model_config = ConfigDict(frozen=True) + kind: Literal["heuristic"] = "heuristic" + + +class LLMClassifier(BaseModel): + model_config = ConfigDict(frozen=True) + kind: Literal["llm"] = "llm" + model: str + timeout_ms: int = 3000 + + +ClassifierChoice = Union[HeuristicClassifier, LLMClassifier] + + +class NoSemanticMatching(BaseModel): + model_config = ConfigDict(frozen=True) + kind: Literal["none"] = "none" + + +class SemanticMatching(BaseModel): + model_config = ConfigDict(frozen=True) + kind: Literal["semantic"] = "semantic" + embedding_model: str + match_threshold: float = 0.5 + + +SemanticMatchingChoice = Union[NoSemanticMatching, SemanticMatching] + +# Satisfies complexity_router's "semantic matching requires non-empty keyword_tier_rules" +# invariant with a sane starting point; the generated config.yaml can be hand-edited afterward. +_DEFAULT_KEYWORD_TIER_RULES: Tuple[Dict[str, JsonValue], ...] = ( + {"keywords": ["hi", "hello", "thanks"], "tier": "SIMPLE"}, + {"keywords": ["explain", "how does"], "tier": "MEDIUM"}, + {"keywords": ["refactor", "implement", "debug"], "tier": "COMPLEX"}, + {"keywords": ["step by step", "think through", "prove"], "tier": "REASONING"}, +) + + +class AutorouteConfig(BaseModel): + model_config = ConfigDict(frozen=True) + + base_url: str + api_key: str + tiers: Dict[str, str] + default_model: str + classifier: ClassifierChoice = Field(default_factory=HeuristicClassifier) + semantic_matching: SemanticMatchingChoice = Field(default_factory=NoSemanticMatching) + adaptive: bool = False + + +def validate_config(config: AutorouteConfig, discovered: Tuple[DiscoveredModel, ...]) -> None: + """Raise ConfigGenerationError if config references a model discovery didn't return.""" + chat_names: FrozenSet[str] = frozenset(m.name for m in chat_models(discovered)) + embedding_names: FrozenSet[str] = frozenset(m.name for m in embedding_models(discovered)) + + for tier, model in config.tiers.items(): + if model not in chat_names: + raise ConfigGenerationError(f"Tier {tier} references unknown chat model '{model}'") + + if config.default_model not in chat_names: + raise ConfigGenerationError(f"default_model '{config.default_model}' is not a known chat model") + + if isinstance(config.classifier, LLMClassifier) and config.classifier.model not in chat_names: + raise ConfigGenerationError(f"classifier model '{config.classifier.model}' is not a known chat model") + + if ( + isinstance(config.semantic_matching, SemanticMatching) + and config.semantic_matching.embedding_model not in embedding_names + ): + raise ConfigGenerationError( + f"embedding model '{config.semantic_matching.embedding_model}' is not a known embedding model" + ) + + +def _litellm_proxy_deployment(name: str, base_url: str, api_key: str) -> Dict[str, JsonValue]: + return { + "model_name": name, + "litellm_params": { + "model": f"litellm_proxy/{name}", + "api_base": base_url, + "api_key": api_key, + }, + } + + +def build_generated_model_list(config: AutorouteConfig) -> List[JsonValue]: + """Build the model_list for the ephemeral proxy's config.yaml. + + Every real model referenced anywhere (tier targets, classifier, embedding) is deduplicated + to exactly one `litellm_proxy/` deployment forwarding to the customer's real proxy, + plus one `auto_router/complexity_router` deployment tying the tiers together. + """ + referenced_names = {*config.tiers.values(), config.default_model} + if isinstance(config.classifier, LLMClassifier): + referenced_names.add(config.classifier.model) + if isinstance(config.semantic_matching, SemanticMatching): + referenced_names.add(config.semantic_matching.embedding_model) + + proxy_deployments = [ + _litellm_proxy_deployment(name, config.base_url, config.api_key) for name in sorted(referenced_names) + ] + + complexity_router_config: Dict[str, JsonValue] = { + "tiers": dict(config.tiers), + "default_model": config.default_model, + } + if isinstance(config.classifier, LLMClassifier): + complexity_router_config["classifier_type"] = "llm" + complexity_router_config["classifier_llm_config"] = { + "model": config.classifier.model, + "timeout_ms": config.classifier.timeout_ms, + } + if isinstance(config.semantic_matching, SemanticMatching): + complexity_router_config["semantic_keyword_matching"] = True + complexity_router_config["embedding_model"] = config.semantic_matching.embedding_model + complexity_router_config["match_threshold"] = config.semantic_matching.match_threshold + complexity_router_config["keyword_tier_rules"] = list(_DEFAULT_KEYWORD_TIER_RULES) + if config.adaptive: + complexity_router_config["adaptive"] = True + + auto_router_deployment: Dict[str, JsonValue] = { + "model_name": "autorouter", + "litellm_params": { + "model": "auto_router/complexity_router", + "complexity_router_config": complexity_router_config, + }, + } + return [*proxy_deployments, auto_router_deployment] + + +def build_generated_proxy_config(config: AutorouteConfig, master_key: str) -> Dict[str, JsonValue]: + """Full config.yaml content for the ephemeral proxy, including its own auth key. + + master_key must live under general_settings, not litellm_settings -- the proxy server + only ever reads general_settings.master_key (proxy_server.py:4530) to authenticate + requests; a key placed under litellm_settings is silently ignored, leaving the proxy + with no real auth at all. + """ + return { + "model_list": build_generated_model_list(config), + "general_settings": {"master_key": master_key}, + } + + +__all__ = [ + "TIER_NAMES", + "ConfigGenerationError", + "DiscoveredModel", + "parse_discovered_models", + "chat_models", + "embedding_models", + "HeuristicClassifier", + "LLMClassifier", + "ClassifierChoice", + "NoSemanticMatching", + "SemanticMatching", + "SemanticMatchingChoice", + "AutorouteConfig", + "validate_config", + "build_generated_model_list", +] diff --git a/litellm/proxy/client/cli/commands/autoroute/process.py b/litellm/proxy/client/cli/commands/autoroute/process.py new file mode 100644 index 00000000000..ce146e95eec --- /dev/null +++ b/litellm/proxy/client/cli/commands/autoroute/process.py @@ -0,0 +1,158 @@ +import contextlib +import json +import os +import signal +import socket +import subprocess +import sys +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +import click +import requests +from pydantic import TypeAdapter + +AUTOROUTE_DIR = Path.home() / ".litellm" / "autorouter" +CONFIG_PATH = AUTOROUTE_DIR / "config.yaml" +LOG_PATH = AUTOROUTE_DIR / "proxy.log" +PID_RECORD_PATH = AUTOROUTE_DIR / "proxy.pid.json" + + +class ProcessLaunchError(Exception): + """Raised when the ephemeral proxy subprocess fails to come up healthy.""" + + +@dataclass(frozen=True, slots=True) +class PidRecord: + pid: int + port: int + config_path: str + log_path: str + + +_PID_RECORD_ADAPTER = TypeAdapter(PidRecord) + + +def allocate_free_port() -> int: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", 0)) + return int(sock.getsockname()[1]) + + +def launch_proxy(config_path: Path, port: int, log_path: Path) -> "subprocess.Popen[bytes]": + log_path.parent.mkdir(parents=True, exist_ok=True) + log_file = open(log_path, "w") + return subprocess.Popen( + [sys.executable, "-m", "litellm.proxy.proxy_cli", "--config", str(config_path), "--port", str(port)], + stdout=log_file, + stderr=subprocess.STDOUT, + ) + + +def _tail(log_path: Path, lines: int = 40) -> str: + if not log_path.exists(): + return "(no log output captured)" + return "\n".join(log_path.read_text(errors="replace").splitlines()[-lines:]) + + +def poll_liveliness(base_url: str, log_path: Path, process: "subprocess.Popen[bytes]", timeout: float = 30.0) -> None: + """Poll /health/liveliness until it responds, the process dies, or timeout elapses.""" + deadline = time.monotonic() + timeout + url = base_url.rstrip("/") + "/health/liveliness" + while time.monotonic() < deadline: + if process.poll() is not None: + raise ProcessLaunchError( + f"Ephemeral proxy exited early (code {process.returncode}). Last log lines:\n{_tail(log_path)}" + ) + with contextlib.suppress(requests.RequestException): + if requests.get(url, timeout=2).status_code == 200: + return + time.sleep(0.5) + raise ProcessLaunchError( + f"Ephemeral proxy never became healthy within {timeout}s. Last log lines:\n{_tail(log_path)}" + ) + + +def write_pid_record(record: PidRecord, path: Optional[Path] = None) -> None: + resolved_path = path if path is not None else PID_RECORD_PATH + resolved_path.parent.mkdir(parents=True, exist_ok=True) + with open(resolved_path, "w") as f: + json.dump( + {"pid": record.pid, "port": record.port, "config_path": record.config_path, "log_path": record.log_path}, + f, + indent=2, + ) + + +def read_pid_record(path: Optional[Path] = None) -> Optional[PidRecord]: + resolved_path = path if path is not None else PID_RECORD_PATH + if not resolved_path.exists(): + return None + with open(resolved_path, "r") as f: + return _PID_RECORD_ADAPTER.validate_json(f.read()) + + +def clear_pid_record(path: Optional[Path] = None) -> None: + resolved_path = path if path is not None else PID_RECORD_PATH + resolved_path.unlink(missing_ok=True) + + +def is_running(pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + except PermissionError: + return True + return True + + +def terminate(pid: int, grace_period: float = 5.0) -> None: + """Terminate a process by pid, escalating from SIGTERM to SIGKILL if needed.""" + if not is_running(pid): + return + with contextlib.suppress(ProcessLookupError): + os.kill(pid, signal.SIGTERM) + deadline = time.monotonic() + grace_period + while time.monotonic() < deadline and is_running(pid): + time.sleep(0.2) + if is_running(pid): + with contextlib.suppress(ProcessLookupError): + os.kill(pid, signal.SIGKILL) + + +def stream_log(log_path: Path, stop_event: threading.Event) -> None: + """Print new lines appended to log_path until stop_event is set. Blocks the calling thread.""" + while not log_path.exists() and not stop_event.is_set(): + time.sleep(0.1) + if stop_event.is_set() or not log_path.exists(): + return + with open(log_path, "r") as f: + while not stop_event.is_set(): + line = f.readline() + if line: + click.echo(line, nl=False) + else: + time.sleep(0.2) + + +__all__ = [ + "AUTOROUTE_DIR", + "CONFIG_PATH", + "LOG_PATH", + "PID_RECORD_PATH", + "ProcessLaunchError", + "PidRecord", + "allocate_free_port", + "launch_proxy", + "poll_liveliness", + "write_pid_record", + "read_pid_record", + "clear_pid_record", + "is_running", + "terminate", + "stream_log", +] diff --git a/litellm/proxy/client/cli/commands/autoroute/settings.py b/litellm/proxy/client/cli/commands/autoroute/settings.py new file mode 100644 index 00000000000..0d7b12d7aaf --- /dev/null +++ b/litellm/proxy/client/cli/commands/autoroute/settings.py @@ -0,0 +1,35 @@ +from typing import Dict + +from pydantic import JsonValue + +ENV_KEY = "env" +API_KEY_HELPER_KEY = "apiKeyHelper" +ANTHROPIC_API_KEY_KEY = "ANTHROPIC_API_KEY" +ANTHROPIC_AUTH_TOKEN_KEY = "ANTHROPIC_AUTH_TOKEN" +ANTHROPIC_BASE_URL_KEY = "ANTHROPIC_BASE_URL" + + +def merge_claude_settings_static_token( + settings: Dict[str, JsonValue], base_url: str, auth_token: str +) -> Dict[str, JsonValue]: + """Return a new settings dict wired to a local ephemeral proxy with a static token. + + Unlike up.py's merge_claude_settings (which sets apiKeyHelper for a long-lived, real + remote proxy needing refreshable SSO tokens), this proxy is ephemeral and its key was just + minted for this session, so a plain env var is simpler and correct. Any existing + apiKeyHelper is cleared so it can't fight with the static token. + """ + raw_env = settings.get(ENV_KEY, {}) + base_env = raw_env if isinstance(raw_env, dict) else {} + env: Dict[str, JsonValue] = { + **base_env, + ANTHROPIC_BASE_URL_KEY: base_url.rstrip("/"), + ANTHROPIC_AUTH_TOKEN_KEY: auth_token, + } + env.pop(ANTHROPIC_API_KEY_KEY, None) + merged: Dict[str, JsonValue] = {**settings, ENV_KEY: env} + merged.pop(API_KEY_HELPER_KEY, None) + return merged + + +__all__ = ["merge_claude_settings_static_token"] diff --git a/litellm/proxy/client/cli/commands/autoroute/wizard.py b/litellm/proxy/client/cli/commands/autoroute/wizard.py new file mode 100644 index 00000000000..e2d1b85dc6f --- /dev/null +++ b/litellm/proxy/client/cli/commands/autoroute/wizard.py @@ -0,0 +1,106 @@ +from pathlib import Path +from typing import Tuple + +import click +import yaml +from rich.console import Console +from rich.table import Table + +from .... import Client +from .config import ( + TIER_NAMES, + AutorouteConfig, + ConfigGenerationError, + DiscoveredModel, + HeuristicClassifier, + LLMClassifier, + NoSemanticMatching, + SemanticMatching, + build_generated_model_list, + chat_models, + embedding_models, + parse_discovered_models, + validate_config, +) +from .process import CONFIG_PATH + + +def _render_and_prompt_for_model(models: Tuple[DiscoveredModel, ...], prompt_label: str) -> str: + console = Console() + table = Table(title=f"Pick a model for {prompt_label}") + table.add_column("Index", style="cyan", no_wrap=True) + table.add_column("Model", style="magenta") + for i, model in enumerate(models): + table.add_row(str(i + 1), model.name) + console.print(table) + + while True: + choice = click.prompt(f"\nSelect a model for {prompt_label} by index", type=str).strip() + try: + index = int(choice) - 1 + except ValueError: + click.echo("Invalid input. Please enter a number.") + continue + if 0 <= index < len(models): + return models[index].name + click.echo(f"Invalid selection. Please enter a number between 1 and {len(models)}") + + +def run_configure_wizard(ctx: click.Context) -> Path: + """Discover the caller's accessible models, walk them through tier assignment, write config.""" + base_url = ctx.obj["base_url"] + api_key = ctx.obj["api_key"] + client = Client(base_url=base_url, api_key=api_key) + + raw_groups = client.model_groups.info() + assert isinstance(raw_groups, list) + discovered = parse_discovered_models(raw_groups) + chat_pool = chat_models(discovered) + embedding_pool = embedding_models(discovered) + + if not chat_pool: + raise click.ClickException("Your key has no chat-capable models available on this proxy.") + + click.echo("Assign a model to each complexity tier (from what your key can access):") + tiers = {tier: _render_and_prompt_for_model(chat_pool, tier) for tier in TIER_NAMES} + default_model = tiers["MEDIUM"] + + classifier = HeuristicClassifier() + if click.confirm("\nUse an LLM classifier instead of the free heuristic scorer?", default=False): + classifier_model = _render_and_prompt_for_model(chat_pool, "LLM classifier") + classifier = LLMClassifier(model=classifier_model) + + semantic_matching = NoSemanticMatching() + if embedding_pool and click.confirm("\nEnable semantic keyword matching?", default=False): + embedding_model = _render_and_prompt_for_model(embedding_pool, "semantic embeddings") + semantic_matching = SemanticMatching(embedding_model=embedding_model) + + adaptive = click.confirm("\nEnable adaptive (bandit) selection on top of tiering?", default=False) + + config = AutorouteConfig( + base_url=base_url, + api_key=api_key, + tiers=tiers, + default_model=default_model, + classifier=classifier, + semantic_matching=semantic_matching, + adaptive=adaptive, + ) + try: + validate_config(config, discovered) + except ConfigGenerationError as e: + raise click.ClickException(str(e)) + + model_list = build_generated_model_list(config) + CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True) + with open(CONFIG_PATH, "w") as f: + yaml.safe_dump({"model_list": model_list}, f, sort_keys=False) + CONFIG_PATH.chmod(0o600) + + click.echo(f"\nWrote {CONFIG_PATH}") + for tier, model in tiers.items(): + click.echo(f" {tier}: {model}") + return CONFIG_PATH + + +__all__ = ["run_configure_wizard"] diff --git a/litellm/proxy/client/cli/commands/model_groups.py b/litellm/proxy/client/cli/commands/model_groups.py new file mode 100644 index 00000000000..629a0b5aaf1 --- /dev/null +++ b/litellm/proxy/client/cli/commands/model_groups.py @@ -0,0 +1,54 @@ +from typing import Literal + +import click +import rich +import rich.table + +from ... import Client + + +def create_client(ctx: click.Context) -> Client: + return Client(base_url=ctx.obj["base_url"], api_key=ctx.obj["api_key"]) + + +@click.group(name="model-groups") +def model_groups() -> None: + """Inspect model groups your key can access on the proxy""" + + +@model_groups.command("list") +@click.option( + "--format", + "output_format", + type=click.Choice(["table", "json"]), + default="table", + help="Output format (table or json)", +) +@click.pass_context +def list_model_groups(ctx: click.Context, output_format: Literal["table", "json"]) -> None: + """List model groups accessible to your key, with mode and pricing""" + client = create_client(ctx) + groups = client.model_groups.info() + assert isinstance(groups, list) + + if output_format == "json": + rich.print_json(data=groups) + return + + table = rich.table.Table(title="Accessible Model Groups") + table.add_column("Model", style="cyan") + table.add_column("Mode", style="green") + table.add_column("Input $/token", style="yellow") + table.add_column("Output $/token", style="yellow") + + for group in groups: + table.add_row( + str(group.get("model_group", "")), + str(group.get("mode", "chat")), + str(group.get("input_cost_per_token", "")), + str(group.get("output_cost_per_token", "")), + ) + rich.print(table) + + +__all__ = ["model_groups"] diff --git a/litellm/proxy/client/cli/commands/up.py b/litellm/proxy/client/cli/commands/up.py index fb9fb2b3fa4..b59a31147e3 100644 --- a/litellm/proxy/client/cli/commands/up.py +++ b/litellm/proxy/client/cli/commands/up.py @@ -68,34 +68,40 @@ def merge_claude_settings( return {**settings, ENV_KEY: env, API_KEY_HELPER_KEY: api_key_helper} -def write_backup(record: BackupRecord) -> None: - BACKUP_PATH.parent.mkdir(exist_ok=True) - with open(BACKUP_PATH, "w") as f: +def write_backup(record: BackupRecord, backup_path: Optional[Path] = None) -> None: + path = backup_path if backup_path is not None else BACKUP_PATH + path.parent.mkdir(exist_ok=True) + with open(path, "w") as f: json.dump({"existed": record.existed, "content": record.content}, f, indent=2) - os.chmod(BACKUP_PATH, 0o600) + os.chmod(path, 0o600) -def read_backup() -> Optional[BackupRecord]: - if not BACKUP_PATH.exists(): +def read_backup(backup_path: Optional[Path] = None) -> Optional[BackupRecord]: + path = backup_path if backup_path is not None else BACKUP_PATH + if not path.exists(): return None - with open(BACKUP_PATH, "r") as f: + with open(path, "r") as f: return _BACKUP_RECORD_ADAPTER.validate_json(f.read()) -def restore_claude_settings() -> Optional[BackupRecord]: - """Restore ~/.claude/settings.json from the backup, then delete the backup. +def restore_claude_settings( + settings_path: Optional[Path] = None, backup_path: Optional[Path] = None +) -> Optional[BackupRecord]: + """Restore settings_path from the backup at backup_path, then delete the backup. Returns the restored record, or None if there was nothing to restore. """ - record = read_backup() + resolved_settings_path = settings_path if settings_path is not None else CLAUDE_SETTINGS_PATH + resolved_backup_path = backup_path if backup_path is not None else BACKUP_PATH + record = read_backup(resolved_backup_path) if record is None: return None if record.existed and record.content is not None: - with open(CLAUDE_SETTINGS_PATH, "w") as f: + with open(resolved_settings_path, "w") as f: json.dump(record.content, f, indent=2) - elif CLAUDE_SETTINGS_PATH.exists(): - CLAUDE_SETTINGS_PATH.unlink() - BACKUP_PATH.unlink() + elif resolved_settings_path.exists(): + resolved_settings_path.unlink() + resolved_backup_path.unlink() return record diff --git a/litellm/proxy/client/cli/main.py b/litellm/proxy/client/cli/main.py index 322ce2b287a..e641956b2c5 100644 --- a/litellm/proxy/client/cli/main.py +++ b/litellm/proxy/client/cli/main.py @@ -9,11 +9,13 @@ from .commands.agents import agent_commands from .commands.auth import auth_group, get_stored_api_key, login, logout, whoami +from .commands.autoroute.commands import autoroute_group from .commands.chat import chat from .commands.credentials import credentials from .commands.encryption import encryption from .commands.http import http from .commands.keys import keys +from .commands.model_groups import model_groups # local imports from .commands.models import models @@ -135,6 +137,10 @@ def version(ctx: click.Context): # Add the up/down commands (route Claude Code through the local LiteLLM proxy) cli.add_command(up) cli.add_command(down) +# Add the model-groups command group (discover models your key can access) +cli.add_command(model_groups) +# Add the autoroute command group (QA auto-routing against your real proxy) +cli.add_command(autoroute_group, name="autoroute") if __name__ == "__main__": diff --git a/tests/test_litellm/proxy/client/cli/autoroute/__init__.py b/tests/test_litellm/proxy/client/cli/autoroute/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py b/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py new file mode 100644 index 00000000000..098659d5e8b --- /dev/null +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py @@ -0,0 +1,175 @@ +import json +from typing import Optional + +import yaml +from click.testing import CliRunner + +from litellm.proxy.client.cli.commands.autoroute import commands as commands_module +from litellm.proxy.client.cli.commands.autoroute import process as process_module +from litellm.proxy.client.cli.commands.autoroute.commands import down, up +from litellm.proxy.client.cli.commands.autoroute.process import PidRecord, ProcessLaunchError, write_pid_record +from litellm.proxy.client.cli.commands.up import BackupRecord as ClaudeBackupRecord +from litellm.proxy.client.cli.commands.up import write_backup + + +class FakeProcess: + def __init__(self, pid: int): + self.pid = pid + self.returncode: Optional[int] = None + + def poll(self) -> Optional[int]: + return self.returncode + + +def _patch_paths(monkeypatch, tmp_path): + config_path = tmp_path / "config.yaml" + log_path = tmp_path / "proxy.log" + claude_settings_path = tmp_path / "claude_settings.json" + backup_path = tmp_path / "backup.json" + pid_record_path = tmp_path / "pid.json" + + monkeypatch.setattr(commands_module, "CONFIG_PATH", config_path) + monkeypatch.setattr(commands_module, "LOG_PATH", log_path) + monkeypatch.setattr(commands_module, "CLAUDE_SETTINGS_PATH", claude_settings_path) + monkeypatch.setattr(commands_module, "AUTOROUTE_BACKUP_PATH", backup_path) + monkeypatch.setattr(process_module, "PID_RECORD_PATH", pid_record_path) + + return config_path, log_path, claude_settings_path, backup_path, pid_record_path + + +def _silence_signal_handling(monkeypatch): + monkeypatch.setattr(commands_module.signal, "signal", lambda *a, **k: None) + monkeypatch.setattr(commands_module.atexit, "register", lambda *a, **k: None) + monkeypatch.setattr(commands_module, "stream_log", lambda *a, **k: None) + + +class TestUpCommand: + def setup_method(self): + self.runner = CliRunner() + + def test_refuses_when_never_configured(self, monkeypatch, tmp_path): + _patch_paths(monkeypatch, tmp_path) + + result = self.runner.invoke(up) + + assert result.exit_code != 0 + assert "lite autoroute configure" in result.output + + def test_refuses_when_pid_record_exists_and_process_still_running(self, monkeypatch, tmp_path): + config_path, _log_path, _settings_path, _backup_path, pid_record_path = _patch_paths(monkeypatch, tmp_path) + config_path.write_text(yaml.safe_dump({"model_list": []})) + write_pid_record( + PidRecord(pid=123, port=4000, config_path=str(config_path), log_path="/tmp/proxy.log"), pid_record_path + ) + monkeypatch.setattr(commands_module, "is_running", lambda pid: True) + + result = self.runner.invoke(up) + + assert result.exit_code != 0 + assert "already running" in result.output + assert "lite autoroute down" in result.output + assert config_path.read_text() == yaml.safe_dump({"model_list": []}) + + def test_happy_path_patches_settings_then_restores_everything_on_stop(self, monkeypatch, tmp_path): + config_path, log_path, claude_settings_path, backup_path, pid_record_path = _patch_paths(monkeypatch, tmp_path) + config_path.write_text(yaml.safe_dump({"model_list": []})) + original_settings = {"theme": "dark"} + claude_settings_path.write_text(json.dumps(original_settings)) + _silence_signal_handling(monkeypatch) + + fake_process = FakeProcess(pid=99999) + terminate_calls = [] + monkeypatch.setattr(commands_module, "launch_proxy", lambda *a, **k: fake_process) + monkeypatch.setattr(commands_module, "poll_liveliness", lambda *a, **k: None) + monkeypatch.setattr(commands_module, "allocate_free_port", lambda: 54321) + monkeypatch.setattr(commands_module, "terminate", lambda pid, **k: terminate_calls.append(pid)) + monkeypatch.setattr(commands_module.secrets, "token_urlsafe", lambda n: "fixed-master-key") + + captured = {} + + def fake_wait(self, timeout=None): + captured["settings"] = json.loads(claude_settings_path.read_text()) + captured["backup_existed"] = backup_path.exists() + return True + + monkeypatch.setattr("threading.Event.wait", fake_wait) + + result = self.runner.invoke(up) + + assert result.exit_code == 0, result.output + assert captured["backup_existed"] is True + assert captured["settings"]["theme"] == "dark" + assert captured["settings"]["env"]["ANTHROPIC_BASE_URL"] == "http://127.0.0.1:54321" + assert captured["settings"]["env"]["ANTHROPIC_AUTH_TOKEN"] == "fixed-master-key" + assert "apiKeyHelper" not in captured["settings"] + + assert terminate_calls == [99999] + assert not pid_record_path.exists() + assert not backup_path.exists() + assert json.loads(claude_settings_path.read_text()) == original_settings + + written_config = yaml.safe_load(config_path.read_text()) + assert written_config["general_settings"]["master_key"] == "fixed-master-key" + + def test_surfaces_clean_error_and_cleans_up_when_health_check_fails(self, monkeypatch, tmp_path): + config_path, _log_path, claude_settings_path, backup_path, pid_record_path = _patch_paths(monkeypatch, tmp_path) + config_path.write_text(yaml.safe_dump({"model_list": []})) + original_settings = {"theme": "dark"} + claude_settings_path.write_text(json.dumps(original_settings)) + + fake_process = FakeProcess(pid=555) + + def _raise_launch_error(*args, **kwargs): + raise ProcessLaunchError("boom: proxy never became healthy") + + monkeypatch.setattr(commands_module, "launch_proxy", lambda *a, **k: fake_process) + monkeypatch.setattr(commands_module, "poll_liveliness", _raise_launch_error) + monkeypatch.setattr(commands_module, "allocate_free_port", lambda: 12345) + monkeypatch.setattr(commands_module.secrets, "token_urlsafe", lambda n: "fixed-master-key") + + result = self.runner.invoke(up) + + assert result.exit_code != 0 + assert "boom" in result.output + assert not pid_record_path.exists() + assert not backup_path.exists() + assert json.loads(claude_settings_path.read_text()) == original_settings + + +class TestDownCommand: + def setup_method(self): + self.runner = CliRunner() + + def test_restores_settings_and_terminates_when_process_still_running(self, monkeypatch, tmp_path): + _config_path, _log_path, claude_settings_path, backup_path, pid_record_path = _patch_paths( + monkeypatch, tmp_path + ) + original_settings = {"theme": "dark"} + write_backup(ClaudeBackupRecord(existed=True, content=original_settings), backup_path) + claude_settings_path.write_text(json.dumps({"env": {"ANTHROPIC_AUTH_TOKEN": "fixed-master-key"}})) + write_pid_record(PidRecord(pid=777, port=1234, config_path="c", log_path="l"), pid_record_path) + + terminate_calls = [] + monkeypatch.setattr(commands_module, "is_running", lambda pid: True) + monkeypatch.setattr(commands_module, "terminate", lambda pid, **k: terminate_calls.append(pid)) + + result = self.runner.invoke(down) + + assert result.exit_code == 0, result.output + assert "Stopped leftover ephemeral proxy" in result.output + assert "Restored" in result.output + assert terminate_calls == [777] + assert not pid_record_path.exists() + assert not backup_path.exists() + assert json.loads(claude_settings_path.read_text()) == original_settings + + def test_is_a_clean_no_op_when_nothing_is_running_and_no_backup_exists(self, monkeypatch, tmp_path): + _config_path, _log_path, claude_settings_path, _backup_path, _pid_record_path = _patch_paths( + monkeypatch, tmp_path + ) + + result = self.runner.invoke(down) + + assert result.exit_code == 0, result.output + assert "Nothing to restore." in result.output + assert not claude_settings_path.exists() diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_config.py b/tests/test_litellm/proxy/client/cli/autoroute/test_config.py new file mode 100644 index 00000000000..a48db0c79e4 --- /dev/null +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_config.py @@ -0,0 +1,167 @@ +from typing import Any, Dict, Tuple + +import pytest + +from litellm.proxy.client.cli.commands.autoroute.config import ( + AutorouteConfig, + ConfigGenerationError, + DiscoveredModel, + HeuristicClassifier, + LLMClassifier, + NoSemanticMatching, + SemanticMatching, + build_generated_model_list, + build_generated_proxy_config, + chat_models, + embedding_models, + parse_discovered_models, + validate_config, +) + +DISCOVERED: Tuple[DiscoveredModel, ...] = ( + DiscoveredModel(name="gpt-4o-mini", mode="chat"), + DiscoveredModel(name="gpt-4o", mode="chat"), + DiscoveredModel(name="o1", mode="chat"), + DiscoveredModel(name="text-embedding-3-small", mode="embedding"), +) + + +def _base_config(**overrides: Any) -> AutorouteConfig: + defaults: Dict[str, Any] = { + "base_url": "http://real-proxy.internal:4000", + "api_key": "sk-real-key", + "tiers": { + "SIMPLE": "gpt-4o-mini", + "MEDIUM": "gpt-4o", + "COMPLEX": "gpt-4o", + "REASONING": "o1", + }, + "default_model": "gpt-4o", + } + defaults.update(overrides) + return AutorouteConfig(**defaults) + + +class TestParseDiscoveredModels: + def test_parses_valid_raw_list_into_typed_tuple(self): + raw = [ + { + "model_group": "gpt-4o", + "mode": "chat", + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + }, + {"model_group": "text-embedding-3-small", "mode": "embedding"}, + ] + result = parse_discovered_models(raw) + assert result == ( + DiscoveredModel(name="gpt-4o", mode="chat", input_cost_per_token=0.01, output_cost_per_token=0.02), + DiscoveredModel(name="text-embedding-3-small", mode="embedding"), + ) + + def test_ignores_unknown_extra_fields(self): + raw = [{"model_group": "gpt-4o", "mode": "chat", "totally_unknown_field": "whatever"}] + result = parse_discovered_models(raw) + assert result == (DiscoveredModel(name="gpt-4o", mode="chat"),) + + def test_missing_mode_defaults_to_chat(self): + raw = [{"model_group": "gpt-4o"}] + result = parse_discovered_models(raw) + assert result[0].mode == "chat" + + +class TestChatAndEmbeddingFiltering: + def test_filters_by_mode(self): + models = ( + DiscoveredModel(name="gpt-4o", mode="chat"), + DiscoveredModel(name="text-embedding-3-small", mode="embedding"), + DiscoveredModel(name="claude", mode="chat"), + ) + assert chat_models(models) == (models[0], models[2]) + assert embedding_models(models) == (models[1],) + + +class TestBuildGeneratedModelList: + def test_dedups_model_used_in_multiple_roles(self): + config = _base_config(classifier=LLMClassifier(model="gpt-4o")) + model_list = build_generated_model_list(config) + gpt4o_entries = [m for m in model_list if m["model_name"] == "gpt-4o"] + assert len(gpt4o_entries) == 1 + + def test_every_proxy_deployment_points_back_at_customer_proxy(self): + config = _base_config() + model_list = build_generated_model_list(config) + proxy_entries = [m for m in model_list if m["model_name"] != "autorouter"] + names = {m["model_name"] for m in proxy_entries} + assert names == {"gpt-4o-mini", "gpt-4o", "o1"} + for entry in proxy_entries: + assert entry["litellm_params"]["model"] == f"litellm_proxy/{entry['model_name']}" + assert entry["litellm_params"]["api_base"] == config.base_url + assert entry["litellm_params"]["api_key"] == config.api_key + + def test_complexity_router_config_reflects_llm_classifier(self): + config = _base_config(classifier=LLMClassifier(model="gpt-4o", timeout_ms=1234)) + autorouter = next(m for m in build_generated_model_list(config) if m["model_name"] == "autorouter") + router_config = autorouter["litellm_params"]["complexity_router_config"] + assert router_config["classifier_type"] == "llm" + assert router_config["classifier_llm_config"] == {"model": "gpt-4o", "timeout_ms": 1234} + assert "semantic_keyword_matching" not in router_config + assert "adaptive" not in router_config + + def test_complexity_router_config_reflects_semantic_matching(self): + config = _base_config( + semantic_matching=SemanticMatching(embedding_model="text-embedding-3-small", match_threshold=0.7) + ) + autorouter = next(m for m in build_generated_model_list(config) if m["model_name"] == "autorouter") + router_config = autorouter["litellm_params"]["complexity_router_config"] + assert router_config["semantic_keyword_matching"] is True + assert router_config["embedding_model"] == "text-embedding-3-small" + assert router_config["match_threshold"] == 0.7 + assert router_config["keyword_tier_rules"] + assert "classifier_type" not in router_config + + def test_complexity_router_config_reflects_adaptive(self): + config = _base_config(adaptive=True) + autorouter = next(m for m in build_generated_model_list(config) if m["model_name"] == "autorouter") + assert autorouter["litellm_params"]["complexity_router_config"]["adaptive"] is True + + def test_default_classifier_and_semantic_matching_add_no_extra_keys(self): + config = _base_config(classifier=HeuristicClassifier(), semantic_matching=NoSemanticMatching()) + autorouter = next(m for m in build_generated_model_list(config) if m["model_name"] == "autorouter") + router_config = autorouter["litellm_params"]["complexity_router_config"] + assert set(router_config.keys()) == {"tiers", "default_model"} + + +class TestBuildGeneratedProxyConfig: + def test_embeds_master_key_under_general_settings(self): + config = _base_config() + proxy_config = build_generated_proxy_config(config, "sk-master-123") + assert proxy_config["general_settings"] == {"master_key": "sk-master-123"} + assert proxy_config["model_list"] == build_generated_model_list(config) + + +class TestValidateConfig: + def test_passes_for_fully_valid_config(self): + validate_config(_base_config(), DISCOVERED) + + def test_raises_for_tier_referencing_unknown_model(self): + config = _base_config( + tiers={"SIMPLE": "unknown-model", "MEDIUM": "gpt-4o", "COMPLEX": "gpt-4o", "REASONING": "o1"} + ) + with pytest.raises(ConfigGenerationError, match="unknown-model"): + validate_config(config, DISCOVERED) + + def test_raises_for_unknown_default_model(self): + config = _base_config(default_model="unknown-model") + with pytest.raises(ConfigGenerationError, match="unknown-model"): + validate_config(config, DISCOVERED) + + def test_raises_for_unknown_llm_classifier_model(self): + config = _base_config(classifier=LLMClassifier(model="unknown-model")) + with pytest.raises(ConfigGenerationError, match="unknown-model"): + validate_config(config, DISCOVERED) + + def test_raises_for_unknown_semantic_embedding_model(self): + config = _base_config(semantic_matching=SemanticMatching(embedding_model="unknown-embedding")) + with pytest.raises(ConfigGenerationError, match="unknown-embedding"): + validate_config(config, DISCOVERED) diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_process.py b/tests/test_litellm/proxy/client/cli/autoroute/test_process.py new file mode 100644 index 00000000000..bffd8b598f4 --- /dev/null +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_process.py @@ -0,0 +1,113 @@ +import os +import socket +from typing import Optional + +import pytest + +from litellm.proxy.client.cli.commands.autoroute import process as process_module +from litellm.proxy.client.cli.commands.autoroute.process import ( + PidRecord, + ProcessLaunchError, + allocate_free_port, + clear_pid_record, + is_running, + poll_liveliness, + read_pid_record, + write_pid_record, +) + + +class FakeProcess: + def __init__(self, returncode: Optional[int] = None): + self.returncode = returncode + + def poll(self) -> Optional[int]: + return self.returncode + + +class FakeResponse: + def __init__(self, status_code: int): + self.status_code = status_code + + +def test_allocate_free_port_returns_a_bindable_port(): + port = allocate_free_port() + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as sock: + sock.bind(("127.0.0.1", port)) + + +class TestPidRecordRoundTrip: + def test_write_then_read_round_trips(self, tmp_path): + path = tmp_path / "pid.json" + record = PidRecord(pid=123, port=4000, config_path="/tmp/config.yaml", log_path="/tmp/proxy.log") + + write_pid_record(record, path) + + assert read_pid_record(path) == record + + def test_read_missing_file_returns_none(self, tmp_path): + assert read_pid_record(tmp_path / "missing.json") is None + + def test_clear_removes_an_existing_record(self, tmp_path): + path = tmp_path / "pid.json" + write_pid_record(PidRecord(pid=1, port=1, config_path="a", log_path="b"), path) + assert path.exists() + + clear_pid_record(path) + + assert not path.exists() + + def test_clear_missing_file_is_a_no_op(self, tmp_path): + clear_pid_record(tmp_path / "missing.json") + + def test_write_creates_parent_directories(self, tmp_path): + path = tmp_path / "nested" / "dir" / "pid.json" + + write_pid_record(PidRecord(pid=1, port=1, config_path="a", log_path="b"), path) + + assert path.exists() + + +class TestIsRunning: + def test_current_process_is_running(self): + assert is_running(os.getpid()) is True + + def test_huge_unlikely_pid_is_not_running(self): + assert is_running(2**30) is False + + def test_permission_error_from_kill_is_treated_as_running(self, monkeypatch): + def fake_kill(pid: int, sig: int) -> None: + raise PermissionError("not permitted to signal this pid") + + monkeypatch.setattr(process_module.os, "kill", fake_kill) + + assert is_running(999) is True + + +class TestPollLiveliness: + def test_succeeds_when_health_check_returns_200_quickly(self, monkeypatch, tmp_path): + monkeypatch.setattr(process_module.requests, "get", lambda url, timeout: FakeResponse(200)) + + poll_liveliness("http://127.0.0.1:4000", tmp_path / "proxy.log", FakeProcess(), timeout=5.0) + + def test_raises_with_log_tail_when_timeout_elapses(self, monkeypatch, tmp_path): + log_path = tmp_path / "proxy.log" + log_path.write_text("line one\nline two\nline three\n") + monkeypatch.setattr(process_module.requests, "get", lambda url, timeout: FakeResponse(500)) + monkeypatch.setattr(process_module.time, "sleep", lambda seconds: None) + + with pytest.raises(ProcessLaunchError) as exc_info: + poll_liveliness("http://127.0.0.1:4000", log_path, FakeProcess(), timeout=0.05) + + assert "never became healthy" in str(exc_info.value) + assert "line three" in str(exc_info.value) + + def test_raises_immediately_when_process_already_exited(self, tmp_path): + log_path = tmp_path / "proxy.log" + log_path.write_text("crash log line") + + with pytest.raises(ProcessLaunchError) as exc_info: + poll_liveliness("http://127.0.0.1:4000", log_path, FakeProcess(returncode=1), timeout=5.0) + + assert "exited early" in str(exc_info.value) + assert "crash log line" in str(exc_info.value) diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_settings.py b/tests/test_litellm/proxy/client/cli/autoroute/test_settings.py new file mode 100644 index 00000000000..203d77b70c1 --- /dev/null +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_settings.py @@ -0,0 +1,36 @@ +from litellm.proxy.client.cli.commands.autoroute.settings import merge_claude_settings_static_token + + +def test_preserves_unrelated_top_level_keys(): + merged = merge_claude_settings_static_token({"theme": "dark"}, "http://127.0.0.1:4000", "token-abc") + assert merged["theme"] == "dark" + + +def test_preserves_unrelated_env_keys(): + settings = {"env": {"SOME_OTHER_VAR": "value"}} + merged = merge_claude_settings_static_token(settings, "http://127.0.0.1:4000", "token-abc") + assert merged["env"]["SOME_OTHER_VAR"] == "value" + + +def test_sets_base_url_and_auth_token(): + merged = merge_claude_settings_static_token({}, "http://127.0.0.1:4000/", "token-abc") + assert merged["env"]["ANTHROPIC_BASE_URL"] == "http://127.0.0.1:4000" + assert merged["env"]["ANTHROPIC_AUTH_TOKEN"] == "token-abc" + + +def test_drops_stray_api_key(): + settings = {"env": {"ANTHROPIC_API_KEY": "leaked-key"}} + merged = merge_claude_settings_static_token(settings, "http://127.0.0.1:4000", "token-abc") + assert "ANTHROPIC_API_KEY" not in merged["env"] + + +def test_removes_existing_api_key_helper(): + settings = {"apiKeyHelper": "/usr/local/bin/lite auth print-token"} + merged = merge_claude_settings_static_token(settings, "http://127.0.0.1:4000", "token-abc") + assert "apiKeyHelper" not in merged + + +def test_does_not_mutate_input(): + settings = {"env": {"FOO": "bar"}, "apiKeyHelper": "old-helper"} + merge_claude_settings_static_token(settings, "http://127.0.0.1:4000", "token-abc") + assert settings == {"env": {"FOO": "bar"}, "apiKeyHelper": "old-helper"} diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py b/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py new file mode 100644 index 00000000000..81f54bf9b78 --- /dev/null +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py @@ -0,0 +1,188 @@ +from typing import Any, Dict, List, Tuple +from unittest.mock import patch + +import click +import yaml +from click.testing import CliRunner + +from litellm.proxy.client.cli.commands.autoroute import wizard as wizard_module +from litellm.proxy.client.cli.commands.autoroute.config import DiscoveredModel +from litellm.proxy.client.cli.commands.autoroute.wizard import ( + _render_and_prompt_for_model, + run_configure_wizard, +) + +CHAT_AND_EMBEDDING_GROUPS: List[Dict[str, Any]] = [ + {"model_group": "gpt-4o-mini", "mode": "chat", "input_cost_per_token": 0.01, "output_cost_per_token": 0.02}, + {"model_group": "gpt-4o", "mode": "chat", "input_cost_per_token": 0.01, "output_cost_per_token": 0.02}, + {"model_group": "claude-opus", "mode": "chat"}, + {"model_group": "o1", "mode": "chat"}, + {"model_group": "text-embedding-3-small", "mode": "embedding"}, +] + +CHAT_ONLY_GROUPS: List[Dict[str, Any]] = [ + {"model_group": "gpt-4o-mini", "mode": "chat"}, + {"model_group": "gpt-4o", "mode": "chat"}, + {"model_group": "claude-opus", "mode": "chat"}, + {"model_group": "o1", "mode": "chat"}, +] + +EMBEDDING_ONLY_GROUPS: List[Dict[str, Any]] = [ + {"model_group": "text-embedding-3-small", "mode": "embedding"}, +] + + +@click.command() +@click.pass_context +def _invoke_wizard(ctx: click.Context) -> None: + run_configure_wizard(ctx) + + +def _run(tmp_path, raw_groups: List[Dict[str, Any]], input_str: str): + config_path = tmp_path / "config.yaml" + runner = CliRunner() + with ( + patch.object(wizard_module, "Client") as mock_client_cls, + patch.object(wizard_module, "CONFIG_PATH", config_path), + ): + mock_client_cls.return_value.model_groups.info.return_value = raw_groups + result = runner.invoke( + _invoke_wizard, + obj={"base_url": "http://localhost:4000", "api_key": "sk-test"}, + input=input_str, + ) + return result, config_path + + +def _router_config(config_path) -> Dict[str, Any]: + written = yaml.safe_load(config_path.read_text()) + autorouter = next(m for m in written["model_list"] if m["model_name"] == "autorouter") + return autorouter["litellm_params"]["complexity_router_config"] + + +class TestRunConfigureWizardHappyPath: + def test_assigns_tiers_and_declines_everything(self, tmp_path): + result, config_path = _run( + tmp_path, + CHAT_AND_EMBEDDING_GROUPS, + input_str="1\n2\n3\n4\nn\nn\nn\n", + ) + + assert result.exit_code == 0, result.output + router_config = _router_config(config_path) + assert router_config["tiers"] == { + "SIMPLE": "gpt-4o-mini", + "MEDIUM": "gpt-4o", + "COMPLEX": "claude-opus", + "REASONING": "o1", + } + assert router_config["default_model"] == "gpt-4o" + assert "classifier_type" not in router_config + assert "classifier_llm_config" not in router_config + assert "semantic_keyword_matching" not in router_config + assert "adaptive" not in router_config + + def test_writes_config_file_with_restricted_permissions(self, tmp_path): + result, config_path = _run( + tmp_path, + CHAT_AND_EMBEDDING_GROUPS, + input_str="1\n2\n3\n4\nn\nn\nn\n", + ) + + assert result.exit_code == 0, result.output + assert config_path.exists() + assert oct(config_path.stat().st_mode)[-3:] == "600" + + def test_no_embedding_pool_skips_semantic_prompt_entirely(self, tmp_path): + result, config_path = _run( + tmp_path, + CHAT_ONLY_GROUPS, + input_str="1\n2\n3\n4\nn\nn\n", + ) + + assert result.exit_code == 0, result.output + router_config = _router_config(config_path) + assert "semantic_keyword_matching" not in router_config + + +class TestRunConfigureWizardLLMClassifier: + def test_accepting_llm_classifier_records_chosen_model(self, tmp_path): + result, config_path = _run( + tmp_path, + CHAT_AND_EMBEDDING_GROUPS, + input_str="1\n2\n3\n4\ny\n2\nn\nn\n", + ) + + assert result.exit_code == 0, result.output + router_config = _router_config(config_path) + assert router_config["classifier_type"] == "llm" + assert router_config["classifier_llm_config"]["model"] == "gpt-4o" + + +class TestRunConfigureWizardSemanticMatching: + def test_accepting_semantic_matching_records_embedding_model(self, tmp_path): + result, config_path = _run( + tmp_path, + CHAT_AND_EMBEDDING_GROUPS, + input_str="1\n2\n3\n4\nn\ny\n1\nn\n", + ) + + assert result.exit_code == 0, result.output + router_config = _router_config(config_path) + assert router_config["semantic_keyword_matching"] is True + assert router_config["embedding_model"] == "text-embedding-3-small" + + +class TestRunConfigureWizardAdaptive: + def test_accepting_adaptive_sets_adaptive_flag(self, tmp_path): + result, config_path = _run( + tmp_path, + CHAT_AND_EMBEDDING_GROUPS, + input_str="1\n2\n3\n4\nn\nn\ny\n", + ) + + assert result.exit_code == 0, result.output + router_config = _router_config(config_path) + assert router_config["adaptive"] is True + + +class TestRunConfigureWizardNoChatModels: + def test_fails_cleanly_without_prompting_when_no_chat_models(self, tmp_path): + result, config_path = _run(tmp_path, EMBEDDING_ONLY_GROUPS, input_str="") + + assert result.exit_code != 0 + assert "no chat-capable models" in result.output.lower() + assert not config_path.exists() + + +class TestRenderAndPromptForModel: + def _models(self) -> Tuple[DiscoveredModel, ...]: + return ( + DiscoveredModel(name="model-a"), + DiscoveredModel(name="model-b"), + ) + + def test_reprompts_on_non_numeric_input(self): + with patch("click.prompt", side_effect=["not-a-number", "2"]): + result = _render_and_prompt_for_model(self._models(), "test tier") + + assert result == "model-b" + + def test_reprompts_on_out_of_range_index(self): + with patch("click.prompt", side_effect=["5", "1"]): + result = _render_and_prompt_for_model(self._models(), "test tier") + + assert result == "model-a" + + def test_reprompts_on_zero_index(self): + with patch("click.prompt", side_effect=["0", "2"]): + result = _render_and_prompt_for_model(self._models(), "test tier") + + assert result == "model-b" + + def test_valid_first_answer_returns_immediately(self): + with patch("click.prompt", return_value="1") as mock_prompt: + result = _render_and_prompt_for_model(self._models(), "test tier") + + assert result == "model-a" + mock_prompt.assert_called_once() diff --git a/tests/test_litellm/proxy/client/cli/test_model_groups_commands.py b/tests/test_litellm/proxy/client/cli/test_model_groups_commands.py new file mode 100644 index 00000000000..d7b5b9dede4 --- /dev/null +++ b/tests/test_litellm/proxy/client/cli/test_model_groups_commands.py @@ -0,0 +1,104 @@ +import json +import os +from typing import Any, Dict, List +from unittest.mock import patch + +import pytest +from click.testing import CliRunner + +from litellm.proxy.client.cli import cli + +SAMPLE_MODEL_GROUPS: List[Dict[str, Any]] = [ + { + "model_group": "gpt-4o", + "mode": "chat", + "input_cost_per_token": 0.01, + "output_cost_per_token": 0.02, + }, + { + "model_group": "text-embedding-3-small", + "mode": "embedding", + "input_cost_per_token": 0.0001, + "output_cost_per_token": None, + }, +] + + +@pytest.fixture +def mock_client(): + with patch("litellm.proxy.client.cli.commands.model_groups.Client") as MockClient: + yield MockClient + + +@pytest.fixture +def cli_runner(): + return CliRunner() + + +@pytest.fixture(autouse=True) +def mock_env(): + with patch.dict( + os.environ, + { + "LITELLM_PROXY_URL": "http://localhost:4000", + "LITELLM_PROXY_API_KEY": "sk-test", + }, + ): + yield + + +def test_list_table_format_shows_model_names_and_modes(mock_client, cli_runner): + mock_client.return_value.model_groups.info.return_value = SAMPLE_MODEL_GROUPS + + result = cli_runner.invoke(cli, ["model-groups", "list"]) + + assert result.exit_code == 0, result.output + assert "gpt-4o" in result.output + assert "chat" in result.output + assert "text-embedding-3-small" in result.output + assert "embedding" in result.output + assert "0.01" in result.output + assert "0.02" in result.output + + mock_client.assert_called_once_with(base_url="http://localhost:4000", api_key="sk-test") + mock_client.return_value.model_groups.info.assert_called_once() + + +def test_list_table_format_defaults_missing_mode_to_chat(mock_client, cli_runner): + mock_client.return_value.model_groups.info.return_value = [{"model_group": "some-model"}] + + result = cli_runner.invoke(cli, ["model-groups", "list"]) + + assert result.exit_code == 0, result.output + assert "some-model" in result.output + assert "chat" in result.output + + +def test_list_json_format_round_trips_raw_data(mock_client, cli_runner): + mock_client.return_value.model_groups.info.return_value = SAMPLE_MODEL_GROUPS + + result = cli_runner.invoke(cli, ["model-groups", "list", "--format", "json"]) + + assert result.exit_code == 0, result.output + assert json.loads(result.output) == SAMPLE_MODEL_GROUPS + + +def test_list_with_custom_base_url_and_api_key(mock_client, cli_runner): + mock_client.return_value.model_groups.info.return_value = [] + + result = cli_runner.invoke( + cli, + ["--base-url", "http://custom.server:8000", "--api-key", "custom-key", "model-groups", "list"], + ) + + assert result.exit_code == 0, result.output + mock_client.assert_called_once_with(base_url="http://custom.server:8000", api_key="custom-key") + + +def test_list_error_handling(mock_client, cli_runner): + mock_client.return_value.model_groups.info.side_effect = Exception("API Error") + + result = cli_runner.invoke(cli, ["model-groups", "list"]) + + assert result.exit_code != 0 + assert "API Error" in str(result.exception) From 159c7ec8dac53883488bf0944caa26cfa8f5dc38 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 14 Jul 2026 12:06:22 -0700 Subject: [PATCH 02/12] feat(cli): allow multiple models per autoroute tier complexity_router already supports a pool of models per tier (randomly picked per request; adaptive mode specifically needs a pool to choose within), but the configure wizard only ever let you assign one. Tiers are now a tuple of model names; the wizard prompt accepts comma-separated indices to pick more than one per tier. --- litellm/proxy/client/cli/README.md | 2 +- .../client/cli/commands/autoroute/config.py | 16 +++-- .../client/cli/commands/autoroute/wizard.py | 59 ++++++++++++++----- .../proxy/client/cli/autoroute/test_config.py | 15 +++-- .../proxy/client/cli/autoroute/test_wizard.py | 20 +++++-- 5 files changed, 80 insertions(+), 32 deletions(-) diff --git a/litellm/proxy/client/cli/README.md b/litellm/proxy/client/cli/README.md index ecd5f3d48e2..86ad9687901 100644 --- a/litellm/proxy/client/cli/README.md +++ b/litellm/proxy/client/cli/README.md @@ -507,7 +507,7 @@ Lists the model groups your key can reach on the proxy, via `/model_group/info`, lite autoroute configure ``` -An interactive wizard. It runs the same model-group discovery as above, splits the results into chat-capable and embedding-capable pools, and asks you to assign a model from the chat pool to each of the four complexity tiers -- SIMPLE, MEDIUM, COMPLEX, REASONING. From there it optionally offers: classifying prompt complexity with an LLM (again picked from your discovered pool) instead of the free built-in heuristic scorer, semantic keyword matching for tier assignment (needs an embedding model from the pool), and adaptive (bandit-based) selection layered on top of tiering. +An interactive wizard. It runs the same model-group discovery as above, splits the results into chat-capable and embedding-capable pools, and asks you to assign one or more models from the chat pool to each of the four complexity tiers -- SIMPLE, MEDIUM, COMPLEX, REASONING (enter comma-separated indices to assign a pool of models to a tier instead of just one; complexity_router picks randomly among a tier's pool per request, and adaptive mode specifically depends on having more than one candidate to choose from). From there it optionally offers: classifying prompt complexity with an LLM (again picked from your discovered pool) instead of the free built-in heuristic scorer, semantic keyword matching for tier assignment (needs an embedding model from the pool), and adaptive (bandit-based) selection layered on top of tiering. The wizard writes the result to `~/.litellm/autorouter/config.yaml` with `0600` permissions, since the file embeds your real proxy API key. Every model referenced anywhere in that config -- tier targets, the classifier model, the embedding model -- becomes its own `litellm_proxy/` deployment whose `api_base` and `api_key` point back at your real proxy. That is the trick that keeps your real proxy's config untouched: every actual network call this generates, whether it is the routed completion, an LLM-classifier call, or an embedding call, forwards transparently through your real, already-running proxy with your real key. diff --git a/litellm/proxy/client/cli/commands/autoroute/config.py b/litellm/proxy/client/cli/commands/autoroute/config.py index 85b5223524b..c0f5f778fd8 100644 --- a/litellm/proxy/client/cli/commands/autoroute/config.py +++ b/litellm/proxy/client/cli/commands/autoroute/config.py @@ -103,7 +103,9 @@ class AutorouteConfig(BaseModel): base_url: str api_key: str - tiers: Dict[str, str] + # Each tier maps to a pool of one or more models; complexity_router picks randomly among + # them per request (or, in adaptive mode, learns which to prefer within the pool). + tiers: Dict[str, Tuple[str, ...]] default_model: str classifier: ClassifierChoice = Field(default_factory=HeuristicClassifier) semantic_matching: SemanticMatchingChoice = Field(default_factory=NoSemanticMatching) @@ -115,9 +117,10 @@ def validate_config(config: AutorouteConfig, discovered: Tuple[DiscoveredModel, chat_names: FrozenSet[str] = frozenset(m.name for m in chat_models(discovered)) embedding_names: FrozenSet[str] = frozenset(m.name for m in embedding_models(discovered)) - for tier, model in config.tiers.items(): - if model not in chat_names: - raise ConfigGenerationError(f"Tier {tier} references unknown chat model '{model}'") + for tier, models in config.tiers.items(): + for model in models: + if model not in chat_names: + raise ConfigGenerationError(f"Tier {tier} references unknown chat model '{model}'") if config.default_model not in chat_names: raise ConfigGenerationError(f"default_model '{config.default_model}' is not a known chat model") @@ -152,7 +155,8 @@ def build_generated_model_list(config: AutorouteConfig) -> List[JsonValue]: to exactly one `litellm_proxy/` deployment forwarding to the customer's real proxy, plus one `auto_router/complexity_router` deployment tying the tiers together. """ - referenced_names = {*config.tiers.values(), config.default_model} + referenced_names = {model for models in config.tiers.values() for model in models} + referenced_names.add(config.default_model) if isinstance(config.classifier, LLMClassifier): referenced_names.add(config.classifier.model) if isinstance(config.semantic_matching, SemanticMatching): @@ -163,7 +167,7 @@ def build_generated_model_list(config: AutorouteConfig) -> List[JsonValue]: ] complexity_router_config: Dict[str, JsonValue] = { - "tiers": dict(config.tiers), + "tiers": {tier: list(models) for tier, models in config.tiers.items()}, "default_model": config.default_model, } if isinstance(config.classifier, LLMClassifier): diff --git a/litellm/proxy/client/cli/commands/autoroute/wizard.py b/litellm/proxy/client/cli/commands/autoroute/wizard.py index e2d1b85dc6f..c3ac2bdd637 100644 --- a/litellm/proxy/client/cli/commands/autoroute/wizard.py +++ b/litellm/proxy/client/cli/commands/autoroute/wizard.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Tuple +from typing import List, Optional, Tuple import click import yaml @@ -25,25 +25,52 @@ from .process import CONFIG_PATH -def _render_and_prompt_for_model(models: Tuple[DiscoveredModel, ...], prompt_label: str) -> str: +def _render_model_table(models: Tuple[DiscoveredModel, ...], prompt_label: str) -> None: console = Console() - table = Table(title=f"Pick a model for {prompt_label}") + table = Table(title=f"Pick model(s) for {prompt_label}") table.add_column("Index", style="cyan", no_wrap=True) table.add_column("Model", style="magenta") for i, model in enumerate(models): table.add_row(str(i + 1), model.name) console.print(table) - while True: - choice = click.prompt(f"\nSelect a model for {prompt_label} by index", type=str).strip() + +def _parse_indices(choice: str, count: int) -> Optional[Tuple[int, ...]]: + raw_parts = [part.strip() for part in choice.split(",") if part.strip()] + if not raw_parts: + return None + indices: List[int] = [] + for part in raw_parts: try: - index = int(choice) - 1 + index = int(part) - 1 except ValueError: - click.echo("Invalid input. Please enter a number.") - continue - if 0 <= index < len(models): - return models[index].name - click.echo(f"Invalid selection. Please enter a number between 1 and {len(models)}") + return None + if not (0 <= index < count): + return None + indices.append(index) + return tuple(dict.fromkeys(indices)) + + +def _render_and_prompt_for_model(models: Tuple[DiscoveredModel, ...], prompt_label: str) -> str: + _render_model_table(models, prompt_label) + while True: + choice = click.prompt(f"\nSelect a model for {prompt_label} by index", type=str).strip() + indices = _parse_indices(choice, len(models)) + if indices is not None and len(indices) == 1: + return models[indices[0]].name + click.echo(f"Invalid selection. Please enter a single number between 1 and {len(models)}") + + +def _render_and_prompt_for_models(models: Tuple[DiscoveredModel, ...], prompt_label: str) -> Tuple[str, ...]: + _render_model_table(models, prompt_label) + while True: + choice = click.prompt( + f"\nSelect model(s) for {prompt_label} by index (comma-separated for multiple)", type=str + ).strip() + indices = _parse_indices(choice, len(models)) + if indices is not None: + return tuple(models[i].name for i in indices) + click.echo(f"Invalid selection. Please enter number(s) between 1 and {len(models)}, comma-separated") def run_configure_wizard(ctx: click.Context) -> Path: @@ -61,9 +88,9 @@ def run_configure_wizard(ctx: click.Context) -> Path: if not chat_pool: raise click.ClickException("Your key has no chat-capable models available on this proxy.") - click.echo("Assign a model to each complexity tier (from what your key can access):") - tiers = {tier: _render_and_prompt_for_model(chat_pool, tier) for tier in TIER_NAMES} - default_model = tiers["MEDIUM"] + click.echo("Assign model(s) to each complexity tier (from what your key can access):") + tiers = {tier: _render_and_prompt_for_models(chat_pool, tier) for tier in TIER_NAMES} + default_model = tiers["MEDIUM"][0] classifier = HeuristicClassifier() if click.confirm("\nUse an LLM classifier instead of the free heuristic scorer?", default=False): @@ -98,8 +125,8 @@ def run_configure_wizard(ctx: click.Context) -> Path: CONFIG_PATH.chmod(0o600) click.echo(f"\nWrote {CONFIG_PATH}") - for tier, model in tiers.items(): - click.echo(f" {tier}: {model}") + for tier, models in tiers.items(): + click.echo(f" {tier}: {', '.join(models)}") return CONFIG_PATH diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_config.py b/tests/test_litellm/proxy/client/cli/autoroute/test_config.py index a48db0c79e4..2335f07a823 100644 --- a/tests/test_litellm/proxy/client/cli/autoroute/test_config.py +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_config.py @@ -31,10 +31,10 @@ def _base_config(**overrides: Any) -> AutorouteConfig: "base_url": "http://real-proxy.internal:4000", "api_key": "sk-real-key", "tiers": { - "SIMPLE": "gpt-4o-mini", - "MEDIUM": "gpt-4o", - "COMPLEX": "gpt-4o", - "REASONING": "o1", + "SIMPLE": ("gpt-4o-mini",), + "MEDIUM": ("gpt-4o",), + "COMPLEX": ("gpt-4o",), + "REASONING": ("o1",), }, "default_model": "gpt-4o", } @@ -146,7 +146,12 @@ def test_passes_for_fully_valid_config(self): def test_raises_for_tier_referencing_unknown_model(self): config = _base_config( - tiers={"SIMPLE": "unknown-model", "MEDIUM": "gpt-4o", "COMPLEX": "gpt-4o", "REASONING": "o1"} + tiers={ + "SIMPLE": ("unknown-model",), + "MEDIUM": ("gpt-4o",), + "COMPLEX": ("gpt-4o",), + "REASONING": ("o1",), + } ) with pytest.raises(ConfigGenerationError, match="unknown-model"): validate_config(config, DISCOVERED) diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py b/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py index 81f54bf9b78..07ad1abd71f 100644 --- a/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py @@ -71,10 +71,10 @@ def test_assigns_tiers_and_declines_everything(self, tmp_path): assert result.exit_code == 0, result.output router_config = _router_config(config_path) assert router_config["tiers"] == { - "SIMPLE": "gpt-4o-mini", - "MEDIUM": "gpt-4o", - "COMPLEX": "claude-opus", - "REASONING": "o1", + "SIMPLE": ["gpt-4o-mini"], + "MEDIUM": ["gpt-4o"], + "COMPLEX": ["claude-opus"], + "REASONING": ["o1"], } assert router_config["default_model"] == "gpt-4o" assert "classifier_type" not in router_config @@ -82,6 +82,18 @@ def test_assigns_tiers_and_declines_everything(self, tmp_path): assert "semantic_keyword_matching" not in router_config assert "adaptive" not in router_config + def test_assigns_multiple_models_to_a_single_tier(self, tmp_path): + result, config_path = _run( + tmp_path, + CHAT_AND_EMBEDDING_GROUPS, + input_str="1,2\n2\n3\n4\nn\nn\nn\n", + ) + + assert result.exit_code == 0, result.output + router_config = _router_config(config_path) + assert router_config["tiers"]["SIMPLE"] == ["gpt-4o-mini", "gpt-4o"] + assert router_config["default_model"] == "gpt-4o" + def test_writes_config_file_with_restricted_permissions(self, tmp_path): result, config_path = _run( tmp_path, From 53f6fdd87470fb6c3b21d113cdfbeaa15acd0dc9 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 14 Jul 2026 19:17:03 -0700 Subject: [PATCH 03/12] feat(cli): fuzzy model picker and auto-route Claude Code to autorouter Numbered-index selection didn't scale past a handful of models, so switch the tier picker to InquirerPy's fzf-style fuzzy search. Also set ANTHROPIC_DEFAULT_{SONNET,HAIKU,OPUS}_MODEL to "autorouter" in Claude Code's settings, since Router resolves auto-router deployments by literal model name with no wildcard support, so a "*" catch-all model_name would never match real traffic. --- litellm/proxy/client/cli/README.md | 4 +- .../client/cli/commands/autoroute/config.py | 23 +- .../client/cli/commands/autoroute/settings.py | 13 ++ .../client/cli/commands/autoroute/wizard.py | 80 ++++--- litellm/proxy/client/cli/commands/up.py | 5 +- pyproject.toml | 4 +- .../proxy/client/cli/autoroute/test_config.py | 11 +- .../client/cli/autoroute/test_settings.py | 22 +- .../proxy/client/cli/autoroute/test_wizard.py | 197 +++++++++++++----- .../proxy/client/cli/test_up_commands.py | 21 ++ uv.lock | 40 +++- 11 files changed, 308 insertions(+), 112 deletions(-) diff --git a/litellm/proxy/client/cli/README.md b/litellm/proxy/client/cli/README.md index 86ad9687901..ab75c569283 100644 --- a/litellm/proxy/client/cli/README.md +++ b/litellm/proxy/client/cli/README.md @@ -507,10 +507,12 @@ Lists the model groups your key can reach on the proxy, via `/model_group/info`, lite autoroute configure ``` -An interactive wizard. It runs the same model-group discovery as above, splits the results into chat-capable and embedding-capable pools, and asks you to assign one or more models from the chat pool to each of the four complexity tiers -- SIMPLE, MEDIUM, COMPLEX, REASONING (enter comma-separated indices to assign a pool of models to a tier instead of just one; complexity_router picks randomly among a tier's pool per request, and adaptive mode specifically depends on having more than one candidate to choose from). From there it optionally offers: classifying prompt complexity with an LLM (again picked from your discovered pool) instead of the free built-in heuristic scorer, semantic keyword matching for tier assignment (needs an embedding model from the pool), and adaptive (bandit-based) selection layered on top of tiering. +An interactive wizard. It runs the same model-group discovery as above, splits the results into chat-capable and embedding-capable pools, and asks you to assign one or more models from the chat pool to each of the four complexity tiers -- SIMPLE, MEDIUM, COMPLEX, REASONING. Each tier's picker is a type-to-filter fuzzy search (fzf-style) rather than a scrollable numbered list, so it stays usable even with hundreds of model groups: type a substring to narrow the list, tab to toggle a model into the selection, enter to confirm (assigning more than one model to a tier is exactly when this matters -- complexity_router picks randomly among a tier's pool per request, and adaptive mode specifically depends on having more than one candidate to choose from). From there it optionally offers: classifying prompt complexity with an LLM (again picked from your discovered pool) instead of the free built-in heuristic scorer, semantic keyword matching for tier assignment (needs an embedding model from the pool), and adaptive (bandit-based) selection layered on top of tiering. The wizard writes the result to `~/.litellm/autorouter/config.yaml` with `0600` permissions, since the file embeds your real proxy API key. Every model referenced anywhere in that config -- tier targets, the classifier model, the embedding model -- becomes its own `litellm_proxy/` deployment whose `api_base` and `api_key` point back at your real proxy. That is the trick that keeps your real proxy's config untouched: every actual network call this generates, whether it is the routed completion, an LLM-classifier call, or an embedding call, forwards transparently through your real, already-running proxy with your real key. +You do not need to tell Claude Code to request `autorouter` by name yourself: `lite autoroute up` also sets `ANTHROPIC_DEFAULT_SONNET_MODEL`, `ANTHROPIC_DEFAULT_HAIKU_MODEL`, and `ANTHROPIC_DEFAULT_OPUS_MODEL` to `autorouter` in `~/.claude/settings.json`, so every one of Claude Code's own model tiers requests it directly regardless of `/model` or whatever it defaults to otherwise. (A bare `model_name: "*"` deployment looks like the obvious way to catch any request instead, but litellm's Router looks up auto-router deployments by the literal requested model string with no wildcard resolution, so a `"*"` entry would never actually match real traffic -- these env var overrides are what makes it work.) + You must run `configure` at least once before `up`; running `up` first fails with a clear error telling you to configure first. #### Launch the Ephemeral Auto-Router Proxy diff --git a/litellm/proxy/client/cli/commands/autoroute/config.py b/litellm/proxy/client/cli/commands/autoroute/config.py index c0f5f778fd8..f23ebff2330 100644 --- a/litellm/proxy/client/cli/commands/autoroute/config.py +++ b/litellm/proxy/client/cli/commands/autoroute/config.py @@ -3,6 +3,7 @@ from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter TIER_NAMES: Tuple[str, ...] = ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING") +AUTOROUTER_MODEL_NAME = "autorouter" class ConfigGenerationError(Exception): @@ -184,14 +185,21 @@ def build_generated_model_list(config: AutorouteConfig) -> List[JsonValue]: if config.adaptive: complexity_router_config["adaptive"] = True - auto_router_deployment: Dict[str, JsonValue] = { - "model_name": "autorouter", - "litellm_params": { - "model": "auto_router/complexity_router", - "complexity_router_config": complexity_router_config, - }, + auto_router_litellm_params: Dict[str, JsonValue] = { + "model": "auto_router/complexity_router", + "complexity_router_config": complexity_router_config, } - return [*proxy_deployments, auto_router_deployment] + # A bare "*" model_name looks like the obvious way to catch every request Claude Code + # might send regardless of which model it thinks it's using, but Router's auto-router + # registry is keyed by the literal requested model string (router.py:10711-10717), not + # resolved through pattern/wildcard matching first -- so a "*" entry here would only ever + # match a client that literally sends model="*", never an actual wildcard catch-all. Callers + # instead need to make Claude Code request this "autorouter" name directly (see + # ANTHROPIC_DEFAULT_*_MODEL in settings.py's merge_claude_settings_static_token). + return [ + *proxy_deployments, + {"model_name": AUTOROUTER_MODEL_NAME, "litellm_params": auto_router_litellm_params}, + ] def build_generated_proxy_config(config: AutorouteConfig, master_key: str) -> Dict[str, JsonValue]: @@ -210,6 +218,7 @@ def build_generated_proxy_config(config: AutorouteConfig, master_key: str) -> Di __all__ = [ "TIER_NAMES", + "AUTOROUTER_MODEL_NAME", "ConfigGenerationError", "DiscoveredModel", "parse_discovered_models", diff --git a/litellm/proxy/client/cli/commands/autoroute/settings.py b/litellm/proxy/client/cli/commands/autoroute/settings.py index 0d7b12d7aaf..811d131f433 100644 --- a/litellm/proxy/client/cli/commands/autoroute/settings.py +++ b/litellm/proxy/client/cli/commands/autoroute/settings.py @@ -2,11 +2,23 @@ from pydantic import JsonValue +from .config import AUTOROUTER_MODEL_NAME + ENV_KEY = "env" API_KEY_HELPER_KEY = "apiKeyHelper" ANTHROPIC_API_KEY_KEY = "ANTHROPIC_API_KEY" ANTHROPIC_AUTH_TOKEN_KEY = "ANTHROPIC_AUTH_TOKEN" ANTHROPIC_BASE_URL_KEY = "ANTHROPIC_BASE_URL" +# Force every one of Claude Code's own model tiers to request the auto-router by name. +# Router's auto-router registry is keyed by the literal requested model string +# (litellm/router.py:10711-10717) with no wildcard/pattern resolution, so a bare "*" +# model_name can never work as a catch-all -- these overrides are what actually makes +# Claude Code send "autorouter" regardless of /model or its own version-specific defaults. +ANTHROPIC_DEFAULT_MODEL_ENV_KEYS = ( + "ANTHROPIC_DEFAULT_SONNET_MODEL", + "ANTHROPIC_DEFAULT_HAIKU_MODEL", + "ANTHROPIC_DEFAULT_OPUS_MODEL", +) def merge_claude_settings_static_token( @@ -25,6 +37,7 @@ def merge_claude_settings_static_token( **base_env, ANTHROPIC_BASE_URL_KEY: base_url.rstrip("/"), ANTHROPIC_AUTH_TOKEN_KEY: auth_token, + **{key: AUTOROUTER_MODEL_NAME for key in ANTHROPIC_DEFAULT_MODEL_ENV_KEYS}, } env.pop(ANTHROPIC_API_KEY_KEY, None) merged: Dict[str, JsonValue] = {**settings, ENV_KEY: env} diff --git a/litellm/proxy/client/cli/commands/autoroute/wizard.py b/litellm/proxy/client/cli/commands/autoroute/wizard.py index c3ac2bdd637..1aaa37a2b7c 100644 --- a/litellm/proxy/client/cli/commands/autoroute/wizard.py +++ b/litellm/proxy/client/cli/commands/autoroute/wizard.py @@ -1,10 +1,11 @@ +import sys from pathlib import Path -from typing import List, Optional, Tuple +from typing import List, Tuple import click import yaml -from rich.console import Console -from rich.table import Table +from InquirerPy import inquirer +from InquirerPy.base.control import Choice from .... import Client from .config import ( @@ -25,52 +26,42 @@ from .process import CONFIG_PATH -def _render_model_table(models: Tuple[DiscoveredModel, ...], prompt_label: str) -> None: - console = Console() - table = Table(title=f"Pick model(s) for {prompt_label}") - table.add_column("Index", style="cyan", no_wrap=True) - table.add_column("Model", style="magenta") - for i, model in enumerate(models): - table.add_row(str(i + 1), model.name) - console.print(table) - - -def _parse_indices(choice: str, count: int) -> Optional[Tuple[int, ...]]: - raw_parts = [part.strip() for part in choice.split(",") if part.strip()] - if not raw_parts: - return None - indices: List[int] = [] - for part in raw_parts: - try: - index = int(part) - 1 - except ValueError: - return None - if not (0 <= index < count): - return None - indices.append(index) - return tuple(dict.fromkeys(indices)) +def _is_interactive() -> bool: + return sys.stdin.isatty() -def _render_and_prompt_for_model(models: Tuple[DiscoveredModel, ...], prompt_label: str) -> str: - _render_model_table(models, prompt_label) +def _fuzzy_pick(models: Tuple[DiscoveredModel, ...], prompt_label: str, multiselect: bool) -> List[str]: + """Type-to-filter picker over a (possibly huge) model pool, using InquirerPy's fzf-style fuzzy prompt. + + A plain numbered table + typed index does not scale past a handful of models -- proxies with + hundreds of model groups made that interaction unusable. This lets the user narrow the pool by + typing a substring instead of scrolling/counting. + + Assumes the caller already checked interactivity (run_configure_wizard does, once, up front) -- + checking here too would check the wrong thing under test, where InquirerPy is driven through its + own injected input/output rather than the real process stdin. + """ + choices = [Choice(value=model.name, name=model.name) for model in models] + toggle_hint = "tab to toggle, " if multiselect else "" while True: - choice = click.prompt(f"\nSelect a model for {prompt_label} by index", type=str).strip() - indices = _parse_indices(choice, len(models)) - if indices is not None and len(indices) == 1: - return models[indices[0]].name - click.echo(f"Invalid selection. Please enter a single number between 1 and {len(models)}") + result = inquirer.fuzzy( + message=f"{prompt_label}: type to filter, {toggle_hint}enter to confirm", + choices=choices, + multiselect=multiselect, + max_height="70%", + ).execute() + selected = result if multiselect else [result] + if selected: + return selected + click.echo("Select at least one model.") + + +def _render_and_prompt_for_model(models: Tuple[DiscoveredModel, ...], prompt_label: str) -> str: + return _fuzzy_pick(models, prompt_label, multiselect=False)[0] def _render_and_prompt_for_models(models: Tuple[DiscoveredModel, ...], prompt_label: str) -> Tuple[str, ...]: - _render_model_table(models, prompt_label) - while True: - choice = click.prompt( - f"\nSelect model(s) for {prompt_label} by index (comma-separated for multiple)", type=str - ).strip() - indices = _parse_indices(choice, len(models)) - if indices is not None: - return tuple(models[i].name for i in indices) - click.echo(f"Invalid selection. Please enter number(s) between 1 and {len(models)}, comma-separated") + return tuple(_fuzzy_pick(models, prompt_label, multiselect=True)) def run_configure_wizard(ctx: click.Context) -> Path: @@ -88,6 +79,9 @@ def run_configure_wizard(ctx: click.Context) -> Path: if not chat_pool: raise click.ClickException("Your key has no chat-capable models available on this proxy.") + if not _is_interactive(): + raise click.ClickException("`lite autoroute configure` requires an interactive terminal.") + click.echo("Assign model(s) to each complexity tier (from what your key can access):") tiers = {tier: _render_and_prompt_for_models(chat_pool, tier) for tier in TIER_NAMES} default_model = tiers["MEDIUM"][0] diff --git a/litellm/proxy/client/cli/commands/up.py b/litellm/proxy/client/cli/commands/up.py index b59a31147e3..a4ed8a6cc5f 100644 --- a/litellm/proxy/client/cli/commands/up.py +++ b/litellm/proxy/client/cli/commands/up.py @@ -48,7 +48,10 @@ def load_json_or_empty(path: Path) -> Dict[str, JsonValue]: if not path.exists(): return {} with open(path, "r") as f: - return _SETTINGS_ADAPTER.validate_json(f.read()) + content = f.read() + if not content.strip(): + return {} + return _SETTINGS_ADAPTER.validate_json(content) def merge_claude_settings( diff --git a/pyproject.toml b/pyproject.toml index 2c796d14c16..cc24ba6743a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ proxy = [ "litellm-enterprise==0.1.49", "RestrictedPython>=8.1,<9.0", "rich>=13.9.4,<14.0", + "InquirerPy>=0.3.4,<1.0", "polars>=1.38.1,<2.0", "soundfile>=0.12.1,<1.0", "pyroscope-io>=0.8.16,<1.0; sys_platform != 'win32'", @@ -74,11 +75,12 @@ proxy = [ ] # Thin client install for the `lite` CLI on developer laptops. The CLI's heavy # imports (fastapi, cryptography, ...) are all guarded, so it runs on the base -# SDK plus just these three; none of the server runtime in `proxy` is pulled in. +# SDK plus just these four; none of the server runtime in `proxy` is pulled in. cli = [ "rich>=13.9.4,<14.0", "pyyaml>=6.0.3,<7.0", "requests>=2.32.0,<3.0", + "InquirerPy>=0.3.4,<1.0", ] extra_proxy = [ "prisma>=0.11.0,<1.0", diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_config.py b/tests/test_litellm/proxy/client/cli/autoroute/test_config.py index 2335f07a823..9fa01524ef3 100644 --- a/tests/test_litellm/proxy/client/cli/autoroute/test_config.py +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_config.py @@ -91,7 +91,7 @@ def test_dedups_model_used_in_multiple_roles(self): def test_every_proxy_deployment_points_back_at_customer_proxy(self): config = _base_config() model_list = build_generated_model_list(config) - proxy_entries = [m for m in model_list if m["model_name"] != "autorouter"] + proxy_entries = [m for m in model_list if m["model_name"] not in ("autorouter", "*")] names = {m["model_name"] for m in proxy_entries} assert names == {"gpt-4o-mini", "gpt-4o", "o1"} for entry in proxy_entries: @@ -99,6 +99,15 @@ def test_every_proxy_deployment_points_back_at_customer_proxy(self): assert entry["litellm_params"]["api_base"] == config.base_url assert entry["litellm_params"]["api_key"] == config.api_key + def test_no_wildcard_deployment_is_generated(self): + # A bare "*" model_name looks like the obvious catch-all, but Router's auto-router + # registry is keyed by the literal requested model string with no wildcard resolution + # (litellm/router.py:10711-10717), so a "*" entry here would silently never match real + # traffic. Regression guard: don't reintroduce it. + config = _base_config() + model_list = build_generated_model_list(config) + assert not any(m["model_name"] == "*" for m in model_list) + def test_complexity_router_config_reflects_llm_classifier(self): config = _base_config(classifier=LLMClassifier(model="gpt-4o", timeout_ms=1234)) autorouter = next(m for m in build_generated_model_list(config) if m["model_name"] == "autorouter") diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_settings.py b/tests/test_litellm/proxy/client/cli/autoroute/test_settings.py index 203d77b70c1..40d3e7f2aee 100644 --- a/tests/test_litellm/proxy/client/cli/autoroute/test_settings.py +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_settings.py @@ -1,4 +1,7 @@ -from litellm.proxy.client.cli.commands.autoroute.settings import merge_claude_settings_static_token +from litellm.proxy.client.cli.commands.autoroute.settings import ( + ANTHROPIC_DEFAULT_MODEL_ENV_KEYS, + merge_claude_settings_static_token, +) def test_preserves_unrelated_top_level_keys(): @@ -34,3 +37,20 @@ def test_does_not_mutate_input(): settings = {"env": {"FOO": "bar"}, "apiKeyHelper": "old-helper"} merge_claude_settings_static_token(settings, "http://127.0.0.1:4000", "token-abc") assert settings == {"env": {"FOO": "bar"}, "apiKeyHelper": "old-helper"} + + +def test_forces_all_claude_code_default_model_tiers_to_the_autorouter(): + # A bare "*" model_name deployment looks like the obvious way to catch every request + # regardless of which model Claude Code thinks it's using, but Router's auto-router + # registry is keyed by the literal requested model string with no wildcard resolution + # (litellm/router.py:10711-10717) -- so the only reliable way to make every one of Claude + # Code's own tiers hit the auto-router is to override the env vars it reads per tier. + merged = merge_claude_settings_static_token({}, "http://127.0.0.1:4000", "token-abc") + for key in ANTHROPIC_DEFAULT_MODEL_ENV_KEYS: + assert merged["env"][key] == "autorouter" + + +def test_overrides_a_preexisting_default_model_env_var(): + settings = {"env": {"ANTHROPIC_DEFAULT_SONNET_MODEL": "claude-opus-4-8"}} + merged = merge_claude_settings_static_token(settings, "http://127.0.0.1:4000", "token-abc") + assert merged["env"]["ANTHROPIC_DEFAULT_SONNET_MODEL"] == "autorouter" diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py b/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py index 07ad1abd71f..e14e93cf5c6 100644 --- a/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py @@ -1,16 +1,19 @@ +import asyncio from typing import Any, Dict, List, Tuple from unittest.mock import patch import click +import pytest import yaml from click.testing import CliRunner +from InquirerPy.base.control import Choice +from prompt_toolkit.application import create_app_session +from prompt_toolkit.input import create_pipe_input +from prompt_toolkit.output import DummyOutput from litellm.proxy.client.cli.commands.autoroute import wizard as wizard_module from litellm.proxy.client.cli.commands.autoroute.config import DiscoveredModel -from litellm.proxy.client.cli.commands.autoroute.wizard import ( - _render_and_prompt_for_model, - run_configure_wizard, -) +from litellm.proxy.client.cli.commands.autoroute.wizard import run_configure_wizard CHAT_AND_EMBEDDING_GROUPS: List[Dict[str, Any]] = [ {"model_group": "gpt-4o-mini", "mode": "chat", "input_cost_per_token": 0.01, "output_cost_per_token": 0.02}, @@ -38,12 +41,37 @@ def _invoke_wizard(ctx: click.Context) -> None: run_configure_wizard(ctx) -def _run(tmp_path, raw_groups: List[Dict[str, Any]], input_str: str): +def _run( + tmp_path, + raw_groups: List[Dict[str, Any]], + tier_picks: Dict[str, Tuple[str, ...]], + input_str: str, + classifier_pick: str = "", + embedding_pick: str = "", +): + """Drives run_configure_wizard's orchestration logic (discovery, validation, config writing, + classifier/semantic/adaptive branching) by mocking the fuzzy picker itself, since that widget + is a real prompt_toolkit application tested separately in TestFuzzyPickWidget. CliRunner's + injected input still drives the plain click.confirm() y/n prompts.""" config_path = tmp_path / "config.yaml" runner = CliRunner() + + def _fake_prompt_for_models(models, prompt_label): + return tier_picks[prompt_label] + + def _fake_prompt_for_model(models, prompt_label): + if prompt_label == "LLM classifier": + return classifier_pick + if prompt_label == "semantic embeddings": + return embedding_pick + raise AssertionError(f"unexpected single-pick prompt_label {prompt_label!r}") + with ( patch.object(wizard_module, "Client") as mock_client_cls, patch.object(wizard_module, "CONFIG_PATH", config_path), + patch.object(wizard_module, "_is_interactive", return_value=True), + patch.object(wizard_module, "_render_and_prompt_for_models", side_effect=_fake_prompt_for_models), + patch.object(wizard_module, "_render_and_prompt_for_model", side_effect=_fake_prompt_for_model), ): mock_client_cls.return_value.model_groups.info.return_value = raw_groups result = runner.invoke( @@ -60,13 +88,17 @@ def _router_config(config_path) -> Dict[str, Any]: return autorouter["litellm_params"]["complexity_router_config"] +_SIMPLE_TIER_PICKS: Dict[str, Tuple[str, ...]] = { + "SIMPLE": ("gpt-4o-mini",), + "MEDIUM": ("gpt-4o",), + "COMPLEX": ("claude-opus",), + "REASONING": ("o1",), +} + + class TestRunConfigureWizardHappyPath: def test_assigns_tiers_and_declines_everything(self, tmp_path): - result, config_path = _run( - tmp_path, - CHAT_AND_EMBEDDING_GROUPS, - input_str="1\n2\n3\n4\nn\nn\nn\n", - ) + result, config_path = _run(tmp_path, CHAT_AND_EMBEDDING_GROUPS, _SIMPLE_TIER_PICKS, input_str="n\nn\nn\n") assert result.exit_code == 0, result.output router_config = _router_config(config_path) @@ -83,11 +115,8 @@ def test_assigns_tiers_and_declines_everything(self, tmp_path): assert "adaptive" not in router_config def test_assigns_multiple_models_to_a_single_tier(self, tmp_path): - result, config_path = _run( - tmp_path, - CHAT_AND_EMBEDDING_GROUPS, - input_str="1,2\n2\n3\n4\nn\nn\nn\n", - ) + tier_picks = {**_SIMPLE_TIER_PICKS, "SIMPLE": ("gpt-4o-mini", "gpt-4o")} + result, config_path = _run(tmp_path, CHAT_AND_EMBEDDING_GROUPS, tier_picks, input_str="n\nn\nn\n") assert result.exit_code == 0, result.output router_config = _router_config(config_path) @@ -95,22 +124,14 @@ def test_assigns_multiple_models_to_a_single_tier(self, tmp_path): assert router_config["default_model"] == "gpt-4o" def test_writes_config_file_with_restricted_permissions(self, tmp_path): - result, config_path = _run( - tmp_path, - CHAT_AND_EMBEDDING_GROUPS, - input_str="1\n2\n3\n4\nn\nn\nn\n", - ) + result, config_path = _run(tmp_path, CHAT_AND_EMBEDDING_GROUPS, _SIMPLE_TIER_PICKS, input_str="n\nn\nn\n") assert result.exit_code == 0, result.output assert config_path.exists() assert oct(config_path.stat().st_mode)[-3:] == "600" def test_no_embedding_pool_skips_semantic_prompt_entirely(self, tmp_path): - result, config_path = _run( - tmp_path, - CHAT_ONLY_GROUPS, - input_str="1\n2\n3\n4\nn\nn\n", - ) + result, config_path = _run(tmp_path, CHAT_ONLY_GROUPS, _SIMPLE_TIER_PICKS, input_str="n\nn\n") assert result.exit_code == 0, result.output router_config = _router_config(config_path) @@ -120,9 +141,7 @@ def test_no_embedding_pool_skips_semantic_prompt_entirely(self, tmp_path): class TestRunConfigureWizardLLMClassifier: def test_accepting_llm_classifier_records_chosen_model(self, tmp_path): result, config_path = _run( - tmp_path, - CHAT_AND_EMBEDDING_GROUPS, - input_str="1\n2\n3\n4\ny\n2\nn\nn\n", + tmp_path, CHAT_AND_EMBEDDING_GROUPS, _SIMPLE_TIER_PICKS, input_str="y\nn\nn\n", classifier_pick="gpt-4o" ) assert result.exit_code == 0, result.output @@ -136,7 +155,9 @@ def test_accepting_semantic_matching_records_embedding_model(self, tmp_path): result, config_path = _run( tmp_path, CHAT_AND_EMBEDDING_GROUPS, - input_str="1\n2\n3\n4\nn\ny\n1\nn\n", + _SIMPLE_TIER_PICKS, + input_str="n\ny\nn\n", + embedding_pick="text-embedding-3-small", ) assert result.exit_code == 0, result.output @@ -147,11 +168,7 @@ def test_accepting_semantic_matching_records_embedding_model(self, tmp_path): class TestRunConfigureWizardAdaptive: def test_accepting_adaptive_sets_adaptive_flag(self, tmp_path): - result, config_path = _run( - tmp_path, - CHAT_AND_EMBEDDING_GROUPS, - input_str="1\n2\n3\n4\nn\nn\ny\n", - ) + result, config_path = _run(tmp_path, CHAT_AND_EMBEDDING_GROUPS, _SIMPLE_TIER_PICKS, input_str="n\nn\ny\n") assert result.exit_code == 0, result.output router_config = _router_config(config_path) @@ -160,41 +177,109 @@ def test_accepting_adaptive_sets_adaptive_flag(self, tmp_path): class TestRunConfigureWizardNoChatModels: def test_fails_cleanly_without_prompting_when_no_chat_models(self, tmp_path): - result, config_path = _run(tmp_path, EMBEDDING_ONLY_GROUPS, input_str="") + result, config_path = _run(tmp_path, EMBEDDING_ONLY_GROUPS, {}, input_str="") assert result.exit_code != 0 assert "no chat-capable models" in result.output.lower() assert not config_path.exists() -class TestRenderAndPromptForModel: +class TestRunConfigureWizardNotInteractive: + def test_fails_cleanly_when_not_a_tty(self, tmp_path): + config_path = tmp_path / "config.yaml" + runner = CliRunner() + with ( + patch.object(wizard_module, "Client") as mock_client_cls, + patch.object(wizard_module, "CONFIG_PATH", config_path), + patch.object(wizard_module, "_is_interactive", return_value=False), + ): + mock_client_cls.return_value.model_groups.info.return_value = CHAT_AND_EMBEDDING_GROUPS + result = runner.invoke(_invoke_wizard, obj={"base_url": "http://localhost:4000", "api_key": "sk-test"}) + + assert result.exit_code != 0 + assert "interactive terminal" in result.output + assert not config_path.exists() + + +def _drive_fuzzy_pick( + models: Tuple[DiscoveredModel, ...], + prompt_label: str, + multiselect: bool, + key_events: List[Tuple[str, float]], +) -> List[str]: + """Drives the real InquirerPy fuzzy prompt through prompt_toolkit's own test input/output, + exercising the actual widget (filtering, tab-to-toggle, enter-to-confirm) rather than mocking + it away. asyncio.to_thread propagates the create_app_session context into the worker thread + running _fuzzy_pick's synchronous .execute() call.""" + + async def _run() -> List[str]: + with create_pipe_input() as pipe_input: + with create_app_session(input=pipe_input, output=DummyOutput()): + task = asyncio.ensure_future( + asyncio.to_thread(wizard_module._fuzzy_pick, models, prompt_label, multiselect) + ) + await asyncio.sleep(0.05) + for text, delay in key_events: + pipe_input.send_text(text) + await asyncio.sleep(delay) + return await task + + return asyncio.run(_run()) + + +class TestFuzzyPickWidget: def _models(self) -> Tuple[DiscoveredModel, ...]: - return ( - DiscoveredModel(name="model-a"), - DiscoveredModel(name="model-b"), + return tuple(DiscoveredModel(name=f"model-{i}") for i in range(20)) + + def test_single_select_filters_and_returns_highlighted_match(self): + result = _drive_fuzzy_pick( + self._models(), "test", multiselect=False, key_events=[("model-13", 0.3), ("\r", 0.1)] ) + assert result == ["model-13"] - def test_reprompts_on_non_numeric_input(self): - with patch("click.prompt", side_effect=["not-a-number", "2"]): - result = _render_and_prompt_for_model(self._models(), "test tier") + def test_multiselect_requires_tab_to_toggle_before_enter(self): + result = _drive_fuzzy_pick( + self._models(), "test", multiselect=True, key_events=[("model-7", 0.3), ("\t", 0.1), ("\r", 0.1)] + ) + assert result == ["model-7"] + + def test_multiselect_can_pick_more_than_one_across_filters(self): + result = _drive_fuzzy_pick( + self._models(), + "test", + multiselect=True, + key_events=[ + ("model-3", 0.3), + ("\t", 0.1), + *[("\x7f", 0.02) for _ in range("model-3".__len__())], + ("model-15", 0.3), + ("\t", 0.1), + ("\r", 0.1), + ], + ) + assert set(result) == {"model-3", "model-15"} - assert result == "model-b" + def test_choice_wraps_name_and_value_to_the_same_model_name(self): + model = DiscoveredModel(name="only-model") + choice = Choice(value=model.name, name=model.name) + assert choice.value == choice.name == "only-model" - def test_reprompts_on_out_of_range_index(self): - with patch("click.prompt", side_effect=["5", "1"]): - result = _render_and_prompt_for_model(self._models(), "test tier") +class TestRenderAndPromptForModelWrappers: + def test_single_pick_wrapper_returns_bare_string(self): + with patch.object(wizard_module, "_fuzzy_pick", return_value=["model-a"]) as mock_pick: + result = wizard_module._render_and_prompt_for_model((), "tier") assert result == "model-a" + mock_pick.assert_called_once_with((), "tier", multiselect=False) - def test_reprompts_on_zero_index(self): - with patch("click.prompt", side_effect=["0", "2"]): - result = _render_and_prompt_for_model(self._models(), "test tier") + def test_multi_pick_wrapper_returns_tuple(self): + with patch.object(wizard_module, "_fuzzy_pick", return_value=["model-a", "model-b"]) as mock_pick: + result = wizard_module._render_and_prompt_for_models((), "tier") + assert result == ("model-a", "model-b") + mock_pick.assert_called_once_with((), "tier", multiselect=True) - assert result == "model-b" - def test_valid_first_answer_returns_immediately(self): - with patch("click.prompt", return_value="1") as mock_prompt: - result = _render_and_prompt_for_model(self._models(), "test tier") - - assert result == "model-a" - mock_prompt.assert_called_once() +@pytest.mark.parametrize("isatty_value", [True, False]) +def test_is_interactive_reflects_stdin_isatty(isatty_value): + with patch.object(wizard_module.sys.stdin, "isatty", return_value=isatty_value): + assert wizard_module._is_interactive() is isatty_value diff --git a/tests/test_litellm/proxy/client/cli/test_up_commands.py b/tests/test_litellm/proxy/client/cli/test_up_commands.py index 7595707ac67..63f1da8facf 100644 --- a/tests/test_litellm/proxy/client/cli/test_up_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_up_commands.py @@ -12,6 +12,7 @@ BackupRecord, UpError, down, + load_json_or_empty, merge_claude_settings, read_backup, resolve_api_key_helper, @@ -66,6 +67,26 @@ def test_does_not_mutate_input(self): assert settings == {"env": {"FOO": "bar"}} +class TestLoadJsonOrEmpty: + def test_returns_empty_dict_when_file_does_not_exist(self, tmp_path): + assert load_json_or_empty(tmp_path / "missing.json") == {} + + def test_returns_empty_dict_when_file_is_empty(self, tmp_path): + path = tmp_path / "settings.json" + path.write_text("") + assert load_json_or_empty(path) == {} + + def test_returns_empty_dict_when_file_is_whitespace_only(self, tmp_path): + path = tmp_path / "settings.json" + path.write_text(" \n") + assert load_json_or_empty(path) == {} + + def test_parses_real_content(self, tmp_path): + path = tmp_path / "settings.json" + path.write_text(json.dumps({"theme": "dark"})) + assert load_json_or_empty(path) == {"theme": "dark"} + + class TestBackupRoundTrip: def test_restores_original_content_when_file_existed(self, monkeypatch, tmp_path): settings_path, backup_path = _patch_paths(monkeypatch, tmp_path) diff --git a/uv.lock b/uv.lock index b120547c536..1d5f688aebc 100644 --- a/uv.lock +++ b/uv.lock @@ -9,7 +9,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-07-10T16:47:58.286372Z" +exclude-newer = "2026-07-11T19:28:02.260785Z" exclude-newer-span = "P3D" [manifest] @@ -2655,6 +2655,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] +[[package]] +name = "inquirerpy" +version = "0.3.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pfzy" }, + { name = "prompt-toolkit" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/64/73/7570847b9da026e07053da3bbe2ac7ea6cde6bb2cbd3c7a5a950fa0ae40b/InquirerPy-0.3.4.tar.gz", hash = "sha256:89d2ada0111f337483cb41ae31073108b2ec1e618a49d7110b0d7ade89fc197e", size = 44431, upload-time = "2022-06-27T23:11:20.598Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ce/ff/3b59672c47c6284e8005b42e84ceba13864aa0f39f067c973d1af02f5d91/InquirerPy-0.3.4-py3-none-any.whl", hash = "sha256:c65fdfbac1fa00e3ee4fb10679f4d3ed7a012abf4833910e63c295827fe2a7d4", size = 67677, upload-time = "2022-06-27T23:11:17.723Z" }, +] + [[package]] name = "isodate" version = "0.7.2" @@ -3305,6 +3318,7 @@ caching = [ { name = "diskcache" }, ] cli = [ + { name = "inquirerpy" }, { name = "pyyaml" }, { name = "requests" }, { name = "rich" }, @@ -3340,6 +3354,7 @@ proxy = [ { name = "fastapi-sso" }, { name = "granian" }, { name = "gunicorn" }, + { name = "inquirerpy" }, { name = "litellm-enterprise" }, { name = "litellm-proxy-extras" }, { name = "mcp" }, @@ -3517,6 +3532,8 @@ requires-dist = [ { name = "gunicorn", marker = "extra == 'proxy'", specifier = ">=23.0.0,<24.0" }, { name = "httpx", specifier = ">=0.28.0,<1.0" }, { name = "importlib-metadata", specifier = ">=8.0.0,<9.0" }, + { name = "inquirerpy", marker = "extra == 'cli'", specifier = ">=0.3.4,<1.0" }, + { name = "inquirerpy", marker = "extra == 'proxy'", specifier = ">=0.3.4,<1.0" }, { name = "jinja2", specifier = ">=3.1.6,<4.0" }, { name = "jsonschema", specifier = ">=4.0.0,<5.0" }, { name = "langfuse", marker = "extra == 'proxy-runtime'", specifier = ">=2.59.7,<3.0" }, @@ -5277,6 +5294,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7d/eb/b6260b31b1a96386c0a880edebe26f89669098acea8e0318bff6adb378fd/pathable-0.4.4-py3-none-any.whl", hash = "sha256:5ae9e94793b6ef5a4cbe0a7ce9dbbefc1eec38df253763fd0aeeacf2762dbbc2", size = 9592, upload-time = "2025-01-10T18:43:11.88Z" }, ] +[[package]] +name = "pfzy" +version = "0.3.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/5a/32b50c077c86bfccc7bed4881c5a2b823518f5450a30e639db5d3711952e/pfzy-0.3.4.tar.gz", hash = "sha256:717ea765dd10b63618e7298b2d98efd819e0b30cd5905c9707223dceeb94b3f1", size = 8396, upload-time = "2022-01-28T02:26:17.946Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8c/d7/8ff98376b1acc4503253b685ea09981697385ce344d4e3935c2af49e044d/pfzy-0.3.4-py3-none-any.whl", hash = "sha256:5f50d5b2b3207fa72e7ec0ef08372ef652685470974a107d0d4999fc5a903a96", size = 8537, upload-time = "2022-01-28T02:26:16.047Z" }, +] + [[package]] name = "pillow" version = "12.3.0" @@ -5469,6 +5495,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c7/98/745b810d822103adca2df8decd4c0bbe839ba7ad3511af3f0d09692fc0f0/prometheus_client-0.20.0-py3-none-any.whl", hash = "sha256:cde524a85bce83ca359cc837f28b8c0db5cac7aa653a588fd7e84ba061c329e7", size = 54474, upload-time = "2024-02-14T15:55:03.957Z" }, ] +[[package]] +name = "prompt-toolkit" +version = "3.0.52" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "wcwidth" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a1/96/06e01a7b38dce6fe1db213e061a4602dd6032a8a97ef6c1a862537732421/prompt_toolkit-3.0.52.tar.gz", hash = "sha256:28cde192929c8e7321de85de1ddbe736f1375148b02f2e17edd840042b1be855", size = 434198, upload-time = "2025-08-27T15:24:02.057Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/84/03/0d3ce49e2505ae70cf43bc5bb3033955d2fc9f932163e84dc0779cc47f48/prompt_toolkit-3.0.52-py3-none-any.whl", hash = "sha256:9aac639a3bbd33284347de5ad8d68ecc044b91a762dc39b7c21095fcd6a19955", size = 391431, upload-time = "2025-08-27T15:23:59.498Z" }, +] + [[package]] name = "propcache" version = "0.5.2" From 99a72131b8d86c358e373547d731767adfd56c63 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 14 Jul 2026 20:36:45 -0700 Subject: [PATCH 04/12] feat(cli): allow installing lite CLI from source via LITELLM_CLI_REF Lets testers try an unreleased branch's CLI changes with the same curl-piped installer, instead of waiting for a PyPI release. --- scripts/install-cli.sh | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/scripts/install-cli.sh b/scripts/install-cli.sh index d147286fcac..a39b73c2e5a 100755 --- a/scripts/install-cli.sh +++ b/scripts/install-cli.sh @@ -11,12 +11,21 @@ # Python itself (honouring litellm's requires-python), downloading a managed one # when the host has no suitable interpreter. # +# To try an unreleased branch instead of the latest PyPI release (for example, to +# QA a CLI feature before it ships), set LITELLM_CLI_REF to a branch, tag, or commit: +# curl -fsSL https://raw.githubusercontent.com/BerriAI/litellm//scripts/install-cli.sh | \ +# LITELLM_CLI_REF= sh +# # NOTE: set -e without pipefail for POSIX sh compatibility (dash on Ubuntu/Debian # ignores the shebang when invoked as `sh` and does not support `pipefail`). set -eu -# NOTE: before merging, this must stay as "litellm[cli]" to install from PyPI. -LITELLM_PACKAGE="litellm[cli]" +# Defaults to the PyPI release; LITELLM_CLI_REF opts into installing from source instead. +if [ -n "${LITELLM_CLI_REF:-}" ]; then + LITELLM_PACKAGE="litellm[cli] @ git+https://github.com/BerriAI/litellm.git@${LITELLM_CLI_REF}" +else + LITELLM_PACKAGE="litellm[cli]" +fi UV_VERSION="0.10.9" # ── colours ──────────────────────────────────────────────────────────────── @@ -90,7 +99,11 @@ fi # otherwise download a managed one. Either way uv honours litellm's requires-python, # so a too-old (3.9) or too-new (3.14+) system Python is skipped, not forced. echo "" -header "Installing litellm[cli]…" +if [ -n "${LITELLM_CLI_REF:-}" ]; then + header "Installing litellm[cli] from ${LITELLM_CLI_REF}…" +else + header "Installing litellm[cli]…" +fi echo "" "$UV_BIN" tool install --python-preference system --force "${LITELLM_PACKAGE}" \ From 0ab46cfd5d76087f3bafba6603ed38e1c667bb07 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 14 Jul 2026 21:12:57 -0700 Subject: [PATCH 05/12] fix(ci): modernize type hints to clear ruff strict-rule budget --- .../client/cli/commands/autoroute/commands.py | 14 +++--- .../client/cli/commands/autoroute/config.py | 46 +++++++++---------- .../client/cli/commands/autoroute/process.py | 17 ++++--- .../client/cli/commands/autoroute/settings.py | 10 ++-- .../client/cli/commands/autoroute/wizard.py | 7 ++- litellm/proxy/client/cli/commands/up.py | 34 +++++++------- 6 files changed, 60 insertions(+), 68 deletions(-) diff --git a/litellm/proxy/client/cli/commands/autoroute/commands.py b/litellm/proxy/client/cli/commands/autoroute/commands.py index aca2f395615..cbc07456ac2 100644 --- a/litellm/proxy/client/cli/commands/autoroute/commands.py +++ b/litellm/proxy/client/cli/commands/autoroute/commands.py @@ -4,15 +4,13 @@ import signal import threading from types import FrameType -from typing import Dict, Optional import click import yaml from pydantic import JsonValue, TypeAdapter -from ..up import CLAUDE_SETTINGS_PATH +from ..up import CLAUDE_SETTINGS_PATH, load_json_or_empty, restore_claude_settings, write_backup from ..up import BackupRecord as ClaudeBackupRecord -from ..up import load_json_or_empty, restore_claude_settings, write_backup from .process import ( AUTOROUTE_DIR, CONFIG_PATH, @@ -34,7 +32,7 @@ AUTOROUTE_BACKUP_PATH = AUTOROUTE_DIR / "claude_settings_backup.json" -_GENERATED_CONFIG_ADAPTER = TypeAdapter(Dict[str, JsonValue]) +_GENERATED_CONFIG_ADAPTER = TypeAdapter(dict[str, JsonValue]) def _mint_and_embed_master_key() -> str: @@ -49,11 +47,11 @@ def _mint_and_embed_master_key() -> str: with open(CONFIG_PATH, "r") as f: generated = _GENERATED_CONFIG_ADAPTER.validate_python(yaml.safe_load(f)) general_settings = generated.get("general_settings") - updated_settings: Dict[str, JsonValue] = { + updated_settings: dict[str, JsonValue] = { **(general_settings if isinstance(general_settings, dict) else {}), "master_key": master_key, } - updated: Dict[str, JsonValue] = {**generated, "general_settings": updated_settings} + updated: dict[str, JsonValue] = {**generated, "general_settings": updated_settings} with open(CONFIG_PATH, "w") as f: yaml.safe_dump(updated, f, sort_keys=False) CONFIG_PATH.chmod(0o600) @@ -122,7 +120,7 @@ def _teardown() -> None: restore_claude_settings(CLAUDE_SETTINGS_PATH, AUTOROUTE_BACKUP_PATH) click.echo("\nStopped ephemeral proxy and restored Claude Code settings.") - def _handle_signal(_signum: int, _frame: Optional[FrameType]) -> None: + def _handle_signal(_signum: int, _frame: FrameType | None) -> None: stop_event.set() signal.signal(signal.SIGINT, _handle_signal) @@ -139,7 +137,7 @@ def _handle_signal(_signum: int, _frame: Optional[FrameType]) -> None: @autoroute_group.command("down") def down() -> None: """Restore Claude Code settings and stop a leftover ephemeral proxy, if any""" - record: Optional[PidRecord] = read_pid_record() + record: PidRecord | None = read_pid_record() if record is not None and is_running(record.pid): terminate(record.pid) click.echo(f"Stopped leftover ephemeral proxy (pid {record.pid}).") diff --git a/litellm/proxy/client/cli/commands/autoroute/config.py b/litellm/proxy/client/cli/commands/autoroute/config.py index f23ebff2330..1d18f497324 100644 --- a/litellm/proxy/client/cli/commands/autoroute/config.py +++ b/litellm/proxy/client/cli/commands/autoroute/config.py @@ -1,8 +1,8 @@ -from typing import Dict, FrozenSet, List, Literal, Tuple, Union +from typing import Literal, Union from pydantic import BaseModel, ConfigDict, Field, JsonValue, TypeAdapter -TIER_NAMES: Tuple[str, ...] = ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING") +TIER_NAMES: tuple[str, ...] = ("SIMPLE", "MEDIUM", "COMPLEX", "REASONING") AUTOROUTER_MODEL_NAME = "autorouter" @@ -32,10 +32,10 @@ class _RawModelGroup(BaseModel): output_cost_per_token: float | None = None -_RAW_MODEL_GROUPS_ADAPTER = TypeAdapter(List[_RawModelGroup]) +_RAW_MODEL_GROUPS_ADAPTER = TypeAdapter(list[_RawModelGroup]) -def parse_discovered_models(raw: List[JsonValue]) -> Tuple[DiscoveredModel, ...]: +def parse_discovered_models(raw: list[JsonValue]) -> tuple[DiscoveredModel, ...]: """Validate a raw `/model_group/info` response into typed models.""" parsed = _RAW_MODEL_GROUPS_ADAPTER.validate_python(raw) return tuple( @@ -52,11 +52,11 @@ def parse_discovered_models(raw: List[JsonValue]) -> Tuple[DiscoveredModel, ...] ) -def chat_models(models: Tuple[DiscoveredModel, ...]) -> Tuple[DiscoveredModel, ...]: +def chat_models(models: tuple[DiscoveredModel, ...]) -> tuple[DiscoveredModel, ...]: return tuple(m for m in models if m.mode == "chat") -def embedding_models(models: Tuple[DiscoveredModel, ...]) -> Tuple[DiscoveredModel, ...]: +def embedding_models(models: tuple[DiscoveredModel, ...]) -> tuple[DiscoveredModel, ...]: return tuple(m for m in models if m.mode == "embedding") @@ -91,7 +91,7 @@ class SemanticMatching(BaseModel): # Satisfies complexity_router's "semantic matching requires non-empty keyword_tier_rules" # invariant with a sane starting point; the generated config.yaml can be hand-edited afterward. -_DEFAULT_KEYWORD_TIER_RULES: Tuple[Dict[str, JsonValue], ...] = ( +_DEFAULT_KEYWORD_TIER_RULES: tuple[dict[str, JsonValue], ...] = ( {"keywords": ["hi", "hello", "thanks"], "tier": "SIMPLE"}, {"keywords": ["explain", "how does"], "tier": "MEDIUM"}, {"keywords": ["refactor", "implement", "debug"], "tier": "COMPLEX"}, @@ -106,17 +106,17 @@ class AutorouteConfig(BaseModel): api_key: str # Each tier maps to a pool of one or more models; complexity_router picks randomly among # them per request (or, in adaptive mode, learns which to prefer within the pool). - tiers: Dict[str, Tuple[str, ...]] + tiers: dict[str, tuple[str, ...]] default_model: str classifier: ClassifierChoice = Field(default_factory=HeuristicClassifier) semantic_matching: SemanticMatchingChoice = Field(default_factory=NoSemanticMatching) adaptive: bool = False -def validate_config(config: AutorouteConfig, discovered: Tuple[DiscoveredModel, ...]) -> None: +def validate_config(config: AutorouteConfig, discovered: tuple[DiscoveredModel, ...]) -> None: """Raise ConfigGenerationError if config references a model discovery didn't return.""" - chat_names: FrozenSet[str] = frozenset(m.name for m in chat_models(discovered)) - embedding_names: FrozenSet[str] = frozenset(m.name for m in embedding_models(discovered)) + chat_names: frozenset[str] = frozenset(m.name for m in chat_models(discovered)) + embedding_names: frozenset[str] = frozenset(m.name for m in embedding_models(discovered)) for tier, models in config.tiers.items(): for model in models: @@ -138,7 +138,7 @@ def validate_config(config: AutorouteConfig, discovered: Tuple[DiscoveredModel, ) -def _litellm_proxy_deployment(name: str, base_url: str, api_key: str) -> Dict[str, JsonValue]: +def _litellm_proxy_deployment(name: str, base_url: str, api_key: str) -> dict[str, JsonValue]: return { "model_name": name, "litellm_params": { @@ -149,7 +149,7 @@ def _litellm_proxy_deployment(name: str, base_url: str, api_key: str) -> Dict[st } -def build_generated_model_list(config: AutorouteConfig) -> List[JsonValue]: +def build_generated_model_list(config: AutorouteConfig) -> list[JsonValue]: """Build the model_list for the ephemeral proxy's config.yaml. Every real model referenced anywhere (tier targets, classifier, embedding) is deduplicated @@ -167,7 +167,7 @@ def build_generated_model_list(config: AutorouteConfig) -> List[JsonValue]: _litellm_proxy_deployment(name, config.base_url, config.api_key) for name in sorted(referenced_names) ] - complexity_router_config: Dict[str, JsonValue] = { + complexity_router_config: dict[str, JsonValue] = { "tiers": {tier: list(models) for tier, models in config.tiers.items()}, "default_model": config.default_model, } @@ -185,7 +185,7 @@ def build_generated_model_list(config: AutorouteConfig) -> List[JsonValue]: if config.adaptive: complexity_router_config["adaptive"] = True - auto_router_litellm_params: Dict[str, JsonValue] = { + auto_router_litellm_params: dict[str, JsonValue] = { "model": "auto_router/complexity_router", "complexity_router_config": complexity_router_config, } @@ -202,7 +202,7 @@ def build_generated_model_list(config: AutorouteConfig) -> List[JsonValue]: ] -def build_generated_proxy_config(config: AutorouteConfig, master_key: str) -> Dict[str, JsonValue]: +def build_generated_proxy_config(config: AutorouteConfig, master_key: str) -> dict[str, JsonValue]: """Full config.yaml content for the ephemeral proxy, including its own auth key. master_key must live under general_settings, not litellm_settings -- the proxy server @@ -217,20 +217,20 @@ def build_generated_proxy_config(config: AutorouteConfig, master_key: str) -> Di __all__ = [ - "TIER_NAMES", "AUTOROUTER_MODEL_NAME", + "TIER_NAMES", + "AutorouteConfig", + "ClassifierChoice", "ConfigGenerationError", "DiscoveredModel", - "parse_discovered_models", - "chat_models", - "embedding_models", "HeuristicClassifier", "LLMClassifier", - "ClassifierChoice", "NoSemanticMatching", "SemanticMatching", "SemanticMatchingChoice", - "AutorouteConfig", - "validate_config", "build_generated_model_list", + "chat_models", + "embedding_models", + "parse_discovered_models", + "validate_config", ] diff --git a/litellm/proxy/client/cli/commands/autoroute/process.py b/litellm/proxy/client/cli/commands/autoroute/process.py index ce146e95eec..40d0585cfed 100644 --- a/litellm/proxy/client/cli/commands/autoroute/process.py +++ b/litellm/proxy/client/cli/commands/autoroute/process.py @@ -9,7 +9,6 @@ import time from dataclasses import dataclass from pathlib import Path -from typing import Optional import click import requests @@ -76,7 +75,7 @@ def poll_liveliness(base_url: str, log_path: Path, process: "subprocess.Popen[by ) -def write_pid_record(record: PidRecord, path: Optional[Path] = None) -> None: +def write_pid_record(record: PidRecord, path: Path | None = None) -> None: resolved_path = path if path is not None else PID_RECORD_PATH resolved_path.parent.mkdir(parents=True, exist_ok=True) with open(resolved_path, "w") as f: @@ -87,7 +86,7 @@ def write_pid_record(record: PidRecord, path: Optional[Path] = None) -> None: ) -def read_pid_record(path: Optional[Path] = None) -> Optional[PidRecord]: +def read_pid_record(path: Path | None = None) -> PidRecord | None: resolved_path = path if path is not None else PID_RECORD_PATH if not resolved_path.exists(): return None @@ -95,7 +94,7 @@ def read_pid_record(path: Optional[Path] = None) -> Optional[PidRecord]: return _PID_RECORD_ADAPTER.validate_json(f.read()) -def clear_pid_record(path: Optional[Path] = None) -> None: +def clear_pid_record(path: Path | None = None) -> None: resolved_path = path if path is not None else PID_RECORD_PATH resolved_path.unlink(missing_ok=True) @@ -144,15 +143,15 @@ def stream_log(log_path: Path, stop_event: threading.Event) -> None: "CONFIG_PATH", "LOG_PATH", "PID_RECORD_PATH", - "ProcessLaunchError", "PidRecord", + "ProcessLaunchError", "allocate_free_port", + "clear_pid_record", + "is_running", "launch_proxy", "poll_liveliness", - "write_pid_record", "read_pid_record", - "clear_pid_record", - "is_running", - "terminate", "stream_log", + "terminate", + "write_pid_record", ] diff --git a/litellm/proxy/client/cli/commands/autoroute/settings.py b/litellm/proxy/client/cli/commands/autoroute/settings.py index 811d131f433..4bed184eb34 100644 --- a/litellm/proxy/client/cli/commands/autoroute/settings.py +++ b/litellm/proxy/client/cli/commands/autoroute/settings.py @@ -1,5 +1,3 @@ -from typing import Dict - from pydantic import JsonValue from .config import AUTOROUTER_MODEL_NAME @@ -22,8 +20,8 @@ def merge_claude_settings_static_token( - settings: Dict[str, JsonValue], base_url: str, auth_token: str -) -> Dict[str, JsonValue]: + settings: dict[str, JsonValue], base_url: str, auth_token: str +) -> dict[str, JsonValue]: """Return a new settings dict wired to a local ephemeral proxy with a static token. Unlike up.py's merge_claude_settings (which sets apiKeyHelper for a long-lived, real @@ -33,14 +31,14 @@ def merge_claude_settings_static_token( """ raw_env = settings.get(ENV_KEY, {}) base_env = raw_env if isinstance(raw_env, dict) else {} - env: Dict[str, JsonValue] = { + env: dict[str, JsonValue] = { **base_env, ANTHROPIC_BASE_URL_KEY: base_url.rstrip("/"), ANTHROPIC_AUTH_TOKEN_KEY: auth_token, **{key: AUTOROUTER_MODEL_NAME for key in ANTHROPIC_DEFAULT_MODEL_ENV_KEYS}, } env.pop(ANTHROPIC_API_KEY_KEY, None) - merged: Dict[str, JsonValue] = {**settings, ENV_KEY: env} + merged: dict[str, JsonValue] = {**settings, ENV_KEY: env} merged.pop(API_KEY_HELPER_KEY, None) return merged diff --git a/litellm/proxy/client/cli/commands/autoroute/wizard.py b/litellm/proxy/client/cli/commands/autoroute/wizard.py index 1aaa37a2b7c..06a6ea7d2b1 100644 --- a/litellm/proxy/client/cli/commands/autoroute/wizard.py +++ b/litellm/proxy/client/cli/commands/autoroute/wizard.py @@ -1,6 +1,5 @@ import sys from pathlib import Path -from typing import List, Tuple import click import yaml @@ -30,7 +29,7 @@ def _is_interactive() -> bool: return sys.stdin.isatty() -def _fuzzy_pick(models: Tuple[DiscoveredModel, ...], prompt_label: str, multiselect: bool) -> List[str]: +def _fuzzy_pick(models: tuple[DiscoveredModel, ...], prompt_label: str, multiselect: bool) -> list[str]: """Type-to-filter picker over a (possibly huge) model pool, using InquirerPy's fzf-style fuzzy prompt. A plain numbered table + typed index does not scale past a handful of models -- proxies with @@ -56,11 +55,11 @@ def _fuzzy_pick(models: Tuple[DiscoveredModel, ...], prompt_label: str, multisel click.echo("Select at least one model.") -def _render_and_prompt_for_model(models: Tuple[DiscoveredModel, ...], prompt_label: str) -> str: +def _render_and_prompt_for_model(models: tuple[DiscoveredModel, ...], prompt_label: str) -> str: return _fuzzy_pick(models, prompt_label, multiselect=False)[0] -def _render_and_prompt_for_models(models: Tuple[DiscoveredModel, ...], prompt_label: str) -> Tuple[str, ...]: +def _render_and_prompt_for_models(models: tuple[DiscoveredModel, ...], prompt_label: str) -> tuple[str, ...]: return tuple(_fuzzy_pick(models, prompt_label, multiselect=True)) diff --git a/litellm/proxy/client/cli/commands/up.py b/litellm/proxy/client/cli/commands/up.py index a4ed8a6cc5f..23cce90992c 100644 --- a/litellm/proxy/client/cli/commands/up.py +++ b/litellm/proxy/client/cli/commands/up.py @@ -9,7 +9,7 @@ from dataclasses import dataclass from pathlib import Path from types import FrameType -from typing import Dict, Mapping, Optional +from typing import Mapping import click from pydantic import JsonValue, TypeAdapter @@ -37,14 +37,14 @@ class BackupRecord: """Snapshot of ~/.claude/settings.json taken right before `lite up` patches it.""" existed: bool - content: Optional[Dict[str, JsonValue]] + content: dict[str, JsonValue] | None -_SETTINGS_ADAPTER = TypeAdapter(Dict[str, JsonValue]) +_SETTINGS_ADAPTER = TypeAdapter(dict[str, JsonValue]) _BACKUP_RECORD_ADAPTER = TypeAdapter(BackupRecord) -def load_json_or_empty(path: Path) -> Dict[str, JsonValue]: +def load_json_or_empty(path: Path) -> dict[str, JsonValue]: if not path.exists(): return {} with open(path, "r") as f: @@ -56,7 +56,7 @@ def load_json_or_empty(path: Path) -> Dict[str, JsonValue]: def merge_claude_settings( settings: Mapping[str, JsonValue], base_url: str, api_key_helper: str -) -> Dict[str, JsonValue]: +) -> dict[str, JsonValue]: """Return a new settings dict wired to route Claude Code through the proxy. Only env.ANTHROPIC_BASE_URL and the top-level apiKeyHelper are overridden; a @@ -71,7 +71,7 @@ def merge_claude_settings( return {**settings, ENV_KEY: env, API_KEY_HELPER_KEY: api_key_helper} -def write_backup(record: BackupRecord, backup_path: Optional[Path] = None) -> None: +def write_backup(record: BackupRecord, backup_path: Path | None = None) -> None: path = backup_path if backup_path is not None else BACKUP_PATH path.parent.mkdir(exist_ok=True) with open(path, "w") as f: @@ -79,7 +79,7 @@ def write_backup(record: BackupRecord, backup_path: Optional[Path] = None) -> No os.chmod(path, 0o600) -def read_backup(backup_path: Optional[Path] = None) -> Optional[BackupRecord]: +def read_backup(backup_path: Path | None = None) -> BackupRecord | None: path = backup_path if backup_path is not None else BACKUP_PATH if not path.exists(): return None @@ -87,9 +87,7 @@ def read_backup(backup_path: Optional[Path] = None) -> Optional[BackupRecord]: return _BACKUP_RECORD_ADAPTER.validate_json(f.read()) -def restore_claude_settings( - settings_path: Optional[Path] = None, backup_path: Optional[Path] = None -) -> Optional[BackupRecord]: +def restore_claude_settings(settings_path: Path | None = None, backup_path: Path | None = None) -> BackupRecord | None: """Restore settings_path from the backup at backup_path, then delete the backup. Returns the restored record, or None if there was nothing to restore. @@ -199,7 +197,7 @@ def up(ctx: click.Context) -> None: stop_event = threading.Event() restored = threading.Lock() - def _handle_signal(_signum: int, _frame: Optional[FrameType]) -> None: + def _handle_signal(_signum: int, _frame: FrameType | None) -> None: stop_event.set() def _restore_once() -> None: @@ -225,16 +223,16 @@ def down() -> None: __all__ = [ - "up", - "down", + "BACKUP_PATH", + "CLAUDE_SETTINGS_PATH", "BackupRecord", + "UpError", + "down", "load_json_or_empty", "merge_claude_settings", - "write_backup", "read_backup", - "restore_claude_settings", "resolve_api_key_helper", - "UpError", - "CLAUDE_SETTINGS_PATH", - "BACKUP_PATH", + "restore_claude_settings", + "up", + "write_backup", ] From d4ad5218454e25f56d872804183a26f1d669988e Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 14 Jul 2026 21:12:57 -0700 Subject: [PATCH 06/12] fix(ci): bump httplib2 and setuptools to patched versions Clears osv-scan findings for PYSEC-2026-3444 and PYSEC-2026-3447. --- uv.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/uv.lock b/uv.lock index 1d5f688aebc..776dc9067a2 100644 --- a/uv.lock +++ b/uv.lock @@ -9,7 +9,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-07-11T19:28:02.260785Z" +exclude-newer = "2026-07-12T04:11:39.831514Z" exclude-newer-span = "P3D" [manifest] @@ -2504,14 +2504,14 @@ wheels = [ [[package]] name = "httplib2" -version = "0.31.2" +version = "0.32.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pyparsing" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/c1/1f/e86365613582c027dda5ddb64e1010e57a3d53e99ab8a72093fa13d565ec/httplib2-0.31.2.tar.gz", hash = "sha256:385e0869d7397484f4eab426197a4c020b606edd43372492337c0b4010ae5d24", size = 250800, upload-time = "2026-01-23T11:04:44.165Z" } +sdist = { url = "https://files.pythonhosted.org/packages/84/f5/ccf58de92d61e3ad921119668f54ed36ca1d0cf5dcc5c1657dfb164fd78b/httplib2-0.32.0.tar.gz", hash = "sha256:48a0ef30a42db65d8f3399045e1d09ab0ba66e3b9efc360d07f80ea55d286025", size = 254283, upload-time = "2026-06-26T10:13:56.265Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/2f/90/fd509079dfcab01102c0fdd87f3a9506894bc70afcf9e9785ef6b2b3aff6/httplib2-0.31.2-py3-none-any.whl", hash = "sha256:dbf0c2fa3862acf3c55c078ea9c0bc4481d7dc5117cae71be9514912cf9f8349", size = 91099, upload-time = "2026-01-23T11:04:42.78Z" }, + { url = "https://files.pythonhosted.org/packages/33/a0/550eec327e5f5c7b732531c489f5307efec41f047b0d703bd4ca1e5ad2db/httplib2-0.32.0-py3-none-any.whl", hash = "sha256:dc6705cacdf3fb0a2aba7629fa33c90fd93e30035db0c157325826be177e4816", size = 93148, upload-time = "2026-06-26T10:13:54.985Z" }, ] [[package]] @@ -7041,11 +7041,11 @@ wheels = [ [[package]] name = "setuptools" -version = "82.0.1" +version = "83.0.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/4f/db/cfac1baf10650ab4d1c111714410d2fbb77ac5a616db26775db562c8fab2/setuptools-82.0.1.tar.gz", hash = "sha256:7d872682c5d01cfde07da7bccc7b65469d3dca203318515ada1de5eda35efbf9", size = 1152316, upload-time = "2026-03-09T12:47:17.221Z" } +sdist = { url = "https://files.pythonhosted.org/packages/34/26/f5d29e25ffdb535afef2d35cdb55b325298f96debd670da4c325e08d70f4/setuptools-83.0.0.tar.gz", hash = "sha256:025bccbbf0fa05b6192bc64ae1e7b16e001fd6d6d4d5de03c97b1c1ade523bef", size = 1154254, upload-time = "2026-07-04T15:31:22.699Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", hash = "sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb", size = 1006223, upload-time = "2026-03-09T12:47:15.026Z" }, + { url = "https://files.pythonhosted.org/packages/5d/40/e1e72872c6354b306daef1703549e8e83b4d43cfea356311bf722a043752/setuptools-83.0.0-py3-none-any.whl", hash = "sha256:29b23c360f22f414dc7336bb39178cc7bcbf6021ed2733cde173f09dba19abb3", size = 1008090, upload-time = "2026-07-04T15:31:20.885Z" }, ] [[package]] From 39f6618f93be572ba3d5edea342345a626363415 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Tue, 14 Jul 2026 21:28:11 -0700 Subject: [PATCH 07/12] fix(cli): write autoroute's secret-bearing files with mode 0600 commands.py wrote config.yaml (embeds the real proxy key) and Claude Code's settings.json (embeds the ephemeral proxy's master key) with plain open(), landing at the umask-derived default (commonly 0644) until a later chmod call caught up. That window, and the missed case where settings.json already exists (chmod never ran at all there), left a credential-bearing file readable by another local account. secure_create() fixes the mode via fchmod on the fd before any content is written, covering both the brand-new-file and already-exists cases, and commands.py/wizard.py now route their sensitive writes through it. --- .../client/cli/commands/autoroute/commands.py | 6 ++--- .../client/cli/commands/autoroute/process.py | 24 +++++++++++++++++++ .../client/cli/commands/autoroute/wizard.py | 5 ++-- .../client/cli/autoroute/test_commands.py | 4 ++++ 4 files changed, 33 insertions(+), 6 deletions(-) diff --git a/litellm/proxy/client/cli/commands/autoroute/commands.py b/litellm/proxy/client/cli/commands/autoroute/commands.py index cbc07456ac2..82ba047ec9d 100644 --- a/litellm/proxy/client/cli/commands/autoroute/commands.py +++ b/litellm/proxy/client/cli/commands/autoroute/commands.py @@ -23,6 +23,7 @@ launch_proxy, poll_liveliness, read_pid_record, + secure_create, stream_log, terminate, write_pid_record, @@ -52,9 +53,8 @@ def _mint_and_embed_master_key() -> str: "master_key": master_key, } updated: dict[str, JsonValue] = {**generated, "general_settings": updated_settings} - with open(CONFIG_PATH, "w") as f: + with secure_create(CONFIG_PATH) as f: yaml.safe_dump(updated, f, sort_keys=False) - CONFIG_PATH.chmod(0o600) return master_key @@ -103,7 +103,7 @@ def up() -> None: ) merged = merge_claude_settings_static_token(original_settings, base_url, master_key) CLAUDE_SETTINGS_PATH.parent.mkdir(parents=True, exist_ok=True) - with open(CLAUDE_SETTINGS_PATH, "w") as f: + with secure_create(CLAUDE_SETTINGS_PATH) as f: json.dump(merged, f, indent=2) click.echo(f"litellm: ephemeral auto-router proxy up at {base_url} (pid {process.pid})") diff --git a/litellm/proxy/client/cli/commands/autoroute/process.py b/litellm/proxy/client/cli/commands/autoroute/process.py index 40d0585cfed..c8de68307b9 100644 --- a/litellm/proxy/client/cli/commands/autoroute/process.py +++ b/litellm/proxy/client/cli/commands/autoroute/process.py @@ -9,6 +9,7 @@ import time from dataclasses import dataclass from pathlib import Path +from typing import IO, Iterator import click import requests @@ -20,6 +21,28 @@ PID_RECORD_PATH = AUTOROUTE_DIR / "proxy.pid.json" +@contextlib.contextmanager +def secure_create(path: Path) -> Iterator[IO[str]]: + """Open path for writing with mode 0600 fixed up before any content is written. + + A plain `open(path, "w")` creates a *new* file at the umask-derived default (commonly 0644) + and leaves it world- or group-readable until a later `chmod` call catches up -- a real window + in which a file holding a credential (a proxy master key, a Claude Code auth token) is readable + by another local account. Passing the mode to `os.open` closes that window for a brand-new + file, but `O_CREAT`'s mode argument is only applied on creation: if the file already exists + (the common case for `~/.claude/settings.json`, which normally predates `lite autoroute up`) + its old, broader permissions carry over untouched. `os.fchmod` right after opening -- before a + single byte of the new content is written -- covers both cases. + """ + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + os.fchmod(fd, 0o600) + f: IO[str] = os.fdopen(fd, "w") + try: + yield f + finally: + f.close() + + class ProcessLaunchError(Exception): """Raised when the ephemeral proxy subprocess fails to come up healthy.""" @@ -151,6 +174,7 @@ def stream_log(log_path: Path, stop_event: threading.Event) -> None: "launch_proxy", "poll_liveliness", "read_pid_record", + "secure_create", "stream_log", "terminate", "write_pid_record", diff --git a/litellm/proxy/client/cli/commands/autoroute/wizard.py b/litellm/proxy/client/cli/commands/autoroute/wizard.py index 06a6ea7d2b1..b9f8a5408b5 100644 --- a/litellm/proxy/client/cli/commands/autoroute/wizard.py +++ b/litellm/proxy/client/cli/commands/autoroute/wizard.py @@ -22,7 +22,7 @@ parse_discovered_models, validate_config, ) -from .process import CONFIG_PATH +from .process import CONFIG_PATH, secure_create def _is_interactive() -> bool: @@ -113,9 +113,8 @@ def run_configure_wizard(ctx: click.Context) -> Path: model_list = build_generated_model_list(config) CONFIG_PATH.parent.mkdir(parents=True, exist_ok=True) - with open(CONFIG_PATH, "w") as f: + with secure_create(CONFIG_PATH) as f: yaml.safe_dump({"model_list": model_list}, f, sort_keys=False) - CONFIG_PATH.chmod(0o600) click.echo(f"\nWrote {CONFIG_PATH}") for tier, models in tiers.items(): diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py b/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py index 098659d5e8b..9737e49c4ae 100644 --- a/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py @@ -1,4 +1,5 @@ import json +import stat from typing import Optional import yaml @@ -90,6 +91,7 @@ def test_happy_path_patches_settings_then_restores_everything_on_stop(self, monk def fake_wait(self, timeout=None): captured["settings"] = json.loads(claude_settings_path.read_text()) captured["backup_existed"] = backup_path.exists() + captured["settings_mode"] = stat.S_IMODE(claude_settings_path.stat().st_mode) return True monkeypatch.setattr("threading.Event.wait", fake_wait) @@ -102,6 +104,7 @@ def fake_wait(self, timeout=None): assert captured["settings"]["env"]["ANTHROPIC_BASE_URL"] == "http://127.0.0.1:54321" assert captured["settings"]["env"]["ANTHROPIC_AUTH_TOKEN"] == "fixed-master-key" assert "apiKeyHelper" not in captured["settings"] + assert captured["settings_mode"] == 0o600 assert terminate_calls == [99999] assert not pid_record_path.exists() @@ -110,6 +113,7 @@ def fake_wait(self, timeout=None): written_config = yaml.safe_load(config_path.read_text()) assert written_config["general_settings"]["master_key"] == "fixed-master-key" + assert stat.S_IMODE(config_path.stat().st_mode) == 0o600 def test_surfaces_clean_error_and_cleans_up_when_health_check_fails(self, monkeypatch, tmp_path): config_path, _log_path, claude_settings_path, backup_path, pid_record_path = _patch_paths(monkeypatch, tmp_path) From 851b38fc4a7f80fd9b8c82daa6275a561a30fe84 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 15 Jul 2026 11:39:45 -0700 Subject: [PATCH 08/12] docs(cli): warn that a stale Claude Code session can leak to a squatted port lite autoroute up's master key is embedded statically (unlike lite up's apiKeyHelper, resolved per request), so a Claude Code session still running after teardown keeps sending it, along with prompt content, to a now-unbound loopback port that another local account can bind. This is the same one-time-patch tradeoff lite up already accepts, just with a static secret instead of a re-resolved one -- document it in the README's Caveats section and surface it in the teardown message itself. --- litellm/proxy/client/cli/README.md | 2 ++ litellm/proxy/client/cli/commands/autoroute/commands.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/litellm/proxy/client/cli/README.md b/litellm/proxy/client/cli/README.md index ab75c569283..f98b433a6f1 100644 --- a/litellm/proxy/client/cli/README.md +++ b/litellm/proxy/client/cli/README.md @@ -546,6 +546,8 @@ lite autoroute down # only needed if `up` was killed uncleanly instead of Ctrl Adaptive mode's learned state does not persist across `lite autoroute up` sessions -- there is no local database, so every session starts adaptive selection cold. A Claude Code session already running before `up` started, or still running when it stops, keeps whatever settings it loaded at its own startup; like `lite up`, this is a one-time file patch and restore, not a live traffic interceptor. Only Claude Code is supported, for the same reason as `lite up`: no other supported agent (for example Cursor) has an equivalent hot-patchable config file. +A session that outlives `up` (or is still running the moment you stop it) keeps sending requests, master key included, to that now-freed loopback port until you restart it. Once the ephemeral proxy process exits, nothing stops another local account on the same machine from binding that same port and receiving those requests instead -- unlike `lite up`'s `apiKeyHelper`, which is re-resolved per request, `autoroute`'s master key is a static value, so whoever receives them gets a live-looking token along with the prompt content. Restart any Claude Code session before you consider the machine clean, run `lite autoroute down` promptly rather than leaving a stopped session's settings patched, and do not run `lite autoroute up` on a shared or multi-tenant host. + ## Environment Variables The CLI respects the following environment variables: diff --git a/litellm/proxy/client/cli/commands/autoroute/commands.py b/litellm/proxy/client/cli/commands/autoroute/commands.py index 82ba047ec9d..b70f7bcfab5 100644 --- a/litellm/proxy/client/cli/commands/autoroute/commands.py +++ b/litellm/proxy/client/cli/commands/autoroute/commands.py @@ -119,6 +119,11 @@ def _teardown() -> None: clear_pid_record() restore_claude_settings(CLAUDE_SETTINGS_PATH, AUTOROUTE_BACKUP_PATH) click.echo("\nStopped ephemeral proxy and restored Claude Code settings.") + click.echo( + f"Restart any Claude Code session still open from this session, or another local account could " + f"bind the now-free port {port} and receive its requests. Do not use `lite autoroute up` on a " + f"shared or multi-tenant host." + ) def _handle_signal(_signum: int, _frame: FrameType | None) -> None: stop_event.set() From fbef12baa98ef6e345bbc932dc3987112786b53d Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 15 Jul 2026 11:58:17 -0700 Subject: [PATCH 09/12] fix(cli): address greptile review feedback on autoroute PR - terminate the ephemeral proxy child process when its health check fails, instead of leaking an orphaned, unrecoverable process bound to the port - replace bare assert isinstance checks (no-ops under python -O) with click.ClickException in the model-groups list and configure wizard code paths - close launch_proxy's log file handle once the child process has inherited its fd, instead of leaking it - add build_generated_proxy_config to config.py's __all__ --- .../proxy/client/cli/commands/autoroute/commands.py | 1 + .../proxy/client/cli/commands/autoroute/config.py | 1 + .../proxy/client/cli/commands/autoroute/process.py | 12 ++++++------ .../proxy/client/cli/commands/autoroute/wizard.py | 5 ++++- litellm/proxy/client/cli/commands/model_groups.py | 5 ++++- .../proxy/client/cli/autoroute/test_commands.py | 3 +++ .../proxy/client/cli/autoroute/test_wizard.py | 8 ++++++++ .../proxy/client/cli/test_model_groups_commands.py | 10 ++++++++++ 8 files changed, 37 insertions(+), 8 deletions(-) diff --git a/litellm/proxy/client/cli/commands/autoroute/commands.py b/litellm/proxy/client/cli/commands/autoroute/commands.py index b70f7bcfab5..88dcb6f554a 100644 --- a/litellm/proxy/client/cli/commands/autoroute/commands.py +++ b/litellm/proxy/client/cli/commands/autoroute/commands.py @@ -92,6 +92,7 @@ def up() -> None: try: poll_liveliness(base_url, LOG_PATH, process) except ProcessLaunchError as e: + terminate(process.pid) clear_pid_record() raise click.ClickException(str(e)) diff --git a/litellm/proxy/client/cli/commands/autoroute/config.py b/litellm/proxy/client/cli/commands/autoroute/config.py index 1d18f497324..fe2c4e467e7 100644 --- a/litellm/proxy/client/cli/commands/autoroute/config.py +++ b/litellm/proxy/client/cli/commands/autoroute/config.py @@ -229,6 +229,7 @@ def build_generated_proxy_config(config: AutorouteConfig, master_key: str) -> di "SemanticMatching", "SemanticMatchingChoice", "build_generated_model_list", + "build_generated_proxy_config", "chat_models", "embedding_models", "parse_discovered_models", diff --git a/litellm/proxy/client/cli/commands/autoroute/process.py b/litellm/proxy/client/cli/commands/autoroute/process.py index c8de68307b9..000ccf49623 100644 --- a/litellm/proxy/client/cli/commands/autoroute/process.py +++ b/litellm/proxy/client/cli/commands/autoroute/process.py @@ -66,12 +66,12 @@ def allocate_free_port() -> int: def launch_proxy(config_path: Path, port: int, log_path: Path) -> "subprocess.Popen[bytes]": log_path.parent.mkdir(parents=True, exist_ok=True) - log_file = open(log_path, "w") - return subprocess.Popen( - [sys.executable, "-m", "litellm.proxy.proxy_cli", "--config", str(config_path), "--port", str(port)], - stdout=log_file, - stderr=subprocess.STDOUT, - ) + with open(log_path, "w") as log_file: + return subprocess.Popen( + [sys.executable, "-m", "litellm.proxy.proxy_cli", "--config", str(config_path), "--port", str(port)], + stdout=log_file, + stderr=subprocess.STDOUT, + ) def _tail(log_path: Path, lines: int = 40) -> str: diff --git a/litellm/proxy/client/cli/commands/autoroute/wizard.py b/litellm/proxy/client/cli/commands/autoroute/wizard.py index b9f8a5408b5..89b53d8f9fd 100644 --- a/litellm/proxy/client/cli/commands/autoroute/wizard.py +++ b/litellm/proxy/client/cli/commands/autoroute/wizard.py @@ -70,7 +70,10 @@ def run_configure_wizard(ctx: click.Context) -> Path: client = Client(base_url=base_url, api_key=api_key) raw_groups = client.model_groups.info() - assert isinstance(raw_groups, list) + if not isinstance(raw_groups, list): + raise click.ClickException( + f"Unexpected response from /model_group/info: expected a list, got {type(raw_groups).__name__}" + ) discovered = parse_discovered_models(raw_groups) chat_pool = chat_models(discovered) embedding_pool = embedding_models(discovered) diff --git a/litellm/proxy/client/cli/commands/model_groups.py b/litellm/proxy/client/cli/commands/model_groups.py index 629a0b5aaf1..7de959a9b78 100644 --- a/litellm/proxy/client/cli/commands/model_groups.py +++ b/litellm/proxy/client/cli/commands/model_groups.py @@ -29,7 +29,10 @@ def list_model_groups(ctx: click.Context, output_format: Literal["table", "json" """List model groups accessible to your key, with mode and pricing""" client = create_client(ctx) groups = client.model_groups.info() - assert isinstance(groups, list) + if not isinstance(groups, list): + raise click.ClickException( + f"Unexpected response from /model_group/info: expected a list, got {type(groups).__name__}" + ) if output_format == "json": rich.print_json(data=groups) diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py b/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py index 9737e49c4ae..26c34138683 100644 --- a/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py @@ -122,6 +122,7 @@ def test_surfaces_clean_error_and_cleans_up_when_health_check_fails(self, monkey claude_settings_path.write_text(json.dumps(original_settings)) fake_process = FakeProcess(pid=555) + terminate_calls = [] def _raise_launch_error(*args, **kwargs): raise ProcessLaunchError("boom: proxy never became healthy") @@ -129,12 +130,14 @@ def _raise_launch_error(*args, **kwargs): monkeypatch.setattr(commands_module, "launch_proxy", lambda *a, **k: fake_process) monkeypatch.setattr(commands_module, "poll_liveliness", _raise_launch_error) monkeypatch.setattr(commands_module, "allocate_free_port", lambda: 12345) + monkeypatch.setattr(commands_module, "terminate", lambda pid, **k: terminate_calls.append(pid)) monkeypatch.setattr(commands_module.secrets, "token_urlsafe", lambda n: "fixed-master-key") result = self.runner.invoke(up) assert result.exit_code != 0 assert "boom" in result.output + assert terminate_calls == [555] assert not pid_record_path.exists() assert not backup_path.exists() assert json.loads(claude_settings_path.read_text()) == original_settings diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py b/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py index e14e93cf5c6..75431a2c588 100644 --- a/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_wizard.py @@ -183,6 +183,14 @@ def test_fails_cleanly_without_prompting_when_no_chat_models(self, tmp_path): assert "no chat-capable models" in result.output.lower() assert not config_path.exists() + def test_surfaces_clean_error_when_response_is_not_a_list(self, tmp_path): + result, config_path = _run(tmp_path, {"data": CHAT_AND_EMBEDDING_GROUPS}, {}, input_str="") + + assert result.exit_code != 0 + assert result.exception is None or not isinstance(result.exception, AssertionError) + assert "Unexpected response from /model_group/info" in result.output + assert not config_path.exists() + class TestRunConfigureWizardNotInteractive: def test_fails_cleanly_when_not_a_tty(self, tmp_path): diff --git a/tests/test_litellm/proxy/client/cli/test_model_groups_commands.py b/tests/test_litellm/proxy/client/cli/test_model_groups_commands.py index d7b5b9dede4..c2809a90ba2 100644 --- a/tests/test_litellm/proxy/client/cli/test_model_groups_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_model_groups_commands.py @@ -102,3 +102,13 @@ def test_list_error_handling(mock_client, cli_runner): assert result.exit_code != 0 assert "API Error" in str(result.exception) + + +def test_list_surfaces_clean_error_when_response_is_not_a_list(mock_client, cli_runner): + mock_client.return_value.model_groups.info.return_value = {"data": SAMPLE_MODEL_GROUPS} + + result = cli_runner.invoke(cli, ["model-groups", "list"]) + + assert result.exit_code != 0 + assert result.exception is None or not isinstance(result.exception, AssertionError) + assert "Unexpected response from /model_group/info" in result.output From cb978a570d128ba8b8460292ad0b6f4d3c6ab7a4 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 15 Jul 2026 12:07:51 -0700 Subject: [PATCH 10/12] fix(cli): close TOCTOU window in lite up's settings backup write write_backup wrote the backup (which can embed the original apiKeyHelper/settings content) with plain open() + a chmod call after the fact -- the same permissive-until-corrected window already fixed for autoroute's config.yaml and Claude settings writes, and missed entirely when the backup file already exists with broader permissions. Moves secure_create (atomic-enough 0600 via fchmod before any content is written) to up.py, the module both lite up and lite autoroute share, and has autoroute/process.py import it from there instead of keeping its own copy. --- .../client/cli/commands/autoroute/process.py | 25 ++--------------- litellm/proxy/client/cli/commands/up.py | 27 ++++++++++++++++--- .../proxy/client/cli/test_up_commands.py | 16 +++++++++++ 3 files changed, 42 insertions(+), 26 deletions(-) diff --git a/litellm/proxy/client/cli/commands/autoroute/process.py b/litellm/proxy/client/cli/commands/autoroute/process.py index 000ccf49623..d1b798c40cf 100644 --- a/litellm/proxy/client/cli/commands/autoroute/process.py +++ b/litellm/proxy/client/cli/commands/autoroute/process.py @@ -9,40 +9,19 @@ import time from dataclasses import dataclass from pathlib import Path -from typing import IO, Iterator import click import requests from pydantic import TypeAdapter +from ..up import secure_create + AUTOROUTE_DIR = Path.home() / ".litellm" / "autorouter" CONFIG_PATH = AUTOROUTE_DIR / "config.yaml" LOG_PATH = AUTOROUTE_DIR / "proxy.log" PID_RECORD_PATH = AUTOROUTE_DIR / "proxy.pid.json" -@contextlib.contextmanager -def secure_create(path: Path) -> Iterator[IO[str]]: - """Open path for writing with mode 0600 fixed up before any content is written. - - A plain `open(path, "w")` creates a *new* file at the umask-derived default (commonly 0644) - and leaves it world- or group-readable until a later `chmod` call catches up -- a real window - in which a file holding a credential (a proxy master key, a Claude Code auth token) is readable - by another local account. Passing the mode to `os.open` closes that window for a brand-new - file, but `O_CREAT`'s mode argument is only applied on creation: if the file already exists - (the common case for `~/.claude/settings.json`, which normally predates `lite autoroute up`) - its old, broader permissions carry over untouched. `os.fchmod` right after opening -- before a - single byte of the new content is written -- covers both cases. - """ - fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) - os.fchmod(fd, 0o600) - f: IO[str] = os.fdopen(fd, "w") - try: - yield f - finally: - f.close() - - class ProcessLaunchError(Exception): """Raised when the ephemeral proxy subprocess fails to come up healthy.""" diff --git a/litellm/proxy/client/cli/commands/up.py b/litellm/proxy/client/cli/commands/up.py index 23cce90992c..edf5a7acc05 100644 --- a/litellm/proxy/client/cli/commands/up.py +++ b/litellm/proxy/client/cli/commands/up.py @@ -1,4 +1,5 @@ import atexit +import contextlib import json import os import shlex @@ -9,7 +10,7 @@ from dataclasses import dataclass from pathlib import Path from types import FrameType -from typing import Mapping +from typing import IO, Iterator, Mapping import click from pydantic import JsonValue, TypeAdapter @@ -71,12 +72,32 @@ def merge_claude_settings( return {**settings, ENV_KEY: env, API_KEY_HELPER_KEY: api_key_helper} +@contextlib.contextmanager +def secure_create(path: Path) -> Iterator[IO[str]]: + """Open path for writing with mode 0600 fixed up before any content is written. + + A plain `open(path, "w")` creates a *new* file at the umask-derived default (commonly 0644) + and leaves it world- or group-readable until a later `chmod` call catches up -- a real window + in which a file holding a credential is readable by another local account. Passing the mode to + `os.open` closes that window for a brand-new file, but `O_CREAT`'s mode argument is only + applied on creation: if the file already exists its old, broader permissions carry over + untouched. `os.fchmod` right after opening -- before a single byte of the new content is + written -- covers both cases. + """ + fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600) + os.fchmod(fd, 0o600) + f: IO[str] = os.fdopen(fd, "w") + try: + yield f + finally: + f.close() + + def write_backup(record: BackupRecord, backup_path: Path | None = None) -> None: path = backup_path if backup_path is not None else BACKUP_PATH path.parent.mkdir(exist_ok=True) - with open(path, "w") as f: + with secure_create(path) as f: json.dump({"existed": record.existed, "content": record.content}, f, indent=2) - os.chmod(path, 0o600) def read_backup(backup_path: Path | None = None) -> BackupRecord | None: diff --git a/tests/test_litellm/proxy/client/cli/test_up_commands.py b/tests/test_litellm/proxy/client/cli/test_up_commands.py index 63f1da8facf..49508c22f47 100644 --- a/tests/test_litellm/proxy/client/cli/test_up_commands.py +++ b/tests/test_litellm/proxy/client/cli/test_up_commands.py @@ -1,5 +1,6 @@ import json import shutil +import stat import sys from unittest.mock import patch @@ -129,6 +130,21 @@ def test_read_backup_missing_file_returns_none(self, monkeypatch, tmp_path): _patch_paths(monkeypatch, tmp_path) assert read_backup() is None + def test_write_backup_restricts_permissions_for_a_new_file(self, monkeypatch, tmp_path): + _settings_path, backup_path = _patch_paths(monkeypatch, tmp_path) + write_backup(BackupRecord(existed=True, content={"a": 1})) + assert stat.S_IMODE(backup_path.stat().st_mode) == 0o600 + + def test_write_backup_restricts_permissions_of_a_preexisting_permissive_file(self, monkeypatch, tmp_path): + _settings_path, backup_path = _patch_paths(monkeypatch, tmp_path) + backup_path.parent.mkdir(parents=True, exist_ok=True) + backup_path.write_text("{}") + backup_path.chmod(0o644) + + write_backup(BackupRecord(existed=True, content={"a": 1})) + + assert stat.S_IMODE(backup_path.stat().st_mode) == 0o600 + def test_backup_file_always_removed_after_restore(self, monkeypatch, tmp_path): _settings_path, backup_path = _patch_paths(monkeypatch, tmp_path) write_backup(BackupRecord(existed=False, content=None)) From 7e49429eca986e08034b6a166de74d2b1c6dc1c5 Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 15 Jul 2026 12:55:04 -0700 Subject: [PATCH 11/12] fix(cli): refuse autoroute up when a stale backup exists from a crash The pid-record check only catches a still-live duplicate process; a SIGKILL'd `up` leaves no live pid but does leave AUTOROUTE_BACKUP_PATH behind. Without this guard, a fresh `up` overwrote that backup with the currently-patched Claude settings instead of the true originals, so `down`/Ctrl-C would restore the wrong content permanently. up.py's `lite up` already guards the analogous case; mirror it here. --- .../client/cli/commands/autoroute/commands.py | 6 ++++++ .../client/cli/autoroute/test_commands.py | 20 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/litellm/proxy/client/cli/commands/autoroute/commands.py b/litellm/proxy/client/cli/commands/autoroute/commands.py index 88dcb6f554a..acfebc0f370 100644 --- a/litellm/proxy/client/cli/commands/autoroute/commands.py +++ b/litellm/proxy/client/cli/commands/autoroute/commands.py @@ -83,6 +83,12 @@ def up() -> None: "Run `lite autoroute down` first." ) + if AUTOROUTE_BACKUP_PATH.exists(): + raise click.ClickException( + f"{AUTOROUTE_BACKUP_PATH} already exists -- `lite autoroute up` looks like it's already " + "running (or crashed without cleanup). Run `lite autoroute down` first." + ) + master_key = _mint_and_embed_master_key() port = allocate_free_port() base_url = f"http://127.0.0.1:{port}" diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py b/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py index 26c34138683..f9a520ff148 100644 --- a/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_commands.py @@ -71,6 +71,26 @@ def test_refuses_when_pid_record_exists_and_process_still_running(self, monkeypa assert "lite autoroute down" in result.output assert config_path.read_text() == yaml.safe_dump({"model_list": []}) + def test_refuses_when_backup_exists_after_an_unclean_crash(self, monkeypatch, tmp_path): + """A prior `up` that was SIGKILL'd leaves no live pid but does leave a stale backup file. + + Without this guard, a fresh `up` would overwrite that backup with the currently-patched + (not original) Claude settings, so `down`/Ctrl-C would restore the wrong content forever. + """ + config_path, _log_path, claude_settings_path, backup_path, _pid_record_path = _patch_paths( + monkeypatch, tmp_path + ) + config_path.write_text(yaml.safe_dump({"model_list": []})) + claude_settings_path.write_text(json.dumps({"env": {"ANTHROPIC_AUTH_TOKEN": "stale-patched-token"}})) + write_backup(ClaudeBackupRecord(existed=True, content={"theme": "dark"}), backup_path) + + result = self.runner.invoke(up) + + assert result.exit_code != 0 + assert "already exists" in result.output + assert "lite autoroute down" in result.output + assert json.loads(backup_path.read_text())["content"] == {"theme": "dark"} + def test_happy_path_patches_settings_then_restores_everything_on_stop(self, monkeypatch, tmp_path): config_path, log_path, claude_settings_path, backup_path, pid_record_path = _patch_paths(monkeypatch, tmp_path) config_path.write_text(yaml.safe_dump({"model_list": []})) From f737392867b7e7cb04f7c3098d0b4b4aec47fb6b Mon Sep 17 00:00:00 2001 From: Krrish Dholakia Date: Wed, 15 Jul 2026 12:57:48 -0700 Subject: [PATCH 12/12] fix(cli): bind the ephemeral autoroute proxy to loopback only proxy_cli.py defaults --host to 0.0.0.0 when not passed explicitly. launch_proxy never passed it, so the ephemeral proxy -- despite every base_url in this module being built from 127.0.0.1 -- was actually reachable from other hosts on the network, including its unauthenticated-until-config-lands routes before the master key is wired in. --- .../client/cli/commands/autoroute/process.py | 12 +++++++++++- .../proxy/client/cli/autoroute/test_process.py | 18 ++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/litellm/proxy/client/cli/commands/autoroute/process.py b/litellm/proxy/client/cli/commands/autoroute/process.py index d1b798c40cf..e1cd38531d5 100644 --- a/litellm/proxy/client/cli/commands/autoroute/process.py +++ b/litellm/proxy/client/cli/commands/autoroute/process.py @@ -47,7 +47,17 @@ def launch_proxy(config_path: Path, port: int, log_path: Path) -> "subprocess.Po log_path.parent.mkdir(parents=True, exist_ok=True) with open(log_path, "w") as log_file: return subprocess.Popen( - [sys.executable, "-m", "litellm.proxy.proxy_cli", "--config", str(config_path), "--port", str(port)], + [ + sys.executable, + "-m", + "litellm.proxy.proxy_cli", + "--config", + str(config_path), + "--port", + str(port), + "--host", + "127.0.0.1", + ], stdout=log_file, stderr=subprocess.STDOUT, ) diff --git a/tests/test_litellm/proxy/client/cli/autoroute/test_process.py b/tests/test_litellm/proxy/client/cli/autoroute/test_process.py index bffd8b598f4..d182ebfffc2 100644 --- a/tests/test_litellm/proxy/client/cli/autoroute/test_process.py +++ b/tests/test_litellm/proxy/client/cli/autoroute/test_process.py @@ -1,6 +1,7 @@ import os import socket from typing import Optional +from unittest.mock import patch import pytest @@ -11,6 +12,7 @@ allocate_free_port, clear_pid_record, is_running, + launch_proxy, poll_liveliness, read_pid_record, write_pid_record, @@ -36,6 +38,22 @@ def test_allocate_free_port_returns_a_bindable_port(): sock.bind(("127.0.0.1", port)) +class TestLaunchProxy: + def test_binds_loopback_only_not_all_interfaces(self, tmp_path): + """proxy_cli.py's own --host default is 0.0.0.0 -- without an explicit override here, the + ephemeral proxy would be reachable from other hosts on the network despite base_url always + being built from 127.0.0.1, exposing its unauthenticated-until-master-key-lands routes.""" + config_path = tmp_path / "config.yaml" + log_path = tmp_path / "proxy.log" + + with patch.object(process_module.subprocess, "Popen") as mock_popen: + launch_proxy(config_path, 12345, log_path) + + args = mock_popen.call_args[0][0] + assert "--host" in args + assert args[args.index("--host") + 1] == "127.0.0.1" + + class TestPidRecordRoundTrip: def test_write_then_read_round_trips(self, tmp_path): path = tmp_path / "pid.json"