From 3ab25d1210e5eb1a7ffc3cfca86142e869ee955b Mon Sep 17 00:00:00 2001 From: Jamie Pine Date: Sun, 9 Aug 2026 18:45:32 -0700 Subject: [PATCH 1/2] fix(llm): verified model defaults and rejected-model fallback - ChatGPT device OAuth verifies the model with a real completion before writing it into routing, walking the provider's default candidates when the requested id is rejected - runtime completion path falls back to siblings from the provider's default routing table when a configured model id is rejected and no explicit fallback chain exists; the terminal error names the rejected ids and asks the user to pick a model and report it - openai-chatgpt defaults move to gpt-5.6-sol with a gpt-5.6/gpt-5.5 chain - provider default models served from GET /providers/default-models; the divergent frontend table and CHATGPT_OAUTH_DEFAULT_MODEL constant are gone - routing editor gains an All Models select that fills every slot at once --- interface/src/api/client.ts | 14 +- .../agent-config/ConfigSectionEditor.tsx | 30 +++ .../src/components/settings/constants.ts | 24 --- interface/src/components/settings/index.ts | 2 +- interface/src/routes/Settings.tsx | 29 ++- src/api/portal.rs | 2 +- src/api/providers.rs | 201 +++++++++++++++--- src/api/server.rs | 1 + src/llm/model.rs | 50 ++++- src/llm/routing.rs | 116 +++++++++- 10 files changed, 381 insertions(+), 88 deletions(-) diff --git a/interface/src/api/client.ts b/interface/src/api/client.ts index 51f4038f8..33a768e69 100644 --- a/interface/src/api/client.ts +++ b/interface/src/api/client.ts @@ -1740,12 +1740,22 @@ export const api = { deployment?: string | null; }>; }, - startOpenAiOAuthBrowser: async (params: {model: string}) => { + providerDefaultModels: async () => { + const response = await fetch(`${getApiBase()}/providers/default-models`); + if (!response.ok) { + throw new Error(`API error: ${response.status}`); + } + return response.json() as Promise<{ + defaults: Record; + chatgpt_oauth: string; + }>; + }, + startOpenAiOAuthBrowser: async (params?: {model?: string}) => { const response = await fetch(`${getApiBase()}/providers/openai/browser-oauth/start`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ - model: params.model, + model: params?.model, }), }); if (!response.ok) { diff --git a/interface/src/components/agent-config/ConfigSectionEditor.tsx b/interface/src/components/agent-config/ConfigSectionEditor.tsx index 64745aadf..64f6577a3 100644 --- a/interface/src/components/agent-config/ConfigSectionEditor.tsx +++ b/interface/src/components/agent-config/ConfigSectionEditor.tsx @@ -218,8 +218,38 @@ export function ConfigSectionEditor({ description: "Model for transcribing audio attachments", }, ]; + const textSlotKeys = [ + "channel", + "branch", + "worker", + "compactor", + "cortex", + ] as const; + const textSlotValues = textSlotKeys.map((key) => localValues[key] ?? ""); + const uniformModel = textSlotValues.every((v) => v === textSlotValues[0]) + ? textSlotValues[0] + : ""; + const applyModelToAllSlots = (model: string) => { + setLocalValues((prev) => ({ + ...prev, + channel: model, + branch: model, + worker: model, + compactor: model, + cortex: model, + })); + setLocalDirty(true); + }; return (
+
+ +
{modelSlots.map(({key, label, description}) => { const modelValue = localValues[key] ?? ""; const thinkingKey = diff --git a/interface/src/components/settings/constants.ts b/interface/src/components/settings/constants.ts index 018bce707..bbe842b17 100644 --- a/interface/src/components/settings/constants.ts +++ b/interface/src/components/settings/constants.ts @@ -87,7 +87,6 @@ export const PROVIDERS = [ description: "Multi-provider gateway with unified API", placeholder: "sk-or-...", envVar: "OPENROUTER_API_KEY", - defaultModel: "openrouter/anthropic/claude-sonnet-4", }, { id: "kilo", @@ -95,7 +94,6 @@ export const PROVIDERS = [ description: "OpenAI-compatible multi-provider gateway", placeholder: "sk-...", envVar: "KILO_API_KEY", - defaultModel: "kilo/anthropic/claude-sonnet-4.5", }, { id: "opencode-zen", @@ -103,7 +101,6 @@ export const PROVIDERS = [ description: "Multi-format gateway (Kimi, GLM, MiniMax, Qwen)", placeholder: "...", envVar: "OPENCODE_ZEN_API_KEY", - defaultModel: "opencode-zen/kimi-k2.5", }, { id: "opencode-go", @@ -111,7 +108,6 @@ export const PROVIDERS = [ description: "Lite OpenCode model catalog and limits", placeholder: "...", envVar: "OPENCODE_GO_API_KEY", - defaultModel: "opencode-go/kimi-k2.5", }, { id: "anthropic", @@ -119,7 +115,6 @@ export const PROVIDERS = [ description: "Claude models (Sonnet, Opus, Haiku)", placeholder: "sk-ant-...", envVar: "ANTHROPIC_API_KEY", - defaultModel: "anthropic/claude-sonnet-4", }, { id: "openai", @@ -127,7 +122,6 @@ export const PROVIDERS = [ description: "GPT models", placeholder: "sk-...", envVar: "OPENAI_API_KEY", - defaultModel: "openai/gpt-4.1", }, { id: "zai-coding-plan", @@ -135,7 +129,6 @@ export const PROVIDERS = [ description: "GLM coding models (glm-4.7, glm-5, glm-4.5-air)", placeholder: "...", envVar: "ZAI_CODING_PLAN_API_KEY", - defaultModel: "zai-coding-plan/glm-5", }, { id: "zhipu", @@ -143,7 +136,6 @@ export const PROVIDERS = [ description: "GLM models (GLM-4, GLM-4-Flash)", placeholder: "...", envVar: "ZHIPU_API_KEY", - defaultModel: "zhipu/glm-4-plus", }, { id: "groq", @@ -151,7 +143,6 @@ export const PROVIDERS = [ description: "Fast inference for Llama, Mixtral models", placeholder: "gsk_...", envVar: "GROQ_API_KEY", - defaultModel: "groq/llama-3.3-70b-versatile", }, { id: "together", @@ -159,7 +150,6 @@ export const PROVIDERS = [ description: "Wide model selection with competitive pricing", placeholder: "...", envVar: "TOGETHER_API_KEY", - defaultModel: "together/meta-llama/Meta-Llama-3.1-405B-Instruct-Turbo", }, { id: "fireworks", @@ -167,7 +157,6 @@ export const PROVIDERS = [ description: "Fast inference for popular OSS models", placeholder: "...", envVar: "FIREWORKS_API_KEY", - defaultModel: "fireworks/accounts/fireworks/models/llama-v3p3-70b-instruct", }, { id: "deepseek", @@ -175,7 +164,6 @@ export const PROVIDERS = [ description: "DeepSeek Chat and Reasoner models", placeholder: "sk-...", envVar: "DEEPSEEK_API_KEY", - defaultModel: "deepseek/deepseek-chat", }, { id: "xai", @@ -183,7 +171,6 @@ export const PROVIDERS = [ description: "Grok models", placeholder: "xai-...", envVar: "XAI_API_KEY", - defaultModel: "xai/grok-2-latest", }, { id: "mistral", @@ -191,7 +178,6 @@ export const PROVIDERS = [ description: "Mistral Large, Small, Codestral models", placeholder: "...", envVar: "MISTRAL_API_KEY", - defaultModel: "mistral/mistral-large-latest", }, { id: "gemini", @@ -199,7 +185,6 @@ export const PROVIDERS = [ description: "Google Gemini experimental and production models", placeholder: "AIza...", envVar: "GEMINI_API_KEY", - defaultModel: "gemini/gemini-2.5-flash", }, { id: "nvidia", @@ -207,7 +192,6 @@ export const PROVIDERS = [ description: "NVIDIA-hosted models via NIM API", placeholder: "nvapi-...", envVar: "NVIDIA_API_KEY", - defaultModel: "nvidia/meta/llama-3.1-405b-instruct", }, { id: "minimax", @@ -215,7 +199,6 @@ export const PROVIDERS = [ description: "MiniMax (Anthropic message format)", placeholder: "sk-...", envVar: "MINIMAX_API_KEY", - defaultModel: "minimax/MiniMax-M2.5", }, { id: "minimax-cn", @@ -223,7 +206,6 @@ export const PROVIDERS = [ description: "MiniMax China (Anthropic message format)", placeholder: "sk-...", envVar: "MINIMAX_CN_API_KEY", - defaultModel: "minimax-cn/MiniMax-M2.5", }, { id: "moonshot", @@ -231,7 +213,6 @@ export const PROVIDERS = [ description: "Kimi models (Kimi K2, Kimi K2.5)", placeholder: "sk-...", envVar: "MOONSHOT_API_KEY", - defaultModel: "moonshot/kimi-k2.5", }, { id: "github-copilot", @@ -239,7 +220,6 @@ export const PROVIDERS = [ description: "GitHub Copilot API (uses GitHub PAT for token exchange)", placeholder: "ghp_... or gh auth token", envVar: "GITHUB_COPILOT_API_KEY", - defaultModel: "github-copilot/claude-sonnet-4", }, { id: "azure", @@ -247,7 +227,6 @@ export const PROVIDERS = [ description: "Azure OpenAI Service with custom deployments", placeholder: "Azure API key (alphanumeric string)", envVar: "AZURE_API_KEY", - defaultModel: "azure/gpt-4o", }, { id: "ollama", @@ -255,12 +234,9 @@ export const PROVIDERS = [ description: "Local or remote Ollama API endpoint", placeholder: "http://localhost:11434", envVar: "OLLAMA_BASE_URL", - defaultModel: "ollama/llama3.2", }, ] as const; -export const CHATGPT_OAUTH_DEFAULT_MODEL = "openai-chatgpt/gpt-5.3-codex"; - export const PERMISSION_OPTIONS = [ { value: "allow", diff --git a/interface/src/components/settings/index.ts b/interface/src/components/settings/index.ts index ec98e9a78..acc6f2110 100644 --- a/interface/src/components/settings/index.ts +++ b/interface/src/components/settings/index.ts @@ -11,7 +11,7 @@ export {ChangelogSection} from "./ChangelogSection"; export {ConfigFileSection} from "./ConfigFileSection"; export {ProviderCard} from "./ProviderCard"; export {ChatGptOAuthDialog} from "./ChatGptOAuthDialog"; -export {SECTIONS, PROVIDERS, CHATGPT_OAUTH_DEFAULT_MODEL, PERMISSION_OPTIONS} from "./constants"; +export {SECTIONS, PROVIDERS, PERMISSION_OPTIONS} from "./constants"; export type { SectionId, Platform, diff --git a/interface/src/routes/Settings.tsx b/interface/src/routes/Settings.tsx index c6dfdded2..c0aba4264 100644 --- a/interface/src/routes/Settings.tsx +++ b/interface/src/routes/Settings.tsx @@ -30,7 +30,6 @@ import { ChatGptOAuthDialog, SECTIONS, PROVIDERS, - CHATGPT_OAUTH_DEFAULT_MODEL, type SectionId, } from "@/components/settings"; @@ -90,10 +89,19 @@ export function Settings() { enabled: activeSection === "providers", }); + // Per-provider default models come from the backend routing defaults so the + // UI can never drift from what the server would actually apply. + const {data: defaultModels} = useQuery({ + queryKey: ["provider-default-models"], + queryFn: api.providerDefaultModels, + staleTime: Infinity, + enabled: activeSection === "providers", + }); + // Fetch agents list and default agent config so we can pre-populate the // model field with the currently active routing model when editing an - // already-configured provider (instead of always showing the hardcoded - // defaultModel). + // already-configured provider (instead of always showing the provider's + // default model). const { data: agentsData } = useQuery({ queryKey: ["agents"], queryFn: api.agents, @@ -208,7 +216,7 @@ export function Settings() { ), }); const startOpenAiBrowserOAuthMutation = useMutation({ - mutationFn: (params: {model: string}) => + mutationFn: (params: {model?: string}) => api.startOpenAiOAuthBrowser(params), }); @@ -451,9 +459,8 @@ export function Settings() { setDeviceCodeInfo(null); setDeviceCodeCopied(false); try { - const result = await startOpenAiBrowserOAuthMutation.mutateAsync({ - model: CHATGPT_OAUTH_DEFAULT_MODEL, - }); + // The backend picks and verifies its own default model for this flow. + const result = await startOpenAiBrowserOAuthMutation.mutateAsync({}); if ( !result.success || !result.user_code || @@ -623,20 +630,20 @@ export function Settings() { name={provider.name} description={provider.description} configured={isConfigured(provider.id)} - defaultModel={provider.defaultModel} + defaultModel={defaultModels?.defaults[provider.id] ?? ""} onEdit={() => { setEditingProvider(provider.id); setKeyInput(""); // When the provider is already configured, pre-populate // the model field with the current routing model so the // user sees what's actually active rather than the - // hardcoded defaultModel placeholder. + // default placeholder. const currentChannel = defaultAgentConfig?.routing?.channel; const currentModel = isConfigured(provider.id) && currentChannel?.startsWith(`${provider.id}/`) ? currentChannel : null; - setModelInput(currentModel ?? provider.defaultModel ?? ""); + setModelInput(currentModel ?? defaultModels?.defaults[provider.id] ?? ""); setTestedSignature(null); setTestResult(null); setMessage(null); @@ -689,7 +696,7 @@ export function Settings() { name="ChatGPT Plus (OAuth)" description="Sign in with your ChatGPT Plus account using a device code." configured={isConfigured("openai-chatgpt")} - defaultModel={CHATGPT_OAUTH_DEFAULT_MODEL} + defaultModel={defaultModels?.chatgpt_oauth ?? ""} onEdit={() => setOpenAiOAuthDialogOpen(true)} onRemove={() => removeMutation.mutate("openai-chatgpt")} removing={removeMutation.isPending} diff --git a/src/api/portal.rs b/src/api/portal.rs index e2f6ec3c7..85ca5418a 100644 --- a/src/api/portal.rs +++ b/src/api/portal.rs @@ -492,7 +492,7 @@ pub(super) async fn conversation_defaults( runtime_configs .get(&query.agent_id) .map(|rc| rc.routing.load().channel.clone()) - .unwrap_or_else(|| "anthropic/claude-sonnet-4".to_string()) + .unwrap_or_else(|| crate::llm::RoutingConfig::default().channel) }; // Build available models from configured providers via the models catalog. diff --git a/src/api/providers.rs b/src/api/providers.rs index 4c2dea73e..59274148c 100644 --- a/src/api/providers.rs +++ b/src/api/providers.rs @@ -113,9 +113,20 @@ pub struct ProviderModelTestResponse { pub sample: Option, } +#[derive(Serialize, utoipa::ToSchema)] +pub struct ProviderDefaultModelsResponse { + /// Default model per provider id, from the backend routing defaults. + pub defaults: HashMap, + /// Default model applied by the ChatGPT device OAuth flow. + pub chatgpt_oauth: String, +} + #[derive(Deserialize, utoipa::ToSchema)] pub(super) struct OpenAiOAuthBrowserStartRequest { - model: String, + /// Model to apply after sign-in. Omitted or empty means the backend's + /// default for the ChatGPT OAuth provider. + #[serde(default)] + model: Option, } #[derive(Serialize, utoipa::ToSchema)] @@ -317,21 +328,96 @@ async fn update_device_oauth_status(state_key: &str, status: DeviceOAuthSessionS } } +/// Run a one-shot completion against the ChatGPT OAuth provider to prove a +/// model id is actually served for this account. +async fn verify_openai_chatgpt_model( + llm_manager: &Arc, + model: &str, +) -> Result<(), String> { + let model = crate::llm::SpacebotModel::make(llm_manager, model); + let agent = AgentBuilder::new(model) + .preamble("You are running a provider connectivity check. Reply with exactly: OK") + .build(); + agent + .prompt("Connection test") + .await + .map(|_| ()) + .map_err(|error| error.to_string()) +} + async fn finalize_openai_oauth( state: &Arc, credentials: &crate::openai_auth::OAuthCredentials, model: &str, -) -> anyhow::Result<()> { +) -> anyhow::Result { let instance_dir = (**state.instance_dir.load()).clone(); crate::openai_auth::save_credentials(&instance_dir, credentials) .context("failed to save OpenAI OAuth credentials")?; - if let Some(llm_manager) = state.llm_manager.read().await.as_ref() { - llm_manager - .set_openai_oauth_credentials(credentials.clone()) - .await; + let llm_manager = match state.llm_manager.read().await.as_ref() { + Some(llm_manager) => { + llm_manager + .set_openai_oauth_credentials(credentials.clone()) + .await; + llm_manager.clone() + } + None => { + // No manager yet (first-run setup) — build one just for verification. + let llm_manager = + Arc::new(crate::llm::LlmManager::new(build_test_llm_config("", "")).await?); + llm_manager + .set_openai_oauth_credentials(credentials.clone()) + .await; + llm_manager + } + }; + + // Verify the requested model before writing it into routing; if the + // provider rejects it, walk the default candidates until one responds. + let mut candidates = vec![model.to_string()]; + for candidate in crate::llm::routing::default_model_candidates("openai-chatgpt") { + if !candidates.contains(&candidate) { + candidates.push(candidate); + } } + let mut verified_model = None; + let mut failures = Vec::new(); + for candidate in &candidates { + match verify_openai_chatgpt_model(&llm_manager, candidate).await { + Ok(()) => { + verified_model = Some(candidate.clone()); + break; + } + Err(error) => { + tracing::warn!(model = %candidate, %error, "ChatGPT OAuth model verification failed"); + failures.push(format!("{candidate}: {error}")); + } + } + } + + let (applied_model, message) = match verified_model { + Some(applied) if applied == model => { + let message = format!( + "OpenAI configured via device OAuth. Model '{applied}' verified and applied to defaults and default agent routing." + ); + (applied, message) + } + Some(applied) => { + let message = format!( + "OpenAI configured via device OAuth. Model '{model}' was rejected by the provider, so verified fallback '{applied}' was applied to defaults and default agent routing instead." + ); + (applied, message) + } + None => ( + model.to_string(), + format!( + "OpenAI configured via device OAuth, but no model could be verified ({}). Applied '{model}' anyway — pick a working model in Settings and please report this as a bug.", + failures.join("; ") + ), + ), + }; + let config_path = state.config_path.read().await.clone(); let content = if config_path.exists() { tokio::fs::read_to_string(&config_path) @@ -342,7 +428,7 @@ async fn finalize_openai_oauth( }; let mut doc: toml_edit::DocumentMut = content.parse().context("failed to parse config.toml")?; - apply_model_routing(&mut doc, model); + apply_model_routing(&mut doc, &applied_model); tokio::fs::write(&config_path, doc.to_string()) .await .context("failed to write config.toml")?; @@ -355,7 +441,7 @@ async fn finalize_openai_oauth( .try_send(crate::ProviderSetupEvent::ProvidersConfigured) .ok(); - Ok(()) + Ok(message) } #[utoipa::path( @@ -561,6 +647,60 @@ pub(super) async fn get_providers( Ok(Json(ProvidersResponse { providers, has_any })) } +#[utoipa::path( + get, + path = "/providers/default-models", + responses( + (status = 200, body = ProviderDefaultModelsResponse), + ), + tag = "providers", +)] +pub(super) async fn get_provider_default_models() -> Json { + const PROVIDER_IDS: &[&str] = &[ + "anthropic", + "openai", + "openai-chatgpt", + "openrouter", + "kilo", + "zhipu", + "groq", + "together", + "fireworks", + "deepseek", + "xai", + "mistral", + "gemini", + "nvidia", + "opencode-zen", + "opencode-go", + "minimax", + "minimax-cn", + "moonshot", + "zai-coding-plan", + "github-copilot", + "ollama", + ]; + + let mut defaults: HashMap = PROVIDER_IDS + .iter() + .map(|id| { + ( + id.to_string(), + crate::llm::routing::defaults_for_provider(id).channel, + ) + }) + .collect(); + // Azure model ids name the user's own deployment, so this is a placeholder + // rather than a routing default. + defaults.insert("azure".to_string(), "azure/gpt-4o".to_string()); + + let chatgpt_oauth = crate::llm::routing::defaults_for_provider("openai-chatgpt").channel; + Json(ProviderDefaultModelsResponse { + defaults, + chatgpt_oauth, + }) +} + #[utoipa::path( post, path = "/providers/openai/browser-oauth/start", @@ -575,26 +715,24 @@ pub(super) async fn start_openai_browser_oauth( State(state): State>, Json(request): Json, ) -> Result, StatusCode> { - if request.model.trim().is_empty() { - return Ok(Json(OpenAiOAuthBrowserStartResponse { - success: false, - message: "Model cannot be empty".to_string(), - user_code: None, - verification_url: None, - state: None, - })); - } - let Some(chatgpt_model) = normalize_openai_chatgpt_model(&request.model) else { - return Ok(Json(OpenAiOAuthBrowserStartResponse { - success: false, - message: format!( - "Model '{}' must use provider 'openai' or 'openai-chatgpt'.", - request.model - ), - user_code: None, - verification_url: None, - state: None, - })); + let requested_model = request.model.unwrap_or_default(); + let chatgpt_model = if requested_model.trim().is_empty() { + crate::llm::routing::defaults_for_provider("openai-chatgpt").channel + } else { + match normalize_openai_chatgpt_model(&requested_model) { + Some(model) => model, + None => { + return Ok(Json(OpenAiOAuthBrowserStartResponse { + success: false, + message: format!( + "Model '{requested_model}' must use provider 'openai' or 'openai-chatgpt'." + ), + user_code: None, + verification_url: None, + state: None, + })); + } + } }; prune_expired_device_oauth_sessions().await; @@ -733,13 +871,10 @@ async fn run_device_oauth_background( }; match finalize_openai_oauth(&state, &credentials, &model).await { - Ok(()) => { + Ok(message) => { update_device_oauth_status( &state_key, - DeviceOAuthSessionStatus::Completed(format!( - "OpenAI configured via device OAuth. Model '{}' applied to defaults and default agent routing.", - model - )), + DeviceOAuthSessionStatus::Completed(message), ) .await; } diff --git a/src/api/server.rs b/src/api/server.rs index 51368fd52..9ceeab3bb 100644 --- a/src/api/server.rs +++ b/src/api/server.rs @@ -202,6 +202,7 @@ pub fn api_router() -> OpenApiRouter> { providers::get_providers, providers::update_provider )) + .routes(routes!(providers::get_provider_default_models)) .routes(routes!(providers::start_openai_browser_oauth)) .routes(routes!(providers::openai_browser_oauth_status)) .routes(routes!(providers::test_provider_model)) diff --git a/src/llm/model.rs b/src/llm/model.rs index ba8ea0198..54382a996 100644 --- a/src/llm/model.rs +++ b/src/llm/model.rs @@ -393,7 +393,10 @@ impl CompletionModel for SpacebotModel { }; let cooldown = routing.rate_limit_cooldown_secs; - let fallbacks = routing.get_fallbacks(&self.full_model_name); + let mut fallbacks: Vec = routing.get_fallbacks(&self.full_model_name).to_vec(); + // Set when the configured model id was rejected outright and the + // fallbacks were derived from the provider's default routing table. + let mut provider_recovery = false; let mut last_error: Option = None; // Try the primary model (with retries) unless it's in rate-limit cooldown @@ -422,14 +425,35 @@ impl CompletionModel for SpacebotModel { .record_rate_limit(&self.full_model_name) .await; } + // A rejected model id (stale default, typo, no access) never + // recovers on its own — try the provider's default models + // when no explicit chain is configured. + if fallbacks.is_empty() + && routing::is_model_not_found_error(&error.to_string()) + { + fallbacks = routing::default_model_candidates(&self.provider) + .into_iter() + .filter(|candidate| candidate != &self.full_model_name) + .collect(); + provider_recovery = !fallbacks.is_empty(); + if provider_recovery { + tracing::warn!( + model = %self.full_model_name, + candidates = ?fallbacks, + "provider rejected configured model, trying its default models" + ); + } + } if fallbacks.is_empty() { // No fallbacks — this is the final error return Err(error); } - tracing::warn!( - model = %self.full_model_name, - "primary model exhausted retries, trying fallbacks" - ); + if !provider_recovery { + tracing::warn!( + model = %self.full_model_name, + "primary model exhausted retries, trying fallbacks" + ); + } last_error = Some(error); } } @@ -472,9 +496,21 @@ impl CompletionModel for SpacebotModel { } } - Err(last_error.unwrap_or_else(|| { + let final_error = last_error.unwrap_or_else(|| { CompletionError::ProviderError("all models in fallback chain failed".into()) - })) + }); + if provider_recovery && routing::is_model_not_found_error(&final_error.to_string()) { + return Err(CompletionError::ProviderError(format!( + "provider '{}' rejected the configured model '{}' and every default \ + candidate ({}). Spacebot's built-in model ids for this provider appear \ + to be stale — pick a working model in Settings → Model Routing and \ + please report this as a bug.", + self.provider, + self.full_model_name, + fallbacks.join(", ") + ))); + } + Err(final_error) } .await; diff --git a/src/llm/routing.rs b/src/llm/routing.rs index fe0d0640e..baa57880e 100644 --- a/src/llm/routing.rs +++ b/src/llm/routing.rs @@ -144,6 +144,56 @@ pub fn is_retriable_error(error_message: &str) -> bool { || lower.contains("error decoding response body") } +/// Whether a completion error indicates the provider rejected the model id +/// itself — a stale default, a typo, or a model the account cannot access. +/// These never succeed on retry with the same id, but a sibling model from +/// the same provider usually works. +pub fn is_model_not_found_error(error_message: &str) -> bool { + let lower = error_message.to_lowercase(); + (lower.contains("not_found") && lower.contains("model")) + || lower.contains("model not found") + || lower.contains("unknown model") + || lower.contains("invalid model") + || lower.contains("unsupported model") + || lower.contains("no such model") + || (lower.contains("model") && lower.contains("does not exist")) + || (lower.contains("model") && lower.contains("do not have access")) + || (lower.contains("404") && lower.contains("model")) +} + +/// Ordered, de-duplicated model ids drawn from a provider's default routing +/// table (slots first, then fallback chains). Used to recover when a +/// configured model id is rejected by the provider. +pub fn default_model_candidates(provider: &str) -> Vec { + let prefix = provider_to_prefix(provider); + if prefix.is_empty() { + return Vec::new(); + } + + let defaults = defaults_for_provider(provider); + let mut candidates: Vec = Vec::new(); + let slots = [ + &defaults.channel, + &defaults.branch, + &defaults.worker, + &defaults.compactor, + &defaults.cortex, + ]; + for model in slots { + if model.starts_with(prefix) && !candidates.contains(model) { + candidates.push(model.clone()); + } + } + for chain in defaults.fallbacks.values() { + for model in chain { + if model.starts_with(prefix) && !candidates.contains(model) { + candidates.push(model.clone()); + } + } + } + candidates +} + /// Whether a completion error indicates context window overflow. /// /// Providers return 400 with various phrasings when the request exceeds @@ -218,17 +268,25 @@ pub fn defaults_for_provider(provider: &str) -> RoutingConfig { } } "openai-chatgpt" => { - let channel: String = "openai-chatgpt/gpt-4.1".into(); - let worker: String = "openai-chatgpt/gpt-4.1-mini".into(); + // ChatGPT OAuth bills against the subscription, not per token, and the + // codex backend only serves a narrow model set — one verified model + // across all slots beats guessing at a cheaper tier that may not exist. + let primary: String = "openai-chatgpt/gpt-5.6-sol".into(); RoutingConfig { - channel: channel.clone(), - branch: channel.clone(), - worker: worker.clone(), - compactor: worker.clone(), - cortex: worker.clone(), + channel: primary.clone(), + branch: primary.clone(), + worker: primary.clone(), + compactor: primary.clone(), + cortex: primary.clone(), voice: String::new(), - task_overrides: HashMap::from([("coding".into(), channel.clone())]), - fallbacks: HashMap::from([(channel, vec![worker])]), + task_overrides: HashMap::from([("coding".into(), primary.clone())]), + fallbacks: HashMap::from([( + primary, + vec![ + "openai-chatgpt/gpt-5.6".into(), + "openai-chatgpt/gpt-5.5".into(), + ], + )]), rate_limit_cooldown_secs: 60, ..RoutingConfig::default() } @@ -401,6 +459,7 @@ pub fn defaults_for_provider(provider: &str) -> RoutingConfig { "minimax-cn" => RoutingConfig::for_model("minimax-cn/MiniMax-M2.5".into()), "moonshot" => RoutingConfig::for_model("moonshot/kimi-k2.5".into()), "zai-coding-plan" => RoutingConfig::for_model("zai-coding-plan/glm-5".into()), + "ollama" => RoutingConfig::for_model("ollama/llama3.2".into()), "github-copilot" => { let channel: String = "github-copilot/claude-sonnet-4".into(); let worker: String = "github-copilot/gpt-4.1-mini".into(); @@ -446,6 +505,8 @@ pub fn provider_to_prefix(provider: &str) -> &str { "moonshot" => "moonshot/", "zai-coding-plan" => "zai-coding-plan/", "github-copilot" => "github-copilot/", + "ollama" => "ollama/", + "azure" => "azure/", _ => "", } } @@ -542,6 +603,43 @@ mod tests { assert!(!is_retriable_error("parse error")); } + #[test] + fn is_model_not_found_error_detection() { + // OpenAI phrasing + assert!(is_model_not_found_error( + "The model `gpt-5.3-codex` does not exist or you do not have access to it." + )); + assert!(is_model_not_found_error("model_not_found")); + // Anthropic phrasing + assert!(is_model_not_found_error( + "not_found_error: model: claude-sonnet-9" + )); + // Generic gateway phrasings + assert!(is_model_not_found_error("unknown model: foo")); + assert!(is_model_not_found_error("Invalid model ID")); + assert!(is_model_not_found_error("404: no such model")); + // Unrelated errors must not match + assert!(!is_model_not_found_error("429 Too Many Requests")); + assert!(!is_model_not_found_error("401 Unauthorized")); + assert!(!is_model_not_found_error("404 Not Found")); + assert!(!is_model_not_found_error("context length exceeded")); + } + + #[test] + fn default_model_candidates_are_provider_prefixed_and_unique() { + let candidates = default_model_candidates("openai-chatgpt"); + assert!(!candidates.is_empty()); + assert!(candidates.iter().all(|m| m.starts_with("openai-chatgpt/"))); + let mut deduped = candidates.clone(); + deduped.dedup(); + assert_eq!(candidates.len(), deduped.len()); + // Fallback chain members are included + assert!(candidates.iter().any(|m| m == "openai-chatgpt/gpt-5.6")); + + // Unknown providers yield no candidates rather than anthropic defaults + assert!(default_model_candidates("not-a-provider").is_empty()); + } + #[test] fn is_rate_limit_error_detection() { assert!(is_rate_limit_error("429 Too Many Requests")); From 219409be2bc67c1d9e5040d891beda79ff682720 Mon Sep 17 00:00:00 2001 From: Jamie Pine Date: Sun, 9 Aug 2026 22:22:01 -0700 Subject: [PATCH 2/2] fix(providers): don't persist unverified models, gate editor on defaults OAuth finalization now fails instead of writing the rejected model when no candidate verifies. Routing keeps whatever it had; credentials stay saved so the sign-in doesn't need repeating. The candidate walk moved into resolve_verified_model, which takes the verifier as a parameter so the requested/fallback/unverified paths are unit tested. Settings disables the provider edit button until the default-models query resolves, so the model field can't open seeded with an empty string. --- .../agent-config/ConfigSectionEditor.tsx | 4 +- .../src/components/settings/ProviderCard.tsx | 17 +- interface/src/components/settings/types.ts | 1 + interface/src/routes/Settings.tsx | 22 ++- src/api/providers.rs | 181 +++++++++++++++--- 5 files changed, 186 insertions(+), 39 deletions(-) diff --git a/interface/src/components/agent-config/ConfigSectionEditor.tsx b/interface/src/components/agent-config/ConfigSectionEditor.tsx index 64f6577a3..ca1b0ee86 100644 --- a/interface/src/components/agent-config/ConfigSectionEditor.tsx +++ b/interface/src/components/agent-config/ConfigSectionEditor.tsx @@ -226,7 +226,9 @@ export function ConfigSectionEditor({ "cortex", ] as const; const textSlotValues = textSlotKeys.map((key) => localValues[key] ?? ""); - const uniformModel = textSlotValues.every((v) => v === textSlotValues[0]) + const uniformModel = textSlotValues.every( + (slotValue) => slotValue === textSlotValues[0], + ) ? textSlotValues[0] : ""; const applyModelToAllSlots = (model: string) => { diff --git a/interface/src/components/settings/ProviderCard.tsx b/interface/src/components/settings/ProviderCard.tsx index daac315fe..ff50b435e 100644 --- a/interface/src/components/settings/ProviderCard.tsx +++ b/interface/src/components/settings/ProviderCard.tsx @@ -13,6 +13,7 @@ export function ProviderCard({ removing, actionLabel, showRemove, + editDisabled, }: ProviderCardProps) { const primaryLabel = actionLabel ?? (configured ? "Update" : "Add key"); const shouldShowRemove = showRemove ?? configured; @@ -34,12 +35,20 @@ export function ProviderCard({ )}

{description}

-

- Default model: {defaultModel} -

+ {defaultModel && ( +

+ Default model:{" "} + {defaultModel} +

+ )}
- {shouldShowRemove && ( diff --git a/interface/src/components/settings/types.ts b/interface/src/components/settings/types.ts index 0f4d29ba0..1970926dc 100644 --- a/interface/src/components/settings/types.ts +++ b/interface/src/components/settings/types.ts @@ -45,6 +45,7 @@ export interface ProviderCardProps { removing: boolean; actionLabel?: string; showRemove?: boolean; + editDisabled?: boolean; } export interface ChatGptOAuthDialogProps { diff --git a/interface/src/routes/Settings.tsx b/interface/src/routes/Settings.tsx index c0aba4264..8b51e65a2 100644 --- a/interface/src/routes/Settings.tsx +++ b/interface/src/routes/Settings.tsx @@ -90,8 +90,14 @@ export function Settings() { }); // Per-provider default models come from the backend routing defaults so the - // UI can never drift from what the server would actually apply. - const {data: defaultModels} = useQuery({ + // UI can never drift from what the server would actually apply. The editor + // seeds its model field from these, so provider editing stays disabled until + // they arrive. + const { + data: defaultModels, + isLoading: defaultModelsLoading, + isError: defaultModelsError, + } = useQuery({ queryKey: ["provider-default-models"], queryFn: api.providerDefaultModels, staleTime: Infinity, @@ -616,7 +622,16 @@ export function Settings() {

- {isLoading ? ( + {defaultModelsError && ( +
+

+ Couldn't load the provider default models. Editing is + disabled until they load — reload the page to try again. +

+
+ )} + + {isLoading || defaultModelsLoading ? (
Loading providers... @@ -631,6 +646,7 @@ export function Settings() { description={provider.description} configured={isConfigured(provider.id)} defaultModel={defaultModels?.defaults[provider.id] ?? ""} + editDisabled={!defaultModels} onEdit={() => { setEditingProvider(provider.id); setKeyInput(""); diff --git a/src/api/providers.rs b/src/api/providers.rs index 59274148c..d1981de03 100644 --- a/src/api/providers.rs +++ b/src/api/providers.rs @@ -328,6 +328,53 @@ async fn update_device_oauth_status(state_key: &str, status: DeviceOAuthSessionS } } +/// Outcome of walking the ChatGPT OAuth model candidates to find one the +/// account actually serves. +#[derive(Debug, PartialEq, Eq)] +enum ModelVerification { + /// The requested model responded. + Requested(String), + /// The requested model was rejected and this default candidate responded. + Fallback(String), + /// No candidate responded. Each entry is `model: error`. + Unverified(Vec), +} + +/// Try the requested model first, then the provider's default candidates, and +/// report the first one that responds. Only a model that reaches this point +/// verified is safe to write into routing. +async fn resolve_verified_model(requested: &str, verify: Verify) -> ModelVerification +where + Verify: Fn(String) -> Fut, + Fut: Future>, +{ + let mut candidates = vec![requested.to_string()]; + for candidate in crate::llm::routing::default_model_candidates("openai-chatgpt") { + if !candidates.contains(&candidate) { + candidates.push(candidate); + } + } + + let mut failures = Vec::new(); + for candidate in candidates { + match verify(candidate.clone()).await { + Ok(()) => { + return if candidate == requested { + ModelVerification::Requested(candidate) + } else { + ModelVerification::Fallback(candidate) + }; + } + Err(error) => { + tracing::warn!(model = %candidate, %error, "ChatGPT OAuth model verification failed"); + failures.push(format!("{candidate}: {error}")); + } + } + } + + ModelVerification::Unverified(failures) +} + /// Run a one-shot completion against the ChatGPT OAuth provider to prove a /// model id is actually served for this account. async fn verify_openai_chatgpt_model( @@ -374,48 +421,34 @@ async fn finalize_openai_oauth( // Verify the requested model before writing it into routing; if the // provider rejects it, walk the default candidates until one responds. - let mut candidates = vec![model.to_string()]; - for candidate in crate::llm::routing::default_model_candidates("openai-chatgpt") { - if !candidates.contains(&candidate) { - candidates.push(candidate); - } - } - - let mut verified_model = None; - let mut failures = Vec::new(); - for candidate in &candidates { - match verify_openai_chatgpt_model(&llm_manager, candidate).await { - Ok(()) => { - verified_model = Some(candidate.clone()); - break; - } - Err(error) => { - tracing::warn!(model = %candidate, %error, "ChatGPT OAuth model verification failed"); - failures.push(format!("{candidate}: {error}")); - } - } - } + let verification = resolve_verified_model(model, |candidate| { + let llm_manager = llm_manager.clone(); + async move { verify_openai_chatgpt_model(&llm_manager, &candidate).await } + }) + .await; - let (applied_model, message) = match verified_model { - Some(applied) if applied == model => { + let (applied_model, message) = match verification { + ModelVerification::Requested(applied) => { let message = format!( "OpenAI configured via device OAuth. Model '{applied}' verified and applied to defaults and default agent routing." ); (applied, message) } - Some(applied) => { + ModelVerification::Fallback(applied) => { let message = format!( "OpenAI configured via device OAuth. Model '{model}' was rejected by the provider, so verified fallback '{applied}' was applied to defaults and default agent routing instead." ); (applied, message) } - None => ( - model.to_string(), - format!( - "OpenAI configured via device OAuth, but no model could be verified ({}). Applied '{model}' anyway — pick a working model in Settings and please report this as a bug.", + ModelVerification::Unverified(failures) => { + // The credentials stay saved so the sign-in doesn't have to be + // repeated, but routing keeps the models it already had rather than + // pointing every slot at an id this account can't serve. + anyhow::bail!( + "can't apply model routing: this ChatGPT account served none of the candidate models ({}). Existing routing is unchanged — set a working model under Settings → Model Routing and report this as a bug.", failures.join("; ") - ), - ), + ); + } }; let config_path = state.config_path.read().await.clone(); @@ -1767,7 +1800,14 @@ pub(super) async fn delete_provider( #[cfg(test)] mod tests { - use super::build_test_llm_config; + use super::{ModelVerification, build_test_llm_config, resolve_verified_model}; + + use std::sync::Mutex; + + /// Records every candidate a verification run attempted, in order. + fn record_attempts() -> Mutex> { + Mutex::new(Vec::new()) + } #[test] fn build_test_llm_config_registers_ollama_provider_from_base_url() { @@ -1780,4 +1820,83 @@ mod tests { assert_eq!(provider.base_url, "http://remote-ollama.local:11434"); assert_eq!(provider.api_key, ""); } + + #[tokio::test] + async fn resolve_verified_model_keeps_the_requested_model_when_it_responds() { + let attempts = record_attempts(); + let verification = resolve_verified_model("openai-chatgpt/gpt-5.6-sol", |candidate| { + attempts.lock().unwrap().push(candidate); + async { Ok(()) } + }) + .await; + + assert_eq!( + verification, + ModelVerification::Requested("openai-chatgpt/gpt-5.6-sol".into()) + ); + assert_eq!(attempts.lock().unwrap().len(), 1); + } + + #[tokio::test] + async fn resolve_verified_model_falls_back_when_the_requested_model_is_rejected() { + let requested = "openai-chatgpt/gpt-5.3-codex"; + let verification = resolve_verified_model(requested, |candidate| async move { + if candidate == requested { + Err("model_not_found: the model does not exist".to_string()) + } else { + Ok(()) + } + }) + .await; + + let ModelVerification::Fallback(applied) = verification else { + panic!("a default candidate should have verified"); + }; + assert_ne!(applied, requested); + assert!(applied.starts_with("openai-chatgpt/"), "got {applied}"); + } + + #[tokio::test] + async fn resolve_verified_model_reports_every_failure_when_nothing_responds() { + let attempts = record_attempts(); + let verification = resolve_verified_model("openai-chatgpt/bogus", |candidate| { + attempts.lock().unwrap().push(candidate.clone()); + async move { Err(format!("model_not_found: {candidate}")) } + }) + .await; + + let ModelVerification::Unverified(failures) = verification else { + panic!("no candidate responded, so nothing should be applied"); + }; + let attempts = attempts.lock().unwrap(); + assert!( + attempts.len() > 1, + "default candidates should be tried after the requested model" + ); + assert_eq!(failures.len(), attempts.len()); + assert!(failures[0].starts_with("openai-chatgpt/bogus: ")); + } + + #[tokio::test] + async fn resolve_verified_model_does_not_retry_a_requested_default() { + let requested = crate::llm::routing::default_model_candidates("openai-chatgpt") + .into_iter() + .next() + .expect("openai-chatgpt has default candidates"); + let attempts = record_attempts(); + resolve_verified_model(&requested, |candidate| { + attempts.lock().unwrap().push(candidate); + async { Err("model_not_found".to_string()) } + }) + .await; + + let attempts = attempts.lock().unwrap(); + assert_eq!( + attempts + .iter() + .filter(|candidate| **candidate == requested) + .count(), + 1 + ); + } }