local inference: stricter GGUF requirements, auto detection of tool calling support, fixed thinking output parsing

crates/goose-cli/src/cli.rs

@@ -1816,7 +1816,7 @@ async fn handle_term_subcommand(command: TermCommand) -> Result<()> {
 async fn handle_local_models_command(command: LocalModelsCommand) -> Result<()> {
     use goose::providers::local_inference::hf_models;
     use goose::providers::local_inference::local_model_registry::{
-        get_registry, model_id_from_repo, LocalModelEntry,
+        get_registry, mmproj_local_path, model_id_from_repo, LocalModelEntry,
     };
 
     match command {
@@ -1853,10 +1853,28 @@ async fn handle_local_models_command(command: LocalModelsCommand) -> Result<()>
         }
         LocalModelsCommand::Download { spec } => {
             println!("Resolving {}...", spec);
-            let (repo_id, file) = hf_models::resolve_model_spec(&spec).await?;
+            let (repo_id, resolved) = hf_models::resolve_model_spec_full(&spec).await?;
+            if resolved.files.len() > 1 {
+                anyhow::bail!(
+                    "Model '{}' is sharded ({} files) — download it from the desktop UI",
+                    spec,
+                    resolved.files.len()
+                );
+            }
+            let mmproj = resolved.mmproj;
+            let file = resolved.files.into_iter().next().unwrap();
             let model_id = model_id_from_repo(&repo_id, &file.quantization);
             let local_path =
                 goose::config::paths::Paths::in_data_dir("models").join(&file.filename);
+            let mmproj_path = mmproj
+                .as_ref()
+                .map(|mmproj| mmproj_local_path(&repo_id, &mmproj.filename));
+            let mmproj_source_url = mmproj.as_ref().map(|mmproj| mmproj.download_url.clone());
+            let mmproj_size_bytes = mmproj.as_ref().map_or(0, |mmproj| mmproj.size_bytes);
+            let mut download_files = vec![(file.download_url.clone(), local_path.clone())];
+            if let Some(mmproj) = mmproj {
+                download_files.push((mmproj.download_url, mmproj_path.clone().unwrap()));
+            }
 
             println!(
                 "Downloading {} ({})...",
@@ -1881,9 +1899,10 @@ async fn handle_local_models_command(command: LocalModelsCommand) -> Result<()>
                 source_url: file.download_url.clone(),
                 settings: Default::default(),
                 size_bytes: file.size_bytes,
-                mmproj_path: None,
-                mmproj_source_url: None,
-                mmproj_size_bytes: 0,
+                mmproj_path,
+                mmproj_source_url,
+                mmproj_size_bytes,
+                mmproj_checked: true,
                 shard_files: vec![],
             };
 
@@ -1897,10 +1916,10 @@ async fn handle_local_models_command(command: LocalModelsCommand) -> Result<()>
             // Download
             let manager = goose::download_manager::get_download_manager();
             manager
-                .download_model(
+                .download_model_sharded(
                     format!("{}-model", model_id),
-                    file.download_url,
-                    local_path,
+                    download_files,
+                    file.size_bytes + mmproj_size_bytes,
                     None,
                 )
                 .await?;

crates/goose-server/src/openapi.rs

@@ -683,6 +683,7 @@ pub struct ApiDoc;
         super::routes::local_inference::list_local_models,
         super::routes::local_inference::sync_featured_models,
         super::routes::local_inference::search_hf_models,
+        super::routes::local_inference::list_builtin_chat_templates,
         super::routes::local_inference::get_repo_files,
         super::routes::local_inference::download_hf_model,
         super::routes::local_inference::get_local_model_download_progress,
@@ -701,7 +702,9 @@ pub struct ApiDoc;
         goose::providers::local_inference::hf_models::HfQuantVariant,
         super::routes::local_inference::RepoVariantsResponse,
         goose::providers::local_inference::local_model_registry::ModelSettings,
+        goose::providers::local_inference::local_model_registry::ChatTemplate,
         goose::providers::local_inference::local_model_registry::SamplingConfig,
+        goose::providers::local_inference::local_model_registry::ToolCallingMode,
     ))
 )]
 pub struct LocalInferenceApiDoc;

crates/goose-server/src/routes/local_inference.rs

@@ -13,10 +13,10 @@ use goose::config::paths::Paths;
 use goose::download_manager::{get_download_manager, DownloadProgress};
 use goose::providers::local_inference::hf_models::{self, HfModelInfo, HfQuantVariant};
 use goose::providers::local_inference::{
-    available_inference_memory_bytes,
-    hf_models::{resolve_model_spec, resolve_model_spec_full, HfGgufFile},
+    available_inference_memory_bytes, builtin_chat_template_names,
+    hf_models::{resolve_model_spec_full, HfGgufFile},
     local_model_registry::{
-        default_settings_for_model, featured_mmproj_spec, get_registry, is_featured_model,
+        default_settings_for_model, get_registry, is_featured_model, mmproj_local_path,
         model_id_from_repo, LocalModelEntry, ModelDownloadStatus as RegistryDownloadStatus,
         ModelSettings, ShardFile, FEATURED_MODELS,
     },
@@ -79,26 +79,18 @@ async fn ensure_featured_models_in_registry() -> Result<(), ErrorResponse> {
                 .lock()
                 .map_err(|_| ErrorResponse::internal("Failed to acquire registry lock"))?;
             if let Some(existing) = registry.get_model(&model_id) {
-                let needs_backfill = existing.mmproj_path.is_none() && featured.mmproj.is_some();
-                let needs_download = existing.is_downloaded()
-                    && featured.mmproj.is_some()
-                    && !existing.mmproj_path.as_ref().is_some_and(|p| p.exists());
-
-                if needs_download {
-                    if let Some(mmproj) = featured.mmproj.as_ref() {
-                        let path = mmproj.local_path();
-                        let url = format!(
-                            "https://huggingface.co/{}/resolve/main/{}",
-                            mmproj.repo, mmproj.filename
-                        );
-                        mmproj_downloads_needed.push((model_id.clone(), url, path));
+                if let Some(path) = &existing.mmproj_path {
+                    if existing.is_downloaded() && !path.exists() {
+                        if let Some(url) = &existing.mmproj_source_url {
+                            mmproj_downloads_needed.push((
+                                model_id.clone(),
+                                url.clone(),
+                                path.clone(),
+                            ));
+                        }
                     }
-                }
-
-                if !needs_backfill {
                     continue;
                 }
-                // Fall through to resolve for backfill
             }
         }
 
@@ -110,36 +102,45 @@ async fn ensure_featured_models_in_registry() -> Result<(), ErrorResponse> {
         });
     }
 
-    let resolved: Vec<(PendingResolve, HfGgufFile)> =
+    let resolved: Vec<(PendingResolve, HfGgufFile, Option<HfGgufFile>)> =
         join_all(to_resolve.into_iter().map(|pending| async move {
-            let hf_file = match resolve_model_spec(pending.spec).await {
-                Ok((_repo, file)) => file,
+            let (hf_file, mmproj) = match resolve_model_spec_full(pending.spec).await {
+                Ok((_repo, resolved)) => (resolved.files[0].clone(), resolved.mmproj),
                 Err(_) => {
                     let filename = format!(
                         "{}-{}.gguf",
                         pending.repo_id.split('/').next_back().unwrap_or("model"),
                         pending.quantization
                     );
-                    HfGgufFile {
-                        filename: filename.clone(),
-                        size_bytes: 0,
-                        quantization: pending.quantization.to_string(),
-                        download_url: format!(
-                            "https://huggingface.co/{}/resolve/main/{}",
-                            pending.repo_id, filename
-                        ),
-                    }
+                    (
+                        HfGgufFile {
+                            filename: filename.clone(),
+                            size_bytes: 0,
+                            quantization: pending.quantization.to_string(),
+                            download_url: format!(
+                                "https://huggingface.co/{}/resolve/main/{}",
+                                pending.repo_id, filename
+                            ),
+                        },
+                        None,
+                    )
                 }
             };
-            (pending, hf_file)
+            (pending, hf_file, mmproj)
         }))
         .await;
 
     let entries_to_add: Vec<LocalModelEntry> = resolved
         .into_iter()
-        .map(|(pending, hf_file)| {
+        .map(|(pending, hf_file, mmproj)| {
             let local_path = Paths::in_data_dir("models").join(&hf_file.filename);
             let settings = default_settings_for_model(&pending.model_id);
+            let mmproj_path = mmproj
+                .as_ref()
+                .map(|mmproj| mmproj_local_path(&pending.repo_id, &mmproj.filename));
+            let mmproj_source_url = mmproj.as_ref().map(|mmproj| mmproj.download_url.clone());
+            let mmproj_size_bytes = mmproj.as_ref().map_or(0, |mmproj| mmproj.size_bytes);
+            let mmproj_checked = mmproj.is_some();
             LocalModelEntry {
                 id: pending.model_id,
                 repo_id: pending.repo_id,
@@ -149,9 +150,10 @@ async fn ensure_featured_models_in_registry() -> Result<(), ErrorResponse> {
                 source_url: hf_file.download_url,
                 settings,
                 size_bytes: hf_file.size_bytes,
-                mmproj_path: None,
-                mmproj_source_url: None,
-                mmproj_size_bytes: 0,
+                mmproj_path,
+                mmproj_source_url,
+                mmproj_size_bytes,
+                mmproj_checked,
                 shard_files: vec![],
             }
         })
@@ -165,20 +167,80 @@ async fn ensure_featured_models_in_registry() -> Result<(), ErrorResponse> {
         if !entries_to_add.is_empty() {
             registry.sync_with_featured(entries_to_add);
         }
+    }
+
+    let to_backfill: Vec<(String, String, String)> = {
+        let registry = get_registry()
+            .lock()
+            .map_err(|_| ErrorResponse::internal("Failed to acquire registry lock"))?;
+
+        registry
+            .list_models()
+            .iter()
+            .filter(|model| model.is_downloaded())
+            .filter(|model| model.mmproj_path.is_none())
+            .filter(|model| !model.mmproj_checked)
+            .map(|model| {
+                (
+                    model.id.clone(),
+                    model.repo_id.clone(),
+                    model.quantization.clone(),
+                )
+            })
+            .collect()
+    };
+
+    let mmproj_backfills: Vec<(String, String, Option<Option<HfGgufFile>>)> = join_all(
+        to_backfill
+            .into_iter()
+            .map(|(id, repo_id, quantization)| async move {
+                let spec = format!("{repo_id}:{quantization}");
+                let mmproj = resolve_model_spec_full(&spec)
+                    .await
+                    .ok()
+                    .map(|(_, resolved)| resolved.mmproj);
+                (id, repo_id, mmproj)
+            }),
+    )
+    .await;
+
+    {
+        let mut registry = get_registry()
+            .lock()
+            .map_err(|_| ErrorResponse::internal("Failed to acquire registry lock"))?;
+
+        for (model_id, repo_id, mmproj_result) in mmproj_backfills {
+            if let Some(model) = registry
+                .list_models_mut()
+                .iter_mut()
+                .find(|model| model.id == model_id)
+            {
+                let Some(mmproj) = mmproj_result else {
+                    continue;
+                };
+
+                model.mmproj_checked = true;
+                if let Some(mmproj) = mmproj {
+                    model.mmproj_path = Some(mmproj_local_path(&repo_id, &mmproj.filename));
+                    model.mmproj_source_url = Some(mmproj.download_url);
+                    model.mmproj_size_bytes = mmproj.size_bytes;
+                }
+                model.refresh_mmproj_metadata();
+            }
+        }
 
-        // Backfill mmproj data for all registry models and collect any
-        // needed mmproj downloads for models already on disk.
         for model in registry.list_models_mut() {
-            model.enrich_with_featured_mmproj();
+            model.refresh_mmproj_metadata();
             if model.is_downloaded() {
-                if let Some(mmproj) = featured_mmproj_spec(&model.id) {
-                    let path = mmproj.local_path();
+                if let Some(path) = &model.mmproj_path {
                     if !path.exists() {
-                        let url = format!(
-                            "https://huggingface.co/{}/resolve/main/{}",
-                            mmproj.repo, mmproj.filename
-                        );
-                        mmproj_downloads_needed.push((model.id.clone(), url, path));
+                        if let Some(url) = &model.mmproj_source_url {
+                            mmproj_downloads_needed.push((
+                                model.id.clone(),
+                                url.clone(),
+                                path.clone(),
+                            ));
+                        }
                     }
                 }
             }
@@ -431,6 +493,20 @@ pub async fn download_hf_model(
         vec![]
     };
 
+    let mmproj_path = resolved
+        .mmproj
+        .as_ref()
+        .map(|mmproj| mmproj_local_path(&repo_id, &mmproj.filename));
+    let mmproj_source_url = resolved
+        .mmproj
+        .as_ref()
+        .map(|mmproj| mmproj.download_url.clone());
+    let mmproj_size_bytes = resolved
+        .mmproj
+        .as_ref()
+        .map_or(0, |mmproj| mmproj.size_bytes);
+    let mmproj_checked = true;
+
     let entry = LocalModelEntry {
         id: model_id.clone(),
         repo_id,
@@ -440,13 +516,13 @@ pub async fn download_hf_model(
         source_url: first_file.download_url.clone(),
         settings: default_settings_for_model(&model_id),
         size_bytes: resolved.total_size,
-        mmproj_path: None,
-        mmproj_source_url: None,
-        mmproj_size_bytes: 0,
+        mmproj_path,
+        mmproj_source_url,
+        mmproj_size_bytes,
+        mmproj_checked,
         shard_files: shard_files.clone(),
     };
 
-    // add_model enriches the entry with mmproj metadata from the featured table
     let mmproj_path = {
         let mut registry = get_registry()
             .lock()
@@ -649,6 +725,17 @@ pub async fn update_model_settings(
     Ok(Json(settings))
 }
 
+#[utoipa::path(
+    get,
+    path = "/local-inference/chat-templates/builtin",
+    responses(
+        (status = 200, description = "llama.cpp built-in chat template names", body = Vec<String>)
+    )
+)]
+pub async fn list_builtin_chat_templates() -> Json<Vec<String>> {
+    Json(builtin_chat_template_names())
+}
+
 pub fn routes(state: Arc<AppState>) -> Router {
     let registered_paths: std::collections::HashSet<std::path::PathBuf> = get_registry()
         .lock()
@@ -672,6 +759,10 @@ pub fn routes(state: Arc<AppState>) -> Router {
         .route("/local-inference/models", get(list_local_models))
         .route("/local-inference/sync-featured", post(sync_featured_models))
         .route("/local-inference/search", get(search_hf_models))
+        .route(
+            "/local-inference/chat-templates/builtin",
+            get(list_builtin_chat_templates),
+        )
         .route(
             "/local-inference/repo/{author}/{repo}/files",
             get(get_repo_files),

crates/goose/Cargo.toml

@@ -25,6 +25,7 @@ local-inference = [
     "dep:candle-nn",
     "dep:candle-transformers",
     "dep:llama-cpp-2",
+    "dep:llama-cpp-sys-2",
     "dep:tokenizers",
     "dep:symphonia",
     "dep:rubato",
@@ -213,6 +214,7 @@ pctx_code_mode = { version = "0.3", default-features = false, optional = true }
 # They are just here to pin the version, and can be removed if PCTX updates temporal_rs
 icu_calendar = { version = "=2.1.1", default-features = false }
 icu_locale = { version = "=2.1.1", default-features = false }
+llama-cpp-sys-2 = { workspace = true, optional = true }
 
 [target.'cfg(target_os = "windows")'.dependencies]
 winapi = { workspace = true }