diff --git a/crates/mesh-llm-host-runtime/src/runtime/startup_models.rs b/crates/mesh-llm-host-runtime/src/runtime/startup_models.rs index d27d0eeec3..1721dd2a90 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/startup_models.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/startup_models.rs @@ -495,6 +495,7 @@ pub(super) fn runtime_options_for_test(args: &[&str]) -> RuntimeOptions { "--require-release-attestation" => options.require_release_attestation = true, "--join" => options.join.push(next_test_arg(&mut iter, arg).to_string()), "--model" => options.model.push(next_test_arg(&mut iter, arg).into()), + "--gguf" => options.gguf.push(next_test_arg(&mut iter, arg).into()), "--ctx-size" => { options.ctx_size = Some( next_test_arg(&mut iter, arg) @@ -583,6 +584,34 @@ pub(super) fn resolve_model_parallel_slots( resolve_model_parallel_override(model_parallel, gpu_config).unwrap_or(default_slots) } +/// Detect the `--gguf --model ` naming form. +/// +/// Returns the alias when the user supplied exactly one local GGUF and exactly +/// one `--model` value that is a plain name rather than another model spec. +/// Anything that could itself resolve — an existing path, a Hugging Face ref, +/// or a URL — keeps the previous behaviour of being served as its own model. +fn gguf_alias_from_cli(options: &RuntimeOptions) -> Option { + if options.gguf.len() != 1 || options.model.len() != 1 { + return None; + } + let candidate = options.model[0].to_str()?; + if candidate.is_empty() || !is_plain_model_alias(candidate) { + return None; + } + if options.model[0].exists() { + return None; + } + Some(candidate.to_string()) +} + +/// A plain alias carries no path, ref, or URL syntax. +fn is_plain_model_alias(candidate: &str) -> bool { + !candidate.contains('/') + && !candidate.contains('\\') + && !candidate.contains(':') + && !candidate.contains('@') +} + pub(super) fn build_startup_model_specs( options: &RuntimeOptions, config: &plugin::MeshConfig, @@ -593,6 +622,32 @@ pub(super) fn build_startup_model_specs( let mut specs = Vec::new(); if cli_has_explicit_models(options) { + // `--gguf --model ` names the local file rather than + // requesting a second model: bind the alias to the GGUF so we never + // try to resolve it against the Hugging Face hub or the catalog. + if let Some(alias) = gguf_alias_from_cli(options) { + let path = &options.gguf[0]; + if !path.exists() { + anyhow::bail!("GGUF file not found: {}", path.display()); + } + specs.push(StartupModelSpec { + model_ref: path.clone(), + declared_ref: Some(alias), + mmproj_ref: options.mmproj.clone(), + ctx_size: options.ctx_size, + gpu_id: None, + config_owned: false, + parallel: None, + cache_type_k: None, + cache_type_v: None, + n_batch: None, + n_ubatch: None, + flash_attention: FlashAttentionType::Auto, + profile: String::new(), + }); + return Ok(specs); + } + for path in &options.gguf { if !path.exists() { anyhow::bail!("GGUF file not found: {}", path.display()); diff --git a/crates/mesh-llm-host-runtime/src/runtime/tests/startup_models.rs b/crates/mesh-llm-host-runtime/src/runtime/tests/startup_models.rs index d8dc55a9f1..8bcee94849 100644 --- a/crates/mesh-llm-host-runtime/src/runtime/tests/startup_models.rs +++ b/crates/mesh-llm-host-runtime/src/runtime/tests/startup_models.rs @@ -705,6 +705,76 @@ fn test_build_startup_model_specs_uses_config_models_when_cli_is_empty() { assert!(specs[1].config_owned); } +#[test] +fn gguf_with_plain_model_name_binds_the_name_to_the_local_file() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let model_path = temp_dir.path().join("deepseek.gguf"); + std::fs::write(&model_path, b"gguf").expect("write model"); + let options = runtime_options_for_test(&[ + "mesh-llm", + "--gguf", + model_path.to_str().expect("model path"), + "--model", + "deepseek-v4-flash", + ]); + + let specs = + build_startup_model_specs(&options, &plugin::MeshConfig::default()).expect("startup specs"); + assert_eq!(specs.len(), 1); + assert_eq!(specs[0].model_ref, model_path); + assert_eq!(specs[0].declared_ref.as_deref(), Some("deepseek-v4-flash")); +} + +#[test] +fn gguf_with_hugging_face_model_ref_still_serves_two_models() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let model_path = temp_dir.path().join("deepseek.gguf"); + std::fs::write(&model_path, b"gguf").expect("write model"); + let options = runtime_options_for_test(&[ + "mesh-llm", + "--gguf", + model_path.to_str().expect("model path"), + "--model", + "unsloth/Qwen3-8B-GGUF:Q4_K_M", + ]); + + let specs = + build_startup_model_specs(&options, &plugin::MeshConfig::default()).expect("startup specs"); + assert_eq!(specs.len(), 2); + assert_eq!(specs[0].declared_ref, None); + assert_eq!( + specs[1].model_ref, + PathBuf::from("unsloth/Qwen3-8B-GGUF:Q4_K_M") + ); + assert_eq!(specs[1].declared_ref, None); +} + +#[tokio::test] +async fn gguf_alias_resolves_without_catalog_lookup() { + let temp_dir = tempfile::tempdir().expect("tempdir"); + let model_path = temp_dir.path().join("deepseek.gguf"); + std::fs::write(&model_path, b"gguf").expect("write model"); + let options = runtime_options_for_test(&[ + "mesh-llm", + "--gguf", + model_path.to_str().expect("model path"), + "--model", + "deepseek-v4-flash", + ]); + let specs = + build_startup_model_specs(&options, &plugin::MeshConfig::default()).expect("startup specs"); + + let plans = resolve_startup_models(&specs, false) + .await + .expect("startup models resolve"); + assert_eq!(plans.len(), 1); + assert_eq!(plans[0].declared_ref, "deepseek-v4-flash"); + assert_eq!( + std::fs::canonicalize(&plans[0].resolved_path).expect("canonical resolved path"), + std::fs::canonicalize(&model_path).expect("canonical model path") + ); +} + #[tokio::test] async fn config_hardware_model_path_loads_local_file_under_logical_model_identity() { let temp_dir = tempfile::tempdir().expect("tempdir");