Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 55 additions & 0 deletions crates/mesh-llm-host-runtime/src/runtime/startup_models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -495,6 +495,7 @@ pub(super) fn runtime_options_for_test(args: &[&str]) -> RuntimeOptions {
"--require-release-attestation" => options.require_release_attestation = true,
"--join" => options.join.push(next_test_arg(&mut iter, arg).to_string()),
"--model" => options.model.push(next_test_arg(&mut iter, arg).into()),
"--gguf" => options.gguf.push(next_test_arg(&mut iter, arg).into()),
"--ctx-size" => {
options.ctx_size = Some(
next_test_arg(&mut iter, arg)
Expand Down Expand Up @@ -583,6 +584,34 @@ pub(super) fn resolve_model_parallel_slots(
resolve_model_parallel_override(model_parallel, gpu_config).unwrap_or(default_slots)
}

/// Detect the `--gguf <path> --model <alias>` naming form.
///
/// Returns the alias when the user supplied exactly one local GGUF and exactly
/// one `--model` value that is a plain name rather than another model spec.
/// Anything that could itself resolve — an existing path, a Hugging Face ref,
/// or a URL — keeps the previous behaviour of being served as its own model.
fn gguf_alias_from_cli(options: &RuntimeOptions) -> Option<String> {
if options.gguf.len() != 1 || options.model.len() != 1 {
return None;
}
let candidate = options.model[0].to_str()?;
if candidate.is_empty() || !is_plain_model_alias(candidate) {
return None;
}
if options.model[0].exists() {
return None;
}
Some(candidate.to_string())
}

/// A plain alias carries no path, ref, or URL syntax.
fn is_plain_model_alias(candidate: &str) -> bool {
!candidate.contains('/')
&& !candidate.contains('\\')
&& !candidate.contains(':')
&& !candidate.contains('@')
}

pub(super) fn build_startup_model_specs(
options: &RuntimeOptions,
config: &plugin::MeshConfig,
Expand All @@ -593,6 +622,32 @@ pub(super) fn build_startup_model_specs(

let mut specs = Vec::new();
if cli_has_explicit_models(options) {
// `--gguf <path> --model <alias>` names the local file rather than
// requesting a second model: bind the alias to the GGUF so we never
// try to resolve it against the Hugging Face hub or the catalog.
if let Some(alias) = gguf_alias_from_cli(options) {
let path = &options.gguf[0];
if !path.exists() {
anyhow::bail!("GGUF file not found: {}", path.display());
}
specs.push(StartupModelSpec {
model_ref: path.clone(),
declared_ref: Some(alias),
mmproj_ref: options.mmproj.clone(),
ctx_size: options.ctx_size,
gpu_id: None,
config_owned: false,
parallel: None,
cache_type_k: None,
cache_type_v: None,
n_batch: None,
n_ubatch: None,
flash_attention: FlashAttentionType::Auto,
profile: String::new(),
});
return Ok(specs);
}

for path in &options.gguf {
if !path.exists() {
anyhow::bail!("GGUF file not found: {}", path.display());
Expand Down
70 changes: 70 additions & 0 deletions crates/mesh-llm-host-runtime/src/runtime/tests/startup_models.rs
Original file line number Diff line number Diff line change
Expand Up @@ -705,6 +705,76 @@ fn test_build_startup_model_specs_uses_config_models_when_cli_is_empty() {
assert!(specs[1].config_owned);
}

#[test]
fn gguf_with_plain_model_name_binds_the_name_to_the_local_file() {
let temp_dir = tempfile::tempdir().expect("tempdir");
let model_path = temp_dir.path().join("deepseek.gguf");
std::fs::write(&model_path, b"gguf").expect("write model");
let options = runtime_options_for_test(&[
"mesh-llm",
"--gguf",
model_path.to_str().expect("model path"),
"--model",
"deepseek-v4-flash",
]);

let specs =
build_startup_model_specs(&options, &plugin::MeshConfig::default()).expect("startup specs");
assert_eq!(specs.len(), 1);
assert_eq!(specs[0].model_ref, model_path);
assert_eq!(specs[0].declared_ref.as_deref(), Some("deepseek-v4-flash"));
}

#[test]
fn gguf_with_hugging_face_model_ref_still_serves_two_models() {
let temp_dir = tempfile::tempdir().expect("tempdir");
let model_path = temp_dir.path().join("deepseek.gguf");
std::fs::write(&model_path, b"gguf").expect("write model");
let options = runtime_options_for_test(&[
"mesh-llm",
"--gguf",
model_path.to_str().expect("model path"),
"--model",
"unsloth/Qwen3-8B-GGUF:Q4_K_M",
]);

let specs =
build_startup_model_specs(&options, &plugin::MeshConfig::default()).expect("startup specs");
assert_eq!(specs.len(), 2);
assert_eq!(specs[0].declared_ref, None);
assert_eq!(
specs[1].model_ref,
PathBuf::from("unsloth/Qwen3-8B-GGUF:Q4_K_M")
);
assert_eq!(specs[1].declared_ref, None);
}

#[tokio::test]
async fn gguf_alias_resolves_without_catalog_lookup() {
let temp_dir = tempfile::tempdir().expect("tempdir");
let model_path = temp_dir.path().join("deepseek.gguf");
std::fs::write(&model_path, b"gguf").expect("write model");
let options = runtime_options_for_test(&[
"mesh-llm",
"--gguf",
model_path.to_str().expect("model path"),
"--model",
"deepseek-v4-flash",
]);
let specs =
build_startup_model_specs(&options, &plugin::MeshConfig::default()).expect("startup specs");

let plans = resolve_startup_models(&specs, false)
.await
.expect("startup models resolve");
assert_eq!(plans.len(), 1);
assert_eq!(plans[0].declared_ref, "deepseek-v4-flash");
assert_eq!(
std::fs::canonicalize(&plans[0].resolved_path).expect("canonical resolved path"),
std::fs::canonicalize(&model_path).expect("canonical model path")
);
}

#[tokio::test]
async fn config_hardware_model_path_loads_local_file_under_logical_model_identity() {
let temp_dir = tempfile::tempdir().expect("tempdir");
Expand Down
Loading