From 33bf0c65858d35730496d92b6a119d4862df40d8 Mon Sep 17 00:00:00 2001 From: Pascal Date: Tue, 28 Jul 2026 22:51:05 +0200 Subject: [PATCH 1/3] common: extract the model resolution into its own unit --- common/CMakeLists.txt | 2 + common/download.cpp | 294 ++---------------------------------- common/download.h | 10 +- common/model-resolution.cpp | 293 +++++++++++++++++++++++++++++++++++ common/model-resolution.h | 77 ++++++++++ 5 files changed, 384 insertions(+), 292 deletions(-) create mode 100644 common/model-resolution.cpp create mode 100644 common/model-resolution.h diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 799d227519f9..af0ad9396676 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -84,6 +84,8 @@ add_library(${TARGET} llguidance.cpp log.cpp log.h + model-resolution.cpp + model-resolution.h ngram-cache.cpp ngram-cache.h ngram-map.cpp diff --git a/common/download.cpp b/common/download.cpp index 3776c6c7eb68..d7252fc87cc0 100644 --- a/common/download.cpp +++ b/common/download.cpp @@ -5,6 +5,7 @@ #include "log.h" #include "download.h" #include "hf-cache.h" +#include "model-resolution.h" #define JSON_ASSERT GGML_ASSERT #include @@ -500,230 +501,7 @@ int common_download_file_single(const std::string & url, return 304; // Not Modified - fake cached response } -struct gguf_split_info { - std::string prefix; // tag included - std::string tag; - int index; - int count; -}; - -static gguf_split_info get_gguf_split_info(const std::string & path) { - static const std::regex re_split("^(.+)-([0-9]{5})-of-([0-9]{5})$", std::regex::icase); - static const std::regex re_tag("[-.]([A-Z0-9_]+)$", std::regex::icase); - std::smatch m; - - std::string prefix = path; - if (!string_remove_suffix(prefix, ".gguf")) { - return {}; - } - - int index = 1; - int count = 1; - - if (std::regex_match(prefix, m, re_split)) { - index = std::stoi(m[2].str()); - count = std::stoi(m[3].str()); - prefix = m[1].str(); - } - - std::string tag; - if (std::regex_search(prefix, m, re_tag)) { - tag = m[1].str(); - for (char & c : tag) { - c = std::toupper((unsigned char)c); - } - } - - return {std::move(prefix), std::move(tag), index, count}; -} - -// Q4_0 -> 4, F16 -> 16, NVFP4 -> 4, Q8_K_M -> 8, etc -static int extract_quant_bits(const std::string & filename) { - auto split = get_gguf_split_info(filename); - - auto pos = split.tag.find_first_of("0123456789"); - if (pos == std::string::npos) { - return 0; - } - - return std::stoi(split.tag.substr(pos)); -} - -static hf_cache::hf_files get_split_files(const hf_cache::hf_files & files, - const hf_cache::hf_file & file) { - auto split = get_gguf_split_info(file.path); - - if (split.count <= 1) { - return {file}; - } - hf_cache::hf_files result; - - for (const auto & f : files) { - auto split_f = get_gguf_split_info(f.path); - if (split_f.count == split.count && split_f.prefix == split.prefix) { - result.push_back(f); - } - } - return result; -} - -// pick the best sibling GGUF whose filename contains `keyword` (e.g. "mmproj" / "mtp"), -// preferring deeper shared directory prefix with the model, then exact `tag` match, -// then closest quantization to the tag when given, or to the model otherwise -static hf_cache::hf_file find_best_sibling(const hf_cache::hf_files & files, - const std::string & model, - const std::string & keyword, - const std::string & tag = "") { - hf_cache::hf_file best; - size_t best_depth = 0; - int best_diff = 0; - bool best_exact = false; - bool found = false; - - std::string tag_upper = tag; - for (char & c : tag_upper) { - c = (char) std::toupper((unsigned char) c); - } - - int model_bits = 0; - if (!tag_upper.empty()) { - auto pos = tag_upper.find_first_of("0123456789"); - model_bits = pos == std::string::npos ? 0 : std::stoi(tag_upper.substr(pos)); - } else { - model_bits = extract_quant_bits(model); - } - auto model_parts = string_split(model, '/'); - auto model_dir = model_parts.end() - 1; - - for (const auto & f : files) { - if (!string_ends_with(f.path, ".gguf") || - f.path.find(keyword) == std::string::npos) { - continue; - } - - auto sib_parts = string_split(f.path, '/'); - auto sib_dir = sib_parts.end() - 1; - - auto [_, dir] = std::mismatch(model_parts.begin(), model_dir, - sib_parts.begin(), sib_dir); - if (dir != sib_dir) { - continue; - } - - size_t depth = dir - sib_parts.begin(); - auto bits = extract_quant_bits(f.path); - auto diff = std::abs(bits - model_bits); - - std::string path_upper = f.path; - for (char & c : path_upper) { - c = (char) std::toupper((unsigned char) c); - } - bool exact = !tag_upper.empty() && path_upper.find("-" + tag_upper + ".") != std::string::npos; - - if (!found || depth > best_depth || - (depth == best_depth && exact && !best_exact) || - (depth == best_depth && exact == best_exact && diff < best_diff)) { - best = f; - best_depth = depth; - best_diff = diff; - best_exact = exact; - found = true; - } - } - return best; -} - -static hf_cache::hf_file find_best_mmproj(const hf_cache::hf_files & files, - const std::string & model) { - return find_best_sibling(files, model, "mmproj"); -} - -static hf_cache::hf_file find_best_mtp(const hf_cache::hf_files & files, - const std::string & model, - const std::string & tag = "") { - return find_best_sibling(files, model, "mtp-", tag); -} - -static hf_cache::hf_file find_best_eagle3(const hf_cache::hf_files & files, - const std::string & model, - const std::string & tag = "") { - return find_best_sibling(files, model, "eagle3-", tag); -} - -static hf_cache::hf_file find_best_dflash(const hf_cache::hf_files & files, - const std::string & model, - const std::string & tag = "") { - return find_best_sibling(files, model, "dflash-", tag); -} - -static bool gguf_filename_is_model(const std::string & filepath) { - if (!string_ends_with(filepath, ".gguf")) { - return false; - } - - std::string filename = filepath; - if (auto pos = filename.rfind('/'); pos != std::string::npos) { - filename = filename.substr(pos + 1); - } - - return filename.find("mmproj") == std::string::npos && - filename.find("imatrix") == std::string::npos && - filename.find("mtp-") == std::string::npos && - filename.find("eagle3-") == std::string::npos && - filename.find("dflash-") == std::string::npos; -} - -static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files, - const std::string & tag) { - std::vector tags; - - if (!tag.empty()) { - tags.push_back(tag); - } else { - tags = {"Q4_K_M", "Q8_0"}; - } - - for (const auto & t : tags) { - std::regex pattern(t + "[.-]", std::regex::icase); - for (const auto & f : files) { - if (gguf_filename_is_model(f.path) && - std::regex_search(f.path, pattern)) { - auto split = get_gguf_split_info(f.path); - if (split.count > 1 && split.index != 1) { - continue; - } - return f; - } - } - } - - // fallback to first available model only if tag is empty - if (tag.empty()) { - for (const auto & f : files) { - if (gguf_filename_is_model(f.path)) { - auto split = get_gguf_split_info(f.path); - if (split.count > 1 && split.index != 1) { - continue; - } - return f; - } - } - } - - return {}; -} - -static void list_available_gguf_files(const hf_cache::hf_files & files) { - LOG_INF("Available GGUF files:\n"); - for (const auto & f : files) { - if (string_ends_with(f.path, ".gguf")) { - LOG_INF(" - %s\n", f.path.c_str()); - } - } -} - common_download_hf_plan common_download_get_hf_plan(const common_params_model & model, const common_download_opts & opts) { - common_download_hf_plan plan; hf_cache::hf_files all; auto [repo, tag] = common_download_split_repo_tag(model.hf_repo); @@ -735,66 +513,16 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model & all = hf_cache::get_cached_files(repo); } if (all.empty()) { - return plan; - } - - // if preset.ini exists in the repo root, download only that file - for (const auto & f : all) { - if (f.path == "preset.ini") { - plan.preset = f; - return plan; - } - } - - hf_cache::hf_file primary; - - if (!model.hf_file.empty()) { - for (const auto & f : all) { - if (f.path == model.hf_file) { - primary = f; - break; - } - } - if (primary.path.empty()) { - LOG_ERR("%s: file '%s' not found in repository\n", __func__, model.hf_file.c_str()); - list_available_gguf_files(all); - return plan; - } - } else { - primary = find_best_model(all, tag); - // a requested sidecar can resolve on its own, without a full model of the same tag - if (primary.path.empty() && !opts.download_mtp && !opts.download_dflash && !opts.download_eagle3) { - LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str()); - list_available_gguf_files(all); - return plan; - } - } - - if (!primary.path.empty()) { - plan.primary = primary; - plan.model_files = get_split_files(all, primary); - } - - if (opts.download_mmproj && !primary.path.empty()) { - plan.mmproj = find_best_mmproj(all, primary.path); - } - if (opts.download_mtp) { - plan.mtp = find_best_mtp(all, primary.path, tag); - } - if (opts.download_dflash) { - plan.dflash = find_best_dflash(all, primary.path, tag); - } - if (opts.download_eagle3) { - plan.eagle3 = find_best_eagle3(all, primary.path, tag); + return {}; } - if (primary.path.empty() && - plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty()) { - LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str()); - list_available_gguf_files(all); - } + model_resolution::opts ropts; + ropts.mmproj = opts.download_mmproj; + ropts.mtp = opts.download_mtp; + ropts.dflash = opts.download_dflash; + ropts.eagle3 = opts.download_eagle3; - return plan; + return model_resolution::resolve(all, repo, tag, model.hf_file, ropts); } void common_download_run_tasks(const std::vector & tasks) { @@ -818,7 +546,7 @@ void common_download_run_tasks(const std::vector & tasks) } std::vector common_download_get_all_parts(const std::string & url) { - auto split = get_gguf_split_info(url); + auto split = model_resolution::get_gguf_split_info(url); if (split.count <= 1) { return {url}; @@ -962,7 +690,7 @@ std::vector common_list_cached_models() { auto files = hf_cache::get_cached_files(); for (const auto & f : files) { - auto split = get_gguf_split_info(f.path); + auto split = model_resolution::get_gguf_split_info(f.path); if (split.index != 1 || split.tag.empty() || split.prefix.find("mmproj") != std::string::npos || split.prefix.find("mtp-") != std::string::npos || @@ -1000,7 +728,7 @@ bool common_download_remove(const std::string & hf_repo_with_tag) { // collect snapshot entries whose tag matches std::vector to_remove; for (const auto & f : files) { - auto split = get_gguf_split_info(f.path); + auto split = model_resolution::get_gguf_split_info(f.path); if (split.tag == tag_upper) { to_remove.emplace_back(f.local_path); } diff --git a/common/download.h b/common/download.h index 3e789e9e9369..848853740638 100644 --- a/common/download.h +++ b/common/download.h @@ -1,6 +1,7 @@ #pragma once #include "hf-cache.h" +#include "model-resolution.h" #include #include @@ -103,13 +104,4 @@ std::string common_docker_resolve_model(const std::string & docker); // returns true if anything was removed bool common_download_remove(const std::string & hf_repo_with_tag); -struct common_download_hf_plan { - hf_cache::hf_file primary; - hf_cache::hf_files model_files; - hf_cache::hf_file mmproj; - hf_cache::hf_file mtp; - hf_cache::hf_file eagle3; - hf_cache::hf_file dflash; - hf_cache::hf_file preset; // if set, only this file is downloaded -}; common_download_hf_plan common_download_get_hf_plan(const common_params_model & model, const common_download_opts & opts); diff --git a/common/model-resolution.cpp b/common/model-resolution.cpp new file mode 100644 index 000000000000..d4e96afd7b4b --- /dev/null +++ b/common/model-resolution.cpp @@ -0,0 +1,293 @@ +#include "model-resolution.h" + +#include "common.h" +#include "log.h" + +#include +#include +#include +#include +#include +#include + +namespace model_resolution { + +gguf_split_info get_gguf_split_info(const std::string & path) { + static const std::regex re_split("^(.+)-([0-9]{5})-of-([0-9]{5})$", std::regex::icase); + static const std::regex re_tag("[-.]([A-Z0-9_]+)$", std::regex::icase); + std::smatch m; + + std::string prefix = path; + if (!string_remove_suffix(prefix, ".gguf")) { + return {}; + } + + int index = 1; + int count = 1; + + if (std::regex_match(prefix, m, re_split)) { + index = std::stoi(m[2].str()); + count = std::stoi(m[3].str()); + prefix = m[1].str(); + } + + std::string tag; + if (std::regex_search(prefix, m, re_tag)) { + tag = m[1].str(); + for (char & c : tag) { + c = std::toupper((unsigned char)c); + } + } + + return {std::move(prefix), std::move(tag), index, count}; +} + +int extract_quant_bits(const std::string & filename) { + auto split = get_gguf_split_info(filename); + + auto pos = split.tag.find_first_of("0123456789"); + if (pos == std::string::npos) { + return 0; + } + + return std::stoi(split.tag.substr(pos)); +} + +bool gguf_filename_is_model(const std::string & filepath) { + if (!string_ends_with(filepath, ".gguf")) { + return false; + } + + std::string filename = filepath; + if (auto pos = filename.rfind('/'); pos != std::string::npos) { + filename = filename.substr(pos + 1); + } + + return filename.find("mmproj") == std::string::npos && + filename.find("imatrix") == std::string::npos && + filename.find("mtp-") == std::string::npos && + filename.find("eagle3-") == std::string::npos && + filename.find("dflash-") == std::string::npos; +} + +hf_cache::hf_files get_split_files(const hf_cache::hf_files & files, + const hf_cache::hf_file & file) { + auto split = get_gguf_split_info(file.path); + + if (split.count <= 1) { + return {file}; + } + hf_cache::hf_files result; + + for (const auto & f : files) { + auto split_f = get_gguf_split_info(f.path); + if (split_f.count == split.count && split_f.prefix == split.prefix) { + result.push_back(f); + } + } + return result; +} + +hf_cache::hf_file find_best_model(const hf_cache::hf_files & files, + const std::string & tag) { + std::vector tags; + + if (!tag.empty()) { + tags.push_back(tag); + } else { + tags = {"Q4_K_M", "Q8_0"}; + } + + for (const auto & t : tags) { + std::regex pattern(t + "[.-]", std::regex::icase); + for (const auto & f : files) { + if (gguf_filename_is_model(f.path) && + std::regex_search(f.path, pattern)) { + auto split = get_gguf_split_info(f.path); + if (split.count > 1 && split.index != 1) { + continue; + } + return f; + } + } + } + + // fallback to first available model only if tag is empty + if (tag.empty()) { + for (const auto & f : files) { + if (gguf_filename_is_model(f.path)) { + auto split = get_gguf_split_info(f.path); + if (split.count > 1 && split.index != 1) { + continue; + } + return f; + } + } + } + + return {}; +} + +hf_cache::hf_file find_best_sibling(const hf_cache::hf_files & files, + const std::string & model, + const std::string & keyword, + const std::string & tag) { + hf_cache::hf_file best; + size_t best_depth = 0; + int best_diff = 0; + bool best_exact = false; + bool found = false; + + std::string tag_upper = tag; + for (char & c : tag_upper) { + c = (char) std::toupper((unsigned char) c); + } + + int model_bits = 0; + if (!tag_upper.empty()) { + auto pos = tag_upper.find_first_of("0123456789"); + model_bits = pos == std::string::npos ? 0 : std::stoi(tag_upper.substr(pos)); + } else { + model_bits = extract_quant_bits(model); + } + auto model_parts = string_split(model, '/'); + auto model_dir = model_parts.end() - 1; + + for (const auto & f : files) { + if (!string_ends_with(f.path, ".gguf") || + f.path.find(keyword) == std::string::npos) { + continue; + } + + auto sib_parts = string_split(f.path, '/'); + auto sib_dir = sib_parts.end() - 1; + + auto [_, dir] = std::mismatch(model_parts.begin(), model_dir, + sib_parts.begin(), sib_dir); + if (dir != sib_dir) { + continue; + } + + size_t depth = dir - sib_parts.begin(); + auto bits = extract_quant_bits(f.path); + auto diff = std::abs(bits - model_bits); + + std::string path_upper = f.path; + for (char & c : path_upper) { + c = (char) std::toupper((unsigned char) c); + } + bool exact = !tag_upper.empty() && path_upper.find("-" + tag_upper + ".") != std::string::npos; + + if (!found || depth > best_depth || + (depth == best_depth && exact && !best_exact) || + (depth == best_depth && exact == best_exact && diff < best_diff)) { + best = f; + best_depth = depth; + best_diff = diff; + best_exact = exact; + found = true; + } + } + return best; +} + + +static hf_cache::hf_file find_best_mmproj(const hf_cache::hf_files & files, + const std::string & model) { + return find_best_sibling(files, model, "mmproj"); +} + +static hf_cache::hf_file find_best_mtp(const hf_cache::hf_files & files, + const std::string & model, + const std::string & tag = "") { + return find_best_sibling(files, model, "mtp-", tag); +} + +static hf_cache::hf_file find_best_eagle3(const hf_cache::hf_files & files, + const std::string & model, + const std::string & tag = "") { + return find_best_sibling(files, model, "eagle3-", tag); +} + +static hf_cache::hf_file find_best_dflash(const hf_cache::hf_files & files, + const std::string & model, + const std::string & tag = "") { + return find_best_sibling(files, model, "dflash-", tag); +} + +static void list_available_gguf_files(const hf_cache::hf_files & files) { + LOG_INF("Available GGUF files:\n"); + for (const auto & f : files) { + if (string_ends_with(f.path, ".gguf")) { + LOG_INF(" - %s\n", f.path.c_str()); + } + } +} + +common_download_hf_plan resolve(const hf_cache::hf_files & files, + const std::string & repo, + const std::string & tag, + const std::string & hf_file, + const opts & o) { + common_download_hf_plan plan; + + // if preset.ini exists in the repo root, download only that file + for (const auto & f : files) { + if (f.path == "preset.ini") { + plan.preset = f; + return plan; + } + } + + hf_cache::hf_file primary; + + if (!hf_file.empty()) { + for (const auto & f : files) { + if (f.path == hf_file) { + primary = f; + break; + } + } + if (primary.path.empty()) { + LOG_ERR("%s: file '%s' not found in repository\n", __func__, hf_file.c_str()); + list_available_gguf_files(files); + return plan; + } + } else { + primary = find_best_model(files, tag); + // a requested sidecar can resolve on its own, without a full model of the same tag + if (primary.path.empty() && !o.mtp && !o.dflash && !o.eagle3) { + LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str()); + list_available_gguf_files(files); + return plan; + } + } + + if (!primary.path.empty()) { + plan.primary = primary; + plan.model_files = get_split_files(files, primary); + } + + if (o.mmproj && !primary.path.empty()) { + plan.mmproj = find_best_mmproj(files, primary.path); + } + if (o.mtp) { + plan.mtp = find_best_mtp(files, primary.path, tag); + } + if (o.dflash) { + plan.dflash = find_best_dflash(files, primary.path, tag); + } + if (o.eagle3) { + plan.eagle3 = find_best_eagle3(files, primary.path, tag); + } + + if (primary.path.empty() && + plan.mtp.local_path.empty() && plan.dflash.local_path.empty() && plan.eagle3.local_path.empty()) { + LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str()); + list_available_gguf_files(files); + } + + return plan; +} + +} // namespace model_resolution diff --git a/common/model-resolution.h b/common/model-resolution.h new file mode 100644 index 000000000000..1964421a9d15 --- /dev/null +++ b/common/model-resolution.h @@ -0,0 +1,77 @@ +#pragma once + +#include "hf-cache.h" + +#include + +// pure resolution of GGUF files from a repo listing, no network access: +// the primary model, its shards and the sidecar files are picked from the +// file paths alone, following the naming conventions of the known vendors + +// the files to download for one model reference +struct common_download_hf_plan { + hf_cache::hf_file primary; + hf_cache::hf_files model_files; + hf_cache::hf_file mmproj; + hf_cache::hf_file mtp; + hf_cache::hf_file eagle3; + hf_cache::hf_file dflash; + hf_cache::hf_file preset; // if set, only this file is downloaded +}; + +namespace model_resolution { + +// the sidecar files requested along the primary model +struct opts { + bool mmproj = false; + bool mtp = false; + bool dflash = false; + bool eagle3 = false; +}; + +// decomposition of a GGUF path into its shard prefix, quant tag and shard +// position, "m-Q8_0-00002-of-00003.gguf" gives {"m-Q8_0", "Q8_0", 2, 3} +struct gguf_split_info { + std::string prefix; // tag included + std::string tag; + int index; + int count; +}; + +gguf_split_info get_gguf_split_info(const std::string & path); + +// Q4_0 -> 4, F16 -> 16, NVFP4 -> 4, Q8_K_M -> 8, etc +int extract_quant_bits(const std::string & filename); + +// a GGUF file that is not an mmproj, imatrix or speculative sidecar +bool gguf_filename_is_model(const std::string & filepath); + +// all the shards of `file`, or `file` alone when it is not sharded +hf_cache::hf_files get_split_files(const hf_cache::hf_files & files, + const hf_cache::hf_file & file); + +// pick the best model for `tag`, or the default quant preference then the +// first model of the listing when the tag is empty +hf_cache::hf_file find_best_model(const hf_cache::hf_files & files, + const std::string & tag); + +// pick the best sibling GGUF whose filename contains `keyword` (e.g. "mmproj" / "mtp-"), +// preferring deeper shared directory prefix with the model, then exact `tag` match, +// then closest quantization to the tag when given, or to the model otherwise +hf_cache::hf_file find_best_sibling(const hf_cache::hf_files & files, + const std::string & model, + const std::string & keyword, + const std::string & tag = ""); + +// build the download plan from a repo listing: a preset.ini short-circuits +// everything, an explicit `hf_file` picks that exact file, otherwise the best +// model for `tag` is picked with the requested sidecars, a requested sidecar +// resolves even without a full model at the tag, `repo` is only for the +// error messages +common_download_hf_plan resolve(const hf_cache::hf_files & files, + const std::string & repo, + const std::string & tag, + const std::string & hf_file, + const opts & o); + +} // namespace model_resolution From c63f955441f45596944b9326f117974b5e14dfa6 Mon Sep 17 00:00:00 2001 From: Pascal Date: Tue, 28 Jul 2026 23:03:37 +0200 Subject: [PATCH 2/3] tests: add model resolution and handler assembly test on a fake on-disk model cache --- tests/CMakeLists.txt | 1 + tests/test-model-resolution.cpp | 432 ++++++++++++++++++++++++++++++++ 2 files changed, 433 insertions(+) create mode 100644 tests/test-model-resolution.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 7a93b19a0765..008ed75e4007 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -258,6 +258,7 @@ llama_build_and_test(test-thread-safety.cpp ARGS -m "${MODEL_DEST}" -ngl 99 -p " set_tests_properties(test-thread-safety PROPERTIES FIXTURES_REQUIRED test-download-model) llama_build_and_test(test-arg-parser.cpp) +llama_build_and_test(test-model-resolution.cpp) if (NOT LLAMA_SANITIZE_ADDRESS AND NOT GGML_SCHED_NO_REALLOC) # TODO: repair known memory leaks diff --git a/tests/test-model-resolution.cpp b/tests/test-model-resolution.cpp new file mode 100644 index 000000000000..2a3cfc8bd6d3 --- /dev/null +++ b/tests/test-model-resolution.cpp @@ -0,0 +1,432 @@ +// tests the model resolution of common/model-resolution on synthetic repo +// listings, then the end-to-end model handler assembly through the real CLI +// parsing, resolving offline against a fake on-disk model cache, no network +// access and no instrumentation of the tested code + +#include "arg.h" +#include "common.h" +#include "hf-cache.h" +#include "model-resolution.h" +#include "speculative.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace fs = std::filesystem; + +// independent of NDEBUG, so the checks stay alive in Release builds +#define REQUIRE(x) do { \ + if (!(x)) { \ + fprintf(stderr, "%s:%d: REQUIRE(%s) failed\n", __FILE__, __LINE__, #x); \ + std::abort(); \ + } \ +} while (0) + +// fixtures mimicking real repo layouts + +// flat layout in the style of ggml-org/gemma-4-31B-it-GGUF +static const std::vector flat = { + "README.md", + "model-BF16.gguf", + "model-Q4_K_M.gguf", + "model-Q8_0.gguf", + "mmproj-model-BF16.gguf", + "mmproj-model-Q8_0.gguf", + "mtp-model-BF16.gguf", + "mtp-model-Q4_0.gguf", + "mtp-model-Q8_0.gguf", + "dflash-model-BF16.gguf", + "dflash-model-Q8_0.gguf", +}; + +// quants in subdirectories with sharded files and root sidecars, +// in the style of stepfun-ai/Step-3.7-Flash-GGUF +static const std::vector subdir = { + "mmproj-model-f16.gguf", + "model-mtp-BF16.gguf", + "model-mtp-Q8_0.gguf", + "Q3_K_M/model-Q3_K_M-00001-of-00003.gguf", + "Q3_K_M/model-Q3_K_M-00002-of-00003.gguf", + "Q3_K_M/model-Q3_K_M-00003-of-00003.gguf", + "Q8_0/model-Q8_0-00001-of-00002.gguf", + "Q8_0/model-Q8_0-00002-of-00002.gguf", +}; + +// sidecar quants exist where the full model quant does not, +// in the style of ggml-org/Qwen3.6-27B-GGUF +static const std::vector hole = { + "model-BF16.gguf", + "model-Q4_K_M.gguf", + "model-Q8_0.gguf", + "mtp-model-BF16.gguf", + "mtp-model-Q4_0.gguf", + "mtp-model-Q8_0.gguf", + "dflash-model-BF16.gguf", + "dflash-model-Q8_0.gguf", +}; + +// unsloth-style naming with UD quants and a suffix MTP file +static const std::vector unsloth = { + "model-UD-Q8_K_XL.gguf", + "mmproj-BF16.gguf", + "model-MTP-BF16.gguf", +}; + +// bartowski-style vendor prefix and mradermacher-style dot quant +static const std::vector vendors = { + "TheDrummer_Model-24B-v4.1-Q8_0.gguf", + "BlackSheep-24B.Q8_0.gguf", +}; + +// every speculative sidecar type at the same quant +static const std::vector trio = { + "model-Q8_0.gguf", + "mtp-model-Q8_0.gguf", + "dflash-model-Q8_0.gguf", + "eagle3-model-Q8_0.gguf", +}; + +static const std::vector dflash_only = { + "model-Q8_0.gguf", + "dflash-model-Q8_0.gguf", +}; + +static const std::vector eagle3_only = { + "model-Q8_0.gguf", + "eagle3-model-Q8_0.gguf", +}; + +// a synthetic listing, the resolution only looks at the paths +static hf_cache::hf_files listing(const std::vector & paths) { + hf_cache::hf_files files; + for (const auto & p : paths) { + hf_cache::hf_file f; + f.path = p; + files.push_back(std::move(f)); + } + return files; +} + +// helpers of the model resolution unit + +static void test_helpers() { + printf("test-model-resolution: resolution helpers\n"); + + // quant bits extraction on real world namings + REQUIRE(model_resolution::extract_quant_bits("model-Q8_0.gguf") == 8); + REQUIRE(model_resolution::extract_quant_bits("model-UD-Q2_K_XL.gguf") == 2); + REQUIRE(model_resolution::extract_quant_bits("model.i1-Q6_K.gguf") == 6); + REQUIRE(model_resolution::extract_quant_bits("model-BF16.gguf") == 16); + REQUIRE(model_resolution::extract_quant_bits("model-MXFP4-00001-of-00002.gguf") == 4); + + // sidecar keywords are never a model, wherever they appear in the name + REQUIRE(model_resolution::gguf_filename_is_model("model-Q8_0.gguf")); + REQUIRE(!model_resolution::gguf_filename_is_model("mtp-model-Q8_0.gguf")); + REQUIRE(!model_resolution::gguf_filename_is_model("model-mtp-Q8_0.gguf")); + REQUIRE(!model_resolution::gguf_filename_is_model("dflash-model-Q8_0.gguf")); + REQUIRE(!model_resolution::gguf_filename_is_model("eagle3-model-Q8_0.gguf")); + REQUIRE(!model_resolution::gguf_filename_is_model("mmproj-model-Q8_0.gguf")); + REQUIRE(!model_resolution::gguf_filename_is_model("model.txt")); + + // the sibling picker honors an exact tag over quant proximity + auto files = listing({"mtp-model-BF16.gguf", "mtp-model-Q4_0.gguf", "mtp-model-Q8_0.gguf"}); + REQUIRE(model_resolution::find_best_sibling(files, "model-Q8_0.gguf", "mtp-", "Q4_0").path == "mtp-model-Q4_0.gguf"); + REQUIRE(model_resolution::find_best_sibling(files, "model-Q8_0.gguf", "mtp-").path == "mtp-model-Q8_0.gguf"); +} + +// table-driven plan resolution, each case replayed on permutations of the +// listing to assert determinism, except the cases that legitimately depend +// on the listing order + +struct plan_case { + const char * name; + const std::vector & files; + const char * tag; + const char * hf_file; + bool sidecars; // request mmproj + mtp + dflash + eagle3 + bool order_dependent; // the expected pick depends on the listing order + const char * primary; + std::vector model_files; + const char * mmproj; + const char * mtp; + const char * dflash; + const char * eagle3; +}; + +static const plan_case plan_cases[] = { + // exact tag picks the matching primary, sidecars follow the tag + {"flat exact tag", flat, "Q8_0", "", true, false, + "model-Q8_0.gguf", {"model-Q8_0.gguf"}, + "mmproj-model-Q8_0.gguf", "mtp-model-Q8_0.gguf", "dflash-model-Q8_0.gguf", ""}, + + // no tag falls back to the default quant preference + {"flat default", flat, "", "", false, false, + "model-Q4_K_M.gguf", {"model-Q4_K_M.gguf"}, + "", "", "", ""}, + + // no tag and no default match falls back to the first model in the listing + {"unsloth fallback", unsloth, "", "", true, true, + "model-UD-Q8_K_XL.gguf", {"model-UD-Q8_K_XL.gguf"}, + "mmproj-BF16.gguf", "", "", ""}, + + // explicit hf_file picks that exact file + {"flat hf_file", flat, "", "model-BF16.gguf", false, false, + "model-BF16.gguf", {"model-BF16.gguf"}, + "", "", "", ""}, + + // missing hf_file resolves nothing + {"flat missing hf_file", flat, "", "nope.gguf", false, false, + "", {}, + "", "", "", ""}, + + // a sharded primary brings all its parts, a subdir primary finds the root sidecar + {"subdir shards", subdir, "Q3_K_M", "", true, false, + "Q3_K_M/model-Q3_K_M-00001-of-00003.gguf", + {"Q3_K_M/model-Q3_K_M-00001-of-00003.gguf", + "Q3_K_M/model-Q3_K_M-00002-of-00003.gguf", + "Q3_K_M/model-Q3_K_M-00003-of-00003.gguf"}, + "mmproj-model-f16.gguf", "model-mtp-Q8_0.gguf", "", ""}, + + // a tag with no matching full model still resolves the requested sidecars + {"hole tag sidecar", hole, "Q4_0", "", true, false, + "", {}, + "", "mtp-model-Q4_0.gguf", "dflash-model-Q8_0.gguf", ""}, + + // the same tag without a requested sidecar resolves nothing + {"hole tag alone", hole, "Q4_0", "", false, false, + "", {}, + "", "", "", ""}, + + // no tag anchors the sidecars on the primary quant + {"hole default anchor", hole, "", "", true, false, + "model-Q4_K_M.gguf", {"model-Q4_K_M.gguf"}, + "", "mtp-model-Q4_0.gguf", "dflash-model-Q8_0.gguf", ""}, + + // the mtp- keyword is case sensitive, a suffix -MTP file is not discovered + {"unsloth suffix mtp", unsloth, "Q8_K_XL", "", true, false, + "model-UD-Q8_K_XL.gguf", {"model-UD-Q8_K_XL.gguf"}, + "mmproj-BF16.gguf", "", "", ""}, + + // vendor prefixes and the dot quant convention both match the tag, + // first match wins between two files at the same quant + {"vendor prefix", vendors, "Q8_0", "", false, true, + "TheDrummer_Model-24B-v4.1-Q8_0.gguf", {"TheDrummer_Model-24B-v4.1-Q8_0.gguf"}, + "", "", "", ""}, + + // every sidecar type resolves at the tag + {"trio exact tag", trio, "Q8_0", "", true, false, + "model-Q8_0.gguf", {"model-Q8_0.gguf"}, + "", "mtp-model-Q8_0.gguf", "dflash-model-Q8_0.gguf", "eagle3-model-Q8_0.gguf"}, +}; + +static void check_plan(const plan_case & c, const hf_cache::hf_files & files) { + model_resolution::opts opts; + opts.mmproj = c.sidecars; + opts.mtp = c.sidecars; + opts.dflash = c.sidecars; + opts.eagle3 = c.sidecars; + + auto plan = model_resolution::resolve(files, "test/repo", c.tag, c.hf_file, opts); + + REQUIRE(plan.primary.path == c.primary); + REQUIRE(plan.mmproj.path == c.mmproj); + REQUIRE(plan.mtp.path == c.mtp); + REQUIRE(plan.dflash.path == c.dflash); + REQUIRE(plan.eagle3.path == c.eagle3); + + // the exact shard set, order insensitive, with the primary as first split + std::vector actual; + for (const auto & f : plan.model_files) { + actual.push_back(f.path); + } + std::sort(actual.begin(), actual.end()); + auto expected = c.model_files; + std::sort(expected.begin(), expected.end()); + REQUIRE(actual == expected); + if (!expected.empty()) { + REQUIRE(plan.primary.path == expected.front()); + } + + // invariant: the primary is never a sidecar file + if (!plan.primary.path.empty()) { + REQUIRE(model_resolution::gguf_filename_is_model(plan.primary.path)); + } +} + +static void test_plan_resolution() { + printf("test-model-resolution: plan resolution on %zu cases\n", sizeof(plan_cases) / sizeof(plan_cases[0])); + + for (const auto & c : plan_cases) { + auto files = listing(c.files); + + // invariant: the resolution is insensitive to the listing order + for (size_t rot = 0; rot < files.size(); ++rot) { + if (c.order_dependent && rot > 0) { + continue; + } + auto permuted = files; + std::rotate(permuted.begin(), permuted.begin() + rot, permuted.end()); + if (rot % 2 == 1) { + std::reverse(permuted.begin(), permuted.end()); + } + check_plan(c, permuted); + } + } +} + +// end-to-end assembly: real CLI parsing and real handler init, --offline +// resolves against the fake on-disk cache below and skips the downloads +// while on_done still wires the params + +// the cache layout expected by hf-cache.cpp, refs/main names a commit hash +// of 40 hex characters and snapshots/ holds the files of that commit +static const fs::path cache_dir = fs::temp_directory_path() / "test-model-resolution-cache"; +static const std::string commit = std::string(40, 'c'); + +static fs::path repo_dir(std::string repo_id) { + string_replace_all(repo_id, "/", "--"); + return cache_dir / ("models--" + repo_id); +} + +static void write_repo(const std::string & repo_id, const std::vector & paths) { + fs::create_directories(repo_dir(repo_id) / "refs"); + std::ofstream(repo_dir(repo_id) / "refs" / "main") << commit; + for (const auto & p : paths) { + fs::path file = repo_dir(repo_id) / "snapshots" / commit / fs::path(p); + fs::create_directories(file.parent_path()); + std::ofstream(file) << ""; + } +} + +// the local path the handler is expected to wire for a cached file +static std::string cached(const std::string & repo_id, const std::string & path) { + return (repo_dir(repo_id) / "snapshots" / commit / fs::path(path)).string(); +} + +static void assemble(std::vector argv, common_params & params) { + argv.push_back("--offline"); + std::vector cargv; + for (auto & a : argv) { + cargv.push_back(a.data()); + } + REQUIRE(common_params_parse((int) cargv.size(), cargv.data(), params, LLAMA_EXAMPLE_SERVER)); + + auto handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER); + common_models_handler_apply(handler, params); +} + +static void test_task_assembly() { + printf("test-model-resolution: end-to-end assembly\n"); + + write_repo("test/main", flat); + write_repo("test/hole", hole); + write_repo("test/trio", trio); + write_repo("test/dflash", dflash_only); + write_repo("test/eagle3", eagle3_only); + write_repo("test/small", {"draft-model-Q4_K_M.gguf"}); + write_repo("test/preset", {"preset.ini", "model-Q8_0.gguf"}); + + { + // plain -hf wires the model and its mmproj, nothing speculative + common_params params; + assemble({"server", "-hf", "test/main:Q8_0"}, params); + REQUIRE(params.model.path == cached("test/main", "model-Q8_0.gguf")); + REQUIRE(params.mmproj.path == cached("test/main", "mmproj-model-Q8_0.gguf")); + REQUIRE(params.speculative.draft.mparams.path.empty()); + } + { + // --no-mmproj disables the mmproj discovery + common_params params; + assemble({"server", "-hf", "test/main:Q8_0", "--no-mmproj"}, params); + REQUIRE(params.mmproj.path.empty()); + } + { + // an explicit --mmproj wins over the discovery + common_params params; + assemble({"server", "-hf", "test/main:Q8_0", "--mmproj", "/local/mmproj.gguf"}, params); + REQUIRE(params.mmproj.path == "/local/mmproj.gguf"); + } + { + // -hf with a spec type wires the sidecar of the main repo as fallback draft + common_params params; + assemble({"server", "-hf", "test/main:Q8_0", "--spec-type", "draft-mtp"}, params); + REQUIRE(params.speculative.draft.mparams.path == cached("test/main", "mtp-model-Q8_0.gguf")); + } + { + // -hfd with a spec type wires the draft repo sidecar at its tag, + // not its full model, and suppresses the main repo fallback + common_params params; + assemble({"server", "-hf", "test/hole:Q8_0", "-hfd", "test/hole:Q4_0", "--spec-type", "draft-mtp"}, params); + REQUIRE(params.speculative.draft.mparams.path == cached("test/hole", "mtp-model-Q4_0.gguf")); + } + { + // an explicit -md file wins over the sidecar resolution + common_params params; + assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/main", "-md", "mtp-model-BF16.gguf", "--spec-type", "draft-mtp"}, params); + REQUIRE(params.speculative.draft.mparams.path == cached("test/main", "mtp-model-BF16.gguf")); + } + { + // -hfd without a spec type auto-selects the type, mtp first when all ship + common_params params; + assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/trio:Q8_0"}, params); + REQUIRE(params.speculative.types == std::vector{COMMON_SPECULATIVE_TYPE_DRAFT_MTP}); + REQUIRE(params.speculative.draft.mparams.path == cached("test/trio", "mtp-model-Q8_0.gguf")); + } + { + // auto-selection with only a dflash sidecar + common_params params; + assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/dflash:Q8_0"}, params); + REQUIRE(params.speculative.types == std::vector{COMMON_SPECULATIVE_TYPE_DRAFT_DFLASH}); + REQUIRE(params.speculative.draft.mparams.path == cached("test/dflash", "dflash-model-Q8_0.gguf")); + } + { + // auto-selection with only an eagle3 sidecar + common_params params; + assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/eagle3:Q8_0"}, params); + REQUIRE(params.speculative.types == std::vector{COMMON_SPECULATIVE_TYPE_DRAFT_EAGLE3}); + REQUIRE(params.speculative.draft.mparams.path == cached("test/eagle3", "eagle3-model-Q8_0.gguf")); + } + { + // -hfd on a repo without sidecars keeps resolving a full model as draft + common_params params; + assemble({"server", "-hf", "test/main:Q8_0", "-hfd", "test/small"}, params); + REQUIRE(params.speculative.types == std::vector{COMMON_SPECULATIVE_TYPE_NONE}); + REQUIRE(params.speculative.draft.mparams.path == cached("test/small", "draft-model-Q4_K_M.gguf")); + } + { + // a preset repo wires the preset and clears the model for router mode + common_params params; + assemble({"server", "-hf", "test/preset"}, params); + REQUIRE(params.models_preset == cached("test/preset", "preset.ini")); + REQUIRE(params.model.path.empty()); + REQUIRE(params.model.hf_repo.empty()); + } +} + +static void set_env(const char * name, const char * value) { +#if defined(_WIN32) + _putenv_s(name, value); +#else + setenv(name, value, 1); +#endif +} + +int main(void) { + // the cache location is read once by hf-cache.cpp, point it at the fake + // cache before anything else touches it + set_env("LLAMA_CACHE", cache_dir.string().c_str()); + fs::remove_all(cache_dir); + + test_helpers(); + test_plan_resolution(); + test_task_assembly(); + + fs::remove_all(cache_dir); + printf("test-model-resolution: all tests OK\n"); + return 0; +} From 2b4c1fafa9e535844464fd73146db8a7bf3e48b3 Mon Sep 17 00:00:00 2001 From: Pascal Date: Tue, 28 Jul 2026 23:09:33 +0200 Subject: [PATCH 3/3] tests: keep the model resolution output down to the reports --- tests/test-model-resolution.cpp | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/tests/test-model-resolution.cpp b/tests/test-model-resolution.cpp index 2a3cfc8bd6d3..cc70770ea2a1 100644 --- a/tests/test-model-resolution.cpp +++ b/tests/test-model-resolution.cpp @@ -6,6 +6,7 @@ #include "arg.h" #include "common.h" #include "hf-cache.h" +#include "log.h" #include "model-resolution.h" #include "speculative.h" @@ -19,12 +20,17 @@ namespace fs = std::filesystem; +// the plan case being checked, named by the failure report below +static const char * current_case = ""; + // independent of NDEBUG, so the checks stay alive in Release builds -#define REQUIRE(x) do { \ - if (!(x)) { \ - fprintf(stderr, "%s:%d: REQUIRE(%s) failed\n", __FILE__, __LINE__, #x); \ - std::abort(); \ - } \ +#define REQUIRE(x) do { \ + if (!(x)) { \ + fprintf(stderr, "%s:%d: REQUIRE(%s) failed%s%s\n", \ + __FILE__, __LINE__, #x, \ + *current_case ? " on case " : "", current_case); \ + std::abort(); \ + } \ } while (0) // fixtures mimicking real repo layouts @@ -225,6 +231,7 @@ static const plan_case plan_cases[] = { }; static void check_plan(const plan_case & c, const hf_cache::hf_files & files) { + current_case = c.name; model_resolution::opts opts; opts.mmproj = c.sidecars; opts.mtp = c.sidecars; @@ -256,6 +263,7 @@ static void check_plan(const plan_case & c, const hf_cache::hf_files & files) { if (!plan.primary.path.empty()) { REQUIRE(model_resolution::gguf_filename_is_model(plan.primary.path)); } + current_case = ""; } static void test_plan_resolution() { @@ -417,6 +425,10 @@ static void set_env(const char * name, const char * value) { } int main(void) { + // the negative cases legitimately log errors on every permutation, + // keep the output down to the printf reports and the failure reports + common_log_pause(common_log_main()); + // the cache location is read once by hf-cache.cpp, point it at the fake // cache before anything else touches it set_env("LLAMA_CACHE", cache_dir.string().c_str());