diff --git a/app/download.cpp b/app/download.cpp index 7227baadcb18..dc4141588bfa 100644 --- a/app/download.cpp +++ b/app/download.cpp @@ -30,10 +30,11 @@ int llama_download(int argc, char ** argv) { return 1; } - const bool has_source = !params.model.hf_repo.empty() || !params.model.url.empty() || - !params.model.path.empty() || !params.model.docker_repo.empty(); + const bool has_source = !params.model.hf_repo.empty() || !params.model.ms_repo.empty() || + !params.model.url.empty() || !params.model.path.empty() || + !params.model.docker_repo.empty(); if (!has_source) { - fprintf(stderr, "error: no model source specified (use --hf-repo, --model-url, --model or --docker-repo)\n"); + fprintf(stderr, "error: no model source specified (use --hf-repo, --ms-repo, --model-url, --model or --docker-repo)\n"); return 1; } diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 36f1e0cd50f1..5ebe17ba2dec 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -77,6 +77,8 @@ add_library(${TARGET} fit.h hf-cache.cpp hf-cache.h + ms-cache.cpp + ms-cache.h http.h imatrix-loader.cpp imatrix-loader.h diff --git a/common/arg.cpp b/common/arg.cpp index 86f8610a56d0..f181901b2550 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -361,6 +361,7 @@ static bool spec_types_is_default(const common_params & params) { common_models_handler common_models_handler_init(const common_params & params, llama_example curr_ex) { common_download_hf_plan plan; common_download_hf_plan plan_spec; + common_download_ms_plan plan_ms; common_download_opts opts; const bool spec_type_draft_mtp = std::find(params.speculative.types.begin(), @@ -413,7 +414,11 @@ common_models_handler common_models_handler_init(const common_params & params, l plan_spec = common_download_get_hf_plan(params.speculative.draft.mparams, opts_spec); } - return common_models_handler{plan, plan_spec, opts}; + if (!params.model.ms_repo.empty()) { + plan_ms = common_download_get_ms_plan(params.model, opts); + } + + return common_models_handler{plan, plan_spec, plan_ms, opts}; } bool common_models_handler_is_preset_repo(const common_models_handler & handler) { @@ -463,6 +468,7 @@ void common_models_handler_apply(common_models_handler & handler, common_params auto & plan = handler.plan; auto & plan_spec = handler.plan_spec; + auto & plan_ms = handler.plan_ms; auto opts = handler.opts; // copy opts.callback = callback; @@ -683,6 +689,77 @@ void common_models_handler_apply(common_models_handler & handler, common_params }); } + + // handle ms_plan tasks + // MS authenticates via a cookie, so attach it only to MS tasks (keep HF tasks on bearer) + common_download_opts ms_opts = opts; + if (!params.model.ms_repo.empty() && !opts.bearer_token.empty()) { + ms_opts.headers.emplace_back("Cookie", "m_session_id=" + opts.bearer_token); + } + auto add_ms_tasks = [&ms_opts, &tasks](const ms_cache::ms_files & model_files, + const ms_cache::ms_file & primary, + common_params_model & model) { + for (size_t i = 0; i < model_files.size(); ++i) { + auto & model_file = model_files[i]; + bool is_primary = (model_file.path == primary.path); + tasks.emplace_back(model_file, ms_opts, [&model_file, &is_primary, &model]() { + if (is_primary) { + // the primary file is the first split (00001-of), use it as model path + model.path = ms_cache::finalize_file(model_file); + } else { + ms_cache::finalize_file(model_file); + } + }); + } + }; + if (!plan_ms.model_files.empty()) { + add_ms_tasks(plan_ms.model_files, plan_ms.primary, params.model); + } + if (!plan_ms.mmproj.local_path.empty()) { + tasks.emplace_back(plan_ms.mmproj, ms_opts, [&]() { + params.mmproj.path = ms_cache::finalize_file(plan_ms.mmproj); + }); + } + if (!plan_ms.mtp.local_path.empty() && !had_spec_url) { + tasks.emplace_back(plan_ms.mtp, ms_opts, [&]() { + // only fall back to the discovered MTP head when no draft was explicitly provided + if (params.speculative.draft.mparams.empty()) { + params.speculative.draft.mparams.path = ms_cache::finalize_file(plan_ms.mtp); + } else { + ms_cache::finalize_file(plan_ms.mtp); + } + }); + } + if (!plan_ms.dflash.local_path.empty() && !had_spec_url) { + tasks.emplace_back(plan_ms.dflash, ms_opts, [&]() { + // only fall back to the discovered DFlash sidecar when no draft was explicitly provided + if (params.speculative.draft.mparams.empty()) { + params.speculative.draft.mparams.path = ms_cache::finalize_file(plan_ms.dflash); + } else { + ms_cache::finalize_file(plan_ms.dflash); + } + }); + } + if (!plan_ms.eagle3.local_path.empty() && !had_spec_url) { + tasks.emplace_back(plan_ms.eagle3, ms_opts, [&]() { + // only fall back to the discovered Eagle3 sidecar when no draft was explicitly provided + if (params.speculative.draft.mparams.empty()) { + params.speculative.draft.mparams.path = ms_cache::finalize_file(plan_ms.eagle3); + } else { + ms_cache::finalize_file(plan_ms.eagle3); + } + }); + } + if (!plan_ms.dspark.local_path.empty() && !had_spec_url) { + tasks.emplace_back(plan_ms.dspark, ms_opts, [&]() { + // only fall back to the discovered DSpark sidecar when no draft was explicitly provided + if (params.speculative.draft.mparams.empty()) { + params.speculative.draft.mparams.path = ms_cache::finalize_file(plan_ms.dspark); + } else { + ms_cache::finalize_file(plan_ms.dspark); + } + }); + } // run all tasks in parallel if (!params.offline) { // if duplicated files are found, only download once (but still call on_done for each task) @@ -803,6 +880,13 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context } } + // ModelScope authentication token (passed as a cookie, reusing hf_token as the carrier) + if (params.hf_token.empty()) { + if (const char * ms_token = std::getenv("MS_TOKEN")) { + params.hf_token = ms_token; + } + } + // handle command line arguments auto check_arg = [&](int i) { if (i+1 >= argc) { @@ -1459,7 +1543,7 @@ common_params_context common_params_parser_init(common_params & params, llama_ex auto models = common_list_cached_models(); printf("number of models in cache: %zu\n", models.size()); for (size_t i = 0; i < models.size(); i++) { - printf("%4zu. %s\n", i + 1, models[i].to_string().c_str()); + printf("%4zu. [%s] %s\n", i + 1, models[i].source.c_str(), models[i].to_string().c_str()); } exit(0); } @@ -3047,6 +3131,16 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.model.hf_file = value; } ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_HF_FILE")); + add_opt(common_arg( + {"-ms", "-msr", "--ms-repo"}, "/[:quant]", + "ModelScope model repository; quant is optional, case-insensitive, default to Q4_K_M, or falls back to the first file in the repo if Q4_K_M doesn't exist.\n" + "mmproj is also downloaded automatically if available. to disable, add --no-mmproj\n" + "example: Qwen/Qwen3-0.6B-GGUF:Q4_K_M\n" + "(default: unused)", + [](common_params & params, const std::string & value) { + params.model.ms_repo = value; + } + ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_DOWNLOAD}).set_env("LLAMA_ARG_MS_REPO")); add_opt(common_arg( {"-hft", "--hf-token"}, "TOKEN", "Hugging Face access token (default: value from HF_TOKEN environment variable)", diff --git a/common/arg.h b/common/arg.h index 421bc295fc21..f121d91c8b35 100644 --- a/common/arg.h +++ b/common/arg.h @@ -138,6 +138,7 @@ void common_params_add_preset_options(std::vector & args); struct common_models_handler { common_download_hf_plan plan; common_download_hf_plan plan_spec; + common_download_ms_plan plan_ms; common_download_opts opts; }; diff --git a/common/common.h b/common/common.h index de49dac9f63a..36092816f65a 100644 --- a/common/common.h +++ b/common/common.h @@ -304,11 +304,15 @@ struct common_params_model { std::string hf_repo = ""; // HF repo std::string hf_file = ""; // HF file std::string docker_repo = ""; // Docker repo + std::string ms_repo = ""; // ModelScope repo std::string get_name() const { if (!hf_repo.empty()) { return hf_repo; } + if (!ms_repo.empty()) { + return ms_repo; + } if (!docker_repo.empty()) { return docker_repo; } diff --git a/common/download.cpp b/common/download.cpp index 4b28a708c86e..687bd3c71a5a 100644 --- a/common/download.cpp +++ b/common/download.cpp @@ -6,6 +6,7 @@ #include "download.h" #include "hf-cache.h" #include "json.h" +#include "ms-cache.h" #include #include @@ -545,14 +546,15 @@ static int extract_quant_bits(const std::string & filename) { return std::stoi(split.tag.substr(pos)); } -static hf_cache::hf_files get_split_files(const hf_cache::hf_files & files, - const hf_cache::hf_file & file) { +template +static std::vector get_split_files(const std::vector & files, + const File & file) { auto split = get_gguf_split_info(file.path); if (split.count <= 1) { return {file}; } - hf_cache::hf_files result; + std::vector result; for (const auto & f : files) { auto split_f = get_gguf_split_info(f.path); @@ -566,11 +568,12 @@ static hf_cache::hf_files get_split_files(const hf_cache::hf_files & files, // pick the best sibling GGUF whose filename contains `keyword` (e.g. "mmproj" / "mtp"), // preferring deeper shared directory prefix with the model, then exact `tag` match, // then closest quantization to the tag when given, or to the model otherwise -static hf_cache::hf_file find_best_sibling(const hf_cache::hf_files & files, - const std::string & model, - const std::string & keyword, - const std::string & tag = "") { - hf_cache::hf_file best; +template +static File find_best_sibling(const std::vector & files, + const std::string & model, + const std::string & keyword, + const std::string & tag = "") { + File best; size_t best_depth = 0; int best_diff = 0; bool best_exact = false; @@ -629,32 +632,37 @@ static hf_cache::hf_file find_best_sibling(const hf_cache::hf_files & files, return best; } -static hf_cache::hf_file find_best_mmproj(const hf_cache::hf_files & files, - const std::string & model) { +template +static File find_best_mmproj(const std::vector & files, + const std::string & model) { return find_best_sibling(files, model, "mmproj"); } -static hf_cache::hf_file find_best_mtp(const hf_cache::hf_files & files, - const std::string & model, - const std::string & tag = "") { +template +static File find_best_mtp(const std::vector & files, + const std::string & model, + const std::string & tag = "") { return find_best_sibling(files, model, "mtp-", tag); } -static hf_cache::hf_file find_best_eagle3(const hf_cache::hf_files & files, - const std::string & model, - const std::string & tag = "") { +template +static File find_best_eagle3(const std::vector & files, + const std::string & model, + const std::string & tag = "") { return find_best_sibling(files, model, "eagle3-", tag); } -static hf_cache::hf_file find_best_dflash(const hf_cache::hf_files & files, - const std::string & model, - const std::string & tag = "") { +template +static File find_best_dflash(const std::vector & files, + const std::string & model, + const std::string & tag = "") { return find_best_sibling(files, model, "dflash-", tag); } -static hf_cache::hf_file find_best_dspark(const hf_cache::hf_files & files, - const std::string & model, - const std::string & tag = "") { +template +static File find_best_dspark(const std::vector & files, + const std::string & model, + const std::string & tag = "") { return find_best_sibling(files, model, "dspark-", tag); } @@ -676,8 +684,9 @@ static bool gguf_filename_is_model(const std::string & filepath) { filename.find("dspark-") == std::string::npos; } -static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files, - const std::string & tag) { +template +static File find_best_model(const std::vector & files, + const std::string & tag) { std::vector tags; if (!tag.empty()) { @@ -716,7 +725,8 @@ static hf_cache::hf_file find_best_model(const hf_cache::hf_files & files, return {}; } -static void list_available_gguf_files(const hf_cache::hf_files & files) { +template +static void list_available_gguf_files(const std::vector & files) { LOG_INF("Available GGUF files:\n"); for (const auto & f : files) { if (string_ends_with(f.path, ".gguf")) { @@ -803,12 +813,73 @@ common_download_hf_plan common_download_get_hf_plan(const common_params_model & return plan; } +common_download_ms_plan common_download_get_ms_plan(const common_params_model & model, const common_download_opts & opts) { + common_download_ms_plan plan; + ms_cache::ms_files all; + + auto [repo, tag] = common_download_split_repo_tag(model.ms_repo); + + if (!opts.offline) { + all = ms_cache::get_repo_files(repo, opts.bearer_token); + } + if (all.empty()) { + all = ms_cache::get_cached_files(repo); + } + if (all.empty()) { + return plan; + } + + ms_cache::ms_file primary; + + if (!model.hf_file.empty()) { + for (const auto & f : all) { + if (f.path == model.hf_file) { + primary = f; + break; + } + } + if (primary.path.empty()) { + LOG_ERR("%s: file '%s' not found in repository\n", __func__, model.hf_file.c_str()); + list_available_gguf_files(all); + return plan; + } + } else { + primary = find_best_model(all, tag); + if (primary.path.empty()) { + LOG_ERR("%s: no GGUF files found in repository %s\n", __func__, repo.c_str()); + list_available_gguf_files(all); + return plan; + } + } + + plan.primary = primary; + plan.model_files = get_split_files(all, primary); + + if (opts.download_mmproj) { + plan.mmproj = find_best_mmproj(all, primary.path); + } + if (opts.download_mtp) { + plan.mtp = find_best_mtp(all, primary.path, tag); + } + if (opts.download_dflash) { + plan.dflash = find_best_dflash(all, primary.path, tag); + } + if (opts.download_eagle3) { + plan.eagle3 = find_best_eagle3(all, primary.path, tag); + } + if (opts.download_dspark) { + plan.dspark = find_best_dspark(all, primary.path, tag); + } + + return plan; +} + void common_download_run_tasks(const std::vector & tasks) { std::vector> futures; for (const auto & task : tasks) { futures.push_back(std::async(std::launch::async, [&task]() { - return common_download_file_single(task.url, task.local_path, task.opts, task.is_hf); + return common_download_file_single(task.url, task.local_path, task.opts, task.is_hf || task.is_ms); } )); } @@ -977,8 +1048,25 @@ std::vector common_list_cached_models() { split.prefix.find("dspark-") != std::string::npos) { continue; } - if (seen.insert(f.repo_id + ":" + split.tag).second) { - result.push_back({f.repo_id, split.tag}); + if (seen.insert("hf:" + f.repo_id + ":" + split.tag).second) { + result.push_back({f.repo_id, split.tag, "hf"}); + } + } + + auto ms_files = ms_cache::get_cached_files(); + + for (const auto & f : ms_files) { + auto split = get_gguf_split_info(f.path); + if (split.index != 1 || split.tag.empty() || + split.prefix.find("mmproj") != std::string::npos || + split.prefix.find("mtp-") != std::string::npos || + split.prefix.find("eagle3-") != std::string::npos || + split.prefix.find("dflash-") != std::string::npos || + split.prefix.find("dspark-") != std::string::npos) { + continue; + } + if (seen.insert("ms:" + f.repo_id + ":" + split.tag).second) { + result.push_back({f.repo_id, split.tag, "ms"}); } } @@ -1011,7 +1099,10 @@ bool common_download_remove(const std::string & hf_repo_with_tag) { auto [repo_id, tag] = common_download_split_repo_tag(hf_repo_with_tag); if (tag.empty()) { - return hf_cache::remove_cached_repo(repo_id); + if (hf_cache::remove_cached_repo(repo_id)) { + return true; + } + return ms_cache::remove_cached_repo(repo_id); } std::string tag_upper = tag; @@ -1019,72 +1110,88 @@ bool common_download_remove(const std::string & hf_repo_with_tag) { c = (char) std::toupper((unsigned char) c); } - auto files = hf_cache::get_cached_files(repo_id); - if (files.empty()) { - return false; - } - - // collect snapshot entries whose tag matches - std::vector to_remove; - for (const auto & f : files) { - auto split = get_gguf_split_info(f.path); - if (split.tag == tag_upper) { - to_remove.emplace_back(f.local_path); + // try removing from a cache source (HF or MS), returns true if any file was removed + auto remove_from_cache = [&tag_upper](const auto & files, const std::string & repo_id, bool is_ms) -> bool { + // collect snapshot entries whose tag matches + std::vector to_remove; + for (const auto & f : files) { + auto split = get_gguf_split_info(f.path); + if (split.tag == tag_upper) { + to_remove.emplace_back(f.local_path); + } } - } - if (to_remove.empty()) { - return false; - } + if (to_remove.empty()) { + return false; + } - // resolve blob paths from symlinks before deleting snapshot entries - std::vector blobs_to_check; - for (const auto & p : to_remove) { - std::error_code ec; - if (fs::is_symlink(p, ec)) { - auto target = fs::read_symlink(p, ec); - if (!ec) { - blobs_to_check.push_back((p.parent_path() / target).lexically_normal()); + // resolve blob paths from symlinks before deleting snapshot entries + std::vector blobs_to_check; + for (const auto & p : to_remove) { + std::error_code ec; + if (fs::is_symlink(p, ec)) { + auto target = fs::read_symlink(p, ec); + if (!ec) { + blobs_to_check.push_back((p.parent_path() / target).lexically_normal()); + } } } - } - // remove snapshot entries - for (const auto & p : to_remove) { - std::error_code ec; - fs::remove(p, ec); - if (ec) { - LOG_WRN("%s: failed to remove %s: %s\n", __func__, p.string().c_str(), ec.message().c_str()); + // remove snapshot entries + for (const auto & p : to_remove) { + std::error_code ec; + fs::remove(p, ec); + if (ec) { + LOG_WRN("%s: failed to remove %s: %s\n", __func__, p.string().c_str(), ec.message().c_str()); + } } - } - if (blobs_to_check.empty()) { - return true; - } + if (blobs_to_check.empty()) { + return true; + } - // collect blobs still referenced by remaining snapshot entries - std::unordered_set still_referenced; - for (const auto & f : hf_cache::get_cached_files(repo_id)) { - fs::path p(f.local_path); - std::error_code ec; - if (fs::is_symlink(p, ec)) { - auto target = fs::read_symlink(p, ec); - if (!ec) { - still_referenced.insert((p.parent_path() / target).lexically_normal().string()); + // collect blobs still referenced by remaining snapshot entries + std::unordered_set still_referenced; + if (is_ms) { + for (const auto & f : ms_cache::get_cached_files(repo_id)) { + fs::path p(f.local_path); + std::error_code ec; + if (fs::is_symlink(p, ec)) { + auto target = fs::read_symlink(p, ec); + if (!ec) { + still_referenced.insert((p.parent_path() / target).lexically_normal().string()); + } + } + } + } else { + for (const auto & f : hf_cache::get_cached_files(repo_id)) { + fs::path p(f.local_path); + std::error_code ec; + if (fs::is_symlink(p, ec)) { + auto target = fs::read_symlink(p, ec); + if (!ec) { + still_referenced.insert((p.parent_path() / target).lexically_normal().string()); + } + } } } - } - // remove orphaned blobs - for (const auto & blob : blobs_to_check) { - if (still_referenced.find(blob.string()) == still_referenced.end()) { - std::error_code ec; - fs::remove(blob, ec); - if (ec) { - LOG_WRN("%s: failed to remove blob %s: %s\n", __func__, blob.string().c_str(), ec.message().c_str()); + // remove orphaned blobs + for (const auto & blob : blobs_to_check) { + if (still_referenced.find(blob.string()) == still_referenced.end()) { + std::error_code ec; + fs::remove(blob, ec); + if (ec) { + LOG_WRN("%s: failed to remove blob %s: %s\n", __func__, blob.string().c_str(), ec.message().c_str()); + } } } - } - return true; + return true; + }; + + bool hf_removed = remove_from_cache(hf_cache::get_cached_files(repo_id), repo_id, false); + bool ms_removed = remove_from_cache(ms_cache::get_cached_files(repo_id), repo_id, true); + + return hf_removed || ms_removed; } diff --git a/common/download.h b/common/download.h index 8c30cfc3eadb..e7e57550dd25 100644 --- a/common/download.h +++ b/common/download.h @@ -1,6 +1,7 @@ #pragma once #include "hf-cache.h" +#include "ms-cache.h" #include #include @@ -45,6 +46,7 @@ std::pair common_download_split_repo_tag(const std::st struct common_cached_model_info { std::string repo; std::string tag; + std::string source; // "hf" or "ms" std::string to_string() const { return repo + ":" + tag; } @@ -69,12 +71,17 @@ struct common_download_task { std::string local_path; std::function on_done; bool is_hf = false; + bool is_ms = false; common_download_task() = default; common_download_task(hf_cache::hf_file f, const common_download_opts & opts, std::function on_done = nullptr) : opts(opts), url(f.url), local_path(f.local_path), on_done(on_done), is_hf(true) {} + common_download_task(ms_cache::ms_file f, + const common_download_opts & opts, + std::function on_done = nullptr) + : opts(opts), url(f.url), local_path(f.local_path), on_done(on_done), is_ms(true) {} }; void common_download_run_tasks(const std::vector & tasks); @@ -119,3 +126,14 @@ struct common_download_hf_plan { hf_cache::hf_file preset; // if set, only this file is downloaded }; common_download_hf_plan common_download_get_hf_plan(const common_params_model & model, const common_download_opts & opts); + +struct common_download_ms_plan { + ms_cache::ms_file primary; + ms_cache::ms_files model_files; + ms_cache::ms_file mmproj; + ms_cache::ms_file mtp; + ms_cache::ms_file dflash; + ms_cache::ms_file eagle3; + ms_cache::ms_file dspark; +}; +common_download_ms_plan common_download_get_ms_plan(const common_params_model & model, const common_download_opts & opts); diff --git a/common/ms-cache.cpp b/common/ms-cache.cpp new file mode 100644 index 000000000000..b12d4fc5bc05 --- /dev/null +++ b/common/ms-cache.cpp @@ -0,0 +1,446 @@ +#include "ms-cache.h" + +#include "build-info.h" +#include "common.h" +#include "log.h" +#include "http.h" + +#define JSON_ASSERT GGML_ASSERT +#include + +#include +#include +#include +#include + +namespace nl = nlohmann; + +#if defined(_WIN32) +#define WIN32_LEAN_AND_MEAN +#ifndef NOMINMAX +#define NOMINMAX +#endif +#define HOME_DIR "USERPROFILE" +#include +#else +#define HOME_DIR "HOME" +#include +#include +#endif + +namespace ms_cache { + +namespace fs = std::filesystem; + +static fs::path get_cache_directory() { + static const fs::path cache = []() { + struct { + const char * var; + fs::path path; + } entries[] = { + {"LLAMA_CACHE", fs::path()}, + {"MODELSCOPE_CACHE", fs::path()}, + {"XDG_CACHE_HOME", fs::path("modelscope") / "hub"}, + {HOME_DIR, fs::path(".cache") / "modelscope" / "hub"} + }; + for (const auto & entry : entries) { + if (auto * p = std::getenv(entry.var); p && *p) { + fs::path base(p); + return entry.path.empty() ? base : base / entry.path; + } + } +#ifndef _WIN32 + const struct passwd * pw = getpwuid(getuid()); + + if (pw && pw->pw_dir && *pw->pw_dir) { + return fs::path(pw->pw_dir) / ".cache" / "modelscope" / "hub"; + } +#endif + throw std::runtime_error("Failed to determine ModelScope cache directory"); + }(); + + return cache; +} + +static std::string folder_name_to_repo(const std::string & folder) { + constexpr std::string_view prefix = "models--"; + if (folder.rfind(prefix, 0)) { + return {}; + } + std::string result = folder.substr(prefix.length()); + string_replace_all(result, "--", "/"); + return result; +} + +static std::string repo_to_folder_name(const std::string & repo_id) { + constexpr std::string_view prefix = "models--"; + std::string result = std::string(prefix) + repo_id; + string_replace_all(result, "/", "--"); + return result; +} + +static fs::path get_repo_path(const std::string & repo_id) { + return get_cache_directory() / repo_to_folder_name(repo_id); +} + +static bool is_hex_char(const char c) { + return (c >= 'A' && c <= 'F') || + (c >= 'a' && c <= 'f') || + (c >= '0' && c <= '9'); +} + +static bool is_hex_string(const std::string & s, size_t expected_len) { + if (s.length() != expected_len) { + return false; + } + for (const char c : s) { + if (!is_hex_char(c)) { + return false; + } + } + return true; +} + +static bool is_alphanum(const char c) { + return (c >= 'A' && c <= 'Z') || + (c >= 'a' && c <= 'z') || + (c >= '0' && c <= '9'); +} + +static bool is_special_char(char c) { + return c == '/' || c == '.' || c == '-'; +} + +static bool is_valid_repo_id(const std::string & repo_id) { + if (repo_id.empty() || repo_id.length() > 256) { + return false; + } + int slash = 0; + bool special = true; + + for (const char c : repo_id) { + if (is_alphanum(c) || c == '_') { + special = false; + } else if (is_special_char(c)) { + if (special) { + return false; + } + slash += (c == '/'); + special = true; + } else { + return false; + } + } + return !special && slash == 1; +} + +static bool is_valid_oid(const std::string & oid) { + return is_hex_string(oid, 64); +} + +static bool is_valid_subpath(const fs::path & path, const fs::path & subpath) { + if (subpath.is_absolute()) { + return false; + } + auto b = fs::absolute(path).lexically_normal(); + auto t = (b / subpath).lexically_normal(); + auto [b_end, _] = std::mismatch(b.begin(), b.end(), t.begin(), t.end()); + + return b_end == b.end(); +} + +static void safe_write_file(const fs::path & path, const std::string & data) { + fs::path path_tmp = path.string() + ".tmp"; + + if (path.has_parent_path()) { + fs::create_directories(path.parent_path()); + } + + std::ofstream file(path_tmp); + file << data; + file.close(); + + std::error_code ec; + + if (!file.fail()) { + fs::rename(path_tmp, path, ec); + } + if (file.fail() || ec) { + fs::remove(path_tmp, ec); + throw std::runtime_error("failed to write file: " + path.string()); + } +} + +static const std::string & get_modelscope_endpoint() { + static const std::string endpoint = []() { + const char * env = std::getenv("MODEL_ENDPOINT"); + std::string ep = env ? env : "https://modelscope.cn/"; + if (ep.back() != '/') { + ep += '/'; + } + return ep; + }(); + return endpoint; +} + +static nl::json api_get(const std::string & url, + const std::string & token) { + auto [cli, parts] = common_http_client(url); + + httplib::Headers headers = { + {"User-Agent", "llama-cpp/" + std::string(llama_build_info())}, + {"Accept", "application/json"} + }; + + if (!token.empty()) { + headers.emplace("Cookie", "m_session_id=" + token); + } + + if (auto res = cli.Get(parts.path, headers)) { + auto body = res->body; + + if (res->status == 200) { + return nl::json::parse(res->body); + } + try { + auto json_error = nl::json::parse(res->body); + if (json_error.contains("Message")) { + body = json_error["Message"].get(); + } else if (json_error.contains("msg")) { + body = json_error["msg"].get(); + } + } catch (...) { } + + throw std::runtime_error("GET failed (" + std::to_string(res->status) + "): " + body); + } else { + throw std::runtime_error("HTTPLIB failed: " + httplib::to_string(res.error())); + } +} + +// ModelScope uses "master" as the snapshot key (no single repo-wide commit like HF) +static const std::string & get_snapshot_ref() { + static const std::string ref = "master"; + return ref; +} + +ms_files get_repo_files(const std::string & repo_id, + const std::string & token) { + if (!is_valid_repo_id(repo_id)) { + LOG_WRN("%s: invalid repository: %s\n", __func__, repo_id.c_str()); + return {}; + } + + const std::string & endpoint = get_modelscope_endpoint(); + std::string api_url = endpoint + "api/v1/models/" + repo_id + "/repo/files?Revision=master&Recursive=true"; + + fs::path blobs_path = get_repo_path(repo_id) / "blobs"; + std::string ref = get_snapshot_ref(); + fs::path commit_path = get_repo_path(repo_id) / "snapshots" / ref; + + ms_files files; + + try { + auto response = api_get(api_url, token); + + if (!response.contains("Data") || !response["Data"].contains("Files")) { + LOG_WRN("%s: unexpected response format for '%s'\n", __func__, repo_id.c_str()); + return {}; + } + + fs::path refs_path = get_repo_path(repo_id) / "refs"; + safe_write_file(refs_path / "master", ref); + + for (const auto & item : response["Data"]["Files"]) { + if (!item.contains("Path") || !item["Path"].is_string()) { + continue; + } + + ms_file file; + file.repo_id = repo_id; + file.path = item["Path"].get(); + + if (!is_valid_subpath(commit_path, file.path)) { + LOG_WRN("%s: skip invalid path: %s\n", __func__, file.path.c_str()); + continue; + } + + if (item.contains("Sha256") && item["Sha256"].is_string()) { + file.oid = item["Sha256"].get(); + } + + if (!file.oid.empty() && !is_valid_oid(file.oid)) { + LOG_WRN("%s: skip invalid oid: %s\n", __func__, file.oid.c_str()); + continue; + } + + file.url = endpoint + "models/" + repo_id + "/resolve/master/" + file.path; + + fs::path final_path = commit_path / file.path; + file.final_path = final_path.string(); + + if (!file.oid.empty() && !fs::exists(final_path)) { + fs::path local_path = blobs_path / file.oid; + file.local_path = local_path.string(); + } else { + file.local_path = file.final_path; + } + + files.push_back(file); + } + } catch (const nl::json::exception & e) { + LOG_ERR("%s: JSON error: %s\n", __func__, e.what()); + } catch (const std::exception & e) { + std::string err_msg = e.what(); + if (err_msg.find("404") != std::string::npos) { + LOG_ERR("%s: repository not found: %s\n", __func__, repo_id.c_str()); + } else if (err_msg.find("401") != std::string::npos || + err_msg.find("403") != std::string::npos) { + if (token.empty()) { + LOG_DBG("%s: remote list failed (no token), relying on cache.\n", __func__); + } else { + LOG_ERR("%s: auth failed: %s\n", __func__, err_msg.c_str()); + } + } else { + LOG_ERR("%s: failed to list files for %s: %s\n", __func__, repo_id.c_str(), err_msg.c_str()); + } + } + return files; +} + +static std::string get_cached_ref(const fs::path & repo_path) { + fs::path refs_path = repo_path / "refs"; + if (!fs::is_directory(refs_path)) { + return {}; + } + + for (const auto & entry : fs::directory_iterator(refs_path)) { + if (!entry.is_regular_file()) { + continue; + } + std::ifstream f(entry.path()); + std::string ref; + if (!f || !std::getline(f, ref) || ref.empty()) { + continue; + } + if (entry.path().filename() == "master") { + return ref; + } + } + return {}; +} + +ms_files get_cached_files(const std::string & repo_id) { + fs::path cache_dir = get_cache_directory(); + if (!fs::exists(cache_dir)) { + return {}; + } + + if (!repo_id.empty() && !is_valid_repo_id(repo_id)) { + LOG_WRN("%s: invalid repository: %s\n", __func__, repo_id.c_str()); + return {}; + } + + ms_files files; + + for (const auto & repo : fs::directory_iterator(cache_dir)) { + if (!repo.is_directory()) { + continue; + } + fs::path snapshots_path = repo.path() / "snapshots"; + + if (!fs::exists(snapshots_path)) { + continue; + } + std::string _repo_id = folder_name_to_repo(repo.path().filename().string()); + + if (!is_valid_repo_id(_repo_id)) { + continue; + } + if (!repo_id.empty() && _repo_id != repo_id) { + continue; + } + std::string ref = get_cached_ref(repo.path()); + fs::path ref_path = snapshots_path / ref; + + if (ref.empty() || !fs::is_directory(ref_path)) { + continue; + } + for (const auto & entry : fs::recursive_directory_iterator(ref_path)) { + if (!entry.is_regular_file() && !entry.is_symlink()) { + continue; + } + fs::path path = entry.path().lexically_relative(ref_path); + + if (!path.empty()) { + ms_file file; + file.repo_id = _repo_id; + file.path = path.generic_string(); + file.local_path = entry.path().string(); + file.final_path = file.local_path; + files.push_back(std::move(file)); + } + } + } + + return files; +} + +std::string finalize_file(const ms_file & file) { + static std::atomic symlinks_disabled{false}; + + std::error_code ec; + fs::path local_path(file.local_path); + fs::path final_path(file.final_path); + + if (local_path == final_path || fs::exists(final_path, ec)) { + return file.final_path; + } + + if (!fs::exists(local_path, ec)) { + return file.final_path; + } + + fs::create_directories(final_path.parent_path(), ec); + + if (!symlinks_disabled) { + fs::path target = fs::relative(local_path, final_path.parent_path(), ec); + if (!ec) { + fs::create_symlink(target, final_path, ec); + } + if (!ec) { + return file.final_path; + } + } + + if (!symlinks_disabled.exchange(true)) { + LOG_WRN("%s: failed to create symlink: %s\n", __func__, ec.message().c_str()); + LOG_WRN("%s: switching to degraded mode\n", __func__); + } + + fs::rename(local_path, final_path, ec); + if (ec) { + LOG_WRN("%s: failed to move file to snapshots: %s\n", __func__, ec.message().c_str()); + fs::copy(local_path, final_path, ec); + if (ec) { + LOG_ERR("%s: failed to copy file to snapshots: %s\n", __func__, ec.message().c_str()); + } + } + return file.final_path; +} + +bool remove_cached_repo(const std::string & repo_id) { + if (!is_valid_repo_id(repo_id)) { + LOG_WRN("%s: invalid repository: %s\n", __func__, repo_id.c_str()); + return false; + } + fs::path repo_path = get_repo_path(repo_id); + std::error_code ec; + auto removed = fs::remove_all(repo_path, ec); + if (ec) { + LOG_ERR("%s: failed to remove repo cache %s: %s\n", __func__, repo_path.string().c_str(), ec.message().c_str()); + return false; + } + return removed > 0; +} + +} // namespace ms_cache diff --git a/common/ms-cache.h b/common/ms-cache.h new file mode 100644 index 000000000000..c9e5273fc0ef --- /dev/null +++ b/common/ms-cache.h @@ -0,0 +1,35 @@ +#pragma once + +#include +#include + +// Ref: https://www.modelscope.cn/docs/models/download + +namespace ms_cache { + +struct ms_file { + std::string path; + std::string url; + std::string local_path; + std::string final_path; + std::string oid; + std::string repo_id; +}; + +using ms_files = std::vector; + +// Get files from ModelScope API +ms_files get_repo_files( + const std::string & repo_id, + const std::string & token +); + +ms_files get_cached_files(const std::string & repo_id = {}); + +// Create snapshot path (link or move/copy) and return it +std::string finalize_file(const ms_file & file); + +// Remove the entire cached directory for a repo, returns true if removed +bool remove_cached_repo(const std::string & repo_id); + +} // namespace ms_cache diff --git a/tools/server/server.cpp b/tools/server/server.cpp index 5fe2729ba1b2..2691440bf078 100644 --- a/tools/server/server.cpp +++ b/tools/server/server.cpp @@ -118,7 +118,7 @@ int llama_server(common_params & params, int argc, char ** argv) { common_models_handler models_handler; // note: router mode also accepts -hf remote-preset, so we need to check that first - if (!is_run_by_cli && !params.model.hf_repo.empty()) { + if (!is_run_by_cli && (!params.model.hf_repo.empty() || !params.model.ms_repo.empty())) { try { models_handler = common_models_handler_init(params, LLAMA_EXAMPLE_SERVER); if (common_models_handler_is_preset_repo(models_handler)) { @@ -134,6 +134,7 @@ int llama_server(common_params & params, int argc, char ** argv) { // router server never loads a model and must not touch the GPU const bool is_router_server = params.model.path.empty() && params.model.hf_repo.empty() + && params.model.ms_repo.empty() && params.model.docker_repo.empty(); // skip device enumeration so the CUDA primary context stays uncreated