diff --git a/common/CMakeLists.txt b/common/CMakeLists.txt index 1a56c25857f..24d12f74ad4 100644 --- a/common/CMakeLists.txt +++ b/common/CMakeLists.txt @@ -77,6 +77,8 @@ add_library(${TARGET} fit.h hf-cache.cpp hf-cache.h + ms.cpp + ms.h http.h json-partial.cpp json-partial.h diff --git a/common/arg.cpp b/common/arg.cpp index 3df8010a2ec..87ec8636b0f 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -4,6 +4,7 @@ #include "chat.h" #include "common.h" #include "download.h" +#include "ms.h" #include "json-schema-to-grammar.h" #include "log.h" #include "sampling.h" @@ -377,6 +378,27 @@ static handle_model_result common_params_handle_model(struct common_params_model result.found_mtp = true; result.mtp.path = download_result.mtp_path; } + } else if (!model.ms_repo.empty()) { + // Handle ModelScope repository + // Split the repo ID to extract clean repo and quantization tag + auto [ms_repo, ms_tag] = common_download_split_repo_tag(model.ms_repo); + + // Use the complete original value for model name + model.name = model.ms_repo; + + auto download_result = ms::download_model(ms_repo, model.hf_file, offline, ms_tag, bearer_token); + + if (download_result.model_path.empty()) { + throw std::runtime_error("failed to download model from ModelScope"); + } + + model.path = download_result.model_path; + + // Set mmproj path if available + if (!download_result.mmproj_path.empty()) { + result.found_mmproj = true; + result.mmproj.path = download_result.mmproj_path; + } } else if (!model.url.empty()) { if (model.path.empty()) { auto f = string_split(model.url, '#').front(); @@ -515,6 +537,14 @@ static bool common_params_parse_ex(int argc, char ** argv, common_params_context } } + // Handle MS_TOKEN environment variable for ModelScope authentication + if (params.hf_token.empty()) { + const char * ms_token = std::getenv("MS_TOKEN"); + if (ms_token) { + params.hf_token = ms_token; + } + } + // handle command line arguments auto check_arg = [&](int i) { if (i+1 >= argc) { @@ -2660,6 +2690,16 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.hf_token = value; } ).set_env("HF_TOKEN")); + + add_opt(common_arg( + {"-ms", "-msr", "--ms-repo"}, "/[:quant]", + "ModelScope model repository; quant is optional, case-insensitive, default to Q4_K_M, or falls back to the first file in the repo if Q4_K_M doesn't exist.\n" + "example: Qwen/Qwen3-0.6B-GGUF:Q8_0\n" + "(default: unused)", + [](common_params & params, const std::string & value) { + params.model.ms_repo = value; + } + ).set_env("LLAMA_ARG_MS_REPO")); add_opt(common_arg( {"--context-file"}, "FNAME", "file to load context from (use comma-separated values to specify multiple files)", diff --git a/common/common.h b/common/common.h index 8a0e5eed5ee..a72d8720d87 100644 --- a/common/common.h +++ b/common/common.h @@ -293,6 +293,7 @@ struct common_params_model { std::string url = ""; // model url to download // NOLINT std::string hf_repo = ""; // HF repo // NOLINT std::string hf_file = ""; // HF file // NOLINT + std::string ms_repo = ""; // ModelScope repo // NOLINT std::string docker_repo = ""; // Docker repo // NOLINT std::string name = ""; // in format /[:] (tag is optional) // NOLINT }; diff --git a/common/download.cpp b/common/download.cpp index 103bc408faf..3f8078fa7a5 100644 --- a/common/download.cpp +++ b/common/download.cpp @@ -499,14 +499,7 @@ int common_download_file_single(const std::string & url, return 304; // Not Modified - fake cached response } -struct gguf_split_info { - std::string prefix; // tag included - std::string tag; - int index; - int count; -}; - -static gguf_split_info get_gguf_split_info(const std::string & path) { +gguf_split_info get_gguf_split_info(const std::string & path) { static const std::regex re_split("^(.+)-([0-9]{5})-of-([0-9]{5})$", std::regex::icase); static const std::regex re_tag("[-.]([A-Z0-9_]+)$", std::regex::icase); std::smatch m; @@ -619,7 +612,7 @@ static hf_cache::hf_file find_best_mtp(const hf_cache::hf_files & files, return find_best_sibling(files, model, "mtp-"); } -static bool gguf_filename_is_model(const std::string & filepath) { +bool gguf_filename_is_model(const std::string & filepath) { if (!string_ends_with(filepath, ".gguf")) { return false; } diff --git a/common/download.h b/common/download.h index 4a169ef7796..e7cff80854d 100644 --- a/common/download.h +++ b/common/download.h @@ -108,3 +108,15 @@ int common_download_file_single(const std::string & url, // resolve and download model from Docker registry // return local path to downloaded model file std::string common_docker_resolve_model(const std::string & docker); + +// shared by HF (download.cpp) and ModelScope (ms.cpp) +bool gguf_filename_is_model(const std::string & filepath); + +struct gguf_split_info { + std::string prefix; // tag included + std::string tag; + int index; + int count; +}; + +gguf_split_info get_gguf_split_info(const std::string & path); diff --git a/common/ms.cpp b/common/ms.cpp new file mode 100644 index 00000000000..82bd0673b9f --- /dev/null +++ b/common/ms.cpp @@ -0,0 +1,484 @@ +/** + * ModelScope Integration Module + * + * Handles model downloading and caching from ModelScope for llama.cpp. + * Key features: + * - Repository file listing & automatic model file selection based on tags + * - Download progress tracking & automatic local caching + * + * Configuration: + * - Endpoint: `MODEL_ENDPOINT` env var (default: https://modelscope.cn/) + * - Authentication: Provide token via `-hft` CLI flag or `MS_TOKEN` env var. + * + * Usage: llama-cli -ms -hff -hft (e.g., "Qwen/Qwen3-0.6B-GGUF") + */ + +#include "ms.h" + +#include "common.h" +#include "log.h" +#include "download.h" + +#define JSON_ASSERT GGML_ASSERT +#include + +#include +#include +#include +#include +#include +#include +#include + +#ifndef _WIN32 +#include +#include +#endif + +namespace nl = nlohmann; + +namespace ms { + +namespace fs = std::filesystem; + +struct ms_file { + std::string path; + std::string url; + std::string local_path; + std::string repo_id; + uint64_t size = 0; +}; + +static fs::path get_cache_directory() { + static const fs::path cache = []() { + if (auto * p = std::getenv("LLAMA_CACHE"); p && *p) return fs::path(p); + if (auto * p = std::getenv("MODELSCOPE_CACHE"); p && *p) return fs::path(p); + + if (auto * p = std::getenv("XDG_CACHE_HOME"); p && *p) { + return fs::path(p) / "modelscope"; + } + +#ifndef _WIN32 + const struct passwd * pw = getpwuid(getuid()); + if (pw && pw->pw_dir && *pw->pw_dir) { + return fs::path(pw->pw_dir) / ".cache" / "modelscope"; + } +#endif + +#if defined(_WIN32) + if (auto * p = std::getenv("USERPROFILE"); p && *p) { + return fs::path(p) / ".cache" / "modelscope"; + } +#endif + + return fs::current_path() / ".cache" / "modelscope"; + }(); + return cache; +} + +// Duplicated from hf-cache.cpp. Cannot be reused because +// those functions are static in hf-cache.cpp +static bool is_alphanum(const char c) { + return (c >= 'A' && c <= 'Z') || + (c >= 'a' && c <= 'z') || + (c >= '0' && c <= '9'); +} + +static bool is_special_char(char c) { + return c == '/' || c == '.' || c == '-'; +} + +static bool is_valid_repo_id(const std::string & repo_id) { + if (repo_id.empty() || repo_id.length() > 256) { + return false; + } + int slash = 0; + bool special = true; + + for (const char c : repo_id) { + if (is_alphanum(c) || c == '_') { + special = false; + } else if (is_special_char(c)) { + if (special) { + return false; + } + slash += (c == '/'); + special = true; + } else { + return false; + } + } + return !special && slash == 1; +} + +static bool is_valid_subpath(const fs::path & base, const fs::path & subpath) { + if (subpath.is_absolute()) { + return false; + } + std::error_code ec; + auto abs_base = fs::absolute(base, ec); + if (ec) return false; + + auto b = abs_base.lexically_normal(); + auto t = (b / subpath).lexically_normal(); + + auto [b_end, _] = std::mismatch(b.begin(), b.end(), t.begin(), t.end()); + return b_end == b.end(); +} + +static nl::json api_get(const std::string & url, const std::string & token = "") { + common_remote_params params; + if (!token.empty()) { + params.headers.emplace_back("Cookie", "m_session_id=" + token); + } + params.timeout = 30; + + auto [status, body] = common_remote_get_content(url, params); + + std::string body_str(body.begin(), body.end()); + + if (status != 200) { + if (!body_str.empty()) { + try { + auto json_error = nl::json::parse(body_str); + if (json_error.contains("Message")) { + body_str = json_error["Message"].get(); + } else if (json_error.contains("msg")) { + body_str = json_error["msg"].get(); + } + } catch (...) { + } + } + throw std::runtime_error("HTTP " + std::to_string(status) + ": " + body_str); + } + + return nl::json::parse(body_str); +} + +static const std::string & get_modelscope_endpoint() { + static const std::string endpoint = []() { + const char * env = std::getenv("MODEL_ENDPOINT"); + std::string ep = env ? env : "https://modelscope.cn/"; + if (ep.back() != '/') { + ep += '/'; + } + return ep; + }(); + return endpoint; +} + +// Similar to hf_cache::get_repo_files() but cannot be shared: +// - Different API endpoint (ModelScope vs HuggingFace) +// - Different auth (Cookie vs Bearer) +static std::vector list_files(const std::string & repo_id, const std::string & token = "") { + std::vector files; + + const std::string & endpoint = get_modelscope_endpoint(); + std::string api_url = endpoint + "api/v1/models/" + repo_id + "/repo/files?Revision=master&Recursive=true"; + + try { + auto response = api_get(api_url, token); + + if (response.contains("Data") && response["Data"].contains("Files")) { + for (const auto & file_json : response["Data"]["Files"]) { + ms_file file; + file.repo_id = repo_id; + file.path = file_json.value("Path", ""); + file.size = file_json.value("Size", 0ULL); + + if (!file.path.empty()) { + file.url = endpoint + "models/" + repo_id + "/resolve/master/" + file.path; + files.push_back(std::move(file)); + } + } + } + } catch (const std::exception & e) { + std::string err_msg = e.what(); + if (err_msg.find("404") != std::string::npos) { + LOG_ERR("%s: repository not found: %s\n", __func__, repo_id.c_str()); + } else if (err_msg.find("401") != std::string::npos || + err_msg.find("403") != std::string::npos) { + if (token.empty()) { + LOG_DBG("%s: remote list failed (no token), relying on cache.\n", __func__); + } else { + LOG_ERR("%s: auth failed: %s\n", __func__, err_msg.c_str()); + } + } else { + LOG_ERR("%s: failed to list files for %s: %s\n", __func__, repo_id.c_str(), err_msg.c_str()); + } + } + + return files; +} + +static bool matches_quant_tag(const std::string & filename, const std::string & quant_tag) { + if (quant_tag.empty()) { + return true; + } + std::regex pattern(quant_tag + "[._-]", std::regex::icase); + return std::regex_search(filename, pattern); +} + +// Similar to download.cpp find_best_model() but cannot be shared: +// - Different data types (ms_file vs hf_file) +static std::string find_best_model(const std::vector & files, const std::string & quant_tag) { + if (files.empty()) { + return ""; + } + + std::vector tags; + if (!quant_tag.empty()) { + tags.push_back(quant_tag); + } else { + tags = {"Q4_K_M", "Q8_0"}; + } + + for (const auto & tag : tags) { + for (const auto & file : files) { + if (gguf_filename_is_model(file.path) && matches_quant_tag(file.path, tag)) { + auto split = get_gguf_split_info(file.path); + if (split.count > 1 && split.index != 1) { + continue; + } + return file.path; + } + } + } + + if (quant_tag.empty()) { + for (const auto & file : files) { + if (gguf_filename_is_model(file.path)) { + auto split = get_gguf_split_info(file.path); + if (split.count > 1 && split.index != 1) { + continue; + } + return file.path; + } + } + } + + return ""; +} + +static std::string get_local_path(const ms_file & file) { + fs::path cache_dir = get_cache_directory(); + fs::path base_path = cache_dir / "hub" / "models" / file.repo_id; + + if (!is_valid_subpath(base_path, file.path)) { + LOG_ERR("%s: security check failed for path: %s\n", __func__, file.path.c_str()); + return ""; + } + + fs::path local_path = base_path / file.path; + return local_path.string(); +} + +static void collect_files_from_dir(const fs::path & base, const fs::path & dir, const std::string & clean_repo_id, std::vector & out) { + std::error_code ec; + for (const auto & entry : fs::directory_iterator(dir, ec)) { + if (ec) continue; + + ec.clear(); + if (entry.is_directory(ec) && !ec) { + collect_files_from_dir(base, entry.path(), clean_repo_id, out); + continue; + } + + ec.clear(); + if (!entry.is_regular_file(ec) || ec) continue; + + const std::string fname = entry.path().filename().string(); + if (fname.size() >= 19 && fname.compare(fname.size() - 19, 19, ".downloadInProgress") == 0) { + continue; + } + + ms_file file; + file.repo_id = clean_repo_id; + file.path = entry.path().lexically_relative(base).generic_string(); + file.local_path = entry.path().string(); + ec.clear(); + file.size = fs::file_size(entry.path(), ec); + out.push_back(std::move(file)); + } +} + +// Similar to hf_cache::get_cached_files() but cannot be shared: +// - Different cache structure (flat dir vs blob/snapshot/symlink) +// - Different directory naming (owner/repo vs models--owner--repo) +static std::vector scan_local_cache(const std::string & clean_repo_id) { + std::vector cached_files; + fs::path base_cache_dir = get_cache_directory() / "hub" / "models" / clean_repo_id; + + std::error_code ec; + if (!fs::exists(base_cache_dir, ec) || !fs::is_directory(base_cache_dir, ec)) { + return cached_files; + } + + collect_files_from_dir(base_cache_dir, base_cache_dir, clean_repo_id, cached_files); + return cached_files; +} + +static bool is_file_valid(const ms_file & file, const fs::path & local_path) { + std::error_code ec; + if (!fs::exists(local_path, ec)) { + return false; + } + auto size = fs::file_size(local_path, ec); + if (ec || size == 0) { + return false; + } + if (file.size > 0 && size != file.size) { + return false; + } + return true; +} + +static std::string download_file_with_common(const ms_file & selected_file, const fs::path & local_path, bool offline, const std::string & token = "") { + common_download_opts opts; + opts.offline = offline; + if (!token.empty()) { + opts.headers.emplace_back("Cookie", "m_session_id=" + token); + } + + int status = common_download_file_single(selected_file.url, local_path.string(), opts, false); + + if (status >= 200 && status < 400) { + return local_path.string(); + } + LOG_ERR("%s: download failed with status: %d\n", __func__, status); + return ""; +} + +static std::string resolve_file(const ms_file & file, bool offline, const std::string & token) { + std::string local_path = file.local_path; + if (local_path.empty()) { + local_path = get_local_path(file); + } + if (local_path.empty()) { + LOG_ERR("%s: failed to determine local path for %s\n", __func__, file.path.c_str()); + return ""; + } + + if (is_file_valid(file, local_path)) { + return local_path; + } + + if (offline || file.url.empty()) { + return ""; + } + + return download_file_with_common(file, local_path, offline, token); +} + +// Similar to common_download_model() but cannot be shared: +// - common_download_model hardcodes HF API via get_hf_plan → hf_cache::get_repo_files +// - Different cache structure (flat vs blob/snapshot/symlink with finalize_file) +// - Different auth mechanism (Cookie vs Bearer) +// - Shared infrastructure: common_download_file_single, gguf_filename_is_model, get_gguf_split_info +download_result download_model(const std::string & clean_repo_id, const std::string & filename, bool offline, const std::string & quant_tag, const std::string & token) { + download_result result; + + if (!is_valid_repo_id(clean_repo_id)) { + LOG_ERR("%s: invalid repository: %s\n", __func__, clean_repo_id.c_str()); + return result; + } + + std::vector all_files = scan_local_cache(clean_repo_id); + + if (!offline) { + auto remote_files = list_files(clean_repo_id, token); + std::map file_map; + for (const auto & f : remote_files) { + file_map[f.path] = f; + } + for (const auto & f : all_files) { + auto it = file_map.find(f.path); + if (it == file_map.end()) { + file_map[f.path] = f; + } else if (!f.local_path.empty()) { + it->second.local_path = f.local_path; + } + } + all_files.clear(); + for (auto & [_, f] : file_map) { + all_files.push_back(std::move(f)); + } + } + + if (all_files.empty()) { + LOG_ERR("%s: no files found for repository %s\n", __func__, clean_repo_id.c_str()); + return result; + } + + std::string model_path; + + if (!filename.empty()) { + bool found = false; + for (const auto & file : all_files) { + if (file.path == filename) { + found = true; + model_path = resolve_file(file, offline, token); + break; + } + } + if (model_path.empty()) { + if (found) { + LOG_ERR("%s: failed to download '%s' from repository %s\n", __func__, filename.c_str(), clean_repo_id.c_str()); + } else { + LOG_ERR("%s: file '%s' not found in repository %s\n", __func__, filename.c_str(), clean_repo_id.c_str()); + for (const auto & file : all_files) { + if (file.path.find(".gguf") != std::string::npos) { + LOG_ERR(" %s\n", file.path.c_str()); + } + } + } + return result; + } + } else { + std::string selected = find_best_model(all_files, quant_tag); + if (selected.empty()) { + LOG_ERR("%s: no suitable GGUF file found in repository %s\n", __func__, clean_repo_id.c_str()); + return result; + } + for (const auto & file : all_files) { + if (file.path == selected) { + model_path = resolve_file(file, offline, token); + break; + } + } + if (model_path.empty()) { + LOG_ERR("%s: failed to download model from repository %s\n", __func__, clean_repo_id.c_str()); + return result; + } + } + + result.model_path = model_path; + + auto split = get_gguf_split_info(fs::path(model_path).filename().string()); + if (split.count > 1) { + for (const auto & file : all_files) { + auto f_split = get_gguf_split_info(file.path); + if (f_split.prefix == split.prefix && f_split.count == split.count && f_split.index != 1) { + resolve_file(file, offline, token); + } + } + } + + for (const auto & file : all_files) { + std::string lower = file.path; + std::transform(lower.begin(), lower.end(), lower.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + + if (lower.find(".gguf") != std::string::npos && lower.find("mmproj") != std::string::npos) { + std::string mmproj_path = resolve_file(file, offline, token); + if (!mmproj_path.empty()) { + result.mmproj_path = mmproj_path; + break; + } + } + } + + return result; +} + +} // namespace ms diff --git a/common/ms.h b/common/ms.h new file mode 100644 index 00000000000..254eefc25c2 --- /dev/null +++ b/common/ms.h @@ -0,0 +1,22 @@ +#pragma once + +#include + +// Ref: https://www.modelscope.cn/docs + +namespace ms { + +struct download_result { + std::string model_path; + std::string mmproj_path; +}; + +// Download a model (and optional mmproj) from ModelScope +// clean_repo_id: format "owner/repo" (without quantization tag) +// filename: specific filename to download (optional, auto-selects best GGUF if empty) +// offline: if true, only check local cache without network requests +// quant_tag: quantization tag extracted from original repo ID (e.g., "Q8_0") +// token: authentication token for private repositories (via MS_TOKEN env or -hft flag) +download_result download_model(const std::string & clean_repo_id, const std::string & filename = "", bool offline = false, const std::string & quant_tag = "", const std::string & token = ""); + +} // namespace ms