Skip to content
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions include/llama.h
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,9 @@ extern "C" {
LLAMA_FTYPE_GUESSED = 1024, // not specified in the model file
};

// Get the model file type (quantization) as a string, e.g. "Q8_0" or "Q4_K - Medium"
LLAMA_API const char * llama_ftype_name(enum llama_ftype ftype);

enum llama_rope_scaling_type {
LLAMA_ROPE_SCALING_TYPE_UNSPECIFIED = -1,
LLAMA_ROPE_SCALING_TYPE_NONE = 0,
Expand Down Expand Up @@ -606,6 +609,9 @@ extern "C" {
// Get a string describing the model type
LLAMA_API int32_t llama_model_desc(const struct llama_model * model, char * buf, size_t buf_size);

// Get the model file type (quantization), e.g. LLAMA_FTYPE_MOSTLY_Q8_0
LLAMA_API enum llama_ftype llama_model_ftype(const struct llama_model * model);

// Returns the total size of all the tensors in the model in bytes
LLAMA_API uint64_t llama_model_size(const struct llama_model * model);

Expand Down
88 changes: 43 additions & 45 deletions src/llama-model-loader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -27,50 +27,48 @@ const char * llama_file_version_name(llama_fver version) {
return "unknown";
}

static std::string llama_model_ftype_name(llama_ftype ftype) {
if (ftype & LLAMA_FTYPE_GUESSED) {
return llama_model_ftype_name((enum llama_ftype) (ftype & ~LLAMA_FTYPE_GUESSED)) + " (guessed)";
}

switch (ftype) {
case LLAMA_FTYPE_ALL_F32: return "all F32";
case LLAMA_FTYPE_MOSTLY_F16: return "F16";
case LLAMA_FTYPE_MOSTLY_BF16: return "BF16";
case LLAMA_FTYPE_MOSTLY_Q1_0: return "Q1_0";
case LLAMA_FTYPE_MOSTLY_Q4_0: return "Q4_0";
case LLAMA_FTYPE_MOSTLY_Q4_1: return "Q4_1";
case LLAMA_FTYPE_MOSTLY_Q5_0: return "Q5_0";
case LLAMA_FTYPE_MOSTLY_Q5_1: return "Q5_1";
case LLAMA_FTYPE_MOSTLY_Q8_0: return "Q8_0";
case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: return "MXFP4 MoE";
case LLAMA_FTYPE_MOSTLY_NVFP4: return "NVFP4";
case LLAMA_FTYPE_MOSTLY_Q2_K: return "Q2_K - Medium";
case LLAMA_FTYPE_MOSTLY_Q2_K_S: return "Q2_K - Small";
case LLAMA_FTYPE_MOSTLY_Q3_K_S: return "Q3_K - Small";
case LLAMA_FTYPE_MOSTLY_Q3_K_M: return "Q3_K - Medium";
case LLAMA_FTYPE_MOSTLY_Q3_K_L: return "Q3_K - Large";
case LLAMA_FTYPE_MOSTLY_Q4_K_S: return "Q4_K - Small";
case LLAMA_FTYPE_MOSTLY_Q4_K_M: return "Q4_K - Medium";
case LLAMA_FTYPE_MOSTLY_Q5_K_S: return "Q5_K - Small";
case LLAMA_FTYPE_MOSTLY_Q5_K_M: return "Q5_K - Medium";
case LLAMA_FTYPE_MOSTLY_Q6_K: return "Q6_K";
case LLAMA_FTYPE_MOSTLY_TQ1_0: return "TQ1_0 - 1.69 bpw ternary";
case LLAMA_FTYPE_MOSTLY_TQ2_0: return "TQ2_0 - 2.06 bpw ternary";
case LLAMA_FTYPE_MOSTLY_IQ2_XXS: return "IQ2_XXS - 2.0625 bpw";
case LLAMA_FTYPE_MOSTLY_IQ2_XS: return "IQ2_XS - 2.3125 bpw";
case LLAMA_FTYPE_MOSTLY_IQ2_S: return "IQ2_S - 2.5 bpw";
case LLAMA_FTYPE_MOSTLY_IQ2_M: return "IQ2_M - 2.7 bpw";
case LLAMA_FTYPE_MOSTLY_IQ3_XS: return "IQ3_XS - 3.3 bpw";
case LLAMA_FTYPE_MOSTLY_IQ3_XXS: return "IQ3_XXS - 3.0625 bpw";
case LLAMA_FTYPE_MOSTLY_IQ1_S: return "IQ1_S - 1.5625 bpw";
case LLAMA_FTYPE_MOSTLY_IQ1_M: return "IQ1_M - 1.75 bpw";
case LLAMA_FTYPE_MOSTLY_IQ4_NL: return "IQ4_NL - 4.5 bpw";
case LLAMA_FTYPE_MOSTLY_IQ4_XS: return "IQ4_XS - 4.25 bpw";
case LLAMA_FTYPE_MOSTLY_IQ3_S: return "IQ3_S - 3.4375 bpw";
case LLAMA_FTYPE_MOSTLY_IQ3_M: return "IQ3_S mix - 3.66 bpw";

default: return "unknown, may not work";
const char * llama_ftype_name(llama_ftype ftype) {
static constexpr size_t guessed_prefix_len = sizeof("(guessed) ") - 1;
const char * name;
switch ((enum llama_ftype) (ftype & ~LLAMA_FTYPE_GUESSED)) {
case LLAMA_FTYPE_ALL_F32: name = "(guessed) all F32"; break;
Comment thread
angt marked this conversation as resolved.
Resolved
case LLAMA_FTYPE_MOSTLY_F16: name = "(guessed) F16"; break;
case LLAMA_FTYPE_MOSTLY_BF16: name = "(guessed) BF16"; break;
case LLAMA_FTYPE_MOSTLY_Q1_0: name = "(guessed) Q1_0"; break;
case LLAMA_FTYPE_MOSTLY_Q4_0: name = "(guessed) Q4_0"; break;
case LLAMA_FTYPE_MOSTLY_Q4_1: name = "(guessed) Q4_1"; break;
case LLAMA_FTYPE_MOSTLY_Q5_0: name = "(guessed) Q5_0"; break;
case LLAMA_FTYPE_MOSTLY_Q5_1: name = "(guessed) Q5_1"; break;
case LLAMA_FTYPE_MOSTLY_Q8_0: name = "(guessed) Q8_0"; break;
case LLAMA_FTYPE_MOSTLY_MXFP4_MOE: name = "(guessed) MXFP4 MoE"; break;
case LLAMA_FTYPE_MOSTLY_NVFP4: name = "(guessed) NVFP4"; break;
case LLAMA_FTYPE_MOSTLY_Q2_K: name = "(guessed) Q2_K - Medium"; break;
case LLAMA_FTYPE_MOSTLY_Q2_K_S: name = "(guessed) Q2_K - Small"; break;
case LLAMA_FTYPE_MOSTLY_Q3_K_S: name = "(guessed) Q3_K - Small"; break;
case LLAMA_FTYPE_MOSTLY_Q3_K_M: name = "(guessed) Q3_K - Medium"; break;
case LLAMA_FTYPE_MOSTLY_Q3_K_L: name = "(guessed) Q3_K - Large"; break;
case LLAMA_FTYPE_MOSTLY_Q4_K_S: name = "(guessed) Q4_K - Small"; break;
case LLAMA_FTYPE_MOSTLY_Q4_K_M: name = "(guessed) Q4_K - Medium"; break;
case LLAMA_FTYPE_MOSTLY_Q5_K_S: name = "(guessed) Q5_K - Small"; break;
case LLAMA_FTYPE_MOSTLY_Q5_K_M: name = "(guessed) Q5_K - Medium"; break;
case LLAMA_FTYPE_MOSTLY_Q6_K: name = "(guessed) Q6_K"; break;
case LLAMA_FTYPE_MOSTLY_TQ1_0: name = "(guessed) TQ1_0 - 1.69 bpw ternary"; break;
case LLAMA_FTYPE_MOSTLY_TQ2_0: name = "(guessed) TQ2_0 - 2.06 bpw ternary"; break;
case LLAMA_FTYPE_MOSTLY_IQ2_XXS: name = "(guessed) IQ2_XXS - 2.0625 bpw"; break;
case LLAMA_FTYPE_MOSTLY_IQ2_XS: name = "(guessed) IQ2_XS - 2.3125 bpw"; break;
case LLAMA_FTYPE_MOSTLY_IQ2_S: name = "(guessed) IQ2_S - 2.5 bpw"; break;
case LLAMA_FTYPE_MOSTLY_IQ2_M: name = "(guessed) IQ2_M - 2.7 bpw"; break;
case LLAMA_FTYPE_MOSTLY_IQ3_XS: name = "(guessed) IQ3_XS - 3.3 bpw"; break;
case LLAMA_FTYPE_MOSTLY_IQ3_XXS: name = "(guessed) IQ3_XXS - 3.0625 bpw"; break;
case LLAMA_FTYPE_MOSTLY_IQ1_S: name = "(guessed) IQ1_S - 1.5625 bpw"; break;
case LLAMA_FTYPE_MOSTLY_IQ1_M: name = "(guessed) IQ1_M - 1.75 bpw"; break;
case LLAMA_FTYPE_MOSTLY_IQ4_NL: name = "(guessed) IQ4_NL - 4.5 bpw"; break;
case LLAMA_FTYPE_MOSTLY_IQ4_XS: name = "(guessed) IQ4_XS - 4.25 bpw"; break;
case LLAMA_FTYPE_MOSTLY_IQ3_S: name = "(guessed) IQ3_S - 3.4375 bpw"; break;
case LLAMA_FTYPE_MOSTLY_IQ3_M: name = "(guessed) IQ3_S mix - 3.66 bpw"; break;
default: name = "(guessed) unknown, may not work"; break;
}
return (ftype & LLAMA_FTYPE_GUESSED) ? name : name + guessed_prefix_len;
}

// return a list of splits for a given path
Expand Down Expand Up @@ -1693,12 +1691,12 @@ bool llama_model_loader::load_all_data(
}

std::string llama_model_loader::ftype_name() const {
return llama_model_ftype_name(ftype);
return llama_ftype_name(ftype);
}

void llama_model_loader::print_info() const {
LLAMA_LOG_INFO("%s: file format = %s\n", __func__, llama_file_version_name(fver));
LLAMA_LOG_INFO("%s: file type = %s\n", __func__, llama_model_ftype_name(ftype).c_str());
LLAMA_LOG_INFO("%s: file type = %s\n", __func__, llama_ftype_name(ftype));
if (n_bytes < GiB) {
LLAMA_LOG_INFO("%s: file size = %.2f MiB (%.2f BPW) \n", __func__, n_bytes/1024.0/1024.0, n_bytes*8.0/n_elements);
} else {
Expand Down
15 changes: 15 additions & 0 deletions src/llama-model.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -987,6 +987,8 @@ struct llama_model::impl {

std::string desc_str;

llama_ftype ftype = LLAMA_FTYPE_ALL_F32;

// model memory mapped files
llama_mmaps mappings;

Expand Down Expand Up @@ -1200,6 +1202,8 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {

pimpl->desc_str = arch_name() + " " + type_name() + " " + ml.ftype_name();

pimpl->ftype = ml.ftype;

if (hparams.f_max_alibi_bias > 0.0f) {
hparams.use_alibi = true;
}
Expand Down Expand Up @@ -1646,6 +1650,10 @@ std::string llama_model::desc() const {
return pimpl->desc_str;
}

llama_ftype llama_model::ftype() const {
return pimpl->ftype;
}

size_t llama_model::size() const {
return pimpl->n_bytes;
}
Expand Down Expand Up @@ -2616,6 +2624,13 @@ int32_t llama_model_desc(const llama_model * model, char * buf, size_t buf_size)
return snprintf(buf, buf_size, "%s", model->desc().c_str());
}

llama_ftype llama_model_ftype(const llama_model * model) {
if (!model) {
return LLAMA_FTYPE_ALL_F32;
}
return model->ftype();
}

Comment thread
angt marked this conversation as resolved.
Resolved
uint64_t llama_model_size(const llama_model * model) {
return model->size();
}
Expand Down
2 changes: 2 additions & 0 deletions src/llama-model.h
Original file line number Diff line number Diff line change
Expand Up @@ -637,6 +637,8 @@ struct llama_model {

std::string desc() const;

llama_ftype ftype() const;

size_t size() const; // file size
size_t n_tensors() const;
size_t n_devices() const;
Expand Down
3 changes: 3 additions & 0 deletions tools/cli/cli.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -448,6 +448,9 @@ int llama_cli(int argc, char ** argv) {
console::log("%s\n", LLAMA_ASCII_LOGO);
console::log("build : %s\n", inf.build_info.c_str());
console::log("model : %s\n", inf.model_name.c_str());
if (!inf.model_ftype.empty()) {
console::log("ftype : %s\n", inf.model_ftype.c_str());
}
console::log("modalities : %s\n", modalities.c_str());
if (!params.system_prompt.empty()) {
console::log("using custom system prompt\n");
Expand Down
4 changes: 4 additions & 0 deletions tools/server/server-context.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3989,6 +3989,8 @@ server_context_meta server_context::get_meta() const {
auto bos_token_str = bos_id != LLAMA_TOKEN_NULL ? common_token_to_piece(impl->ctx_tgt, bos_id, true) : "";
auto eos_token_str = eos_id != LLAMA_TOKEN_NULL ? common_token_to_piece(impl->ctx_tgt, eos_id, true) : "";

const char * ftype_name = llama_ftype_name(llama_model_ftype(impl->model_tgt));

return server_context_meta {
/* build_info */ std::string(llama_build_info()),
/* model_name */ impl->model_name,
Expand Down Expand Up @@ -4023,6 +4025,7 @@ server_context_meta server_context::get_meta() const {
/* model_n_embd_inp */ llama_model_n_embd(impl->model_tgt),
/* model_n_params */ llama_model_n_params(impl->model_tgt),
/* model_size */ llama_model_size(impl->model_tgt),
/* model_ftype */ ftype_name,
};
}

Expand Down Expand Up @@ -5118,6 +5121,7 @@ json server_routes::get_model_info() const {
{"n_embd", meta->model_n_embd_inp},
{"n_params", meta->model_n_params},
{"size", meta->model_size},
{"ftype", meta->model_ftype},
}},
};
}
Expand Down
1 change: 1 addition & 0 deletions tools/server/server-context.h
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,7 @@ struct server_context_meta {
int32_t model_n_embd_inp;
uint64_t model_n_params;
uint64_t model_size;
std::string model_ftype;
};

enum server_state {
Expand Down