Skip to content

ggml : update ggml_prec specification #26675

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
ggerganov wants to merge 10 commits into
base: master
Choose a base branch
from
+134 −22
Open

ggml : update ggml_prec specification #26675

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
83d5ed8
ggml : update ggml_prec specification
ggerganov Aug 6, 2026
9b970f6
cont : add GGML_PREC_BF16
ggerganov Aug 26, 2026
7f05174
cont : rework API
ggerganov Aug 26, 2026
76c0f1e
cont : use new API
ggerganov Aug 26, 2026
bddf805
cont : swap arg order
ggerganov Aug 26, 2026
9372029
cont : support for MUL_MAT_ID
ggerganov Aug 27, 2026
7886f52
cont : fix accidental remove of "break;"
ggerganov Aug 27, 2026
085ae2d
cont : return bools, add doc TAG_GGML_PREC, clean-up
ggerganov Aug 28, 2026
81cce78
cont : add search tag
ggerganov Aug 28, 2026
5851394
cont : ws
ggerganov Aug 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 56 additions & 7 deletions ggml/include/ggml.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -433,10 +433,21 @@ extern "C" {
GGML_TYPE_COUNT = 43,
};

// precision
// [TAG_GGML_PREC]
// this enum is used to declare the allowed numerical precision/data-types types that can be used during the compute of an op
// the declared types can be:
// - result accumulation type
// - source tensor data representation type
// - etc.
// the precision parameters are stored as ggml_tensor.op_params to the respective ops
enum ggml_prec {
GGML_PREC_DEFAULT = 0, // stored as ggml_tensor.op_params, 0 by default
GGML_PREC_F32 = 10,
GGML_PREC_UNDEFINED = 0,
GGML_PREC_DEFAULT = 0, // note: deprecated, use GGML_PREC_UNDEFINED
GGML_PREC_F32 = 10,
GGML_PREC_BF16 = 15,
GGML_PREC_F16 = 20,
GGML_PREC_Q8 = 30,
GGML_PREC_Q4 = 40,
};

// op hint
Expand Down Expand Up @@ -1422,6 +1433,42 @@ extern "C" {
struct ggml_tensor * b,
float eps);

// [TAG_GGML_PREC]
// set the minimum required accumulator type for the implementation to use during the compute
// for example:
// - GGML_PREC_F32 - requires accumulation of the results in F32
// - GGML_PREC_BF16 - can accumulate the results in BF16, F32
// - GGML_PREC_F16 - can accumulate the results in F16, F32
// - GGML_PREC_Q8 - not allowed
// - GGML_PREC_Q4 - not allowed
//
// return false on faliure
GGML_API bool ggml_prec_set_acc(
struct ggml_tensor * a,
enum ggml_prec prec);

// [TAG_GGML_PREC]
// set the smallest rank that the implementation can use to internally convert the src[idx] data to
// ranks in decreasing order:
// - GGML_PREC_F32 - GGML_TYPE_F32
// - GGML_PREC_BF16 - GGML_TYPE_BF16
// - GGML_PREC_F16 - GGML_TYPE_F16,
// - GGML_PREC_Q8 - GGML_TYPE_Q8_0, GGML_TYPE_Q8_1, GGML_TYPE_Q8_K, etc.
// - GGML_PREC_Q4 - GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_NVFP4, GGML_TYPE_MXFP4, etc.
//
// for example:
// - ggml_prec_set_src(a, GGML_PREC_Q8, 1):
// - allows the implementation to quantize F32, BF16, F16 data of src[1] down to GGML_TYPE_Q8_0
// - cannot quantize it down to GGML_TYPE_Q4_0 or GGML_TYPE_NVFP4
// - ggml_prec_set_src(a, GGML_PREC_Q4, 1):
// - allows the implementation to quantize F32, BF16, F16 data of src[1] down to 4-bit datatypes such as GGML_TYPE_Q4_K, GGML_TYPE_NVFP4 etc.
//
// return false on faliure
GGML_API bool ggml_prec_set_src(
struct ggml_tensor * a,
enum ggml_prec prec,
int idx);

// A: k columns, n rows => [ne03, ne02, n, k]
// B: k columns, m rows (i.e. we transpose it internally) => [ne03 * x, ne02 * y, m, k]
// result is n columns, m rows => [ne03 * x, ne02 * y, m, n]
Expand All @@ -1432,9 +1479,10 @@ extern "C" {

// change the precision of a matrix multiplication
// set to GGML_PREC_F32 for higher precision (useful for phi-2)
GGML_API void ggml_mul_mat_set_prec(
GGML_DEPRECATED(GGML_API void ggml_mul_mat_set_prec(
struct ggml_tensor * a,
enum ggml_prec prec);
enum ggml_prec prec),
"use ggml_prec_set_acc() instead");

// change the hint of a matrix multiplication
GGML_API void ggml_mul_mat_set_hint(
Expand Down Expand Up @@ -2439,9 +2487,10 @@ extern "C" {
float max_bias,
float logit_softcap);

GGML_API void ggml_flash_attn_ext_set_prec(
GGML_DEPRECATED(GGML_API void ggml_flash_attn_ext_set_prec(
struct ggml_tensor * a,
enum ggml_prec prec);
enum ggml_prec prec),
"use ggml_prec_set_acc() instead");

GGML_API enum ggml_prec ggml_flash_attn_ext_get_prec(
const struct ggml_tensor * a);
Expand Down
12 changes: 12 additions & 0 deletions ggml/src/ggml-impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -160,6 +160,18 @@ static float ggml_get_op_params_f32(const struct ggml_tensor * tensor, uint32_t
return ((const float *)(tensor->op_params))[i];
}

// [TAG_GGML_PREC]
// - GGML_OP_MUL_MAT
// 0 - acc
// 1 - hint
// 2 - src0 precision
// 3 - src1 precision
//
// - GGML_OP_MUL_MAT_ID
// 0 - acc
// 1 - hint
// 2 - src0 precision
// 3 - src1 precision
static void ggml_set_op_params_i32(struct ggml_tensor * tensor, uint32_t i, int32_t value) {
assert(i < GGML_MAX_OP_PARAMS / sizeof(int32_t));
((int32_t *)(tensor->op_params))[i] = value;
Expand Down
51 changes: 51 additions & 0 deletions ggml/src/ggml.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3265,6 +3265,57 @@ struct ggml_tensor * ggml_l2_norm_inplace(
return ggml_l2_norm_impl(ctx, a, eps, true);
}

// ggml_prec

bool ggml_prec_set_acc(
struct ggml_tensor * a,
enum ggml_prec prec) {
switch (a->op) {

@ORippler ORippler Aug 28, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In the docs we say only

    //  - GGML_PREC_F32  - requires accumulation of the results in F32
    //  - GGML_PREC_BF16 - can accumulate the results in BF16, F32
    //  - GGML_PREC_F16  - can accumulate the results in F16, F32

are supported. If the other values are truly not allowed, we should check prec here and error if the other values are supplied

@ggerganov ggerganov Aug 28, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I was think that we should actually make this function return a bool and return false on failure. Also return false when applied to unsupported op, etc.

case GGML_OP_MUL_MAT:
case GGML_OP_MUL_MAT_ID:
{
const int32_t prec_i32 = (int32_t) prec;
ggml_set_op_params_i32(a, 0, prec_i32);
}
break;
case GGML_OP_FLASH_ATTN_EXT:
{
const int32_t prec_i32 = (int32_t) prec;
ggml_set_op_params_i32(a, 3, prec_i32);

@ORippler ORippler Aug 28, 2026

Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Suggested change
ggml_set_op_params_i32(a, 3, prec_i32);
ggml_set_op_params_i32(a, 3, prec_i32); // scale is on first pos, max_bias on second

@ggerganov ggerganov Aug 28, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added a search tag [TAG_GGML_PREC] that should make looking up this information more simple.

}
break;
default:

@ynankani ynankani Aug 27, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we also handle MUL_MAT_ID here?

@ggerganov ggerganov Aug 27, 2026

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updated

return false;
};

return true;
}

bool ggml_prec_set_src(
struct ggml_tensor * a,
enum ggml_prec prec,
int idx) {
GGML_ASSERT(idx >= 0 && idx < GGML_MAX_SRC);

switch (a->op) {
case GGML_OP_MUL_MAT:
case GGML_OP_MUL_MAT_ID:
{
if (idx != 1) {
return false;
}

Comment thread
ggerganov marked this conversation as resolved.
const int32_t prec_i32 = (int32_t) prec;
ggml_set_op_params_i32(a, 2 + idx, prec_i32);
}
break;
default:

@ynankani ynankani Aug 27, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same, can we handle MUL_MAT_ID here as well

return false;
};

return true;
}

// ggml_mul_mat

static inline bool ggml_can_mul_mat(const struct ggml_tensor * t0, const struct ggml_tensor * t1) {
Expand Down
12 changes: 6 additions & 6 deletions src/llama-graph.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1874,7 +1874,7 @@ ggml_tensor * llm_graph_context::build_ffn(
cur = build_lora_mm(down, cur);
if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {
// GLM4, GLM4_MOE, and JAIS2 seem to have numerical issues with half-precision accumulators
ggml_mul_mat_set_prec(cur, GGML_PREC_F32);
ggml_prec_set_acc(cur, GGML_PREC_F32);
}
}

Expand Down Expand Up @@ -1972,7 +1972,7 @@ ggml_tensor * llm_graph_context::build_moe_ffn(
if (probs_in == nullptr) {
logits = build_lora_mm(gate_inp, cur); // [n_expert, n_tokens]
if (gating_op == LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS) {
ggml_mul_mat_set_prec(logits, GGML_PREC_F32);
ggml_prec_set_acc(logits, GGML_PREC_F32);
}
cb(logits, "ffn_moe_logits", il);
} else {
Expand Down Expand Up @@ -2583,7 +2583,7 @@ ggml_tensor * llm_graph_context::build_attn_mha(
res->add_fused_node({LLM_FUSED_OP_FLASH_ATTN, cur, il});

ggml_flash_attn_ext_add_sinks(cur, sinks);
ggml_flash_attn_ext_set_prec (cur, GGML_PREC_F32);
ggml_prec_set_acc(cur, GGML_PREC_F32);

if (v_mla) {
#if 0
Expand All @@ -2609,7 +2609,7 @@ ggml_tensor * llm_graph_context::build_attn_mha(

// note: this op tends to require high floating point range
// while for some models F16 is enough, for others it is not, so we default to F32 here
ggml_mul_mat_set_prec(kq, GGML_PREC_F32);
ggml_prec_set_acc(kq, GGML_PREC_F32);

if (arch == LLM_ARCH_GROK) {
// need to do the following:
Expand Down Expand Up @@ -2842,7 +2842,7 @@ ggml_tensor * llm_graph_context::build_attn(
if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE || arch == LLM_ARCH_JAIS2) {
// GLM4, GLM4_MOE, and JAIS2 seem to have numerical issues with half-precision accumulators
cur = build_lora_mm(wo, cur);
ggml_mul_mat_set_prec(cur, GGML_PREC_F32);
ggml_prec_set_acc(cur, GGML_PREC_F32);
if (wo_s) {
cur = ggml_mul(ctx0, cur, wo_s);
}
Expand Down Expand Up @@ -2929,7 +2929,7 @@ ggml_tensor * llm_graph_context::build_attn(
if (arch == LLM_ARCH_GLM4 || arch == LLM_ARCH_GLM4_MOE) {
// GLM4 and GLM4_MOE seem to have numerical issues with half-precision accumulators
cur = build_lora_mm(wo, cur);
ggml_mul_mat_set_prec(cur, GGML_PREC_F32);
ggml_prec_set_acc(cur, GGML_PREC_F32);
if (wo_s) {
cur = ggml_mul(ctx0, cur, wo_s);
}
Expand Down
6 changes: 3 additions & 3 deletions src/models/minimax-m3.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,7 @@ ggml_tensor * llama_model_minimax_m3::graph::build_attn_msa_fa(

ggml_tensor * o = ggml_flash_attn_ext(ctx0, q, k, v, mask, kq_scale,
hparams.f_max_alibi_bias, 0.0f);
ggml_flash_attn_ext_set_prec(o, GGML_PREC_F32);
ggml_prec_set_acc(o, GGML_PREC_F32);
cb(o, "msa_fattn", il);

// [D, Gp, R, C] -> [D, Gp, C, R] -> [n_embd, T]
Expand Down Expand Up @@ -389,7 +389,7 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_
ggml_tensor * iq4 = ggml_reshape_4d(ctx0, iq, n_idx_dim, Hd, 1, ns);
ggml_tensor * sc = ggml_mul_mat(ctx0,
ggml_reshape_4d(ctx0, ikp, n_idx_dim, n_ps, 1, ns), iq4);
ggml_mul_mat_set_prec(sc, GGML_PREC_F32);
ggml_prec_set_acc(sc, GGML_PREC_F32);
// unmapped positions come out -inf, so they can never rank into the top-k
sc = ggml_add_inplace(ctx0, sc,
ggml_reshape_4d(ctx0, msa->pos_mask, n_ps, 1, 1, ns));
Expand Down Expand Up @@ -471,7 +471,7 @@ llama_model_minimax_m3::graph::graph(const llama_model & model, const llm_graph_
ggml_tensor * sc = ggml_mul_mat(ctx0, ikp,
ggml_reshape_2d(ctx0, iq_s, n_idx_dim, Hd*n_tps));
// indexer scores run in F32
ggml_mul_mat_set_prec(sc, GGML_PREC_F32);
ggml_prec_set_acc(sc, GGML_PREC_F32);
sc = ggml_reshape_3d(ctx0, sc, n_ps, Hd, n_tps);
// unmapped positions (holes, padding, empty cells) come out -inf
sc = ggml_add_inplace(ctx0, sc, pm_s);
Expand Down
2 changes: 1 addition & 1 deletion tests/test-backend-ops.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7174,7 +7174,7 @@ struct test_flash_attn_ext : public test_case {

ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f/sqrtf(hsk), max_bias, logit_softcap);
ggml_flash_attn_ext_add_sinks(out, s);
ggml_flash_attn_ext_set_prec (out, prec);
ggml_prec_set_acc(out, prec);
ggml_set_name(out, "out");

return out;
Expand Down
4 changes: 2 additions & 2 deletions tools/mtmd/clip.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -780,7 +780,7 @@ ggml_tensor * clip_graph::build_attn(
}

cur = ggml_flash_attn_ext(ctx0, q, k, v, kq_mask, kq_scale, 0.0f, 0.0f);
ggml_flash_attn_ext_set_prec(cur, GGML_PREC_F32);
ggml_prec_set_acc(cur, GGML_PREC_F32);
if (sinks != nullptr) {
ggml_flash_attn_ext_add_sinks(cur, sinks);
}
Expand All @@ -793,7 +793,7 @@ ggml_tensor * clip_graph::build_attn(

ggml_tensor * kq = ggml_mul_mat(ctx0, k, q);
// F32 may not needed for vision encoders?
// ggml_mul_mat_set_prec(kq, GGML_PREC_F32);
// ggml_prec_set_acc(kq, GGML_PREC_F32);

kq = ggml_soft_max_ext(ctx0, kq, kq_mask, kq_scale, 0.0f);
if (sinks != nullptr) {
Expand Down
2 changes: 1 addition & 1 deletion tools/mtmd/models/mimovl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

ggml_tensor * clip_graph_mimovl::build_mm(ggml_tensor * w, ggml_tensor * x) const {
ggml_tensor * cur = ggml_mul_mat(ctx0, w, x);
ggml_mul_mat_set_prec(cur, GGML_PREC_F32);
ggml_prec_set_acc(cur, GGML_PREC_F32);
return cur;
}

Expand Down
2 changes: 1 addition & 1 deletion tools/mtmd/models/qwen3tts-spkenc.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ ggml_tensor * clip_graph_qwen3tts_spkenc::conv1d_same(ggml_tensor * x, ggml_tens

ggml_tensor * w2d = ggml_reshape_2d(ctx0, w, (int64_t) K * IC, OC);
ggml_tensor * y = ggml_mul_mat(ctx0, w2d, col); // [OC, T_out]
ggml_mul_mat_set_prec(y, GGML_PREC_F32);
ggml_prec_set_acc(y, GGML_PREC_F32);

ggml_tensor * b2d = ggml_reshape_2d(ctx0, b, OC, 1);
y = ggml_add(ctx0, y, b2d);
Expand Down
2 changes: 1 addition & 1 deletion tools/tuning/fa-vec.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ static ggml_tensor * fa_build_graph(ggml_context * ctx, const fa_shape & s) {
ggml_set_name(m, "m");

ggml_tensor * out = ggml_flash_attn_ext(ctx, q, k, v, m, 1.0f / sqrtf((float) s.dk), 0.0f, 0.0f);
ggml_flash_attn_ext_set_prec(out, GGML_PREC_F32);
ggml_prec_set_acc(out, GGML_PREC_F32);
ggml_set_name(out, "out");

return out;
Expand Down
Loading