Skip to content

GGML/llama.cpp: Add scaled GEMMs for more robust NVFP4 support #23484

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
ORippler wants to merge 4 commits into from
Closed

GGML/llama.cpp: Add scaled GEMMs for more robust NVFP4 support #23484

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7ea0c17
Add support for derived tensors to mul_mat and mul_mat_id
ORippler May 13, 2026
906c396
llama: start routeing derived matmul scales via ext API
ORippler May 15, 2026
811110c
Rename is_derived -> needs_scale, add comment
ORippler May 28, 2026
4c03b5a
Add error logs to ggml_mul_mat and ggml_mult_mat_ext
ORippler May 28, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions ggml/include/ggml.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -756,6 +756,7 @@ extern "C" {
GGML_API size_t ggml_element_size(const struct ggml_tensor * tensor);

GGML_API bool ggml_is_quantized(enum ggml_type type);
GGML_API bool ggml_needs_scale_quantized(enum ggml_type type);

// TODO: temporary until model loading of ggml examples is refactored
GGML_API enum ggml_type ggml_ftype_to_ggml_type(enum ggml_ftype ftype);
Expand Down Expand Up @@ -1419,6 +1420,13 @@ extern "C" {
struct ggml_tensor * a,
struct ggml_tensor * b);

GGML_API struct ggml_tensor * ggml_mul_mat_ext(
struct ggml_context * ctx,
struct ggml_tensor * a,
struct ggml_tensor * b,
struct ggml_tensor * scale_weight,
struct ggml_tensor * scale_activations);

// change the precision of a matrix multiplication
// set to GGML_PREC_F32 for higher precision (useful for phi-2)
GGML_API void ggml_mul_mat_set_prec(
Expand All @@ -1437,6 +1445,14 @@ extern "C" {
struct ggml_tensor * b,
struct ggml_tensor * ids);

GGML_API struct ggml_tensor * ggml_mul_mat_id_ext(
struct ggml_context * ctx,
struct ggml_tensor * as,
struct ggml_tensor * b,
struct ggml_tensor * ids,
struct ggml_tensor * scale_weight,
struct ggml_tensor * scale_activations);

// A: m columns, n rows,
// B: p columns, n rows,
// result is m columns, p rows
Expand Down Expand Up @@ -2802,6 +2818,7 @@ extern "C" {
int64_t blck_size_interleave; // interleave elements in blocks
size_t type_size;
bool is_quantized;
bool needs_scale; // whether the quantization type needs a scale factor for valid dequantization
ggml_to_float_t to_float;
ggml_from_float_t from_float_ref;
};
Expand Down
84 changes: 84 additions & 0 deletions ggml/src/ggml.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -746,6 +746,7 @@ static const struct ggml_type_traits type_traits[GGML_TYPE_COUNT] = {
.blck_size = QK_NVFP4,
.type_size = sizeof(block_nvfp4),
.is_quantized = true,
.needs_scale = true,
.to_float = (ggml_to_float_t) dequantize_row_nvfp4,
.from_float_ref = (ggml_from_float_t)quantize_row_nvfp4_ref,
},
Expand Down Expand Up @@ -1335,6 +1336,13 @@ bool ggml_is_quantized(enum ggml_type type) {
return type_traits[type].is_quantized;
}

bool ggml_needs_scale_quantized(enum ggml_type type) {
assert(type >= 0);
assert(type < GGML_TYPE_COUNT);
assert(!type_traits[type].needs_scale || type_traits[type].is_quantized);
return type_traits[type].needs_scale;
}

const char * ggml_op_name(enum ggml_op op) {
return GGML_OP_NAME[op];
}
Expand Down Expand Up @@ -3241,22 +3249,62 @@ struct ggml_tensor * ggml_mul_mat(
struct ggml_context * ctx,
struct ggml_tensor * a,
struct ggml_tensor * b) {
if (ggml_needs_scale_quantized(a->type) || ggml_needs_scale_quantized(b->type)) {
GGML_LOG_ERROR("%s: tensor types %s or %s requires explicit dequantization scales; use ggml_mul_mat_ext instead\n",
__func__, ggml_type_name(a->type), ggml_type_name(b->type));
GGML_ABORT("fatal error");
}

return ggml_mul_mat_ext(ctx, a, b, NULL, NULL);
}

struct ggml_tensor * ggml_mul_mat_ext(
struct ggml_context * ctx,
struct ggml_tensor * a,
struct ggml_tensor * b,
struct ggml_tensor * scale_weight,
struct ggml_tensor * scale_activations) {
GGML_ASSERT(ggml_can_mul_mat(a, b));
GGML_ASSERT(!ggml_is_transposed(a));
if (ggml_needs_scale_quantized(a->type) && scale_weight == NULL) {
GGML_LOG_ERROR("%s: tensor type %s requires explicit dequantization scales; pass scale_weight to ggml_mul_mat_ext\n",
__func__, ggml_type_name(a->type));
GGML_ABORT("fatal error");
}
if (ggml_needs_scale_quantized(b->type)) {
GGML_LOG_ERROR("%s: scaled tensor type %s currently cannot be used as the activation tensor\n",
__func__, ggml_type_name(b->type));
GGML_ABORT("fatal error");
}
GGML_ASSERT(scale_weight == NULL || scale_weight->type == GGML_TYPE_F32);
GGML_ASSERT(scale_activations == NULL || scale_activations->type == GGML_TYPE_F32);

const int64_t ne[4] = { a->ne[1], b->ne[1], b->ne[2], b->ne[3] };
struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);

result->op = GGML_OP_MUL_MAT;
result->src[0] = a;
result->src[1] = b;
// TODO: decide during review whether scale_weight should be attached as matmul metadata
// or inferred only from the post-matmul multiply.
result->src[2] = scale_weight;
result->src[3] = scale_activations;
Comment on lines +3288 to +3291

@JohannesGaessler JohannesGaessler May 28, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My opinion is that it is preferable not to change GGML_OP_MUL_MAT and to instead construct the ggml_graph around it in such a way that prevents an accidental misuse of NVFP4.

@ORippler ORippler May 28, 2026

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

My opinion is that it is preferable not to change GGML_OP_MUL_MAT and to instead construct the ggml_graph around it in such a way that prevents an accidental misuse of NVFP4.

Agreed. I tried, but failed to make this work with the optional input_scale consumption we would like to have available for W4A4 on NVGPUs. Afaik, GGML doesn't prescribe to a backend whether to promote or downcast its inputs during GEMMs. Here, only backends that use NVFP4 hardware-accelerators will need a pre-div/post-mul with input_scale for the W4A4 path to maintain accuracy. Hence, this needs to be optional:

graph LR
A[NVFP4 weights] --> B{GGML_OP_MUL_MAT}
C[FP32 activations] --> B
D[F32 weights_post_scale] --> B
E[F32 activations_pre_post_scale] -.-> B
B --> F{non-linearity, <br> e.g. SwiGLU}
Loading

Definitely open to suggestions on how to fully represent this optionality at the ggml_cgraph level. I refrained from always pre-div/post-muling, as I thought it to be the more intrusive and thus worse approach - maybe you disagree on this?


if (scale_weight) {
GGML_ASSERT(ggml_can_repeat(scale_weight, result));
result = ggml_mul(ctx, result, scale_weight);
}

return result;
}

void ggml_mul_mat_set_prec(
struct ggml_tensor * a,
enum ggml_prec prec) {
if (a->op == GGML_OP_MUL && a->src[0] && a->src[0]->op == GGML_OP_MUL_MAT) {
a = a->src[0];
}

GGML_ASSERT(a->op == GGML_OP_MUL_MAT);

const int32_t prec_i32 = (int32_t) prec;
Expand All @@ -3267,6 +3315,10 @@ void ggml_mul_mat_set_prec(
void ggml_mul_mat_set_hint(
struct ggml_tensor * a,
enum ggml_op_hint hint) {
if (a->op == GGML_OP_MUL && a->src[0] && a->src[0]->op == GGML_OP_MUL_MAT) {
a = a->src[0];
}

GGML_ASSERT(a->op == GGML_OP_MUL_MAT);

const int32_t hint_i32 = (int32_t) hint;
Expand All @@ -3293,8 +3345,24 @@ struct ggml_tensor * ggml_mul_mat_id(
struct ggml_tensor * as,
struct ggml_tensor * b,
struct ggml_tensor * ids) {
GGML_ASSERT(!ggml_needs_scale_quantized(as->type) && !ggml_needs_scale_quantized(b->type));

return ggml_mul_mat_id_ext(ctx, as, b, ids, NULL, NULL);
}

struct ggml_tensor * ggml_mul_mat_id_ext(
struct ggml_context * ctx,
struct ggml_tensor * as,
struct ggml_tensor * b,
struct ggml_tensor * ids,
struct ggml_tensor * scale_weight,
struct ggml_tensor * scale_activations) {
GGML_ASSERT(!ggml_is_transposed(as));
GGML_ASSERT(ids->type == GGML_TYPE_I32);
GGML_ASSERT(!ggml_needs_scale_quantized(as->type) || scale_weight != NULL);
GGML_ASSERT(!ggml_needs_scale_quantized(b->type));
GGML_ASSERT(scale_weight == NULL || scale_weight->type == GGML_TYPE_F32);
GGML_ASSERT(scale_activations == NULL || scale_activations->type == GGML_TYPE_F32);

GGML_ASSERT(as->ne[3] == 1); // as is 3d (one matrix per expert)
GGML_ASSERT(b->ne[3] == 1); // b is 3d
Expand All @@ -3310,6 +3378,22 @@ struct ggml_tensor * ggml_mul_mat_id(
result->src[0] = as;
result->src[1] = b;
result->src[2] = ids;
// TODO: decide during review whether scale_weight should be attached as matmul metadata
// or inferred only from the post-matmul multiply.
result->src[3] = scale_weight;
result->src[4] = scale_activations;

if (scale_weight) {
struct ggml_tensor * s = scale_weight;
if (s->ne[0] == as->ne[2] && s->ne[1] == 1 && s->ne[2] == 1 && s->ne[3] == 1) {
s = ggml_reshape_3d(ctx, s, 1, as->ne[2], 1);
s = ggml_repeat_4d(ctx, s, 1, as->ne[2], b->ne[2], 1);
s = ggml_get_rows(ctx, s, ids);
}

GGML_ASSERT(ggml_can_repeat(s, result));
result = ggml_mul(ctx, result, s);
}

return result;
}
Expand Down
Loading
Loading