Skip to content

[GGUF FE] Support Q2_0 quantization and add missing op translators #37380

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mvafin merged 5 commits into from
Aug 13, 2026
Merged

[GGUF FE] Support Q2_0 quantization and add missing op translators #37380

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
47d2f22
[GGUF FE] Support the Q2_0 (ternary) quantization type
mvafin Aug 11, 2026
7dc2143
[GGUF FE] Add LOG/SIN/COS/TOP_K/RELU/ELU/GELU_QUICK and name sliced V…
mvafin Aug 11, 2026
4c16171
[GGUF FE] Emit MVN for GGML_OP_NORM instead of the manual decomposition
mvafin Aug 11, 2026
b0dcd4e
[GGUF FE] Keep the output port when passing a value between translators
mvafin Aug 12, 2026
d9464b3
Merge branch 'master' into mvafin/gguf/q2_0-and-view-naming
mvafin Aug 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/frontends/gguf/src/op/argsort.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ OutputVector translate_argsort(const NodeContext& context) {
// ggml ARGSORT sorts ne[0] == the OV last axis; derive it from the rank (rank-4 keeps axis 3).
const auto& in_ps = input.get_partial_shape();
const int64_t axis = in_ps.rank().is_static() ? in_ps.rank().get_length() - 1 : 3;
auto k = std::make_shared<ov::op::v0::Squeeze>(get_dimensions(input.get_node_shared_ptr(), {(int)axis}),
auto k = std::make_shared<ov::op::v0::Squeeze>(get_dimensions(input, {(int)axis}),
ov::op::v0::Constant::create(ov::element::i64, {1}, {0}));
auto topk = std::make_shared<ov::op::v11::TopK>(input,
k,
Expand Down
19 changes: 4 additions & 15 deletions src/frontends/gguf/src/op/norm.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,11 @@
//

#include <memory>
#include "openvino/op/add.hpp"
#include "openvino/op/constant.hpp"
#include "openvino/op/divide.hpp"
#include "openvino/op/multiply.hpp"
#include "openvino/op/reduce_mean.hpp"
#include "openvino/op/sqrt.hpp"
#include "openvino/op/subtract.hpp"

#include "node_context.hpp"
#include "op_table.hpp"
#include "openvino/op/constant.hpp"
#include "openvino/op/mvn.hpp"
#include "utils.hpp"

namespace ov {
Expand All @@ -27,14 +22,8 @@ OutputVector translate_norm(const NodeContext& context) {
auto input_node = context.get_input(0);
float eps = context.get_attribute<float>("eps");

auto axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1});
auto mean = std::make_shared<ov::op::v1::ReduceMean>(input_node, axis, true);
auto centered = std::make_shared<ov::op::v1::Subtract>(input_node, mean);
auto squared = std::make_shared<ov::op::v1::Multiply>(centered, centered);
auto variance = std::make_shared<ov::op::v1::ReduceMean>(squared, axis, true);
auto std_dev = std::make_shared<ov::op::v0::Sqrt>(std::make_shared<ov::op::v1::Add>(
variance, ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {eps})));
auto res = std::make_shared<ov::op::v1::Divide>(centered, std_dev);
auto axes = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1});
auto res = std::make_shared<ov::op::v6::MVN>(input_node, axes, true, eps, ov::op::MVNEpsMode::INSIDE_SQRT);

return rename_outputs_with_suffix({res}, context.get_name());
}
Expand Down
26 changes: 13 additions & 13 deletions src/frontends/gguf/src/op/rope.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ OutputVector translate_rope(const NodeContext& context) {

ov::Output<Node> res;

auto data_node = context.get_input(0).get_node_shared_ptr();
auto data = context.get_input(0);
auto output_shape = context.get_output_shape().to_shape();
auto rope_config = context.get_attribute<RopeConfig>("rope_config");
const int mode = (op_case & 0xFFFF0000) >> 16;
Expand Down Expand Up @@ -80,8 +80,8 @@ OutputVector translate_rope(const NodeContext& context) {
if (op_case == 2) {
// The input comes from a VIEW
int slice_len = static_cast<int>(output_shape[2] * output_shape[3]);
data_node = process_view_input(context, 0, slice_len).get_node_shared_ptr();
data_node = std::make_shared<ov::op::v1::Reshape>(data_node, make_bhsd_shape(), false);
data = process_view_input(context, 0, slice_len);
data = std::make_shared<ov::op::v1::Reshape>(data, make_bhsd_shape(), false);
}

if (mode == TYPE_NORMAL) {
Expand All @@ -98,14 +98,14 @@ OutputVector translate_rope(const NodeContext& context) {
// incoming rank in element order, so no separate rank lift is needed).
auto paired_shape = ov::op::v0::Constant::create(
ov::element::i64, {5}, std::vector<int64_t>{1, -1, n_heads, half, 2});
auto x_paired = std::make_shared<ov::op::v1::Reshape>(data_node, paired_shape, false);
auto x_paired = std::make_shared<ov::op::v1::Reshape>(data, paired_shape, false);

auto split_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {-1LL});
auto data_split = std::make_shared<ov::op::v1::Split>(x_paired, split_axis, 2);
auto x0 = data_split->output(0);
auto x1 = data_split->output(1);

auto neg_one_f = ov::op::v0::Constant::create(data_node->get_element_type(), ov::Shape{}, {-1.0f});
auto neg_one_f = ov::op::v0::Constant::create(data.get_element_type(), ov::Shape{}, {-1.0f});
auto x1_neg = std::make_shared<ov::op::v1::Multiply>(x1, neg_one_f);

auto x_rotated_paired = std::make_shared<ov::op::v0::Concat>(ov::OutputVector{x1_neg, x0}, -1);
Expand All @@ -126,7 +126,7 @@ OutputVector translate_rope(const NodeContext& context) {
auto cos_full = expand_cos_sin(cos_theta_node);
auto sin_full = expand_cos_sin(sin_theta_node);

auto y1 = std::make_shared<ov::op::v1::Multiply>(data_node, cos_full);
auto y1 = std::make_shared<ov::op::v1::Multiply>(data, cos_full);
auto y2 = std::make_shared<ov::op::v1::Multiply>(x_rotated, sin_full);
res = std::make_shared<ov::op::v1::Add>(y1, y2);
} else if (mode == TYPE_NEOX) {
Expand All @@ -137,16 +137,16 @@ OutputVector translate_rope(const NodeContext& context) {
const int64_t n_rot = rope_config.n_dims > 0 ? rope_config.n_dims : head_dim;

// Rotate only the first n_rot elements of every head and concatenate the untouched tail.
Output<Node> rotary_in = data_node;
Output<Node> rotary_in = data;
Output<Node> pass_through;
if (n_rot < head_dim) {
auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
auto n_rot_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_rot});
auto head_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {head_dim});
rotary_in = std::make_shared<ov::op::v8::Slice>(data_node, zero, n_rot_c, one, neg_one);
pass_through = std::make_shared<ov::op::v8::Slice>(data_node, n_rot_c, head_c, one, neg_one);
rotary_in = std::make_shared<ov::op::v8::Slice>(data, zero, n_rot_c, one, neg_one);
pass_through = std::make_shared<ov::op::v8::Slice>(data, n_rot_c, head_c, one, neg_one);
}

// Core split-halves RoPE via the shared decomposition helper: it emits the exact
Expand All @@ -162,21 +162,21 @@ OutputVector translate_rope(const NodeContext& context) {
// rotated, the tail is passed through unchanged -- e.g. qwen3.5 has head_dim 256 but
// rope.dimension_count 64. cos/sin carry width n_rot/2, so the rotated block must be
// exactly n_rot wide; using the full head here rotates the pass-through tail and corrupts
// every full-attention layer. (Use output_shape, not data_node->get_shape() which throws
// every full-attention layer. (Use output_shape, not data.get_shape() which throws
// on a dynamic dim.)
const int64_t head_dim = static_cast<int64_t>(output_shape[3]);
const int64_t n_rot = rope_config.n_dims > 0 ? rope_config.n_dims : head_dim;

Output<Node> rotary_in = data_node;
Output<Node> rotary_in = data;
Output<Node> pass_through;
if (n_rot < head_dim) {
auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1});
auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
auto n_rot_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_rot});
auto head_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {head_dim});
rotary_in = std::make_shared<ov::op::v8::Slice>(data_node, zero, n_rot_c, one, neg_one);
pass_through = std::make_shared<ov::op::v8::Slice>(data_node, n_rot_c, head_c, one, neg_one);
rotary_in = std::make_shared<ov::op::v8::Slice>(data, zero, n_rot_c, one, neg_one);
pass_through = std::make_shared<ov::op::v8::Slice>(data, n_rot_c, head_c, one, neg_one);
}

auto cos_sin_shape = std::make_shared<ov::op::v0::Constant>(ov::element::i64,
Expand Down
5 changes: 2 additions & 3 deletions src/frontends/gguf/src/op/softmax.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,6 @@ OutputVector translate_soft_max(const NodeContext& context) {
num_inputs_check(context, 1, 3);

auto input0 = context.get_input(0);
auto input_node = input0.get_node_shared_ptr();
ov::Output<Node> res;

// ggml SOFT_MAX reduces ne[0] == the OV last axis (rank-3 attention -> 2, rank-4 router -> 3);
Expand All @@ -94,7 +93,7 @@ OutputVector translate_soft_max(const NodeContext& context) {
float max_bias = context.get_attribute<float>("max_bias", 0.0f);

auto scale_node = std::make_shared<ov::op::v0::Constant>(ov::element::f32, ov::Shape{}, std::vector<float>{scale});
ov::Output<ov::Node> scaled_input = std::make_shared<ov::op::v1::Multiply>(input_node, scale_node);
ov::Output<ov::Node> scaled_input = std::make_shared<ov::op::v1::Multiply>(input0, scale_node);

// Disambiguate a 2nd input: it is either the additive mask or (gpt-oss) the attention sinks.
const bool second_input_is_sinks =
Expand All @@ -117,7 +116,7 @@ OutputVector translate_soft_max(const NodeContext& context) {
if (context.has_input("KQ_mask_sliced")) {
mask_node_sliced = context.get_input("KQ_mask_sliced");
} else {
auto token_len = get_dimensions(input_node, {1});
auto token_len = get_dimensions(input0, {1});
auto mask_node = context.get_input(1);
auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0});
auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1});
Expand Down
38 changes: 38 additions & 0 deletions src/frontends/gguf/src/op/top_k.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
// Copyright (C) 2018-2026 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//

#include "node_context.hpp"
#include "op_table.hpp"
#include "openvino/core/node_output.hpp"
#include "openvino/op/constant.hpp"
#include "openvino/op/topk.hpp"
#include "utils.hpp"

namespace ov {
namespace frontend {
namespace gguf {
namespace op {

// ggml_top_k(a, k): the indices of the k largest values along ne[0] (the OV last axis),
// ordered by descending value, as i32. k is the extent of that axis on the output.
OutputVector translate_top_k(const NodeContext& context) {
num_inputs_check(context, 1, 1);

auto input = context.get_input(0);
const int64_t k = context.get_output_shape()[context.get_output_shape().size() - 1].get_length();

@bumbosiepsak bumbosiepsak Aug 13, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What: context.get_output_shape() is constructed twice on one line. More importantly, Dimension::get_length() throws if the output's last (k) dimension is dynamic. This differs from translate_argsort, which derives its length from the input shape at runtime via get_dimensions(...) and tolerates dynamic rank.
Why it matters: If a model ever produces a TOP_K whose extent is not statically known, conversion aborts with a hard throw rather than degrading. It's also a minor readability/efficiency nit.
Fix: Cache the shape in a local; if a dynamic k is reachable, derive it dynamically as ARGSORT does:

Suggested change
const int64_t k = context.get_output_shape()[context.get_output_shape().size() - 1].get_length();
const auto& out_ps = context.get_output_shape();
const int64_t k = out_ps[out_ps.size() - 1].get_length();

auto k_node = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {k});
auto topk = std::make_shared<ov::op::v11::TopK>(input,

@bumbosiepsak bumbosiepsak Aug 13, 2026

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[nitpick] Near-duplicate v11::TopK construction across top_k.cpp and argsort.cpp.

translate_top_k (top_k.cpp:25–32) and translate_argsort (argsort.cpp:47–55) both
build a v11::TopK, thread the output_type attribute through as the index element
type, and return topk->output(1) via rename_outputs_with_suffix. Only the axis/k
derivation and the sort Mode differ.

Why it matters: the "indices come from output(1), index type follows the decoder's
output_type" contract is now duplicated in two places. If that contract changes (e.g.
a different port or a default index type), both sites must be kept in sync, and one can
silently drift.

Fix (optional): extract a small shared helper in utils, e.g.

// returns the indices port (output(1)) of a TopK over `axis`
ov::Output<ov::Node> make_topk_indices(const ov::Output<ov::Node>& input,
                                        const ov::Output<ov::Node>& k,
                                        int64_t axis,
                                        ov::op::v11::TopK::Mode mode,
                                        const ov::element::Type& index_type);

and call it from both translators. Non-blocking — the two ops are distinct enough that
keeping them separate is also defensible.

k_node,
-1,
ov::op::v11::TopK::Mode::MAX,
ov::op::v11::TopK::SortType::SORT_VALUES,
context.get_attribute<ov::element::Type>("output_type"));

return rename_outputs_with_suffix({topk->output(1)}, context.get_name());
}

} // namespace op
} // namespace gguf
} // namespace frontend
} // namespace ov
29 changes: 29 additions & 0 deletions src/frontends/gguf/src/op/unary_elu.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
// Copyright (C) 2018-2026 Intel Corporation
// SPDX-License-Identifier: Apache-2.0
//

#include "node_context.hpp"
#include "op_table.hpp"
#include "openvino/core/node_output.hpp"
#include "openvino/op/elu.hpp"
#include "utils.hpp"

namespace ov {
namespace frontend {
namespace gguf {
namespace op {

OutputVector translate_unary_elu(const NodeContext& context) {
num_inputs_check(context, 1, 1);

auto input = context.get_input(0);
// ggml's op_elu is `x > 0 ? x : expm1(x)`, i.e. ELU with alpha fixed at 1; it takes no param.
auto res = std::make_shared<ov::op::v0::Elu>(input, 1.0);

return rename_outputs_with_suffix({res}, context.get_name());
}

} // namespace op
} // namespace gguf
} // namespace frontend
} // namespace ov
16 changes: 16 additions & 0 deletions src/frontends/gguf/src/op/unary_gelu.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,10 @@
#include "utils.hpp"

#include "openvino/core/node_output.hpp"
#include "openvino/op/constant.hpp"
#include "openvino/op/gelu.hpp"
#include "openvino/op/multiply.hpp"
#include "openvino/op/sigmoid.hpp"

namespace ov {
namespace frontend {
Expand All @@ -24,6 +27,19 @@ OutputVector translate_unary_gelu(const NodeContext & context) {
return rename_outputs_with_suffix({res}, context.get_name());
}

OutputVector translate_unary_gelu_quick(const NodeContext& context) {
num_inputs_check(context, 1, 1);

auto input = context.get_input(0);
// ggml_gelu_quick_f32: x * (1 / (1 + exp(-1.702 * x))) == x * sigmoid(1.702 * x).
// A different approximation from GGML_UNARY_OP_GELU; the two are not interchangeable.
auto coef = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{}, {1.702f});
auto scaled = std::make_shared<ov::op::v1::Multiply>(input, coef);
auto res = std::make_shared<ov::op::v1::Multiply>(input, std::make_shared<ov::op::v0::Sigmoid>(scaled));

return rename_outputs_with_suffix({res}, context.get_name());
}

} // namespace op
} // namespace gguf
} // namespace frontend
Expand Down
8 changes: 7 additions & 1 deletion src/frontends/gguf/src/op/view.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,13 @@ OutputVector translate_view(const NodeContext & context) {
result = std::make_shared<ov::op::v1::Reshape>(
result, ov::op::v0::Constant::create(ov::element::i64, {tgt.size()}, tgt), false);
}
return {result};
// A view that neither restores rank, slices nor reshapes is a pass-through: the value is
// still the producer's output, so renaming it here would rename a node owned by another
// ggml tensor (and the suffix compounds, since the helper appends).
if (result == context.get_input(0)) {
return {result};
}
return rename_outputs_with_suffix({result}, context.get_name());
Comment thread
mvafin marked this conversation as resolved.
}
return {context.get_input(0)};
}
Expand Down
11 changes: 11 additions & 0 deletions src/frontends/gguf/src/op_table.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,17 @@
#include "op_table.hpp"

#include "openvino/op/add.hpp"
#include "openvino/op/cos.hpp"
#include "openvino/op/divide.hpp"
#include "openvino/op/exp.hpp"
#include "openvino/op/gather.hpp"
#include "openvino/op/log.hpp"
#include "openvino/op/matmul.hpp"
#include "openvino/op/multiply.hpp"
#include "openvino/op/negative.hpp"
#include "openvino/op/relu.hpp"
#include "openvino/op/sigmoid.hpp"
#include "openvino/op/sin.hpp"
#include "openvino/op/softplus.hpp"
#include "openvino/op/subtract.hpp"
#include "openvino/op/tanh.hpp"
Expand All @@ -35,6 +39,7 @@ std::unordered_map<std::string, CreatorFunction> get_supported_ops() {
{"GGML_OP_CLAMP", op::translate_clamp},
{"GGML_OP_CONCAT", op::translate_concat},
{"GGML_OP_CONT", op::translate_cont},
{"GGML_OP_COS", op::translate_1to1_match_1_input<v0::Cos>},
{"GGML_OP_CPY", op::translate_cpy},
{"GGML_OP_CUMSUM", op::translate_cumsum},
{"GGML_OP_DIAG", op::translate_diag},
Expand All @@ -45,6 +50,7 @@ std::unordered_map<std::string, CreatorFunction> get_supported_ops() {
{"GGML_OP_GET_ROWS", op::translate_get_rows},
{"GGML_OP_IM2COL", op::translate_im2col},
{"GGML_OP_L2_NORM", op::translate_l2_norm},
{"GGML_OP_LOG", op::translate_1to1_match_1_input<v0::Log>},
{"GGML_OP_MUL", op::translate_1to1_match_2_inputs<v1::Multiply>},
{"GGML_OP_MUL_MAT", op::translate_mulmat},
{"GGML_OP_MUL_MAT_ID", op::translate_mul_mat_id},
Expand All @@ -60,18 +66,23 @@ std::unordered_map<std::string, CreatorFunction> get_supported_ops() {
{"GGML_OP_SCALE", op::translate_scale},
{"GGML_OP_SET", op::translate_set},
{"GGML_OP_SET_ROWS", op::translate_set_rows},
{"GGML_OP_SIN", op::translate_1to1_match_1_input<v0::Sin>},
{"GGML_OP_SOFT_MAX", op::translate_soft_max},
{"GGML_OP_SQR", op::translate_sqr},
{"GGML_OP_SQRT", op::translate_sqrt},
{"GGML_OP_SSM_CONV", op::translate_ssm_conv},
{"GGML_OP_SUB", op::translate_1to1_match_2_inputs<v1::Subtract>},
{"GGML_OP_SUM_ROWS", op::translate_sum_rows},
{"GGML_OP_TOP_K", op::translate_top_k},
{"GGML_OP_TRI", op::translate_tri},
{"GGML_OP_TRANSPOSE", op::translate_transpose},
{"GGML_OP_VIEW", op::translate_view},
{"GGML_UNARY_OP_ELU", op::translate_unary_elu},
{"GGML_UNARY_OP_EXP", op::translate_1to1_match_1_input<v0::Exp>},
{"GGML_UNARY_OP_GELU", op::translate_unary_gelu},
{"GGML_UNARY_OP_GELU_QUICK", op::translate_unary_gelu_quick},
{"GGML_UNARY_OP_NEG", op::translate_1to1_match_1_input<v0::Negative>},
{"GGML_UNARY_OP_RELU", op::translate_1to1_match_1_input<v0::Relu>},
{"GGML_UNARY_OP_SIGMOID", op::translate_1to1_match_1_input<v0::Sigmoid>},
{"GGML_UNARY_OP_SILU", op::translate_unary_silu},
{"GGML_UNARY_OP_SOFTPLUS", op::translate_1to1_match_1_input<v4::SoftPlus>},
Expand Down
3 changes: 3 additions & 0 deletions src/frontends/gguf/src/op_table.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,10 +41,13 @@ GGUF_OP_CONVERTER(translate_scale);
GGUF_OP_CONVERTER(translate_set);
GGUF_OP_CONVERTER(translate_sqr);
GGUF_OP_CONVERTER(translate_sqrt);
GGUF_OP_CONVERTER(translate_top_k);
GGUF_OP_CONVERTER(translate_tri);
GGUF_OP_CONVERTER(translate_sum_rows);
GGUF_OP_CONVERTER(translate_unary_silu);
GGUF_OP_CONVERTER(translate_unary_gelu);
GGUF_OP_CONVERTER(translate_unary_gelu_quick);
GGUF_OP_CONVERTER(translate_unary_elu);
GGUF_OP_CONVERTER(translate_soft_max);
GGUF_OP_CONVERTER(translate_transpose);
GGUF_OP_CONVERTER(translate_view);
Expand Down
5 changes: 5 additions & 0 deletions src/frontends/gguf/src/quant/gguf.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ enum gguf_tensor_type {
GGUF_TYPE_F64 = 28,
GGUF_TYPE_BF16 = 30,
GGUF_TYPE_MXFP4 = 39, // 4-bit microscaling (gpt-oss): 1-byte E8M0 scale + 32x E2M1
GGUF_TYPE_Q2_0 = 42, // ternary: f16 scale + 64x 2-bit codes, value = (code - 1) * scale
GGUF_TYPE_COUNT,
};

Expand Down Expand Up @@ -111,6 +112,10 @@ void gguf_fill_asym(const gguf_tensor& tensor, ov::Tensor& weights, ov::Tensor&
// Fill pre-allocated f4e2m1 weights and f8e8m0 scales from an MXFP4 GGUF tensor.
void gguf_fill_mxfp4(const gguf_tensor& tensor, ov::Tensor& weights, ov::Tensor& scales);

// Fill pre-allocated u2 weights, f16 scales and zero-points from a Q2_0 (ternary) tensor.
// The zero-point is the constant 1 for every block.
void gguf_fill_q2_0(const gguf_tensor& tensor, ov::Tensor& weights, ov::Tensor& scales, ov::Tensor& zp);

// Fused bit-exact ggml dequant + channel-wise Q8_0_C requant for the token_embd/output/Q6_K/Q5_K
// requant path. Streams one row at a time (never materializes the full f32 weight). Fills i8
// weights [rows,cols] + f16 scales [rows,1]; matches upstream's to_float->quantize_q8_0 exactly so
Expand Down
Loading
Loading