diff --git a/src/frontends/gguf/src/op/argsort.cpp b/src/frontends/gguf/src/op/argsort.cpp index 79d57d4bc4d496..c601238ffc5cdd 100644 --- a/src/frontends/gguf/src/op/argsort.cpp +++ b/src/frontends/gguf/src/op/argsort.cpp @@ -42,7 +42,7 @@ OutputVector translate_argsort(const NodeContext& context) { // ggml ARGSORT sorts ne[0] == the OV last axis; derive it from the rank (rank-4 keeps axis 3). const auto& in_ps = input.get_partial_shape(); const int64_t axis = in_ps.rank().is_static() ? in_ps.rank().get_length() - 1 : 3; - auto k = std::make_shared(get_dimensions(input.get_node_shared_ptr(), {(int)axis}), + auto k = std::make_shared(get_dimensions(input, {(int)axis}), ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); auto topk = std::make_shared(input, k, diff --git a/src/frontends/gguf/src/op/norm.cpp b/src/frontends/gguf/src/op/norm.cpp index 8f5d484fa84ea3..98eb80d1f735a0 100644 --- a/src/frontends/gguf/src/op/norm.cpp +++ b/src/frontends/gguf/src/op/norm.cpp @@ -3,16 +3,11 @@ // #include -#include "openvino/op/add.hpp" -#include "openvino/op/constant.hpp" -#include "openvino/op/divide.hpp" -#include "openvino/op/multiply.hpp" -#include "openvino/op/reduce_mean.hpp" -#include "openvino/op/sqrt.hpp" -#include "openvino/op/subtract.hpp" #include "node_context.hpp" #include "op_table.hpp" +#include "openvino/op/constant.hpp" +#include "openvino/op/mvn.hpp" #include "utils.hpp" namespace ov { @@ -27,14 +22,8 @@ OutputVector translate_norm(const NodeContext& context) { auto input_node = context.get_input(0); float eps = context.get_attribute("eps"); - auto axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}); - auto mean = std::make_shared(input_node, axis, true); - auto centered = std::make_shared(input_node, mean); - auto squared = std::make_shared(centered, centered); - auto variance = std::make_shared(squared, axis, true); - auto std_dev = std::make_shared(std::make_shared( - variance, ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {eps}))); - auto res = std::make_shared(centered, std_dev); + auto axes = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}); + auto res = std::make_shared(input_node, axes, true, eps, ov::op::MVNEpsMode::INSIDE_SQRT); return rename_outputs_with_suffix({res}, context.get_name()); } diff --git a/src/frontends/gguf/src/op/rope.cpp b/src/frontends/gguf/src/op/rope.cpp index 989e84ab012db3..ec5cabf6ed848f 100644 --- a/src/frontends/gguf/src/op/rope.cpp +++ b/src/frontends/gguf/src/op/rope.cpp @@ -42,7 +42,7 @@ OutputVector translate_rope(const NodeContext& context) { ov::Output res; - auto data_node = context.get_input(0).get_node_shared_ptr(); + auto data = context.get_input(0); auto output_shape = context.get_output_shape().to_shape(); auto rope_config = context.get_attribute("rope_config"); const int mode = (op_case & 0xFFFF0000) >> 16; @@ -80,8 +80,8 @@ OutputVector translate_rope(const NodeContext& context) { if (op_case == 2) { // The input comes from a VIEW int slice_len = static_cast(output_shape[2] * output_shape[3]); - data_node = process_view_input(context, 0, slice_len).get_node_shared_ptr(); - data_node = std::make_shared(data_node, make_bhsd_shape(), false); + data = process_view_input(context, 0, slice_len); + data = std::make_shared(data, make_bhsd_shape(), false); } if (mode == TYPE_NORMAL) { @@ -98,14 +98,14 @@ OutputVector translate_rope(const NodeContext& context) { // incoming rank in element order, so no separate rank lift is needed). auto paired_shape = ov::op::v0::Constant::create( ov::element::i64, {5}, std::vector{1, -1, n_heads, half, 2}); - auto x_paired = std::make_shared(data_node, paired_shape, false); + auto x_paired = std::make_shared(data, paired_shape, false); auto split_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {-1LL}); auto data_split = std::make_shared(x_paired, split_axis, 2); auto x0 = data_split->output(0); auto x1 = data_split->output(1); - auto neg_one_f = ov::op::v0::Constant::create(data_node->get_element_type(), ov::Shape{}, {-1.0f}); + auto neg_one_f = ov::op::v0::Constant::create(data.get_element_type(), ov::Shape{}, {-1.0f}); auto x1_neg = std::make_shared(x1, neg_one_f); auto x_rotated_paired = std::make_shared(ov::OutputVector{x1_neg, x0}, -1); @@ -126,7 +126,7 @@ OutputVector translate_rope(const NodeContext& context) { auto cos_full = expand_cos_sin(cos_theta_node); auto sin_full = expand_cos_sin(sin_theta_node); - auto y1 = std::make_shared(data_node, cos_full); + auto y1 = std::make_shared(data, cos_full); auto y2 = std::make_shared(x_rotated, sin_full); res = std::make_shared(y1, y2); } else if (mode == TYPE_NEOX) { @@ -137,7 +137,7 @@ OutputVector translate_rope(const NodeContext& context) { const int64_t n_rot = rope_config.n_dims > 0 ? rope_config.n_dims : head_dim; // Rotate only the first n_rot elements of every head and concatenate the untouched tail. - Output rotary_in = data_node; + Output rotary_in = data; Output pass_through; if (n_rot < head_dim) { auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); @@ -145,8 +145,8 @@ OutputVector translate_rope(const NodeContext& context) { auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); auto n_rot_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_rot}); auto head_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {head_dim}); - rotary_in = std::make_shared(data_node, zero, n_rot_c, one, neg_one); - pass_through = std::make_shared(data_node, n_rot_c, head_c, one, neg_one); + rotary_in = std::make_shared(data, zero, n_rot_c, one, neg_one); + pass_through = std::make_shared(data, n_rot_c, head_c, one, neg_one); } // Core split-halves RoPE via the shared decomposition helper: it emits the exact @@ -162,12 +162,12 @@ OutputVector translate_rope(const NodeContext& context) { // rotated, the tail is passed through unchanged -- e.g. qwen3.5 has head_dim 256 but // rope.dimension_count 64. cos/sin carry width n_rot/2, so the rotated block must be // exactly n_rot wide; using the full head here rotates the pass-through tail and corrupts - // every full-attention layer. (Use output_shape, not data_node->get_shape() which throws + // every full-attention layer. (Use output_shape, not data.get_shape() which throws // on a dynamic dim.) const int64_t head_dim = static_cast(output_shape[3]); const int64_t n_rot = rope_config.n_dims > 0 ? rope_config.n_dims : head_dim; - Output rotary_in = data_node; + Output rotary_in = data; Output pass_through; if (n_rot < head_dim) { auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); @@ -175,8 +175,8 @@ OutputVector translate_rope(const NodeContext& context) { auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); auto n_rot_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_rot}); auto head_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {head_dim}); - rotary_in = std::make_shared(data_node, zero, n_rot_c, one, neg_one); - pass_through = std::make_shared(data_node, n_rot_c, head_c, one, neg_one); + rotary_in = std::make_shared(data, zero, n_rot_c, one, neg_one); + pass_through = std::make_shared(data, n_rot_c, head_c, one, neg_one); } auto cos_sin_shape = std::make_shared(ov::element::i64, diff --git a/src/frontends/gguf/src/op/softmax.cpp b/src/frontends/gguf/src/op/softmax.cpp index 5eba537072f087..3bd58e032357c4 100644 --- a/src/frontends/gguf/src/op/softmax.cpp +++ b/src/frontends/gguf/src/op/softmax.cpp @@ -82,7 +82,6 @@ OutputVector translate_soft_max(const NodeContext& context) { num_inputs_check(context, 1, 3); auto input0 = context.get_input(0); - auto input_node = input0.get_node_shared_ptr(); ov::Output res; // ggml SOFT_MAX reduces ne[0] == the OV last axis (rank-3 attention -> 2, rank-4 router -> 3); @@ -94,7 +93,7 @@ OutputVector translate_soft_max(const NodeContext& context) { float max_bias = context.get_attribute("max_bias", 0.0f); auto scale_node = std::make_shared(ov::element::f32, ov::Shape{}, std::vector{scale}); - ov::Output scaled_input = std::make_shared(input_node, scale_node); + ov::Output scaled_input = std::make_shared(input0, scale_node); // Disambiguate a 2nd input: it is either the additive mask or (gpt-oss) the attention sinks. const bool second_input_is_sinks = @@ -117,7 +116,7 @@ OutputVector translate_soft_max(const NodeContext& context) { if (context.has_input("KQ_mask_sliced")) { mask_node_sliced = context.get_input("KQ_mask_sliced"); } else { - auto token_len = get_dimensions(input_node, {1}); + auto token_len = get_dimensions(input0, {1}); auto mask_node = context.get_input(1); auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); diff --git a/src/frontends/gguf/src/op/top_k.cpp b/src/frontends/gguf/src/op/top_k.cpp new file mode 100644 index 00000000000000..3df1fdce5aef27 --- /dev/null +++ b/src/frontends/gguf/src/op/top_k.cpp @@ -0,0 +1,38 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "node_context.hpp" +#include "op_table.hpp" +#include "openvino/core/node_output.hpp" +#include "openvino/op/constant.hpp" +#include "openvino/op/topk.hpp" +#include "utils.hpp" + +namespace ov { +namespace frontend { +namespace gguf { +namespace op { + +// ggml_top_k(a, k): the indices of the k largest values along ne[0] (the OV last axis), +// ordered by descending value, as i32. k is the extent of that axis on the output. +OutputVector translate_top_k(const NodeContext& context) { + num_inputs_check(context, 1, 1); + + auto input = context.get_input(0); + const int64_t k = context.get_output_shape()[context.get_output_shape().size() - 1].get_length(); + auto k_node = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {k}); + auto topk = std::make_shared(input, + k_node, + -1, + ov::op::v11::TopK::Mode::MAX, + ov::op::v11::TopK::SortType::SORT_VALUES, + context.get_attribute("output_type")); + + return rename_outputs_with_suffix({topk->output(1)}, context.get_name()); +} + +} // namespace op +} // namespace gguf +} // namespace frontend +} // namespace ov diff --git a/src/frontends/gguf/src/op/unary_elu.cpp b/src/frontends/gguf/src/op/unary_elu.cpp new file mode 100644 index 00000000000000..9a7954e1efe097 --- /dev/null +++ b/src/frontends/gguf/src/op/unary_elu.cpp @@ -0,0 +1,29 @@ +// Copyright (C) 2018-2026 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 +// + +#include "node_context.hpp" +#include "op_table.hpp" +#include "openvino/core/node_output.hpp" +#include "openvino/op/elu.hpp" +#include "utils.hpp" + +namespace ov { +namespace frontend { +namespace gguf { +namespace op { + +OutputVector translate_unary_elu(const NodeContext& context) { + num_inputs_check(context, 1, 1); + + auto input = context.get_input(0); + // ggml's op_elu is `x > 0 ? x : expm1(x)`, i.e. ELU with alpha fixed at 1; it takes no param. + auto res = std::make_shared(input, 1.0); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + +} // namespace op +} // namespace gguf +} // namespace frontend +} // namespace ov diff --git a/src/frontends/gguf/src/op/unary_gelu.cpp b/src/frontends/gguf/src/op/unary_gelu.cpp index 70ba83b0ba0cb9..51ea9c14a957ae 100644 --- a/src/frontends/gguf/src/op/unary_gelu.cpp +++ b/src/frontends/gguf/src/op/unary_gelu.cpp @@ -7,7 +7,10 @@ #include "utils.hpp" #include "openvino/core/node_output.hpp" +#include "openvino/op/constant.hpp" #include "openvino/op/gelu.hpp" +#include "openvino/op/multiply.hpp" +#include "openvino/op/sigmoid.hpp" namespace ov { namespace frontend { @@ -24,6 +27,19 @@ OutputVector translate_unary_gelu(const NodeContext & context) { return rename_outputs_with_suffix({res}, context.get_name()); } +OutputVector translate_unary_gelu_quick(const NodeContext& context) { + num_inputs_check(context, 1, 1); + + auto input = context.get_input(0); + // ggml_gelu_quick_f32: x * (1 / (1 + exp(-1.702 * x))) == x * sigmoid(1.702 * x). + // A different approximation from GGML_UNARY_OP_GELU; the two are not interchangeable. + auto coef = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{}, {1.702f}); + auto scaled = std::make_shared(input, coef); + auto res = std::make_shared(input, std::make_shared(scaled)); + + return rename_outputs_with_suffix({res}, context.get_name()); +} + } // namespace op } // namespace gguf } // namespace frontend diff --git a/src/frontends/gguf/src/op/view.cpp b/src/frontends/gguf/src/op/view.cpp index eabca06f072b7e..c9c3ac3f92cfc9 100644 --- a/src/frontends/gguf/src/op/view.cpp +++ b/src/frontends/gguf/src/op/view.cpp @@ -199,7 +199,13 @@ OutputVector translate_view(const NodeContext & context) { result = std::make_shared( result, ov::op::v0::Constant::create(ov::element::i64, {tgt.size()}, tgt), false); } - return {result}; + // A view that neither restores rank, slices nor reshapes is a pass-through: the value is + // still the producer's output, so renaming it here would rename a node owned by another + // ggml tensor (and the suffix compounds, since the helper appends). + if (result == context.get_input(0)) { + return {result}; + } + return rename_outputs_with_suffix({result}, context.get_name()); } return {context.get_input(0)}; } diff --git a/src/frontends/gguf/src/op_table.cpp b/src/frontends/gguf/src/op_table.cpp index ed2bb8cca5d807..f3e6cc3f040acf 100644 --- a/src/frontends/gguf/src/op_table.cpp +++ b/src/frontends/gguf/src/op_table.cpp @@ -5,13 +5,17 @@ #include "op_table.hpp" #include "openvino/op/add.hpp" +#include "openvino/op/cos.hpp" #include "openvino/op/divide.hpp" #include "openvino/op/exp.hpp" #include "openvino/op/gather.hpp" +#include "openvino/op/log.hpp" #include "openvino/op/matmul.hpp" #include "openvino/op/multiply.hpp" #include "openvino/op/negative.hpp" +#include "openvino/op/relu.hpp" #include "openvino/op/sigmoid.hpp" +#include "openvino/op/sin.hpp" #include "openvino/op/softplus.hpp" #include "openvino/op/subtract.hpp" #include "openvino/op/tanh.hpp" @@ -35,6 +39,7 @@ std::unordered_map get_supported_ops() { {"GGML_OP_CLAMP", op::translate_clamp}, {"GGML_OP_CONCAT", op::translate_concat}, {"GGML_OP_CONT", op::translate_cont}, + {"GGML_OP_COS", op::translate_1to1_match_1_input}, {"GGML_OP_CPY", op::translate_cpy}, {"GGML_OP_CUMSUM", op::translate_cumsum}, {"GGML_OP_DIAG", op::translate_diag}, @@ -45,6 +50,7 @@ std::unordered_map get_supported_ops() { {"GGML_OP_GET_ROWS", op::translate_get_rows}, {"GGML_OP_IM2COL", op::translate_im2col}, {"GGML_OP_L2_NORM", op::translate_l2_norm}, + {"GGML_OP_LOG", op::translate_1to1_match_1_input}, {"GGML_OP_MUL", op::translate_1to1_match_2_inputs}, {"GGML_OP_MUL_MAT", op::translate_mulmat}, {"GGML_OP_MUL_MAT_ID", op::translate_mul_mat_id}, @@ -60,18 +66,23 @@ std::unordered_map get_supported_ops() { {"GGML_OP_SCALE", op::translate_scale}, {"GGML_OP_SET", op::translate_set}, {"GGML_OP_SET_ROWS", op::translate_set_rows}, + {"GGML_OP_SIN", op::translate_1to1_match_1_input}, {"GGML_OP_SOFT_MAX", op::translate_soft_max}, {"GGML_OP_SQR", op::translate_sqr}, {"GGML_OP_SQRT", op::translate_sqrt}, {"GGML_OP_SSM_CONV", op::translate_ssm_conv}, {"GGML_OP_SUB", op::translate_1to1_match_2_inputs}, {"GGML_OP_SUM_ROWS", op::translate_sum_rows}, + {"GGML_OP_TOP_K", op::translate_top_k}, {"GGML_OP_TRI", op::translate_tri}, {"GGML_OP_TRANSPOSE", op::translate_transpose}, {"GGML_OP_VIEW", op::translate_view}, + {"GGML_UNARY_OP_ELU", op::translate_unary_elu}, {"GGML_UNARY_OP_EXP", op::translate_1to1_match_1_input}, {"GGML_UNARY_OP_GELU", op::translate_unary_gelu}, + {"GGML_UNARY_OP_GELU_QUICK", op::translate_unary_gelu_quick}, {"GGML_UNARY_OP_NEG", op::translate_1to1_match_1_input}, + {"GGML_UNARY_OP_RELU", op::translate_1to1_match_1_input}, {"GGML_UNARY_OP_SIGMOID", op::translate_1to1_match_1_input}, {"GGML_UNARY_OP_SILU", op::translate_unary_silu}, {"GGML_UNARY_OP_SOFTPLUS", op::translate_1to1_match_1_input}, diff --git a/src/frontends/gguf/src/op_table.hpp b/src/frontends/gguf/src/op_table.hpp index a8eb61730e9f66..39ceb369046891 100644 --- a/src/frontends/gguf/src/op_table.hpp +++ b/src/frontends/gguf/src/op_table.hpp @@ -41,10 +41,13 @@ GGUF_OP_CONVERTER(translate_scale); GGUF_OP_CONVERTER(translate_set); GGUF_OP_CONVERTER(translate_sqr); GGUF_OP_CONVERTER(translate_sqrt); +GGUF_OP_CONVERTER(translate_top_k); GGUF_OP_CONVERTER(translate_tri); GGUF_OP_CONVERTER(translate_sum_rows); GGUF_OP_CONVERTER(translate_unary_silu); GGUF_OP_CONVERTER(translate_unary_gelu); +GGUF_OP_CONVERTER(translate_unary_gelu_quick); +GGUF_OP_CONVERTER(translate_unary_elu); GGUF_OP_CONVERTER(translate_soft_max); GGUF_OP_CONVERTER(translate_transpose); GGUF_OP_CONVERTER(translate_view); diff --git a/src/frontends/gguf/src/quant/gguf.hpp b/src/frontends/gguf/src/quant/gguf.hpp index 234deebbd4a8a1..e0b0a6b03360cf 100644 --- a/src/frontends/gguf/src/quant/gguf.hpp +++ b/src/frontends/gguf/src/quant/gguf.hpp @@ -42,6 +42,7 @@ enum gguf_tensor_type { GGUF_TYPE_F64 = 28, GGUF_TYPE_BF16 = 30, GGUF_TYPE_MXFP4 = 39, // 4-bit microscaling (gpt-oss): 1-byte E8M0 scale + 32x E2M1 + GGUF_TYPE_Q2_0 = 42, // ternary: f16 scale + 64x 2-bit codes, value = (code - 1) * scale GGUF_TYPE_COUNT, }; @@ -111,6 +112,10 @@ void gguf_fill_asym(const gguf_tensor& tensor, ov::Tensor& weights, ov::Tensor& // Fill pre-allocated f4e2m1 weights and f8e8m0 scales from an MXFP4 GGUF tensor. void gguf_fill_mxfp4(const gguf_tensor& tensor, ov::Tensor& weights, ov::Tensor& scales); +// Fill pre-allocated u2 weights, f16 scales and zero-points from a Q2_0 (ternary) tensor. +// The zero-point is the constant 1 for every block. +void gguf_fill_q2_0(const gguf_tensor& tensor, ov::Tensor& weights, ov::Tensor& scales, ov::Tensor& zp); + // Fused bit-exact ggml dequant + channel-wise Q8_0_C requant for the token_embd/output/Q6_K/Q5_K // requant path. Streams one row at a time (never materializes the full f32 weight). Fills i8 // weights [rows,cols] + f16 scales [rows,1]; matches upstream's to_float->quantize_q8_0 exactly so diff --git a/src/frontends/gguf/src/quant/gguf_quants.cpp b/src/frontends/gguf/src/quant/gguf_quants.cpp index 17edd3b24c7f14..c89b8692f9d7b1 100644 --- a/src/frontends/gguf/src/quant/gguf_quants.cpp +++ b/src/frontends/gguf/src/quant/gguf_quants.cpp @@ -640,6 +640,32 @@ void fill_q8_k(const gguf_tensor& tensor, ov::Tensor& weights_arr, ov::Tensor& s }); } +// Block = |f16 d|u2 qs[64]| (18 bytes / 64 weights). ggml packs the codes 4 per byte LSB-first, +// the same order an OpenVINO u2 Constant reads, so the code bytes are copied verbatim. +void gguf_fill_q2_0(const gguf_tensor& tensor, ov::Tensor& weights_arr, ov::Tensor& scales_arr, ov::Tensor& zp_arr) { + const uint64_t bytes_per_block = 18; + const uint64_t bytes_per_block_codes = 16; + auto data = static_cast(tensor.weights_data); + auto weights = static_cast(weights_arr.data()); + auto scales = scales_arr.data::value_type>(); + const bool zp_is_f16 = zp_arr.get_element_type() == ov::element::f16; + auto zp_u8 = zp_is_f16 ? nullptr : static_cast(zp_arr.data()); + auto zp_f16 = zp_is_f16 ? zp_arr.data::value_type>() : nullptr; + + ov::parallel_for(scales_arr.get_size(), [&](size_t i) { + const uint8_t* block = data + i * bytes_per_block; + uint16_t scale_bits; + std::memcpy(&scale_bits, block, sizeof(scale_bits)); + scales[i] = ov::float16::from_bits(scale_bits); + std::memcpy(weights + i * bytes_per_block_codes, block + 2, bytes_per_block_codes); + if (zp_is_f16) { + zp_f16[i] = ov::float16(1.0f); + } else { + zp_u8[i] = 1; + } + }); +} + // Symmetric types (Q8_0, Q5_0, Q6_K, Q3_K): fill weights + scales (f16), no zero-point. // Q8_K uses f32 scales and is handled by a separate overload dispatched on tensor.type. void gguf_fill_sym(const gguf_tensor& tensor, ov::Tensor& weights, ov::Tensor& scales) { diff --git a/src/frontends/gguf/src/quant/weights.cpp b/src/frontends/gguf/src/quant/weights.cpp index 256ff316b8da60..1599c03c43aa38 100644 --- a/src/frontends/gguf/src/quant/weights.cpp +++ b/src/frontends/gguf/src/quant/weights.cpp @@ -362,7 +362,7 @@ std::vector dequant_extracted_to_f32(const std::unordered_map(q[r * cols + c])); } else if (et == ov::element::u2) { - // Q2_K: u2 weights, 4 per byte LSB-first, raw [0..3] with a zero-point. + // Q2_K / Q2_0: u2 weights, 4 per byte LSB-first, raw [0..3] with a zero-point. const auto* bytes = static_cast(weight.data()); const size_t per_row_bytes = cols / 4; for (size_t r = 0; r < rows; ++r) @@ -420,6 +420,7 @@ std::shared_ptr make_weight_node(const std::string& base, node = make_int4(base, weights); break; case GGUF_TYPE_Q2_K: + case GGUF_TYPE_Q2_0: node = make_int2(base, weights); break; case GGUF_TYPE_Q5_K: @@ -468,7 +469,8 @@ gguf_tensor_type gguf_type_from_name(const std::string& quant_type) { {"Q5_K", GGUF_TYPE_Q5_K}, {"Q6_K", GGUF_TYPE_Q6_K}, {"Q8_K", GGUF_TYPE_Q8_K}, - {"MXFP4", GGUF_TYPE_MXFP4}}; + {"MXFP4", GGUF_TYPE_MXFP4}, + {"Q2_0", GGUF_TYPE_Q2_0}}; // Accept ggml's lowercase type names ("q4_0", "q6_K", "f16", ...) as well as the // canonical uppercase form by upper-casing the prefix before the "_K"/"_0" suffix. std::string key = quant_type; @@ -526,12 +528,13 @@ std::shared_ptr make_weight_node(const ov::Tensor& data, // zp is an INTEGER (u8) low-precision constant; a fractional f16 zp leaves a standalone // dequant MatMul (~2x slower prefill). Q4_K is the asymmetric type that appears as MatMul // weights in modern models (Q4_K_M = Q4_K + symmetric Q6_K), so it uses integer zp to match - // the original ggml-openvino backend. The legacy Q4_1/Q5_1/Q2_K types keep a faithful f16 zp: + // the original ggml-openvino backend; Q2_0's zp is the exact integer 1, so it does too. + // The legacy Q4_1/Q5_1/Q2_K types keep a faithful f16 zp: // they are not perf-critical here, and their zp = -min/scale can fall outside u8 range. The // requant path (token_embd/output) also keeps f16 -- its dequant feeds channel-wise Q8_0_C. const bool requant = needs_q8_0_c_requant(name, qtype); const ov::element::Type zp_type = - (!requant && qtype == GGUF_TYPE_Q4_K) ? ov::element::u8 : ov::element::f16; + (!requant && (qtype == GGUF_TYPE_Q4_K || qtype == GGUF_TYPE_Q2_0)) ? ov::element::u8 : ov::element::f16; // K-quant requant sources: the fused dequant -> Q8_0_C streams from the raw bytes, so skip the // full-tensor gguf_fill_* extraction below (it would be discarded) and return before the switch. @@ -616,6 +619,17 @@ std::shared_ptr make_weight_node(const ov::Tensor& data, w[base + ".zp"] = zp; break; } + case GGUF_TYPE_Q2_0: { + // Ternary: u2 weights + f16 scales + a zero-point that is the constant 1 (group 64). + ov::Tensor weights(ov::element::u2, ov::Shape{rows, cols}); + ov::Tensor scales(ov::element::f16, ov::Shape{rows, sub_blocks_per_row(64)}); + ov::Tensor zp(zp_type, ov::Shape{rows, sub_blocks_per_row(64)}); + gguf_fill_q2_0(tensor, weights, scales, zp); + w[base + ".weight"] = weights; + w[base + ".scales"] = scales; + w[base + ".zp"] = zp; + break; + } default: OPENVINO_THROW("[ggml] unsupported weight quant type: ", quant_type); } diff --git a/src/frontends/gguf/src/utils.cpp b/src/frontends/gguf/src/utils.cpp index e3d612893606aa..32f99fd37b144b 100644 --- a/src/frontends/gguf/src/utils.cpp +++ b/src/frontends/gguf/src/utils.cpp @@ -42,8 +42,8 @@ std::shared_ptr get_dimensions(const std::shared_ptr(shape, dims_const, zero); } -std::shared_ptr get_dimensions(const std::shared_ptr& node, const std::vector& dims) { - return get_dimensions(std::make_shared(node), dims); +std::shared_ptr get_dimensions(const ov::Output& output, const std::vector& dims) { + return get_dimensions(std::make_shared(output), dims); } OutputVector rename_outputs_with_suffix(const OutputVector& outputs, const std::string& suffix) { diff --git a/src/frontends/gguf/src/utils.hpp b/src/frontends/gguf/src/utils.hpp index bd576d726d1fa4..8ce6c9b76ddc6c 100644 --- a/src/frontends/gguf/src/utils.hpp +++ b/src/frontends/gguf/src/utils.hpp @@ -25,7 +25,8 @@ void num_inputs_check(const NodeContext& context, size_t min_inputs, size_t max_ std::shared_ptr get_dimensions(const std::shared_ptr& shape, const std::vector& dims); -std::shared_ptr get_dimensions(const std::shared_ptr& node, const std::vector& dims); +// Takes the Output rather than the node so a producer with several outputs keeps the right port. +std::shared_ptr get_dimensions(const ov::Output& output, const std::vector& dims); OutputVector rename_outputs_with_suffix(const OutputVector& outputs, const std::string& suffix); diff --git a/src/frontends/gguf/tests/CMakeLists.txt b/src/frontends/gguf/tests/CMakeLists.txt index a55c3cee305fd5..4eb970883a1806 100644 --- a/src/frontends/gguf/tests/CMakeLists.txt +++ b/src/frontends/gguf/tests/CMakeLists.txt @@ -49,9 +49,11 @@ set(FRONTEND_SRCS "${FE_SRC_DIR}/op/softmax.cpp" "${FE_SRC_DIR}/op/sqr.cpp" "${FE_SRC_DIR}/op/ssm_conv.cpp" + "${FE_SRC_DIR}/op/top_k.cpp" "${FE_SRC_DIR}/op/tri.cpp" "${FE_SRC_DIR}/op/sum_rows.cpp" "${FE_SRC_DIR}/op/transpose.cpp" + "${FE_SRC_DIR}/op/unary_elu.cpp" "${FE_SRC_DIR}/op/unary_gelu.cpp" "${FE_SRC_DIR}/op/unary_silu.cpp" "${FE_SRC_DIR}/op/view.cpp" diff --git a/src/frontends/gguf/tests/test_data/q2_0_deq.npy b/src/frontends/gguf/tests/test_data/q2_0_deq.npy new file mode 100644 index 00000000000000..24b188c067e8ce Binary files /dev/null and b/src/frontends/gguf/tests/test_data/q2_0_deq.npy differ diff --git a/src/frontends/gguf/tests/test_data/q2_0_qbytes.npy b/src/frontends/gguf/tests/test_data/q2_0_qbytes.npy new file mode 100644 index 00000000000000..88e1ad9ceb1fb1 Binary files /dev/null and b/src/frontends/gguf/tests/test_data/q2_0_qbytes.npy differ diff --git a/src/frontends/gguf/tests/test_dequant_vs_ggml.cpp b/src/frontends/gguf/tests/test_dequant_vs_ggml.cpp index b949994063e811..77c1f473a12fd7 100644 --- a/src/frontends/gguf/tests/test_dequant_vs_ggml.cpp +++ b/src/frontends/gguf/tests/test_dequant_vs_ggml.cpp @@ -74,6 +74,8 @@ const char* type_name(uint32_t type) { return "Q5_K"; case GGUF_TYPE_Q6_K: return "Q6_K"; + case GGUF_TYPE_Q2_0: + return "Q2_0"; default: return ""; } @@ -115,6 +117,8 @@ constexpr float kTolRequant = 1.5e-2f; // channel-wise Q8_0_C requant round-off // (matching the original ggml-openvino backend). The integer zp rounds min to a multiple of // scale, so the dequant diverges from ggml's faithful to_float by up to ~0.045 per weight. constexpr float kTolIntZp = 5e-2f; +// Q2_0: (code - 1) * d on both sides and the zero-point of 1 is exact, so hold it to bit-equality. +constexpr float kTolExact = 0.0f; } // namespace @@ -178,7 +182,8 @@ INSTANTIATE_TEST_SUITE_P(AllQuantTypes, DeqCase{"q3_k", GGUF_TYPE_Q3_K, kTolFaithful}, DeqCase{"q4_k", GGUF_TYPE_Q4_K, kTolIntZp}, DeqCase{"q5_k", GGUF_TYPE_Q5_K, kTolRequant}, - DeqCase{"q6_k", GGUF_TYPE_Q6_K, kTolRequant}), + DeqCase{"q6_k", GGUF_TYPE_Q6_K, kTolRequant}, + DeqCase{"q2_0", GGUF_TYPE_Q2_0, kTolExact}), [](const ::testing::TestParamInfo& i) { return std::string(i.param.stem); }); diff --git a/src/frontends/gguf/tests/test_ops.cpp b/src/frontends/gguf/tests/test_ops.cpp index 4af080b66f9673..2deae757bb0fea 100644 --- a/src/frontends/gguf/tests/test_ops.cpp +++ b/src/frontends/gguf/tests/test_ops.cpp @@ -13,6 +13,8 @@ #include #include "op_test_utils.hpp" +#include "openvino/op/topk.hpp" +#include "utils.hpp" using namespace ov_gguf_test; @@ -101,12 +103,76 @@ INSTANTIATE_TEST_SUITE_P( [](float x) { return 0.5f * x * (1.0f + std::erf(x / std::sqrt(2.0f))); }, 1e-3f}, UnaryCase{"tanh", "GGML_UNARY_OP_TANH", [](float x) { return std::tanh(x); }, 1e-4f}, + UnaryCase{"relu", "GGML_UNARY_OP_RELU", [](float x) { return x > 0.0f ? x : 0.0f; }, 1e-4f}, + UnaryCase{"elu", "GGML_UNARY_OP_ELU", [](float x) { return x > 0.0f ? x : std::expm1(x); }, 1e-4f}, + UnaryCase{"gelu_quick", + "GGML_UNARY_OP_GELU_QUICK", + [](float x) { return x * (1.0f / (1.0f + std::exp(-1.702f * x))); }, + 1e-4f}, + UnaryCase{"sin", "GGML_OP_SIN", [](float x) { return std::sin(x); }, 1e-4f}, + UnaryCase{"cos", "GGML_OP_COS", [](float x) { return std::cos(x); }, 1e-4f}, UnaryCase{"softplus", "GGML_UNARY_OP_SOFTPLUS", [](float x) { return std::log1p(std::exp(-std::abs(x))) + std::max(x, 0.0f); }, 1e-3f}), [](const ::testing::TestParamInfo& i) { return std::string(i.param.name); }); +// Log is only defined for x > 0, so it gets its own inputs rather than the shared range above. +TEST(GGUFOps, Log) { + auto model = SingleOpBuilder() + .op("GGML_OP_LOG") + .input("x", ov::element::f32, {2, 3}) + .output("out", ov::element::f32, {2, 3}) + .build(); + + std::vector x{0.25f, 0.5f, 1, 2, 4, 10}; + auto out = run_on_cpu(model, {{"x", make_f32_tensor({2, 3}, x)}}); + + std::vector expected(x.size()); + for (size_t i = 0; i < x.size(); ++i) + expected[i] = std::log(x[i]); + expect_near(out, expected, 1e-4f); +} + +// get_dimensions must read the shape of the given output, not of the producing node's output 0. +// TOP_K and ARGSORT hand downstream translators output(1) of a TopK, so taking the node instead +// of the output would silently measure the values port. +TEST(GGUFOps, GetDimensionsKeepsOutputPort) { + auto data = std::make_shared(ov::element::f32, ov::PartialShape{2, 4}); + auto k = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {2}); + auto topk = std::make_shared(data, + k, + -1, + ov::op::v11::TopK::Mode::MAX, + ov::op::v11::TopK::SortType::SORT_VALUES, + ov::element::i32); + + auto dims = ov::frontend::gguf::get_dimensions(topk->output(1), {0}); + auto shape_of = dims->input_value(0).get_node_shared_ptr(); + ASSERT_EQ(shape_of->get_input_size(), 1u); + EXPECT_EQ(shape_of->input_value(0).get_index(), 1u) << "get_dimensions measured the wrong port"; +} + +// ggml_top_k: indices of the k largest values along ne[0], ordered by descending value. +TEST(GGUFOps, TopK) { + auto model = SingleOpBuilder() + .op("GGML_OP_TOP_K") + .input("x", ov::element::f32, {1, 1, 2, 4}) + .output("out", ov::element::i32, {1, 1, 2, 2}) + .build(); + + std::vector x{4, 1, 3, 2, 10, 40, 20, 30}; + auto out = run_on_cpu(model, {{"x", make_f32_tensor({1, 1, 2, 4}, x)}}); + + ASSERT_EQ(out.get_element_type(), ov::element::i32); + ASSERT_EQ(out.get_size(), 4u); + const int32_t* a = out.data(); + // Row 0: 4,1,3,2 -> top2 are 4 (idx 0) then 3 (idx 2). Row 1: 10,40,20,30 -> 40 (1), 30 (3). + std::vector expected{0, 2, 1, 3}; + for (size_t i = 0; i < expected.size(); ++i) + EXPECT_EQ(a[i], expected[i]) << "mismatch at index " << i; +} + // Scale: out = in * scale + bias (scale/bias in op-params slots 0,1). TEST(GGUFOps, Scale) { const float scale = 2.5f; @@ -331,6 +397,71 @@ TEST(GGUFOps, Norm) { expect_near(out, expected, 1e-4f); } +// The index type comes from the decoder, not a hardcoded i32: an i64 TOP_K output must produce +// an i64 tensor, otherwise the model signature and the actual tensor disagree. +TEST(GGUFOps, TopKIndexTypeFollowsOutput) { + auto model = SingleOpBuilder() + .op("GGML_OP_TOP_K") + .input("x", ov::element::f32, {1, 1, 2, 4}) + .output("out", ov::element::i64, {1, 1, 2, 2}) + .build(); + + std::vector x{4, 1, 3, 2, 10, 40, 20, 30}; + auto out = run_on_cpu(model, {{"x", make_f32_tensor({1, 1, 2, 4}, x)}}); + + ASSERT_EQ(out.get_element_type(), ov::element::i64); + const int64_t* a = out.data(); + std::vector expected{0, 2, 1, 3}; + for (size_t i = 0; i < expected.size(); ++i) + EXPECT_EQ(a[i], expected[i]) << "mismatch at index " << i; +} + +// A pass-through VIEW must not rename its producer: that node belongs to another ggml tensor +// (here a model input), and the helper appends, so the name would compound on every such view. +TEST(GGUFOps, ViewPassThroughKeepsProducerName) { + auto model = SingleOpBuilder() + .op("GGML_OP_VIEW") + .input("x", ov::element::f32, {1, 1, 2, 4}) + .output("myview", ov::element::f32, {1, 1, 2, 4}) + .attr("op_case", 3) + .attr("input_ggml_shape", ov::Shape{1, 1, 2, 4}) + .build(); + + ASSERT_EQ(model->get_parameters().size(), 1u); + EXPECT_EQ(model->get_parameters()[0]->get_friendly_name(), "x"); +} + +// The norm axis is the literal -1, so a dynamic token dim must still convert and run. +TEST(GGUFOps, NormDynamicShape) { + const float eps = 1e-5f; + const size_t cols = 4; + auto model = SingleOpBuilder() + .op("GGML_OP_NORM") + .input("x", ov::element::f32, ov::PartialShape{-1, static_cast(cols)}) + .output("out", ov::element::f32, ov::PartialShape{-1, static_cast(cols)}) + .attr("eps", eps) + .build(); + + std::vector x{1, 2, 3, 4, -2, 0, 2, 8}; + auto out = run_on_cpu(model, {{"x", make_f32_tensor({2, cols}, x)}}); + + std::vector expected(x.size()); + for (size_t r = 0; r < 2; ++r) { + float mean = 0.f; + for (size_t c = 0; c < cols; ++c) + mean += x[r * cols + c]; + mean /= cols; + float var = 0.f; + for (size_t c = 0; c < cols; ++c) + var += (x[r * cols + c] - mean) * (x[r * cols + c] - mean); + var /= cols; + float inv = 1.0f / std::sqrt(var + eps); + for (size_t c = 0; c < cols; ++c) + expected[r * cols + c] = (x[r * cols + c] - mean) * inv; + } + expect_near(out, expected, 1e-4f); +} + // L2 norm over the last axis: x / max(sqrt(sum(x^2)), eps). TEST(GGUFOps, L2Norm) { const float eps = 1e-12f; diff --git a/src/frontends/gguf/tests/test_weights.cpp b/src/frontends/gguf/tests/test_weights.cpp index d7b48316fc0a95..1685b34f48cfc0 100644 --- a/src/frontends/gguf/tests/test_weights.cpp +++ b/src/frontends/gguf/tests/test_weights.cpp @@ -86,11 +86,36 @@ INSTANTIATE_TEST_SUITE_P(AllQuantTypes, WeightCase{"q3_k", "Q3_K", kTolFaithful}, WeightCase{"q4_k", "Q4_K", kTolIntZp}, WeightCase{"q5_k", "Q5_K", kTolRequant}, - WeightCase{"q6_k", "Q6_K", kTolRequant}), + WeightCase{"q6_k", "Q6_K", kTolRequant}, + WeightCase{"q2_0", "Q2_0", kTolFaithful}), [](const ::testing::TestParamInfo& i) { return std::string(i.param.stem); }); +// token_embd / output are requantized to channel-wise Q8_0_C, and that path reads the zero-point +// as f16 -- Q2_0 used to hard-code u8 here, which threw for every ternary model. +TEST(GGUFWeightRequant, Q2_0AsTokenEmbd) { + const auto qbytes = load_npy("q2_0_qbytes"); + const auto ref = load_npy("q2_0_deq"); + ASSERT_EQ(ref.size(), kRows * kCols); + + auto model = SingleOpBuilder() + .op("GGML_OP_NONE") + .output("token_embd.weight", ov::element::f32, {kRows, kCols}) + .attr("data", bytes_to_u8_tensor(qbytes)) + .attr("quant_type", "Q2_0") + .build(); + + auto out = run_on_cpu(model, {}); + ASSERT_EQ(out.get_size(), ref.size()); + + const float* a = out.data(); + float max_diff = 0.f; + for (size_t i = 0; i < ref.size(); ++i) + max_diff = std::max(max_diff, std::fabs(a[i] - ref[i])); + EXPECT_LE(max_diff, kTolRequant) << "Q2_0 token_embd requant diverges from ggml to_float"; +} + // An F16 weight is wrapped directly as a constant (no dequant); round-trips the raw bytes. TEST(GGUFWeightPlain, F16) { std::vector vals{1.0f, -2.0f, 3.5f, -4.25f, 0.0f, 7.0f};