From 84ee5c726bfc4ff40866d66e89553c95805ffa8b Mon Sep 17 00:00:00 2001 From: "Yu, Zijun" Date: Fri, 27 Mar 2026 16:00:37 +0800 Subject: [PATCH 1/5] Enable softmax --- ggml/src/ggml-openvino/ggml-decoder.cpp | 21 +++++++++++++++---- ggml/src/ggml-openvino/ggml-openvino.cpp | 10 ++++----- .../src/ggml-openvino/openvino/op/softmax.cpp | 10 ++++----- 3 files changed, 26 insertions(+), 15 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 69c1f9919d41..15314f2d2a89 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -255,13 +255,17 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr for (int i = 0; i < cgraph->n_nodes; i++) { auto * node = cgraph->nodes[i]; std::string name = std::string(node->name); - if (node->op == GGML_OP_FLASH_ATTN_EXT) { + if (node->op == GGML_OP_FLASH_ATTN_EXT || node->op == GGML_OP_SOFT_MAX) { model_params.n_heads = node->src[0]->ne[2]; - model_params.n_heads_kv = node->src[1]->ne[2]; model_params.head_size = node->src[0]->ne[0]; compute_params.input_len = node->src[0]->ne[1]; auto * cache_k_perm = node->src[1]; + if (node->op == GGML_OP_SOFT_MAX) { + cache_k_perm = node->src[0]->src[0]; + } + model_params.n_heads_kv = cache_k_perm->ne[2]; + if (cache_k_perm->op == GGML_OP_CPY) { cache_k_perm = cache_k_perm->src[0]; } @@ -271,7 +275,11 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr auto * cache_k = cache_k_view->src[0]; int layer = extract_layer_from_name(cache_k->name); + auto * mask = node->src[3]; + if (node->op == GGML_OP_SOFT_MAX) { + mask = node->src[1]; + } std::string mask_name(mask->name); model_params.kv_buffer_ctx_id = ggml_backend_openvino_buffer_get_ctx_id(cache_k->buffer); @@ -288,7 +296,12 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr size_t offset; memcpy(&offset, cache_k_view->op_params, sizeof(size_t)); compute_params.seq_active_start = offset / seq_size; - compute_params.token_len_per_seq = node->ne[2]; + + auto * q_perm = node->src[0]; + if (node->op == GGML_OP_SOFT_MAX) { + q_perm = node->src[0]->src[1]; + } + compute_params.token_len_per_seq = q_perm->ne[1]; if (mask_name.find("swa") != std::string::npos) { compute_params.attention_size_swa = mask->ne[0]; @@ -1248,4 +1261,4 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { std::cout << std::endl; } } -} \ No newline at end of file +} diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index b613058206c5..c7b858d068d2 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -940,12 +940,10 @@ static bool ggml_backend_openvino_device_supports_op(ggml_backend_dev_t dev, con GGML_TYPE_I32, GGML_TYPE_Q4_0, GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K}; - static const std::set supported_ops{GGML_OP_NONE, GGML_OP_ADD, GGML_OP_MUL, GGML_OP_MUL_MAT, GGML_OP_VIEW, - GGML_OP_CONT, GGML_OP_RESHAPE, GGML_OP_PERMUTE, GGML_OP_TRANSPOSE, - GGML_OP_GET_ROWS, GGML_OP_ROPE, GGML_OP_RMS_NORM, GGML_OP_SCALE, - // softmax is not updated due to replaced by flash_attn_ext - // GGML_OP_SOFT_MAX, - GGML_OP_SET_ROWS, GGML_OP_FLASH_ATTN_EXT, GGML_OP_CPY}; + static const std::set supported_ops{ + GGML_OP_NONE, GGML_OP_ADD, GGML_OP_MUL, GGML_OP_MUL_MAT, GGML_OP_VIEW, GGML_OP_CONT, + GGML_OP_RESHAPE, GGML_OP_PERMUTE, GGML_OP_TRANSPOSE, GGML_OP_GET_ROWS, GGML_OP_ROPE, GGML_OP_RMS_NORM, + GGML_OP_SCALE, GGML_OP_SOFT_MAX, GGML_OP_SET_ROWS, GGML_OP_FLASH_ATTN_EXT, GGML_OP_CPY}; static const std::set supported_unary_ops{ GGML_UNARY_OP_SILU, }; diff --git a/ggml/src/ggml-openvino/openvino/op/softmax.cpp b/ggml/src/ggml-openvino/openvino/op/softmax.cpp index 9f6330862be4..f9300e36631b 100644 --- a/ggml/src/ggml-openvino/openvino/op/softmax.cpp +++ b/ggml/src/ggml-openvino/openvino/op/softmax.cpp @@ -23,7 +23,6 @@ namespace ggml { namespace op { OutputVector translate_soft_max(const NodeContext & context) { - // TODO code is outdated num_inputs_check(context, 1, 2); auto input_node = context.get_input(0).get_node_shared_ptr(); @@ -48,7 +47,7 @@ OutputVector translate_soft_max(const NodeContext & context) { auto scaled_input = std::make_shared(input_node, scale_node); if (context.get_input_size() < 2) { - res = std::make_shared(scaled_input, 2); + res = std::make_shared(scaled_input, 3); return rename_outputs_with_suffix({res}, context.get_name()); } @@ -56,11 +55,12 @@ OutputVector translate_soft_max(const NodeContext & context) { if (context.has_input("KQ_mask_sliced")) { mask_node_sliced = context.get_input("KQ_mask_sliced"); } else { - auto token_len = get_dimensions(input_node, {1}); + auto token_len = get_dimensions(input_node, {2}); auto mask_node = context.get_input(1); auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - mask_node_sliced = std::make_shared(mask_node, zero, token_len, one, one); + auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {2}); + mask_node_sliced = std::make_shared(mask_node, zero, token_len, one, axis); } if (mask_node_sliced.get_element_type() != context.get_output_type()) { @@ -78,7 +78,7 @@ OutputVector translate_soft_max(const NodeContext & context) { auto input_slope_mask_node = std::make_shared(scaled_input, slope_mask); - res = std::make_shared(input_slope_mask_node, 2); + res = std::make_shared(input_slope_mask_node, 3); return rename_outputs_with_suffix({res}, context.get_name()); } From 5bb36dc20badfbb6f7cce2d904423d33c9b7e15f Mon Sep 17 00:00:00 2001 From: "Yu, Zijun" Date: Tue, 31 Mar 2026 16:09:47 +0800 Subject: [PATCH 2/5] -fa off run with accuracy issue --- ggml/src/ggml-openvino/ggml-decoder.cpp | 25 ++++++++++++++++--- .../src/ggml-openvino/openvino/op/permute.cpp | 11 +++++--- .../src/ggml-openvino/openvino/op/reshape.cpp | 10 ++++++-- .../ggml-openvino/openvino/op/set_rows.cpp | 4 +-- ggml/src/ggml-openvino/utils.cpp | 5 ++-- 5 files changed, 41 insertions(+), 14 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 9f8ed1169a89..912cf6ed5560 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -173,14 +173,31 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { // kv cache tensor std::string src_name(node->view_src->name); int layer = extract_layer_from_name(src_name); - if (!is_swa_layer(layer)) { - op_case = 2; + if (ggml_is_contiguous(node->src[0])) { + // - 19: [ 64, 8, 256, 1] VIEW cache_k_l0 (view) [ 2, 128, 1024, 1048576] + // [ 512, 1024, 1, 1] 0: NONE cache_k_l0 [ 2, 1024, 1048576, 1048576] + // - 20: [ 64, 256, 8, 1] PERMUTE cache_k_l0 (view) (permuted) [ 2, 1024, 128, 1048576] + // [ 64, 8, 256, 1] 0: VIEW cache_k_l0 (view) [ 2, 128, 1024, 1048576] + if (!is_swa_layer(layer)) { + op_case = 3; + } else { + op_case = 4; + } } else { - op_case = 3; + // special case of cache v when `-fa off` + // - 17: [ 256, 8, 64, 1] VIEW cache_v_l0 (view) [ 2, 131072, 2048, 1048576] + // [ 512, 1024, 1, 1] 0: NONE cache_v_l0 [ 2, 1024, 1048576, 1048576] + // - 18: [ 256, 64, 8, 1] PERMUTE cache_v_l0 (view) (permuted) [ 2, 2048, 131072, 1048576] + // [ 256, 8, 64, 1] 0: VIEW cache_v_l0 (view) [ 2, 131072, 2048, 1048576] + if (!is_swa_layer(layer)) { + op_case = 5; + } else { + op_case = 6; + } } } else { // rope'ed query tensor - op_case = 4; + op_case = 2; } break; } diff --git a/ggml/src/ggml-openvino/openvino/op/permute.cpp b/ggml/src/ggml-openvino/openvino/op/permute.cpp index 269fd99f36fb..be035237b6e4 100644 --- a/ggml/src/ggml-openvino/openvino/op/permute.cpp +++ b/ggml/src/ggml-openvino/openvino/op/permute.cpp @@ -23,8 +23,7 @@ OutputVector translate_permute(const NodeContext & context) { num_inputs_check(context, 1, 1); int op_case = context.get_op_case(); - FRONT_END_CHECK_IMPLEMENTED(op_case == 1 || op_case == 2 || op_case == 3 || op_case == 4, - "Unsupported PERMUTE case"); + FRONT_END_CHECK_IMPLEMENTED(op_case != 0, "Unsupported PERMUTE case"); ov::Output res; auto src = context.get_input(0); @@ -39,7 +38,7 @@ OutputVector translate_permute(const NodeContext & context) { if (op_case == 1 || context.is_stateful()) { res = std::make_shared(src, perm); - } else if (op_case == 4) { + } else if (op_case == 2) { auto output_shape = context.get_output_shape().to_shape(); auto n_heads = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[1]}); auto head_size = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[3]}); @@ -62,13 +61,17 @@ OutputVector translate_permute(const NodeContext & context) { auto output_shape = context.get_output_shape().to_shape(); int64_t head_size = output_shape[3]; int64_t n_heads = output_shape[1]; + if (op_case == 5 || op_case == 6) { + head_size = output_shape[2]; + n_heads = output_shape[1]; + } int64_t ctx_per_seq = cache_shape[2].is_static() ? cache_shape[2].get_length() : -1; int64_t n_seq = cache_shape[1].get_length(); Output attention_size; if (!context.has_input("attention_size")) { attention_size = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[2]}); - } else if (op_case == 2) { + } else if (op_case == 3 || op_case == 5) { attention_size = context.get_input("attention_size"); } else { attention_size = context.get_input("attention_size_swa"); diff --git a/ggml/src/ggml-openvino/openvino/op/reshape.cpp b/ggml/src/ggml-openvino/openvino/op/reshape.cpp index efd9a5a860ab..2a1a082d8630 100644 --- a/ggml/src/ggml-openvino/openvino/op/reshape.cpp +++ b/ggml/src/ggml-openvino/openvino/op/reshape.cpp @@ -10,7 +10,6 @@ #include #include #include -#include #include namespace ov { @@ -47,7 +46,14 @@ OutputVector translate_reshape(const NodeContext & context) { std::vector{(int64_t) output_shape[0], (int64_t) output_shape[1], -1, (int64_t) output_shape[3]}); } else if (op_case == 3) { - throw std::runtime_error("might be outdated RESHAPE case"); + // - 14: [ 1, 1024, 1, 1] RESHAPE Vcur-0 (reshaped) (reshaped) + // [ 512, 2, 1, 1] 0: RESHAPE Vcur-0 (reshaped) + // - 15: [ 1, 524288, 1, 1] RESHAPE cache_v_l0 (reshaped) + // [ 512, 1024, 1, 1] 0: NONE cache_v_l0 + // - 16: [ 1, 524288, 1, 1] SET_ROWS cache_v_l0 (reshaped) (view) + // [ 1, 1024, 1, 1] 0: RESHAPE Vcur-0 (reshaped) (reshaped) + // [ 1024, 1, 1, 1] 1: NONE leaf_11 + // [ 1, 524288, 1, 1] 2: RESHAPE cache_v_l0 (reshaped) new_shape_node = ov::op::v0::Constant::create( ov::element::i64, {4}, std::vector{(int64_t) output_shape[0], (int64_t) output_shape[1], -1, 1}); diff --git a/ggml/src/ggml-openvino/openvino/op/set_rows.cpp b/ggml/src/ggml-openvino/openvino/op/set_rows.cpp index 136e4265b429..9f2b841b19c1 100644 --- a/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +++ b/ggml/src/ggml-openvino/openvino/op/set_rows.cpp @@ -34,14 +34,14 @@ OutputVector translate_set_rows(const NodeContext & context) { data = std::make_shared(data, context.get_output_type()); - auto dst_shape = context.get_output_shape().to_shape(); + auto row_size = context.get_input_shape(2)[3].get_length(); auto ind_squeezed = std::make_shared(indices, ov::op::v0::Constant::create(ov::element::i64, {3}, {0, 1, 2})); auto data_reshaped = std::make_shared( data, ov::op::v0::Constant::create(ov::element::i64, {4}, - {(int64_t) 1, (int64_t) 1, (int64_t) -1, (int64_t) dst_shape[3]}), + {(int64_t) 1, (int64_t) 1, (int64_t) -1, (int64_t) row_size}), false); auto axes = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {2}); diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index 1100b14ebd6e..d3822dac50e9 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -187,8 +187,9 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, std::shared_ptr< std::shared_ptr model; auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph); - ggml_decoder = std::make_shared(cgraph, m_params, c_params, model_weights, is_static, stateful, model_is_splitted); - decoder_end_time = ggml_time_us(); + ggml_decoder = std::make_shared(cgraph, m_params, c_params, model_weights, is_static, + stateful, model_is_splitted); + decoder_end_time = ggml_time_us(); auto input_model = std::make_shared(ggml_decoder); model = ov::frontend::ggml::FrontEnd::convert(input_model); From 0fae70273fb0d229a6fc324a6ec18df6f9705411 Mon Sep 17 00:00:00 2001 From: "Yu, Zijun" Date: Wed, 1 Apr 2026 15:06:08 +0800 Subject: [PATCH 3/5] Remove mask_slice in stateless; Fix swa --- ggml/src/ggml-openvino/ggml-decoder.cpp | 14 ++++------ ggml/src/ggml-openvino/ggml-decoder.h | 3 +- .../openvino/op/flash_attn_ext.cpp | 18 +++++------- .../src/ggml-openvino/openvino/op/softmax.cpp | 20 +++++++++---- .../openvino/translate_session.cpp | 28 +++++++++++-------- 5 files changed, 45 insertions(+), 38 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 912cf6ed5560..8c2a236a80a8 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -291,15 +291,18 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr auto * node = cgraph->nodes[i]; std::string name = std::string(node->name); if (node->op == GGML_OP_FLASH_ATTN_EXT || node->op == GGML_OP_SOFT_MAX) { - model_params.n_heads = node->src[0]->ne[2]; - model_params.head_size = node->src[0]->ne[0]; compute_params.input_len = node->src[0]->ne[1]; + auto * q_perm = node->src[0]; auto * cache_k_perm = node->src[1]; if (node->op == GGML_OP_SOFT_MAX) { + q_perm = node->src[0]->src[1]; cache_k_perm = node->src[0]->src[0]; } + model_params.head_size = cache_k_perm->ne[0]; model_params.n_heads_kv = cache_k_perm->ne[2]; + model_params.n_heads = q_perm->ne[2]; + compute_params.token_len_per_seq = q_perm->ne[1]; if (cache_k_perm->op == GGML_OP_CPY) { cache_k_perm = cache_k_perm->src[0]; @@ -332,12 +335,6 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr memcpy(&offset, cache_k_view->op_params, sizeof(size_t)); compute_params.seq_active_start = offset / seq_size; - auto * q_perm = node->src[0]; - if (node->op == GGML_OP_SOFT_MAX) { - q_perm = node->src[0]->src[1]; - } - compute_params.token_len_per_seq = q_perm->ne[1]; - if (mask_name.find("swa") != std::string::npos) { compute_params.attention_size_swa = mask->ne[0]; } else { @@ -348,7 +345,6 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr compute_params.attention_size_swa = model_params.ctx_per_seq_swa; compute_params.token_len_per_seq = 1; } - break; } // if the node op is TRANSPOSE and its input is PERMUTE and the source of the PERMUTE is VIEW, then get the attention size with the TRANSPOSE node ne[0] (in case no GGML_OP_FLASH_ATTN_EXT) if (node->op == GGML_OP_TRANSPOSE && node->src[0]->op == GGML_OP_PERMUTE && diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index 1a7849c52516..9ab8efe3963c 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -239,7 +239,8 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { } inline static bool is_inp_mask(const ggml_tensor * tensor, const ggml_tensor * op) { - return op->op == GGML_OP_CPY || (op->op == GGML_OP_FLASH_ATTN_EXT && tensor == op->src[3]); + return op->op == GGML_OP_CPY || (op->op == GGML_OP_FLASH_ATTN_EXT && tensor == op->src[3]) || + (op->op == GGML_OP_SOFT_MAX && tensor == op->src[1]); } inline static bool is_rope_freqs_weight(const ggml_tensor * tensor, const ggml_tensor * op) { diff --git a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp index 42602a730a4f..059556107efd 100644 --- a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +++ b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp @@ -34,23 +34,19 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) { auto q = std::make_shared(q_f32, ov::element::f16); auto scale_node = std::make_shared(ov::element::f16, ov::Shape{}, std::vector{scale}); - ov::Output mask_sliced, res; + ov::Output res; + + // For stateful std::string mask_name = "KQ_mask_sliced"; if (context.get_input_names()[3].find("swa") != std::string::npos) { mask_name = "KQ_mask_swa_sliced"; } if (context.has_input(mask_name)) { - mask_sliced = context.get_input(mask_name); - } else { - auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto two = ov::op::v0::Constant::create(ov::element::i64, {1}, {2}); - auto token_len = get_dimensions(q, {2}); - mask_sliced = std::make_shared(mask, zero, token_len, one, two); + mask = context.get_input(mask_name); } - if (mask_sliced.get_element_type() != ov::element::f16) { - mask_sliced = std::make_shared(mask_sliced, ov::element::f16); + if (mask.get_element_type() != ov::element::f16) { + mask = std::make_shared(mask, ov::element::f16); } auto tile_kv = [&](int64_t num_heads, int64_t num_heads_kv, int64_t head_size, ov::Output kv) { @@ -77,7 +73,7 @@ OutputVector translate_flash_attn_ext(const NodeContext & context) { k = tile_kv(q_shape[1], k_shape[1], q_shape[3], k); v = tile_kv(q_shape[1], k_shape[1], q_shape[3], v); - auto sdpa = std::make_shared(q, k, v, mask_sliced, scale_node, false); + auto sdpa = std::make_shared(q, k, v, mask, scale_node, false); res = std::make_shared(sdpa, ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3})); res = std::make_shared(res, ov::element::f32); diff --git a/ggml/src/ggml-openvino/openvino/op/softmax.cpp b/ggml/src/ggml-openvino/openvino/op/softmax.cpp index 6b3a679c6db2..dcdce3a8ce21 100644 --- a/ggml/src/ggml-openvino/openvino/op/softmax.cpp +++ b/ggml/src/ggml-openvino/openvino/op/softmax.cpp @@ -43,14 +43,24 @@ OutputVector translate_soft_max(const NodeContext & context) { FRONT_END_CHECK_IMPLEMENTED(!(max_bias > 0.0f && context.get_input_size() < 2), "OpenVINO softmax ALiBi path requires mask input"); + ov::Output mask = context.get_input(1); + + // For stateful + std::string mask_name = "KQ_mask_sliced"; + if (context.get_input_names()[1].find("swa") != std::string::npos) { + mask_name = "KQ_mask_swa_sliced"; + } + if (context.has_input(mask_name)) { + mask = context.get_input(mask_name); + } + + if (mask.get_element_type() != logits.get_element_type()) { + mask = std::make_shared(mask, logits.get_element_type()); + } + // Optional mask add: logits += mask // For max_bias > 0 (ALiBi), apply per-head slope to mask before adding. if (context.get_input_size() > 1) { - ov::Output mask = context.get_input(1); - if (mask.get_element_type() != logits.get_element_type()) { - mask = std::make_shared(mask, logits.get_element_type()); - } - if (max_bias > 0.0f) { auto out_shape = context.get_output_shape().to_shape(); FRONT_END_CHECK_IMPLEMENTED(out_shape.size() == 4, diff --git a/ggml/src/ggml-openvino/openvino/translate_session.cpp b/ggml/src/ggml-openvino/openvino/translate_session.cpp index 0f68a1f50623..8283777cdd00 100644 --- a/ggml/src/ggml-openvino/openvino/translate_session.cpp +++ b/ggml/src/ggml-openvino/openvino/translate_session.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -88,19 +89,22 @@ void add_sliced_mask(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) { if (is_static) { mask_sliced = mask; } else if (ggml_model_decoder.is_stateful()) { - auto zero_2d = ov::op::v0::Constant::create(ov::element::i64, {2}, {0,0}); - auto one_2d = ov::op::v0::Constant::create(ov::element::i64, {2}, {1,1}); - auto zero_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto three_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); - auto neg_one_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - auto axes = ov::op::v0::Constant::create(ov::element::i64, {2}, {-2,-1}); + auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); + auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); + auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); + + auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); + auto axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); + auto inp_pos = tensor_map.at("inp_pos").get_node_shared_ptr(); - auto gather_inp_pos = std::make_shared(inp_pos, neg_one_1d, three_1d); - auto reshaped_inp_pos = std::make_shared(gather_inp_pos, ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), false); - auto inp_pos_incremented = std::make_shared(reshaped_inp_pos, ov::op::v0::Constant::create(ov::element::i32, ov::Shape{1}, {1})); - auto stop = std::make_shared(ov::OutputVector{token_len_per_seq, std::make_shared(inp_pos_incremented, token_len_per_seq)}, 0); - mask_sliced = - std::make_shared(mask, zero_2d, stop, one_2d, axes); + auto last_inp_pos = std::make_shared(inp_pos, neg_one, three); + auto last_inp_pos_1d = std::make_shared( + last_inp_pos, ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), false); + auto last_inp_pos_cvt = std::make_shared(last_inp_pos_1d, ov::element::i64); + auto last_inp_pos_inc = std::make_shared(last_inp_pos_cvt, one); + + mask_sliced = std::make_shared(mask, zero, last_inp_pos_inc, step, axes); mask_sliced = std::make_shared(mask_sliced, ov::element::f16); mask_sliced->set_friendly_name(sliced_name); } else { From 26b3799c2f2a6d1412dc2aebc92f8dd5946ee712 Mon Sep 17 00:00:00 2001 From: "Yu, Zijun" Date: Thu, 2 Apr 2026 11:42:11 +0800 Subject: [PATCH 4/5] Fix -fa off accuracy --- ggml/src/ggml-openvino/ggml-decoder.cpp | 12 +++---- ggml/src/ggml-openvino/ggml-decoder.h | 7 +++-- ggml/src/ggml-openvino/openvino/op/mulmat.cpp | 12 ++++--- .../src/ggml-openvino/openvino/op/permute.cpp | 31 ++++++++++++++----- 4 files changed, 39 insertions(+), 23 deletions(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index 8c2a236a80a8..ddbb58f780d9 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -1,20 +1,15 @@ #include "ggml-decoder.h" -#include "ggml-backend-impl.h" -#include "ggml-backend.h" +#include "ggml-impl.h" #include "ggml-openvino-extra.h" #include "ggml-openvino.h" #include "ggml-quants.h" -#include -#include - #include #include #include #include #include -#include #include #include #include @@ -30,12 +25,10 @@ #include #include #include -#include #include #include #include #include -#include #include GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph, @@ -204,6 +197,9 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { case GGML_OP_MUL_MAT: { if (node->src[0]->op == GGML_OP_VIEW && node->src[1]->op == GGML_OP_VIEW) { op_case = 3; + } else if (node->src[1]->op == GGML_OP_SOFT_MAX) { + // In the case of `-fa off`, softmax is used, v_trans=true, the dynamic dim is ne[0] for cache_v + op_case = 2; } break; } diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index 9ab8efe3963c..ff8f81e8ae6b 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -1,6 +1,7 @@ #pragma once -#include "ggml-quants.h" +#include "ggml-backend-impl.h" +#include "ggml-backend.h" #include "ggml.h" #include "openvino/decoder.h" @@ -9,7 +10,6 @@ #include #include #include -#include #include struct ModelParams { @@ -248,7 +248,8 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { } inline static bool is_kvcache(const ggml_tensor * tensor, const ggml_tensor * op) { - return op->op == GGML_OP_SET_ROWS && op->src[2] == tensor; + return (op->op == GGML_OP_SET_ROWS && op->src[2] == tensor) || + tensor->buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY; } inline static bool is_kv_idx(const ggml_tensor * tensor, const ggml_tensor * op) { diff --git a/ggml/src/ggml-openvino/openvino/op/mulmat.cpp b/ggml/src/ggml-openvino/openvino/op/mulmat.cpp index 38edec85ddf7..71cf1fd17aa2 100644 --- a/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +++ b/ggml/src/ggml-openvino/openvino/op/mulmat.cpp @@ -34,10 +34,7 @@ OutputVector translate_mulmat(const NodeContext & context) { ov::Output A = context.get_input(1); bool transpose_b = true; - if (op_case == 2) { - B = B.get_node_shared_ptr()->input_value(0); - transpose_b = false; - } else if (op_case == 3) { + if (op_case == 3) { B = process_view_input(context, 0); A = process_view_input(context, 1); } @@ -55,6 +52,7 @@ OutputVector translate_mulmat(const NodeContext & context) { auto batch_small = A_batch_larger ? B_batch : A_batch; Output Z = A_batch_larger ? B : A; + auto Z_shape = A_batch_larger ? B_shape : A_shape; int64_t factor = batch_large / batch_small; if (factor > 1 && batch_small > 1) { auto batch_large_node = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector{batch_large}); @@ -67,7 +65,11 @@ OutputVector translate_mulmat(const NodeContext & context) { auto broadcast_shape = ov::op::v0::Constant::create( ov::element::i64, {5}, {(int64_t) 1, (int64_t) 1, factor, (int64_t) 1, (int64_t) 1}); auto new_Z_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, - {(int64_t) 0, batch_large, (int64_t) -1, (int64_t) A_shape[3]}); + {(int64_t) 0, batch_large, (int64_t) -1, (int64_t) Z_shape[3]}); + if (op_case == 2) { + new_Z_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, + {(int64_t) 0, batch_large, (int64_t) Z_shape[2], (int64_t) -1}); + } auto Z_broadcasted = std::make_shared(Z_unsqueezed, broadcast_shape, ov::op::BroadcastType::BIDIRECTIONAL); diff --git a/ggml/src/ggml-openvino/openvino/op/permute.cpp b/ggml/src/ggml-openvino/openvino/op/permute.cpp index be035237b6e4..a9a3800e663d 100644 --- a/ggml/src/ggml-openvino/openvino/op/permute.cpp +++ b/ggml/src/ggml-openvino/openvino/op/permute.cpp @@ -24,6 +24,10 @@ OutputVector translate_permute(const NodeContext & context) { int op_case = context.get_op_case(); FRONT_END_CHECK_IMPLEMENTED(op_case != 0, "Unsupported PERMUTE case"); + // op_case 1 is trivial permute + // op_case 2 is to permute Q. It has a preceding VIEW that reshapes Q to restore the sequqence dimension + // op_case 3 4 it to permute KV cache in the default layout + // op_case 5 6 is to permute V cache when `-fa off`, where v_trans=true ov::Output res; auto src = context.get_input(0); @@ -91,18 +95,31 @@ OutputVector translate_permute(const NodeContext & context) { seq_active_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {seq_active_end_val}); } - // 1. reshape to [n_seq, ctx_per_seq, n_heads, head_size] + // 1. reshape to [n_seq, ctx_per_seq, n_heads, head_size] (for `-fa off` [n_seq, n_heads, head_size, ctx_per_seq]) // 2. slice out the active sequences // 3. slice out the attention part in each sequence - // 4. permute + // 4. permute (skip for `-fa off`) auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto src_reshaped = std::make_shared( - src, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, ctx_per_seq, n_heads, head_size}), false); - auto slice1 = std::make_shared(src_reshaped, seq_active_start, seq_active_end, one, zero); - auto slice2 = std::make_shared(slice1, zero, attention_size, one, one); - res = std::make_shared(slice2, perm); + if (op_case == 3 || op_case == 4) { + auto src_reshaped = std::make_shared( + src, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, ctx_per_seq, n_heads, head_size}), + false); + auto slice1 = + std::make_shared(src_reshaped, seq_active_start, seq_active_end, one, zero); + auto slice2 = std::make_shared(slice1, zero, attention_size, one, one); + res = std::make_shared(slice2, perm); + } else { + auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); + auto src_reshaped = std::make_shared( + src, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, n_heads, head_size, ctx_per_seq}), + false); + auto slice1 = + std::make_shared(src_reshaped, seq_active_start, seq_active_end, one, zero); + auto slice2 = std::make_shared(slice1, zero, attention_size, one, three); + res = slice2; + } } return rename_outputs_with_suffix({res}, context.get_name()); } From f7db28febcfb18dc2fe1d84a09495b8a82b74a44 Mon Sep 17 00:00:00 2001 From: "Yu, Zijun" Date: Thu, 2 Apr 2026 13:15:48 +0800 Subject: [PATCH 5/5] Fix llama-perplexity -fa off --- ggml/src/ggml-openvino/ggml-decoder.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index ddbb58f780d9..9d2cf60cf608 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -152,7 +152,7 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { if (src->ne[2] * src->ne[3] == node->ne[1]) { op_case = 5; } - } else if (src->ne[0] * src->ne[1] == node->ne[1]) { + } else if (src->ne[0] * src->ne[1] * src->ne[2] == node->ne[1]) { op_case = 3; } else if (src->ne[1] * src->ne[2] == node->ne[1]) { op_case = 6;