From de56c35dcdf8c902e620f91605aabdd6c0bb5095 Mon Sep 17 00:00:00 2001 From: chraac Date: Sun, 22 Mar 2026 23:14:27 +0800 Subject: [PATCH 01/22] optimize hmx_mat_mul functions by calculating row and column tiles upfront --- ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c index a56356bee9f..1635b79dd43 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c @@ -714,6 +714,7 @@ int hmx_mat_mul_permuted_w16a32_batched(struct htp_context *ctx, const hmx_matmu for (size_t mr = 0; mr < (size_t) params->m; mr += m_chunk_n_rows) { const size_t n_rows = hex_smin((size_t) params->m - mr, m_chunk_n_rows); + const size_t n_row_tiles = hmx_ceil_div((int) n_rows, HMX_FP16_TILE_N_ROWS); // Pre-load activations for all heads in the group (once per m_chunk). // When the source is strided (permuted Q), use 2D DMA to gather @@ -755,6 +756,7 @@ int hmx_mat_mul_permuted_w16a32_batched(struct htp_context *ctx, const hmx_matmu for (size_t nc = 0; nc < (size_t) params->n; nc += n_chunk_n_cols) { const size_t n_cols = hex_smin((size_t) params->n - nc, n_chunk_n_cols); + const size_t n_col_tiles = hmx_ceil_div((int) n_cols, HMX_FP16_TILE_N_COLS); TIMER_START(weight_load); { @@ -779,8 +781,6 @@ int hmx_mat_mul_permuted_w16a32_batched(struct htp_context *ctx, const hmx_matmu TIMER_START(hmx_core); { const __fp16 *vtcm_act_g = vtcm_activation + (size_t) g * act_head_stride; - const int n_row_tiles = hmx_ceil_div((int) n_rows, HMX_FP16_TILE_N_ROWS); - const int n_col_tiles = hmx_ceil_div((int) n_cols, HMX_FP16_TILE_N_COLS); core_dot_chunk_fp16(vtcm_output, vtcm_act_g, vtcm_weight, vtcm_scales, n_row_tiles, n_col_tiles, params->k / 32); } @@ -883,7 +883,8 @@ int hmx_mat_mul_permuted_w16a32(struct htp_context *ctx, float *restrict dst, co for (size_t mr = 0; mr < m; mr += m_chunk_n_rows) { // transfer activation matrix chunk into VTCM - size_t n_rows = hex_smin(m - mr, m_chunk_n_rows); + const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows); + const size_t n_row_tiles = hmx_ceil_div(n_rows, HMX_FP16_TILE_N_ROWS); TIMER_START(activation_load); { @@ -921,7 +922,8 @@ int hmx_mat_mul_permuted_w16a32(struct htp_context *ctx, float *restrict dst, co } for (size_t nc = 0; nc < n; nc += n_chunk_n_cols) { - size_t n_cols = hex_smin(n - nc, n_chunk_n_cols); + const size_t n_cols = hex_smin(n - nc, n_chunk_n_cols); + const size_t n_col_tiles = hmx_ceil_div(n_cols, HMX_FP16_TILE_N_COLS); TIMER_START(weight_load); { @@ -946,8 +948,6 @@ int hmx_mat_mul_permuted_w16a32(struct htp_context *ctx, float *restrict dst, co TIMER_START(hmx_core); { - const int n_row_tiles = hmx_ceil_div(n_rows, HMX_FP16_TILE_N_ROWS); - const int n_col_tiles = hmx_ceil_div(n_cols, HMX_FP16_TILE_N_COLS); core_dot_chunk_fp16(vtcm_output, vtcm_activation, vtcm_weight, vtcm_scales, n_row_tiles, n_col_tiles, k / 32); } TIMER_STOP(hmx_core); @@ -1087,7 +1087,8 @@ int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, float *restrict ds if (!use_pipeline) { for (size_t mr = 0; mr < m; mr += m_chunk_n_rows) { // transfer activation matrix chunk into VTCM - size_t n_rows = hex_smin(m - mr, m_chunk_n_rows); + const size_t n_rows = hex_smin(m - mr, m_chunk_n_rows); + const size_t n_row_tiles = hmx_ceil_div(n_rows, HMX_FP16_TILE_N_ROWS); TIMER_START(activation_load); { @@ -1108,7 +1109,8 @@ int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, float *restrict ds } for (size_t nc = 0; nc < n; nc += n_chunk_n_cols) { - size_t n_cols = hex_smin(n - nc, n_chunk_n_cols); + const size_t n_cols = hex_smin(n - nc, n_chunk_n_cols); + const size_t n_col_tiles = hmx_ceil_div(n_cols, HMX_FP16_TILE_N_COLS); TIMER_START(weight_load); { @@ -1133,8 +1135,6 @@ int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, float *restrict ds TIMER_START(hmx_core); { - const int n_row_tiles = hmx_ceil_div(n_rows, HMX_FP16_TILE_N_ROWS); - const int n_col_tiles = hmx_ceil_div(n_cols, HMX_FP16_TILE_N_COLS); core_dot_chunk_fp16(vtcm_output, vtcm_activation, vtcm_weight, vtcm_scales, n_row_tiles, n_col_tiles, k / 32); } TIMER_STOP(hmx_core); From b2b21a364a8cead610dd904881a2c3708c41dffb Mon Sep 17 00:00:00 2001 From: chraac Date: Sun, 22 Mar 2026 23:57:01 +0800 Subject: [PATCH 02/22] refactor core_dot_chunk_fp16 to use size_t for tile counts and improve readability --- ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c index 1635b79dd43..7753a9ca5be 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c @@ -480,19 +480,19 @@ static void dequantize_x4x2_weight_chunk_to_fp16_tiles( // requires external HMX lock static void core_dot_chunk_fp16(__fp16 *output, const __fp16 *activation, const __fp16 *weight, const __fp16 *scales, - int n_row_tiles, int n_col_tiles, int n_dot_tiles) { + size_t n_row_tiles, size_t n_col_tiles, size_t n_dot_tiles) { hmx_set_output_scales(scales); - for (int r = 0; r < n_row_tiles; ++r) { - for (int c = 0; c < n_col_tiles; ++c) { - Q6_mxclracc_hf(); + for (size_t r = 0; r < n_row_tiles; ++r) { + const __fp16 *row_tiles = activation + r * n_dot_tiles * HMX_FP16_TILE_N_ELMS; - const __fp16 *row_tiles = activation + r * n_dot_tiles * HMX_FP16_TILE_N_ELMS; + for (size_t c = 0; c < n_col_tiles; ++c) { + Q6_mxclracc_hf(); const __fp16 *col_tiles = weight + c * n_dot_tiles * HMX_FP16_TILE_N_ELMS; - - for (int k = 0; k < n_dot_tiles; ++k) { - int offset = k * HMX_FP16_TILE_N_ELMS; - hmx_load_tile_pair_fp16(row_tiles + offset, col_tiles + offset); + for (size_t k = 0; k < n_dot_tiles; ++k) { + hmx_load_tile_pair_fp16(row_tiles, col_tiles); + row_tiles += HMX_FP16_TILE_N_ELMS; + col_tiles += HMX_FP16_TILE_N_ELMS; } __fp16 *out_tile = output + (r * n_col_tiles + c) * HMX_FP16_TILE_N_ELMS; @@ -559,7 +559,7 @@ static void transfer_output_chunk_threaded(struct htp_context *ctx, float *dst, assert(n_cols % HMX_FP16_TILE_N_COLS == 0); size_t n_tot_chunks = n_rows; - size_t n_chunks_per_task = 32; // must be multiple of HMX_FP16_TILE_N_ROWS (32) + size_t n_chunks_per_task = HMX_FP16_TILE_N_ROWS; // must be multiple of HMX_FP16_TILE_N_ROWS (32) output_transfer_task_state_t state; state.n_tasks = (n_tot_chunks + n_chunks_per_task - 1) / n_chunks_per_task; From 5e18f4ed9248566d915ce386dbf0f9e16ce3a9c6 Mon Sep 17 00:00:00 2001 From: chraac Date: Mon, 23 Mar 2026 11:28:07 +0800 Subject: [PATCH 03/22] wip --- ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c index 7753a9ca5be..4d02aca45df 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c @@ -456,9 +456,9 @@ static void dequantize_x4x2_weight_chunk_to_fp16_tiles( assert(n_cols % HMX_FP16_TILE_N_COLS == 0); assert(k_block % HMX_FP16_TILE_N_COLS == 0); - int n_col_tiles = n_cols / HMX_FP16_TILE_N_COLS; - int n_k_tiles = k_block / HMX_FP16_TILE_N_COLS; - int n_tot_tiles = n_col_tiles * n_k_tiles; + size_t n_col_tiles = n_cols / HMX_FP16_TILE_N_COLS; + size_t n_k_tiles = k_block / HMX_FP16_TILE_N_COLS; + size_t n_tot_tiles = n_col_tiles * n_k_tiles; size_t n_tiles_per_task = hmx_ceil_div(n_tot_tiles, ctx->n_threads); From a2628321114cb849e2303cfb6f5125aceb6b56ec Mon Sep 17 00:00:00 2001 From: chraac Date: Mon, 23 Mar 2026 12:11:40 +0800 Subject: [PATCH 04/22] set scale outside of loop --- ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c index 4d02aca45df..3898676260c 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c @@ -481,8 +481,6 @@ static void dequantize_x4x2_weight_chunk_to_fp16_tiles( // requires external HMX lock static void core_dot_chunk_fp16(__fp16 *output, const __fp16 *activation, const __fp16 *weight, const __fp16 *scales, size_t n_row_tiles, size_t n_col_tiles, size_t n_dot_tiles) { - hmx_set_output_scales(scales); - for (size_t r = 0; r < n_row_tiles; ++r) { const __fp16 *row_tiles = activation + r * n_dot_tiles * HMX_FP16_TILE_N_ELMS; @@ -753,6 +751,7 @@ int hmx_mat_mul_permuted_w16a32_batched(struct htp_context *ctx, const hmx_matmu } HAP_compute_res_hmx_lock(ctx->vtcm_rctx); + hmx_set_output_scales(vtcm_scales); for (size_t nc = 0; nc < (size_t) params->n; nc += n_chunk_n_cols) { const size_t n_cols = hex_smin((size_t) params->n - nc, n_chunk_n_cols); @@ -880,6 +879,7 @@ int hmx_mat_mul_permuted_w16a32(struct htp_context *ctx, float *restrict dst, co TIMER_START(total); HAP_compute_res_hmx_lock(ctx->vtcm_rctx); + hmx_set_output_scales(vtcm_scales); for (size_t mr = 0; mr < m; mr += m_chunk_n_rows) { // transfer activation matrix chunk into VTCM @@ -1083,6 +1083,7 @@ int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, float *restrict ds (size_t)(vtcm_ptr - (uint8_t *)ctx->vtcm_base), vtcm_budget); HAP_compute_res_hmx_lock(ctx->vtcm_rctx); + hmx_set_output_scales(vtcm_scales); if (!use_pipeline) { for (size_t mr = 0; mr < m; mr += m_chunk_n_rows) { From ee95d92a3a60f387fafe5bdbb493dd6051ec804f Mon Sep 17 00:00:00 2001 From: chraac Date: Mon, 23 Mar 2026 12:14:54 +0800 Subject: [PATCH 05/22] wip --- ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c | 41 ++++++++++++++-------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c index 3898676260c..abeba30be25 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c @@ -479,27 +479,35 @@ static void dequantize_x4x2_weight_chunk_to_fp16_tiles( // --- End x4x2 dequantizers --- // requires external HMX lock -static void core_dot_chunk_fp16(__fp16 *output, const __fp16 *activation, const __fp16 *weight, const __fp16 *scales, - size_t n_row_tiles, size_t n_col_tiles, size_t n_dot_tiles) { +static void core_dot_chunk_fp16(__fp16 * output, + const __fp16 * activation, + const __fp16 * weight, + size_t n_row_tiles, + size_t n_col_tiles, + size_t n_dot_tiles) { for (size_t r = 0; r < n_row_tiles; ++r) { - const __fp16 *row_tiles = activation + r * n_dot_tiles * HMX_FP16_TILE_N_ELMS; + const __fp16 * row_tiles = activation + r * n_dot_tiles * HMX_FP16_TILE_N_ELMS; for (size_t c = 0; c < n_col_tiles; ++c) { Q6_mxclracc_hf(); - const __fp16 *col_tiles = weight + c * n_dot_tiles * HMX_FP16_TILE_N_ELMS; + const __fp16 * col_tiles = weight + c * n_dot_tiles * HMX_FP16_TILE_N_ELMS; for (size_t k = 0; k < n_dot_tiles; ++k) { hmx_load_tile_pair_fp16(row_tiles, col_tiles); row_tiles += HMX_FP16_TILE_N_ELMS; col_tiles += HMX_FP16_TILE_N_ELMS; } - __fp16 *out_tile = output + (r * n_col_tiles + c) * HMX_FP16_TILE_N_ELMS; + __fp16 * out_tile = output + (r * n_col_tiles + c) * HMX_FP16_TILE_N_ELMS; hmx_consume_accumulator_fp16(out_tile); } } } -static void transfer_output_chunk_fp16_to_fp32(float *restrict dst, const __fp16 *restrict vtcm_src, int n_rows, int n_cols, int n) { +static void transfer_output_chunk_fp16_to_fp32(float * restrict dst, + const __fp16 * restrict vtcm_src, + int n_rows, + int n_cols, + int n) { assert(n_cols % HMX_FP16_TILE_N_COLS == 0); const int n_col_tiles = n_cols / HMX_FP16_TILE_N_COLS; @@ -779,9 +787,9 @@ int hmx_mat_mul_permuted_w16a32_batched(struct htp_context *ctx, const hmx_matmu for (int g = 0; g < group_size; ++g) { TIMER_START(hmx_core); { - const __fp16 *vtcm_act_g = vtcm_activation + (size_t) g * act_head_stride; - core_dot_chunk_fp16(vtcm_output, vtcm_act_g, vtcm_weight, vtcm_scales, - n_row_tiles, n_col_tiles, params->k / 32); + const __fp16 * vtcm_act_g = vtcm_activation + (size_t) g * act_head_stride; + core_dot_chunk_fp16(vtcm_output, vtcm_act_g, vtcm_weight, n_row_tiles, n_col_tiles, + params->k / 32); } TIMER_STOP(hmx_core); @@ -948,7 +956,7 @@ int hmx_mat_mul_permuted_w16a32(struct htp_context *ctx, float *restrict dst, co TIMER_START(hmx_core); { - core_dot_chunk_fp16(vtcm_output, vtcm_activation, vtcm_weight, vtcm_scales, n_row_tiles, n_col_tiles, k / 32); + core_dot_chunk_fp16(vtcm_output, vtcm_activation, vtcm_weight, n_row_tiles, n_col_tiles, k / 32); } TIMER_STOP(hmx_core); @@ -1136,7 +1144,7 @@ int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, float *restrict ds TIMER_START(hmx_core); { - core_dot_chunk_fp16(vtcm_output, vtcm_activation, vtcm_weight, vtcm_scales, n_row_tiles, n_col_tiles, k / 32); + core_dot_chunk_fp16(vtcm_output, vtcm_activation, vtcm_weight, n_row_tiles, n_col_tiles, k / 32); } TIMER_STOP(hmx_core); @@ -1196,8 +1204,9 @@ int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, float *restrict ds } // C0 - core_dot_chunk_fp16((__fp16 *) vtcm_output_bufs[0], (__fp16 *) vtcm_activation, (__fp16 *) vtcm_weight_bufs[0], vtcm_scales, - hmx_ceil_div(n_rows, HMX_FP16_TILE_N_ROWS), hmx_ceil_div(n_cols_A0, HMX_FP16_TILE_N_COLS), k / HMX_FP16_TILE_N_ROWS); + core_dot_chunk_fp16((__fp16 *) vtcm_output_bufs[0], (__fp16 *) vtcm_activation, + (__fp16 *) vtcm_weight_bufs[0], hmx_ceil_div(n_rows, HMX_FP16_TILE_N_ROWS), + hmx_ceil_div(n_cols_A0, HMX_FP16_TILE_N_COLS), k / HMX_FP16_TILE_N_ROWS); // B1 if (1 < n_chunk_cnt) { @@ -1228,8 +1237,10 @@ int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, float *restrict ds // issue C_{i+1} if (i + 1 < n_chunk_cnt) { - core_dot_chunk_fp16((__fp16 *) vtcm_output_bufs[(i + 1) % 2], (__fp16 *) vtcm_activation, (__fp16 *) vtcm_weight_bufs[(i + 1) % 2], vtcm_scales, - hmx_ceil_div(n_rows, HMX_FP16_TILE_N_ROWS), hmx_ceil_div(n_cols_p1, HMX_FP16_TILE_N_COLS), k / HMX_FP16_TILE_N_ROWS); + core_dot_chunk_fp16((__fp16 *) vtcm_output_bufs[(i + 1) % 2], (__fp16 *) vtcm_activation, + (__fp16 *) vtcm_weight_bufs[(i + 1) % 2], + hmx_ceil_div(n_rows, HMX_FP16_TILE_N_ROWS), + hmx_ceil_div(n_cols_p1, HMX_FP16_TILE_N_COLS), k / HMX_FP16_TILE_N_ROWS); } // compute D_{i} From 33d943163a1cb8293909a362e804925bdf75f065 Mon Sep 17 00:00:00 2001 From: chraac Date: Mon, 23 Mar 2026 22:06:56 +0800 Subject: [PATCH 06/22] refactor core_mma_chunk_fp16 and mat_mul_qk_0_d16a32 to use size_t for tile counts --- ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c index abeba30be25..d6179666988 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c @@ -1276,12 +1276,12 @@ int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, float *restrict ds // C += AB void core_mma_chunk_fp16(__fp16 *c, const __fp16 *a, const __fp16 *b, const __fp16 *col_scales, const __fp16 *eye_tile, - int n_row_tiles, int n_col_tiles, int n_dot_tiles, bool zero_init) { + size_t n_row_tiles, size_t n_col_tiles, size_t n_dot_tiles, bool zero_init) { hmx_set_output_scales(col_scales); - for (int i = 0; i < n_row_tiles; ++i) { - for (int j = 0; j < n_col_tiles; ++j) { + for (size_t i = 0; i < n_row_tiles; ++i) { + for (size_t j = 0; j < n_col_tiles; ++j) { Q6_mxclracc_hf(); const __fp16 *row_tiles = a + i * n_dot_tiles * HMX_FP16_TILE_N_ELMS; @@ -1292,8 +1292,8 @@ void core_mma_chunk_fp16(__fp16 *c, const __fp16 *a, const __fp16 *b, const __fp hmx_load_tile_pair_fp16(accum_tile, eye_tile); } - for (int k = 0; k < n_dot_tiles; ++k) { - int offset = k * HMX_FP16_TILE_N_ELMS; + for (size_t k = 0; k < n_dot_tiles; ++k) { + size_t offset = k * HMX_FP16_TILE_N_ELMS; hmx_load_tile_pair_fp16(row_tiles + offset, col_tiles + offset); } @@ -1372,7 +1372,7 @@ void transfer_activation_chunk_threaded(struct htp_context *ctx, __fp16 *dst, co } int mat_mul_qk_0_d16a32_out_stationary(struct htp_context *ctx, float *restrict out, const float *restrict x, const uint8_t *restrict w, int m, - int k, int n, int weight_type) { + int k, int n, const int weight_type) { // Runtime check -- k >= 16384 exceeds 2D DMA limit if (k >= 16384) { FARF(HIGH, "%s: k=%d exceeds 2D DMA limit", __func__, k); @@ -1448,7 +1448,7 @@ int mat_mul_qk_0_d16a32_out_stationary(struct htp_context *ctx, float *restrict const int n_col_tiles = hmx_ceil_div(n_blk_sz, HMX_FP16_TILE_N_COLS); for (size_t kk = 0; kk < k; kk += K_BLOCK_SIZE) { - size_t k_blk_sz = hex_smin(k - kk, K_BLOCK_SIZE); + const size_t k_blk_sz = hex_smin(k - kk, K_BLOCK_SIZE); TIMER_START(fetch); // fetch activation block into VTCM @@ -1464,6 +1464,7 @@ int mat_mul_qk_0_d16a32_out_stationary(struct htp_context *ctx, float *restrict } // fetch weight block into VTCM (x4x2 sub-block: quants + scales) + const size_t sub_row_stride = get_x4x2_row_stride(weight_type, k_blk_sz); { qweight_fetch_task_state_t s; @@ -1471,7 +1472,6 @@ int mat_mul_qk_0_d16a32_out_stationary(struct htp_context *ctx, float *restrict const int blk_start = kk / QK_Q4_0x4x2; const int nb_sub = (k_blk_sz + QK_Q4_0x4x2 - 1) / QK_Q4_0x4x2; const int full_qrow = is_q4 ? (k / 2) : k; - const size_t sub_row_stride = get_x4x2_row_stride(weight_type, k_blk_sz); s.dst = vtcm_scratch0; s.src = w + nc * row_stride; @@ -1507,7 +1507,6 @@ int mat_mul_qk_0_d16a32_out_stationary(struct htp_context *ctx, float *restrict dma_queue_pop(ctx->dma[0]); // vtcm_scratch0 is used to store the qweight chunk // worker_pool_run_func already returned, so fetch is done - const size_t sub_row_stride = get_x4x2_row_stride(weight_type, k_blk_sz); dequantize_x4x2_weight_chunk_to_fp16_tiles(ctx, vtcm_weight, vtcm_scratch0, n_blk_sz, k_blk_sz, sub_row_stride, weight_type); } From 3a97015f81c2ac0f7168436027cba09362b448fa Mon Sep 17 00:00:00 2001 From: chraac Date: Mon, 23 Mar 2026 22:16:35 +0800 Subject: [PATCH 07/22] wip --- ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c index d6179666988..67a66ed895e 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c @@ -1281,20 +1281,22 @@ void core_mma_chunk_fp16(__fp16 *c, const __fp16 *a, const __fp16 *b, const __fp hmx_set_output_scales(col_scales); for (size_t i = 0; i < n_row_tiles; ++i) { + const __fp16 *row_base = a + i * n_dot_tiles * HMX_FP16_TILE_N_ELMS; + __fp16 *res_base = c + i * n_col_tiles * HMX_FP16_TILE_N_ELMS; for (size_t j = 0; j < n_col_tiles; ++j) { Q6_mxclracc_hf(); - const __fp16 *row_tiles = a + i * n_dot_tiles * HMX_FP16_TILE_N_ELMS; const __fp16 *col_tiles = b + j * n_dot_tiles * HMX_FP16_TILE_N_ELMS; - - __fp16 *accum_tile = c + (i * n_col_tiles + j) * HMX_FP16_TILE_N_ELMS; + const __fp16 *row_tiles = row_base; + __fp16 *accum_tile = res_base + j * HMX_FP16_TILE_N_ELMS; if (!zero_init) { hmx_load_tile_pair_fp16(accum_tile, eye_tile); } for (size_t k = 0; k < n_dot_tiles; ++k) { - size_t offset = k * HMX_FP16_TILE_N_ELMS; - hmx_load_tile_pair_fp16(row_tiles + offset, col_tiles + offset); + hmx_load_tile_pair_fp16(row_tiles, col_tiles); + row_tiles += HMX_FP16_TILE_N_ELMS; + col_tiles += HMX_FP16_TILE_N_ELMS; } hmx_consume_accumulator_fp16(accum_tile); From 6e291d8e8cbb21f943d128a463b2f90f4a1061e7 Mon Sep 17 00:00:00 2001 From: chraac Date: Mon, 23 Mar 2026 22:20:02 +0800 Subject: [PATCH 08/22] wip --- ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c index 67a66ed895e..ff12b61e8a3 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c @@ -486,10 +486,11 @@ static void core_dot_chunk_fp16(__fp16 * output, size_t n_col_tiles, size_t n_dot_tiles) { for (size_t r = 0; r < n_row_tiles; ++r) { - const __fp16 * row_tiles = activation + r * n_dot_tiles * HMX_FP16_TILE_N_ELMS; + const __fp16 * row_base = activation + r * n_dot_tiles * HMX_FP16_TILE_N_ELMS; for (size_t c = 0; c < n_col_tiles; ++c) { Q6_mxclracc_hf(); + const __fp16 * row_tiles = row_base; const __fp16 * col_tiles = weight + c * n_dot_tiles * HMX_FP16_TILE_N_ELMS; for (size_t k = 0; k < n_dot_tiles; ++k) { hmx_load_tile_pair_fp16(row_tiles, col_tiles); From f43d68cac7b7c27f163e5cc4b03f979e653ab124 Mon Sep 17 00:00:00 2001 From: chraac Date: Tue, 24 Mar 2026 10:54:34 +0800 Subject: [PATCH 09/22] refactor transfer_output_chunk_fp16_to_fp32 to use size_t for dimensions --- ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c | 30 +++++++++++----------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c index ff12b61e8a3..4522b6afdea 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c @@ -506,29 +506,29 @@ static void core_dot_chunk_fp16(__fp16 * output, static void transfer_output_chunk_fp16_to_fp32(float * restrict dst, const __fp16 * restrict vtcm_src, - int n_rows, - int n_cols, - int n) { + size_t n_rows, + size_t n_cols, + size_t n) { assert(n_cols % HMX_FP16_TILE_N_COLS == 0); - const int n_col_tiles = n_cols / HMX_FP16_TILE_N_COLS; + const size_t n_col_tiles = n_cols / HMX_FP16_TILE_N_COLS; const HVX_Vector one = hvx_vec_splat_f16(1.0); - for (int r = 0; r < n_rows; r += 2) { - int r0 = r / HMX_FP16_TILE_N_ROWS; - int r1 = r % HMX_FP16_TILE_N_ROWS; + for (size_t r = 0; r < n_rows; r += 2) { + const size_t r0 = r / HMX_FP16_TILE_N_ROWS; + const size_t r1 = (r % HMX_FP16_TILE_N_ROWS) / 2; // index of the row pair within the tile + const __fp16 *row_base = vtcm_src + r0 * n_col_tiles * HMX_FP16_TILE_N_ELMS; + float *output_row_base = dst + r * n; // global memory row base for row r (and r+1) #pragma unroll(4) - for (int c = 0; c < n_cols; c += HMX_FP16_TILE_N_COLS) { - int c0 = c / HMX_FP16_TILE_N_COLS; - - const __fp16 *tile = vtcm_src + (r0 * n_col_tiles + c0) * HMX_FP16_TILE_N_ELMS; - - HVX_Vector v = ((const HVX_Vector *) tile)[r1 / 2]; + for (size_t c = 0; c < n_cols; c += HMX_FP16_TILE_N_COLS) { + const size_t c0 = c / HMX_FP16_TILE_N_COLS; + const __fp16 *tile = row_base + c0 * HMX_FP16_TILE_N_ELMS; + HVX_Vector v = ((const HVX_Vector *) tile)[r1]; HVX_VectorPair vp = Q6_Wqf32_vmpy_VhfVhf(v, one); - volatile HVX_Vector *pv_out0 = (volatile HVX_Vector *) (dst + (r * n + c + 0)); - volatile HVX_Vector *pv_out1 = (volatile HVX_Vector *) (dst + (r * n + c + n)); // next row in global memory + volatile HVX_Vector *pv_out0 = (volatile HVX_Vector *) (output_row_base + c + 0); + volatile HVX_Vector *pv_out1 = (volatile HVX_Vector *) (output_row_base + c + n); // next row in global memory *pv_out0 = Q6_Vsf_equals_Vqf32(Q6_V_lo_W(vp)); if (r + 1 < n_rows) { From 42bd08ca5c1175a1614e4e84126b81d04cebc791 Mon Sep 17 00:00:00 2001 From: chraac Date: Tue, 24 Mar 2026 10:56:56 +0800 Subject: [PATCH 10/22] refactor core_dot_chunk_fp16 to use size_t for tile row stride calculation --- ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c index 4522b6afdea..4bcd2e4d1d0 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c @@ -485,13 +485,14 @@ static void core_dot_chunk_fp16(__fp16 * output, size_t n_row_tiles, size_t n_col_tiles, size_t n_dot_tiles) { + const size_t tile_row_stride = n_dot_tiles * HMX_FP16_TILE_N_ELMS; for (size_t r = 0; r < n_row_tiles; ++r) { - const __fp16 * row_base = activation + r * n_dot_tiles * HMX_FP16_TILE_N_ELMS; + const __fp16 * row_base = activation + r * tile_row_stride; for (size_t c = 0; c < n_col_tiles; ++c) { Q6_mxclracc_hf(); const __fp16 * row_tiles = row_base; - const __fp16 * col_tiles = weight + c * n_dot_tiles * HMX_FP16_TILE_N_ELMS; + const __fp16 * col_tiles = weight + c * tile_row_stride; for (size_t k = 0; k < n_dot_tiles; ++k) { hmx_load_tile_pair_fp16(row_tiles, col_tiles); row_tiles += HMX_FP16_TILE_N_ELMS; From ee951463c7bf9da4da90649c593d42b8c8091020 Mon Sep 17 00:00:00 2001 From: chraac Date: Tue, 24 Mar 2026 23:06:44 +0800 Subject: [PATCH 11/22] wip --- ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c index 4bcd2e4d1d0..e82c0b7bce7 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c @@ -511,14 +511,14 @@ static void transfer_output_chunk_fp16_to_fp32(float * restrict dst, size_t n_cols, size_t n) { assert(n_cols % HMX_FP16_TILE_N_COLS == 0); - const size_t n_col_tiles = n_cols / HMX_FP16_TILE_N_COLS; + const size_t tile_row_stride = (n_cols / HMX_FP16_TILE_N_COLS) * HMX_FP16_TILE_N_ELMS; const HVX_Vector one = hvx_vec_splat_f16(1.0); for (size_t r = 0; r < n_rows; r += 2) { const size_t r0 = r / HMX_FP16_TILE_N_ROWS; const size_t r1 = (r % HMX_FP16_TILE_N_ROWS) / 2; // index of the row pair within the tile - const __fp16 *row_base = vtcm_src + r0 * n_col_tiles * HMX_FP16_TILE_N_ELMS; + const __fp16 *row_base = vtcm_src + r0 * tile_row_stride; float *output_row_base = dst + r * n; // global memory row base for row r (and r+1) #pragma unroll(4) From 91d88a35cfcb36de47e0a843d1854d0a55f3232e Mon Sep 17 00:00:00 2001 From: chraac Date: Fri, 27 Mar 2026 22:02:02 +0800 Subject: [PATCH 12/22] refactor hmx_mat_mul functions to use hvx_vec_splat_f16 for column scales initialization --- ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c index e82c0b7bce7..54816e4d3e0 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c @@ -698,7 +698,7 @@ int hmx_mat_mul_permuted_w16a32_batched(struct htp_context *ctx, const hmx_matmu return hmx_mat_mul_permuted_w16a32_batched_legacy(ctx, params); } - hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // fp16: 1.0 + hmx_init_column_scales(vtcm_scales, hvx_vec_splat_f16(1.0f)); // fp16: 1.0 FARF(MEDIUM, "%s: grouped path m=%d k=%d n=%d group=%d streams=%d mc=%zu nc=%zu vtcm=%zu/%zu", __func__, params->m, params->k, params->n, group_size, params->ne13, @@ -874,7 +874,7 @@ int hmx_mat_mul_permuted_w16a32(struct htp_context *ctx, float *restrict dst, co return -1; } - hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // fp16: 1.0 + hmx_init_column_scales(vtcm_scales, hvx_vec_splat_f16(1.0f)); // fp16: 1.0 FARF(MEDIUM, "%s: m=%d k=%d n=%d mc=%zu nc=%zu vtcm=%zu/%zu", __func__, m, k, n, m_chunk_n_rows, n_chunk_n_cols, @@ -1073,7 +1073,7 @@ int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, float *restrict ds return -1; } - hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // fp16: 1.0 + hmx_init_column_scales(vtcm_scales, hvx_vec_splat_f16(1.0f)); // fp16: 1.0 FARF(MEDIUM, "%s: m=%d k=%d n=%d wtype=%d pipe=%d mc=%zu nc=%zu vtcm=%zu/%zu", __func__, m, k, n, weight_type, use_pipeline, @@ -1434,7 +1434,7 @@ int mat_mul_qk_0_d16a32_out_stationary(struct htp_context *ctx, float *restrict v = Q6_V_vror_VR(v, VLEN - 8); } } - hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // fp16: 1.0 + hmx_init_column_scales(vtcm_scales, hvx_vec_splat_f16(1.0f)); // fp16: 1.0 TIMER_DEFINE(fetch); TIMER_DEFINE(act_load); From 55d7258194f9394ea64aad31ecbb52b05f7c922d Mon Sep 17 00:00:00 2001 From: chraac Date: Fri, 27 Mar 2026 23:32:09 +0800 Subject: [PATCH 13/22] refactor hmx_mat_mul_permuted_w16a32_batched to streamline scale setting and locking --- ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c index 54816e4d3e0..2ff031785de 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c @@ -716,6 +716,9 @@ int hmx_mat_mul_permuted_w16a32_batched(struct htp_context *ctx, const hmx_matmu const size_t fp16_row_bytes = (size_t) params->k * sizeof(__fp16); const size_t weight_row_bytes = (size_t) params->weight_stride * sizeof(__fp16); + HAP_compute_res_hmx_lock(ctx->vtcm_rctx); + hmx_set_output_scales(vtcm_scales); + for (int b3 = 0; b3 < params->ne13; ++b3) { for (int b2_base = 0; b2_base < params->ne12; b2_base += group_size) { const __fp16 *weight_group = hmx_matmul_weight_batch_ptr(params, b2_base, b3); @@ -760,9 +763,6 @@ int hmx_mat_mul_permuted_w16a32_batched(struct htp_context *ctx, const hmx_matmu fp16_row_bytes, weight_row_bytes, fp16_row_bytes, n_cols_first); } - HAP_compute_res_hmx_lock(ctx->vtcm_rctx); - hmx_set_output_scales(vtcm_scales); - for (size_t nc = 0; nc < (size_t) params->n; nc += n_chunk_n_cols) { const size_t n_cols = hex_smin((size_t) params->n - nc, n_chunk_n_cols); const size_t n_col_tiles = hmx_ceil_div((int) n_cols, HMX_FP16_TILE_N_COLS); @@ -803,12 +803,12 @@ int hmx_mat_mul_permuted_w16a32_batched(struct htp_context *ctx, const hmx_matmu TIMER_STOP(output_store); } } - - HAP_compute_res_hmx_unlock(ctx->vtcm_rctx); } } } + HAP_compute_res_hmx_unlock(ctx->vtcm_rctx); + TIMER_STOP(total); #if defined(ENABLE_PROFILE_TIMERS) From 362c62ce0c4147f03dee0f4fdc411bbd9dd3cc3f Mon Sep 17 00:00:00 2001 From: chraac Date: Mon, 30 Mar 2026 12:18:31 +0800 Subject: [PATCH 14/22] refactor core_dot_chunk_fp16 to improve tile stride calculations for output --- ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c index 2ff031785de..33835c24d40 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c @@ -486,8 +486,10 @@ static void core_dot_chunk_fp16(__fp16 * output, size_t n_col_tiles, size_t n_dot_tiles) { const size_t tile_row_stride = n_dot_tiles * HMX_FP16_TILE_N_ELMS; + const size_t tile_col_stride = n_col_tiles * HMX_FP16_TILE_N_ELMS; for (size_t r = 0; r < n_row_tiles; ++r) { const __fp16 * row_base = activation + r * tile_row_stride; + __fp16 * out_tile = output + r * tile_col_stride; for (size_t c = 0; c < n_col_tiles; ++c) { Q6_mxclracc_hf(); @@ -499,8 +501,8 @@ static void core_dot_chunk_fp16(__fp16 * output, col_tiles += HMX_FP16_TILE_N_ELMS; } - __fp16 * out_tile = output + (r * n_col_tiles + c) * HMX_FP16_TILE_N_ELMS; hmx_consume_accumulator_fp16(out_tile); + out_tile += HMX_FP16_TILE_N_ELMS; } } } From 7c1a5a398342a82b5f24c69ec5ecb4e84ef9ece8 Mon Sep 17 00:00:00 2001 From: chraac Date: Sat, 4 Apr 2026 21:33:00 +0800 Subject: [PATCH 15/22] refactor hmx_mat_mul functions to use Q6_V_vsplat_R for column scales initialization --- ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c index fecfaf3911c..f65e5568482 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c @@ -874,7 +874,7 @@ int hmx_mat_mul_permuted_w16a32_batched(struct htp_context *ctx, const hmx_matmu return hmx_mat_mul_permuted_w16a32_batched_legacy(ctx, params); } - hmx_init_column_scales(vtcm_scales, hvx_vec_splat_f16(1.0f)); // fp16: 1.0 + hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // scale: 1.0, bias: 0.0 in FP16 FARF(MEDIUM, "%s: grouped path m=%d k=%d n=%d group=%d streams=%d mc=%zu nc=%zu vtcm=%zu/%zu", __func__, params->m, params->k, params->n, group_size, params->ne13, @@ -1050,7 +1050,7 @@ int hmx_mat_mul_permuted_w16a32(struct htp_context *ctx, float *restrict dst, co return -1; } - hmx_init_column_scales(vtcm_scales, hvx_vec_splat_f16(1.0f)); // fp16: 1.0 + hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // scale: 1.0, bias: 0.0 in FP16 FARF(MEDIUM, "%s: m=%d k=%d n=%d mc=%zu nc=%zu vtcm=%zu/%zu", __func__, m, k, n, m_chunk_n_rows, n_chunk_n_cols, @@ -1249,7 +1249,7 @@ int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, float *restrict ds return -1; } - hmx_init_column_scales(vtcm_scales, hvx_vec_splat_f16(1.0f)); // fp16: 1.0 + hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // scale: 1.0, bias: 0.0 in FP16 FARF(MEDIUM, "%s: m=%d k=%d n=%d wtype=%d pipe=%d mc=%zu nc=%zu vtcm=%zu/%zu", __func__, m, k, n, weight_type, use_pipeline, @@ -1610,7 +1610,8 @@ int mat_mul_qk_0_d16a32_out_stationary(struct htp_context *ctx, float *restrict v = Q6_V_vror_VR(v, VLEN - 8); } } - hmx_init_column_scales(vtcm_scales, hvx_vec_splat_f16(1.0f)); // fp16: 1.0 + + hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // scale: 1.0, bias: 0.0 in FP16 TIMER_DEFINE(fetch); TIMER_DEFINE(act_load); From 2f37db78c1b9063db839d2e4156f4484282bf4a8 Mon Sep 17 00:00:00 2001 From: chraac Date: Wed, 15 Apr 2026 09:19:50 +0800 Subject: [PATCH 16/22] fix compiling error --- ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c index dba8d62ab46..8ff3f3fee61 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c @@ -678,7 +678,7 @@ static void core_dot_chunk_fp16(__fp16 *restrict output, const __fp16 *restrict __builtin_assume(n_dot_tiles > 0); Q6_bias_mxmem2_A((void *)scales); - + for (int r = 0; r < n_row_tiles; ++r) { for (size_t c = 0; c < n_col_tiles; ++c) { Q6_mxclracc_hf(); @@ -944,7 +944,6 @@ int hmx_mat_mul_permuted_w16a32_batched(struct htp_context *ctx, const hmx_matmu const size_t weight_row_bytes = (size_t) params->weight_stride * sizeof(__fp16); HAP_compute_res_hmx_lock(ctx->vtcm_rctx); - hmx_set_output_scales(vtcm_scales); for (int b3 = 0; b3 < params->ne13; ++b3) { for (int b2_base = 0; b2_base < params->ne12; b2_base += group_size) { @@ -1017,7 +1016,7 @@ int hmx_mat_mul_permuted_w16a32_batched(struct htp_context *ctx, const hmx_matmu TIMER_START(hmx_core); { const __fp16 * vtcm_act_g = vtcm_activation + (size_t) g * act_head_stride; - core_dot_chunk_fp16(vtcm_output, vtcm_act_g, vtcm_weight, n_row_tiles, n_col_tiles, + core_dot_chunk_fp16(vtcm_output, vtcm_act_g, vtcm_weight, vtcm_scales, n_row_tiles, n_col_tiles, params->k / 32); } TIMER_STOP(hmx_core); @@ -1118,7 +1117,6 @@ int hmx_mat_mul_permuted_w16a32(struct htp_context *ctx, float *restrict dst, co TIMER_START(total); HAP_compute_res_hmx_lock(ctx->vtcm_rctx); - hmx_set_output_scales(vtcm_scales); for (size_t mr = 0; mr < m; mr += m_chunk_n_rows) { // transfer activation matrix chunk into VTCM @@ -1187,7 +1185,7 @@ int hmx_mat_mul_permuted_w16a32(struct htp_context *ctx, float *restrict dst, co TIMER_START(hmx_core); { - core_dot_chunk_fp16(vtcm_output, vtcm_activation, vtcm_weight, n_row_tiles, n_col_tiles, k / 32); + core_dot_chunk_fp16(vtcm_output, vtcm_activation, vtcm_weight, vtcm_scales, n_row_tiles, n_col_tiles, k / 32); } TIMER_STOP(hmx_core); @@ -1376,7 +1374,7 @@ int hmx_mat_mul_permuted_qk_0_d16a32(struct htp_context *ctx, float *restrict ds TIMER_START(hmx_core); { - core_dot_chunk_fp16(vtcm_output, vtcm_activation, vtcm_weight, n_row_tiles, n_col_tiles, k / 32); + core_dot_chunk_fp16(vtcm_output, vtcm_activation, vtcm_weight, vtcm_scales, n_row_tiles, n_col_tiles, k / 32); } TIMER_STOP(hmx_core); From 87ab485cd5bf5b6848ede3b892de1d34d7319abf Mon Sep 17 00:00:00 2001 From: chraac Date: Wed, 15 Apr 2026 21:44:06 +0800 Subject: [PATCH 17/22] wip --- ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c index 8ff3f3fee61..376a6b27ab5 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c @@ -1697,7 +1697,6 @@ int mat_mul_qk_0_d16a32_out_stationary(struct htp_context *ctx, float *restrict v = Q6_V_vror_VR(v, VLEN - 8); } } - hmx_init_column_scales(vtcm_scales, Q6_V_vsplat_R(0x3c00)); // scale: 1.0, bias: 0.0 in FP16 TIMER_DEFINE(fetch); @@ -1739,7 +1738,6 @@ int mat_mul_qk_0_d16a32_out_stationary(struct htp_context *ctx, float *restrict const int blk_start = kk / QK_Q4_0x4x2; const int nb_sub = (k_blk_sz + QK_Q4_0x4x2 - 1) / QK_Q4_0x4x2; const int full_qrow = (weight_type == HTP_TYPE_Q8_0) ? k : (k / 2); - const size_t sub_row_stride = get_x4x2_row_stride(weight_type, k_blk_sz); const int scale_blk_size = (weight_type == HTP_TYPE_MXFP4) ? HMX_X4X2_MXFP4_EBLK_SIZE : HMX_X4X2_DBLK_SIZE; From 2a12467cf8fce2e45f177917503067fb89cef874 Mon Sep 17 00:00:00 2001 From: chraac Date: Wed, 15 Apr 2026 21:51:51 +0800 Subject: [PATCH 18/22] optimize row and column tile indexing in core_mma_chunk_fp16 function --- ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c index 376a6b27ab5..dbca8220fab 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c @@ -1520,13 +1520,14 @@ void core_mma_chunk_fp16(__fp16 *restrict c, const __fp16 *restrict a, const __f Q6_bias_mxmem2_A((void *)col_scales); + const size_t dot_tile_stride = n_dot_tiles * HMX_FP16_TILE_N_ELMS; for (size_t i = 0; i < n_row_tiles; ++i) { - const __fp16 *row_base = a + i * n_dot_tiles * HMX_FP16_TILE_N_ELMS; + const __fp16 *row_base = a + i * dot_tile_stride; __fp16 *res_base = c + i * n_col_tiles * HMX_FP16_TILE_N_ELMS; for (size_t j = 0; j < n_col_tiles; ++j) { Q6_mxclracc_hf(); - const __fp16 *col_tiles = b + j * n_dot_tiles * HMX_FP16_TILE_N_ELMS; + const __fp16 *col_tiles = b + j * dot_tile_stride; const __fp16 *row_tiles = row_base; __fp16 *accum_tile = res_base + j * HMX_FP16_TILE_N_ELMS; if (!zero_init) { From cde679eff79c4a28dd2d89d32f710015e09592b6 Mon Sep 17 00:00:00 2001 From: chraac Date: Wed, 15 Apr 2026 21:56:52 +0800 Subject: [PATCH 19/22] wip --- ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c index dbca8220fab..e802abfcf7c 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c @@ -943,8 +943,6 @@ int hmx_mat_mul_permuted_w16a32_batched(struct htp_context *ctx, const hmx_matmu const size_t fp16_row_bytes = (size_t) params->k * sizeof(__fp16); const size_t weight_row_bytes = (size_t) params->weight_stride * sizeof(__fp16); - HAP_compute_res_hmx_lock(ctx->vtcm_rctx); - for (int b3 = 0; b3 < params->ne13; ++b3) { for (int b2_base = 0; b2_base < params->ne12; b2_base += group_size) { const __fp16 *weight_group = hmx_matmul_weight_batch_ptr(params, b2_base, b3); @@ -989,6 +987,8 @@ int hmx_mat_mul_permuted_w16a32_batched(struct htp_context *ctx, const hmx_matmu fp16_row_bytes, weight_row_bytes, fp16_row_bytes, n_cols_first); } + HAP_compute_res_hmx_lock(ctx->vtcm_rctx); + for (size_t nc = 0; nc < (size_t) params->n; nc += n_chunk_n_cols) { const size_t n_cols = hex_smin((size_t) params->n - nc, n_chunk_n_cols); const size_t n_col_tiles = hmx_ceil_div((int) n_cols, HMX_FP16_TILE_N_COLS); @@ -1029,12 +1029,12 @@ int hmx_mat_mul_permuted_w16a32_batched(struct htp_context *ctx, const hmx_matmu TIMER_STOP(output_store); } } + + HAP_compute_res_hmx_unlock(ctx->vtcm_rctx); } } } - HAP_compute_res_hmx_unlock(ctx->vtcm_rctx); - TIMER_STOP(total); #if defined(ENABLE_PROFILE_TIMERS) From 086ccf53f0a89ebef49e2fae6a61f3ffc44877cc Mon Sep 17 00:00:00 2001 From: chraac Date: Wed, 15 Apr 2026 22:33:50 +0800 Subject: [PATCH 20/22] Revert "wip" This reverts commit cde679eff79c4a28dd2d89d32f710015e09592b6. --- ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c index e802abfcf7c..dbca8220fab 100644 --- a/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c +++ b/ggml/src/ggml-hexagon/htp/hmx-matmul-ops.c @@ -943,6 +943,8 @@ int hmx_mat_mul_permuted_w16a32_batched(struct htp_context *ctx, const hmx_matmu const size_t fp16_row_bytes = (size_t) params->k * sizeof(__fp16); const size_t weight_row_bytes = (size_t) params->weight_stride * sizeof(__fp16); + HAP_compute_res_hmx_lock(ctx->vtcm_rctx); + for (int b3 = 0; b3 < params->ne13; ++b3) { for (int b2_base = 0; b2_base < params->ne12; b2_base += group_size) { const __fp16 *weight_group = hmx_matmul_weight_batch_ptr(params, b2_base, b3); @@ -987,8 +989,6 @@ int hmx_mat_mul_permuted_w16a32_batched(struct htp_context *ctx, const hmx_matmu fp16_row_bytes, weight_row_bytes, fp16_row_bytes, n_cols_first); } - HAP_compute_res_hmx_lock(ctx->vtcm_rctx); - for (size_t nc = 0; nc < (size_t) params->n; nc += n_chunk_n_cols) { const size_t n_cols = hex_smin((size_t) params->n - nc, n_chunk_n_cols); const size_t n_col_tiles = hmx_ceil_div((int) n_cols, HMX_FP16_TILE_N_COLS); @@ -1029,12 +1029,12 @@ int hmx_mat_mul_permuted_w16a32_batched(struct htp_context *ctx, const hmx_matmu TIMER_STOP(output_store); } } - - HAP_compute_res_hmx_unlock(ctx->vtcm_rctx); } } } + HAP_compute_res_hmx_unlock(ctx->vtcm_rctx); + TIMER_STOP(total); #if defined(ENABLE_PROFILE_TIMERS) From 36b9bb1a879ee1a52bce1497fadc675ec8c4aa38 Mon Sep 17 00:00:00 2001 From: chraac Date: Thu, 16 Apr 2026 21:21:41 +0800 Subject: [PATCH 21/22] Add size limit check for HAP_mmap in htp_iface_mmap and drop_mmap functions --- ggml/src/ggml-hexagon/htp/main.c | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index d71c97ed292..86c3bc88e3e 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -118,7 +118,11 @@ AEEResult htp_iface_close(remote_handle64 handle) { // release the mmaps (if any) for (uint32_t i=0; immap[i].size) { +#if __HVX_ARCH__ > 73 HAP_munmap2((void *) ctx->mmap[i].base, ctx->mmap[i].size); +#else + HAP_munmap((void *) ctx->mmap[i].base, ctx->mmap[i].size); +#endif ctx->mmap[i].size = 0; ctx->mmap[i].base = NULL; ctx->mmap[i].fd = -1; @@ -173,8 +177,16 @@ AEEResult htp_iface_mmap(remote_handle64 handle, int fd, uint32_t size, uint32_t struct htp_mmap *m = &ctx->mmap[i]; if (!m->size) { FARF(HIGH, "mmap : fd %u size %u pinned %u", fd, size, pinned); - +#if __HVX_ARCH__ > 73 void *va = HAP_mmap2(NULL, size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0); +#else + if (size > 2147483648u) { // HAP_mmap has a size limit of 2GB + FARF(ERROR, "mmap failed : size %u exceeds 2GB limit for HAP_mmap", (uint32_t) size); + abort(); // can't do much else at this point + } + + void *va = HAP_mmap(NULL, size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0); +#endif if (va == (void*)-1) { FARF(ERROR, "mmap failed : va %p fd %u size %u", va, fd, (uint32_t) size); return AEE_EFAILED; @@ -202,7 +214,11 @@ AEEResult htp_iface_munmap(remote_handle64 handle, int fd) { struct htp_mmap *m = &ctx->mmap[i]; if (fd < 0 || m->fd == fd) { FARF(HIGH, "unmmap : base %p fd %u size %u", (void*) m->base, m->fd, (uint32_t) m->size); +#if __HVX_ARCH__ > 73 HAP_munmap2((void *) m->base, m->size); +#else + HAP_munmap((void *) m->base, m->size); +#endif m->size = 0; m->base = NULL; m->fd = -1; @@ -526,7 +542,11 @@ static inline bool reuse_buf(struct htp_context *ctx, uint32_t *m_reuse, struct static inline void drop_mmap(struct htp_context *ctx, struct htp_mmap *m) { if (m->size && !m->pinned) { FARF(HIGH, "unmap : fd %u base %p size %u pinned %u", m->fd, (void*) m->base, (uint32_t) m->size, m->pinned); +#if __HVX_ARCH__ > 73 HAP_munmap2((void *) m->base, m->size); +#else + HAP_munmap((void *) m->base, m->size); +#endif m->size = 0; m->base = 0; m->fd = -1; @@ -540,7 +560,16 @@ static inline void mmap_buf(struct htp_context *ctx, struct htp_buf_desc *b) { for (uint32_t i=0; i < HTP_MAX_MMAPS; i++) { struct htp_mmap *m = &ctx->mmap[i]; if (!m->size) { +#if __HVX_ARCH__ > 73 void *va = HAP_mmap2(NULL, b->size, HAP_PROT_READ | HAP_PROT_WRITE, 0, b->fd, 0); +#else + if (b->size > 2147483648u) { // HAP_mmap has a size limit of 2GB + FARF(ERROR, "mmap failed : size %u exceeds 2GB limit for HAP_mmap", (uint32_t) b->size); + abort(); // can't do much else at this point + } + + void *va = HAP_mmap(NULL, b->size, HAP_PROT_READ | HAP_PROT_WRITE, 0, b->fd, 0); +#endif if (va == (void*)-1) { FARF(ERROR, "mmap failed : va %p fd %u size %u", va, b->fd, (uint32_t) b->size); abort(); // can't do much else at this point From 37a2797d2cd36f65184156118841f7cd4476b1e1 Mon Sep 17 00:00:00 2001 From: chraac Date: Thu, 16 Apr 2026 21:36:07 +0800 Subject: [PATCH 22/22] wip --- ggml/src/ggml-hexagon/htp/htp-ops.h | 2 ++ ggml/src/ggml-hexagon/htp/main.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/htp-ops.h b/ggml/src/ggml-hexagon/htp/htp-ops.h index fa84b674cd2..79b5ecd2270 100644 --- a/ggml/src/ggml-hexagon/htp/htp-ops.h +++ b/ggml/src/ggml-hexagon/htp/htp-ops.h @@ -98,6 +98,8 @@ enum htp_op_code { #define HTP_OP_MAX_VMEM (3221225472u) #endif +#define HTP_MMAP_MAX_VMEM (2147483648u) + enum htp_tensor_flags { HTP_TENSOR_COMPUTE = (1U << 0), // Tensor buffer temporal compute data (not weights) HTP_TENSOR_FLUSHED = (1U << 1) // Tensor buffer has been flushed (set by the NPU) diff --git a/ggml/src/ggml-hexagon/htp/main.c b/ggml/src/ggml-hexagon/htp/main.c index 86c3bc88e3e..5091623a653 100644 --- a/ggml/src/ggml-hexagon/htp/main.c +++ b/ggml/src/ggml-hexagon/htp/main.c @@ -180,7 +180,7 @@ AEEResult htp_iface_mmap(remote_handle64 handle, int fd, uint32_t size, uint32_t #if __HVX_ARCH__ > 73 void *va = HAP_mmap2(NULL, size, HAP_PROT_READ | HAP_PROT_WRITE, 0, fd, 0); #else - if (size > 2147483648u) { // HAP_mmap has a size limit of 2GB + if (size > HTP_MMAP_MAX_VMEM) { // HAP_mmap has a size limit of 2GB FARF(ERROR, "mmap failed : size %u exceeds 2GB limit for HAP_mmap", (uint32_t) size); abort(); // can't do much else at this point } @@ -563,7 +563,7 @@ static inline void mmap_buf(struct htp_context *ctx, struct htp_buf_desc *b) { #if __HVX_ARCH__ > 73 void *va = HAP_mmap2(NULL, b->size, HAP_PROT_READ | HAP_PROT_WRITE, 0, b->fd, 0); #else - if (b->size > 2147483648u) { // HAP_mmap has a size limit of 2GB + if (b->size > HTP_MMAP_MAX_VMEM) { // HAP_mmap has a size limit of 2GB FARF(ERROR, "mmap failed : size %u exceeds 2GB limit for HAP_mmap", (uint32_t) b->size); abort(); // can't do much else at this point }