From 5532a6451df64dca7c43394c42148f4028624033 Mon Sep 17 00:00:00 2001 From: Alex Qin Date: Thu, 10 Sep 2026 00:20:38 -0400 Subject: [PATCH] HIP : route Q6_K through MMQ up to ne11 = 1024 on RDNA 3.5 On gfx1151 the Q6_K MMQ kernel runs at a flat 16-17 TFLOPS while the dequantize + hipBLASLt path drops to 11-12 TFLOPS once K >= 8192 (the ffn_down shapes of 8B models) and pays a full f16 dequantize round trip per prefill pass. At ne11 = 2048 the two paths trade places per shape (K = 4096 favours hipBLASLt, K >= 12288 favours MMQ), so the gate stops at 1024. Keep RDNA 3.0 and other targets on the existing threshold. Assisted-by: Claude --- ggml/src/ggml-cuda/mmq.cu | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu index 70d59cee82e0..6e6b77b85ca4 100644 --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -553,6 +553,11 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t case GGML_TYPE_Q2_K: return ne11 <= 128; case GGML_TYPE_Q6_K: + // RDNA 3.5 (gfx1151): dequantize + hipBLASLt loses to MMQ up to ne11 = 1024; at 2048 the two + // paths trade places per shape (K = 4096 favours hipBLASLt, K >= 12288 favours MMQ), see PR. + if (GGML_CUDA_CC_IS_RDNA3_5(cc)) { + return ne11 <= 1024; + } return ne11 <= (GGML_CUDA_CC_IS_RDNA3_0(cc) ? 128 : 256); case GGML_TYPE_IQ2_XS: case GGML_TYPE_IQ2_S: