From af8a1ad5bca74e8890d1f45d8576ca3707b606d2 Mon Sep 17 00:00:00 2001 From: David Friehs Date: Wed, 29 Jul 2026 06:21:04 +0200 Subject: [PATCH] cuda: unblock mmq for MoE on sm_60 --- ggml/src/ggml-cuda/mmq.cu | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ggml/src/ggml-cuda/mmq.cu b/ggml/src/ggml-cuda/mmq.cu index 707437ea3e52..ea05584a9f9d 100644 --- a/ggml/src/ggml-cuda/mmq.cu +++ b/ggml/src/ggml-cuda/mmq.cu @@ -314,6 +314,22 @@ bool ggml_cuda_should_use_mmq(enum ggml_type type, int cc, int64_t ne11, int64_t } if (ggml_cuda_highest_compiled_arch(cc) < GGML_CUDA_CC_DP4A) { + // for MoE, mmq can be faster even without native dp4a + // TODO: check if cards older than pascal might benefit from this as well + if (cc == GGML_CUDA_CC_PASCAL && n_experts > 0) { + // TODO: figure out why these are so slow on sm_60 + // ref: https://github.com/ggml-org/llama.cpp/pull/26264#issuecomment-5141087554 + switch (type) { + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + case GGML_TYPE_Q6_K: + return false; + default: + return true; + } + } + return false; }