Skip to content
Merged
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions ggml/src/ggml-cpu/kleidiai/kleidiai.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1719,6 +1719,7 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {

return true;
}

return false;
}

Expand All @@ -1727,6 +1728,20 @@ class extra_buffer_type : ggml::cpu::extra_buffer_type {
if (op->src[0]->buffer && op->src[0]->buffer->buft == ggml_backend_cpu_kleidiai_buffer_type()) {
return (ggml::cpu::tensor_traits *) op->src[0]->extra;
} else {
// KleidiAI only has kernels for Q4_0 and Q8_0. For a quantized weight of any
// other type (K-quants, IQ) it declines the op and returns nullptr below, so
// KleidiAI does not accelerate it. Another CPU backend may still take the op,
// and this can run during graph planning, so the message says what KleidiAI
// did rather than what ends up executing. Warn once per process.
if (ggml_is_quantized(op->src[0]->type) &&
op->src[0]->type != GGML_TYPE_Q4_0 && op->src[0]->type != GGML_TYPE_Q8_0) {
static std::atomic<bool> warned(false);
if (!warned.exchange(true)) {
GGML_LOG_WARN("kleidiai: no kernel for tensor type %s, not accelerated by KleidiAI "
"(kernels available for Q4_0 and Q8_0)\n",
ggml_type_name(op->src[0]->type));
}
}
if (op->src[0]->type != GGML_TYPE_F16) {
return nullptr;
}
Expand Down
Loading