Skip to content

opencl: tune the quant paths for Intel Xe-LP GPUs to improve its TG and PP performance #26438

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lhez merged 4 commits into from
Aug 31, 2026
Merged

opencl: tune the quant paths for Intel Xe-LP GPUs to improve its TG and PP performance #26438

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
d87f431
opencl: Q4_K/Q5_K mul_mv N_DST 4->8 on Intel for 2x activation reuse
wanghqc Jul 4, 2026
df11be2
opencl: Q4_K mul_mm 8x8 tile fot Intel
wanghqc Jul 4, 2026
2fee355
opencl: Q5_K mul_mm 8x8 tile for Intel
wanghqc Jul 4, 2026
b0e0fd1
opencl: Q4_K mul_mv N_DST 8->16 for Intel
wanghqc Jul 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions ggml/src/ggml-opencl/ggml-opencl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19947,7 +19947,8 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
}

kernel = backend_ctx->kernel_mul_mm_q4_k_f32_l4_lm;
nth0 = 128; // calculated as (BM*BN)/(TM*TN)
// (BM*BN)/(TM*TN): Intel uses an 8x8 microtile (WG=64), others 4x8 (WG=128)
nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128;

int batch_stride_a = ne00*ne01;
int batch_stride_b = ne10*ne11;
Expand Down Expand Up @@ -19991,7 +19992,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
}

kernel = backend_ctx->kernel_mul_mm_q5_k_f32_l4_lm;
nth0 = 128; // calculated as (BM*BN)/(TM*TN)
nth0 = (backend_ctx->gpu_family == INTEL) ? 64 : 128; // Intel 8x8 microtile

int batch_stride_a = ne00*ne01;
int batch_stride_b = ne10*ne11;
Expand Down Expand Up @@ -20805,7 +20806,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
if (backend_ctx->gpu_family == INTEL) {
nth0 = 16;
nth1 = 1;
ndst = 4;
ndst = 16; // 8->16 rows per subgroup — matches N_DST in mul_mv_q4_k_f32_flat.cl (32 spills)
} else if (backend_ctx->gpu_family == ADRENO) {
nth0 = 64;
nth1 = 2;
Expand Down Expand Up @@ -20879,7 +20880,7 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
if (backend_ctx->gpu_family == INTEL) {
nth0 = 16;
nth1 = 1;
ndst = 4;
ndst = 8; // 4->8 rows per subgroup (2x activation reuse)
} else if (backend_ctx->gpu_family == ADRENO) {
nth0 = 64;
nth1 = 2;
Expand Down
10 changes: 10 additions & 0 deletions ggml/src/ggml-opencl/kernels/mul_mm_q4_k_f32_l4_lm.cl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,23 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable

#ifdef cl_intel_required_subgroup_size
#define INTEL_GPU 1
#endif

#define LOAD_VEC_A 4
#define LOAD_VEC_B 4

#define BM 64
#define BN 64
#define BK 32
#ifdef INTEL_GPU
// Intel Xe iGPU: 8x8 microtile (WG = BM*BN/(TM*TN) = 64) — ~+12% pp512 vs 4x8
#define TM 8
#define TN 8
#else
#define TM 4
#define TN 8
#endif

kernel void kernel_mul_mm_q4_k_f32_l4_lm(
global uchar4 * src0_q,
Expand Down
10 changes: 10 additions & 0 deletions ggml/src/ggml-opencl/kernels/mul_mm_q5_k_f32_l4_lm.cl
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,23 @@
#pragma OPENCL EXTENSION cl_khr_fp16 : enable

#ifdef cl_intel_required_subgroup_size
#define INTEL_GPU 1
#endif

#define LOAD_VEC_A 4
#define LOAD_VEC_B 4

#define BM 64
#define BN 64
#define BK 32
#ifdef INTEL_GPU
// Intel Xe iGPU: 8x8 microtile (WG=64)
#define TM 8
#define TN 8
#else
#define TM 4
#define TN 8
#endif

kernel void kernel_mul_mm_q5_k_f32_l4_lm(
global uchar4 * src0_q,
Expand Down
2 changes: 1 addition & 1 deletion ggml/src/ggml-opencl/kernels/mul_mv_q4_k_f32_flat.cl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ typedef struct {
#undef N_SIMDWIDTH

#ifdef INTEL_GPU
#define N_DST 4 // number of rows each SIMD group works on
#define N_DST 16 // number of rows each SIMD group works on (Intel: 8->16, 2x further activation reuse; 32 spills registers)
#define N_SIMDGROUP 1 // number of SIMD groups in a thread group
#define N_SIMDWIDTH 16 // SIMD group size
#elif defined (ADRENO_GPU)
Expand Down
2 changes: 1 addition & 1 deletion ggml/src/ggml-opencl/kernels/mul_mv_q5_k_f32_flat.cl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ typedef struct {
#undef N_SIMDWIDTH

#ifdef INTEL_GPU
#define N_DST 4
#define N_DST 8 // Intel: 4->8 for 2x activation reuse (see mul_mv_q4_k_f32_flat.cl)
#define N_SIMDGROUP 1
#define N_SIMDWIDTH 16
#elif defined(ADRENO_GPU)
Expand Down
Loading