diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index eedd03e48d54..be38bf00a274 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -5275,6 +5275,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_NVFP4], "dequant_nvfp4", dequant_nvfp4_len, dequant_nvfp4_data, "main", 2, 5 * sizeof(uint32_t), {256 * 16, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_TQ3_1S], "dequant_tq3_1s", dequant_tq3_1s_len, dequant_tq3_1s_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_TQ4_1S], "dequant_tq4_1s", dequant_tq4_1s_len, dequant_tq4_1s_data, "main", 2, 5 * sizeof(uint32_t), {256 * 32, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_dequant[GGML_TYPE_TURBO3_0], "dequant_turbo3_0", dequant_turbo3_0_len, dequant_turbo3_0_data, "main", 2, 5 * sizeof(uint32_t), {128, 1, 1}, {}, 1); // get_rows ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_F32 ], "get_rows_f32", get_rows_f32_len, get_rows_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); @@ -5303,6 +5304,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_IQ4_NL], "get_rows_iq4_nl", get_rows_iq4_nl_len, get_rows_iq4_nl_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_MXFP4], "get_rows_mxfp4", get_rows_mxfp4_len, get_rows_mxfp4_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_NVFP4], "get_rows_nvfp4", get_rows_nvfp4_len, get_rows_nvfp4_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_TURBO3_0], "get_rows_turbo3_0", get_rows_turbo3_0_len, get_rows_turbo3_0_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows[GGML_TYPE_I32], "get_rows_i32", get_rows_i32_len, get_rows_i32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_F32 ], "get_rows_f32_f32", get_rows_f32_f32_len, get_rows_f32_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), { 512, 1, 1}, {}, 1); @@ -5331,6 +5333,7 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_IQ4_NL], "get_rows_iq4_nl_f32", get_rows_iq4_nl_f32_len, get_rows_iq4_nl_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_MXFP4], "get_rows_mxfp4_f32", get_rows_mxfp4_f32_len, get_rows_mxfp4_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_NVFP4], "get_rows_nvfp4_f32", get_rows_nvfp4_f32_len, get_rows_nvfp4_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_get_rows_f32[GGML_TYPE_TURBO3_0], "get_rows_turbo3_0_f32", get_rows_turbo3_0_f32_len, get_rows_turbo3_0_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {1024, 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_get_rows_back_f32, "get_rows_back_f32", get_rows_back_f32_len, get_rows_back_f32_data, "main", 3, sizeof(vk_op_binary_push_constants), {256, 1, 1}, {}, 1, true); ggml_vk_create_pipeline(device, device->pipeline_matmul_split_k_reduce, "split_k_reduce", split_k_reduce_len, split_k_reduce_data, "main", 2, 2 * sizeof(uint32_t), {256 * 4, 1, 1}, {}, 1); @@ -5416,7 +5419,8 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_IQ4_NL], "set_rows_" #src "_iq4_nl" #itype, set_rows_ ## src ## _iq4_nl ## itype ## _len, set_rows_ ## src ## _iq4_nl ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true); \ ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_TURBO2_0], "set_rows_" #src "_turbo2_0" #itype, set_rows_ ## src ## _turbo2_0 ## itype ## _len, set_rows_ ## src ## _turbo2_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true, true, 32u); \ ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_TURBO3_0], "set_rows_" #src "_turbo3_0" #itype, set_rows_ ## src ## _turbo3_0 ## itype ## _len, set_rows_ ## src ## _turbo3_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true, true, 32u); \ - ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_TURBO4_0], "set_rows_" #src "_turbo4_0" #itype, set_rows_ ## src ## _turbo4_0 ## itype ## _len, set_rows_ ## src ## _turbo4_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true, true, 32u); + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_TURBO4_0], "set_rows_" #src "_turbo4_0" #itype, set_rows_ ## src ## _turbo4_0 ## itype ## _len, set_rows_ ## src ## _turbo4_0 ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true, true, 32u); \ + ggml_vk_create_pipeline(device, device->pipeline_set_rows ## itype [src_idx][GGML_TYPE_TQ4_1S], "set_rows_" #src "_tq4_1s" #itype, set_rows_ ## src ## _tq4_1s ## itype ## _len, set_rows_ ## src ## _tq4_1s ## itype ## _data, "main", 3, sizeof(vk_op_binary_push_constants), {1, 1, 1}, {1}, 1, true, true, 32u); SET_ROWS(0, f32, _i32) SET_ROWS(0, f32, _i64) @@ -5433,6 +5437,8 @@ static void ggml_vk_load_shaders(vk_device& device, vk_pipeline requested) { ggml_vk_create_pipeline(device, device->pipeline_cpy_quant_f32[GGML_TYPE_Q5_1], "cpy_q5_1_f32", cpy_q5_1_f32_len, cpy_q5_1_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {(uint32_t)ggml_blck_size(GGML_TYPE_Q5_1), 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_cpy_quant_f32[GGML_TYPE_Q8_0], "cpy_q8_0_f32", cpy_q8_0_f32_len, cpy_q8_0_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {(uint32_t)ggml_blck_size(GGML_TYPE_Q8_0), 1, 1}, {}, 1); ggml_vk_create_pipeline(device, device->pipeline_cpy_quant_f32[GGML_TYPE_IQ4_NL], "cpy_iq4_nl_f32", cpy_iq4_nl_f32_len, cpy_iq4_nl_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {(uint32_t)ggml_blck_size(GGML_TYPE_IQ4_NL), 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_quant_f32[GGML_TYPE_TURBO3_0], "cpy_turbo3_0_f32", cpy_turbo3_0_f32_len, cpy_turbo3_0_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {(uint32_t)ggml_blck_size(GGML_TYPE_TURBO3_0), 1, 1}, {}, 1); + ggml_vk_create_pipeline(device, device->pipeline_cpy_quant_f32[GGML_TYPE_TQ4_1S], "cpy_tq4_1s_f32", cpy_tq4_1s_f32_len, cpy_tq4_1s_f32_data, "main", 2, sizeof(vk_op_unary_push_constants), {(uint32_t)ggml_blck_size(GGML_TYPE_TQ4_1S), 1, 1}, {}, 1); auto get_suffix = [](bool src0_f16, bool src1_f16, bool dst_f16) { std::string s; @@ -7520,9 +7526,9 @@ static vk_pipeline ggml_vk_get_to_fp16(ggml_backend_vk_context * ctx, ggml_type break; default: return nullptr; - } + } - return ctx->device->pipeline_dequant[type]; + return ctx->device->pipeline_dequant[type]; } static vk_matmul_pipeline ggml_vk_get_mul_mat_mat_pipeline(ggml_backend_vk_context * ctx, ggml_type src0_type, ggml_type src1_type, ggml_prec prec) { @@ -8872,6 +8878,8 @@ static vk_pipeline ggml_vk_get_cpy_pipeline(ggml_backend_vk_context * ctx, const case GGML_TYPE_Q5_0: case GGML_TYPE_Q5_1: case GGML_TYPE_Q8_0: + case GGML_TYPE_TURBO3_0: + case GGML_TYPE_TQ4_1S: case GGML_TYPE_IQ4_NL: return ctx->device->pipeline_cpy_quant_f32[src->type]; default: @@ -12070,7 +12078,9 @@ static void ggml_vk_op_f32(ggml_backend_vk_context * ctx, vk_context& subctx, co case GGML_OP_SET_ROWS: { uint32_t ne = ggml_nelements(src0); - if (dst->type == GGML_TYPE_TURBO2_0 || dst->type == GGML_TYPE_TURBO3_0 || dst->type == GGML_TYPE_TURBO4_0) { + if (dst->type == GGML_TYPE_TQ4_1S) { + ne = ne / 32; + } else if (dst->type == GGML_TYPE_TURBO2_0 || dst->type == GGML_TYPE_TURBO3_0 || dst->type == GGML_TYPE_TURBO4_0) { // turbo set_rows shaders: 128 threads per WG, one full QK=128 block per WG ne = ne / 128; } else if (ggml_is_quantized(dst->type)) { @@ -17973,6 +17983,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_TYPE_IQ4_NL: case GGML_TYPE_MXFP4: case GGML_TYPE_NVFP4: + case GGML_TYPE_TURBO3_0: case GGML_TYPE_I32: return true; default: @@ -17992,6 +18003,10 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm && (op->src[0]->ne[0] % 128 != 0)) { return false; } + // TQ4_1S uses 32-element blocks + if (op->type == GGML_TYPE_TQ4_1S && (op->src[0]->ne[0] % 32 != 0)) { + return false; + } switch (op->type) { case GGML_TYPE_F32: case GGML_TYPE_F16: @@ -18007,6 +18022,7 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_TYPE_TURBO2_0: case GGML_TYPE_TURBO3_0: case GGML_TYPE_TURBO4_0: + case GGML_TYPE_TQ4_1S: return true; default: return false; @@ -18049,6 +18065,8 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm case GGML_TYPE_Q5_1: case GGML_TYPE_Q8_0: case GGML_TYPE_IQ4_NL: + case GGML_TYPE_TURBO3_0: + case GGML_TYPE_TQ4_1S: return true; default: break; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp b/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp index 2795a0de4a3e..4643757dee65 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/copy_to_quant.comp @@ -415,13 +415,16 @@ void main() { const uint gpr = p.ne00 / 128; if (gpr == 0) return; - if (g >= p.ne / 128) return; + if (g >= gpr * p.ne21 * p.ne22 * p.ne23) return; uint tmp = g; const uint ig = tmp % gpr; tmp /= gpr; const uint i01 = tmp % p.ne01; tmp /= p.ne01; - const uint i02 = tmp % p.ne12; - const uint i03 = tmp / p.ne12; + const uint i02 = tmp % p.ne02; + const uint i03 = tmp / p.ne02; + + // Bail if dispatch slot maps past source extents (broadcast expansion) + if (i01 >= p.ne01 || i02 >= p.ne02 || i03 >= p.ne03) return; const uint sb = src0_idx(ig * 128, i01, i02, i03) + get_aoffset(); const uint i1 = data_i[src1_idx(i01, fastmod(i02, p.ne11), fastmod(i03, p.ne12), 0) + get_boffset()] DATA_I_SWIZZLE; @@ -546,13 +549,16 @@ void main() { const uint gpr = p.ne00 / 128; if (gpr == 0) return; - if (g >= p.ne / 128) return; + if (g >= gpr * p.ne21 * p.ne22 * p.ne23) return; uint tmp = g; const uint ig = tmp % gpr; tmp /= gpr; const uint i01 = tmp % p.ne01; tmp /= p.ne01; - const uint i02 = tmp % p.ne12; - const uint i03 = tmp / p.ne12; + const uint i02 = tmp % p.ne02; + const uint i03 = tmp / p.ne02; + + // Bail if dispatch slot maps past source extents (broadcast expansion) + if (i01 >= p.ne01 || i02 >= p.ne02 || i03 >= p.ne03) return; const uint sb = src0_idx(ig * 128, i01, i02, i03) + get_aoffset(); const uint i1 = data_i[src1_idx(i01, fastmod(i02, p.ne11), fastmod(i03, p.ne12), 0) + get_boffset()] DATA_I_SWIZZLE; @@ -667,13 +673,16 @@ void main() { const uint gpr = p.ne00 / 128; if (gpr == 0) return; - if (g >= p.ne / 128) return; + if (g >= gpr * p.ne21 * p.ne22 * p.ne23) return; uint tmp = g; const uint ig = tmp % gpr; tmp /= gpr; const uint i01 = tmp % p.ne01; tmp /= p.ne01; - const uint i02 = tmp % p.ne12; - const uint i03 = tmp / p.ne12; + const uint i02 = tmp % p.ne02; + const uint i03 = tmp / p.ne02; + + // Bail if dispatch slot maps past source extents (broadcast expansion) + if (i01 >= p.ne01 || i02 >= p.ne02 || i03 >= p.ne03) return; const uint sb = src0_idx(ig * 128, i01, i02, i03) + get_aoffset(); const uint i1 = data_i[src1_idx(i01, fastmod(i02, p.ne11), fastmod(i03, p.ne12), 0) + get_boffset()] DATA_I_SWIZZLE; @@ -747,13 +756,16 @@ void main() { const uint gpr = p.ne00 / 32; // blocks per row if (gpr == 0) return; - if (g >= p.ne / 32) return; + if (g >= gpr * p.ne21 * p.ne22 * p.ne23) return; uint tmp = g; const uint ig = tmp % gpr; tmp /= gpr; const uint i01 = tmp % p.ne01; tmp /= p.ne01; - const uint i02 = tmp % p.ne12; - const uint i03 = tmp / p.ne12; + const uint i02 = tmp % p.ne02; + const uint i03 = tmp / p.ne02; + + // Bail if dispatch slot maps past source extents (broadcast expansion) + if (i01 >= p.ne01 || i02 >= p.ne02 || i03 >= p.ne03) return; const uint sb = src0_idx(ig * 32, i01, i02, i03) + get_aoffset(); const uint i1 = data_i[src1_idx(i01, fastmod(i02, p.ne11), fastmod(i03, p.ne12), 0) + get_boffset()] DATA_I_SWIZZLE; diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp index 51b601691505..7f3f2a7ecf6f 100644 --- a/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp +++ b/ggml/src/ggml-vulkan/vulkan-shaders/vulkan-shaders-gen.cpp @@ -590,6 +590,7 @@ void matmul_shaders(bool fp16, MatMulIdType matmul_id_type, bool coopmat, bool c if (tname == "bf16") { continue; } + // TQ4_1S uses dedicated mul_mat_vec kernel; no generic matmul needed std::string data_a_key = "DATA_A_" + to_uppercase(tname); // For aligned matmul loads @@ -746,7 +747,7 @@ void process_shaders() { for (const auto& tname : type_names) { // mul mat vec std::string data_a_key = "DATA_A_" + to_uppercase(tname); - std::string shader = (string_ends_with(tname, "_k") || string_starts_with(tname, "iq1_") || string_starts_with(tname, "iq2_") || string_starts_with(tname, "iq3_")) ? "mul_mat_vec_" + tname + ".comp" : "mul_mat_vec.comp"; + std::string shader = (string_ends_with(tname, "_k") || string_starts_with(tname, "iq1_") || string_starts_with(tname, "iq2_") || string_starts_with(tname, "iq3_") ) ? "mul_mat_vec_" + tname + ".comp" : "mul_mat_vec.comp"; string_to_spv("mul_mat_vec_" + tname + "_f32_f32", shader, merge_maps(base_dict, {{data_a_key, "1"}, {"B_TYPE", "float"}, {"B_TYPEV2", "vec2"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}})); string_to_spv("mul_mat_vec_" + tname + "_f16_f32", shader, merge_maps(base_dict, {{data_a_key, "1"}, {"B_TYPE", "float16_t"}, {"B_TYPEV2", "f16vec2"}, {"B_TYPEV4", "f16vec4"}, {"D_TYPE", "float"}})); @@ -803,6 +804,11 @@ void process_shaders() { string_to_spv("get_rows_" + tname + "_f32", shader, merge_maps(base_dict, {{"TEMP_TYPE", "FLOAT_TYPE"}, {data_a_key, "1"}, {"B_TYPE", "int"}, {"D_TYPE", "float"}})); } + // TurboQuant3 KV-cache dequant and get_rows (KV-only type, not in type_names) + string_to_spv("dequant_turbo3_0", "dequant_turbo3_0.comp", merge_maps(base_dict, {{"DATA_A_TURBO3_0", "1"}, {"D_TYPE", "float16_t"}})); + string_to_spv("get_rows_turbo3_0", "get_rows_quant.comp", merge_maps(base_dict, {{"TEMP_TYPE", "FLOAT_TYPE"}, {"DATA_A_TURBO3_0", "1"}, {"B_TYPE", "int"}, {"D_TYPE", "float16_t"}})); + string_to_spv("get_rows_turbo3_0_f32", "get_rows_quant.comp", merge_maps(base_dict, {{"TEMP_TYPE", "FLOAT_TYPE"}, {"DATA_A_TURBO3_0", "1"}, {"B_TYPE", "int"}, {"D_TYPE", "float"}})); + string_to_spv("get_rows_i32", "get_rows.comp", {{"TEMP_TYPE", "uint"}, {"A_TYPE", "uint"}, {"B_TYPE", "int"}, {"D_TYPE", "uint"}}); string_to_spv("mul_mat_vec_p021_f16_f32_subgroup_add", "mul_mat_vec_p021.comp", {{"A_TYPE", "float16_t"}, {"A_TYPEV4", "f16vec4"}, {"B_TYPE", "float"}, {"B_TYPEV4", "vec4"}, {"D_TYPE", "float"}, {"USE_SUBGROUP_ADD", "1"}}); @@ -843,9 +849,13 @@ void process_shaders() { string_to_spv("cpy_f32_" + t, "copy_to_quant.comp", {{"DATA_A_" + to_uppercase(t), "1"}, {"S_TYPE", "float"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}}); string_to_spv("cpy_" + t + "_f32", "copy_from_quant.comp", {{"DATA_A_" + to_uppercase(t), "1"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}}); } + // turbo3_0 copy-from-quant only; copy-to-quant (cpy_f32_turbo3_0) omitted because the non-SET_ROWS quantize() path lacks the WHT transform + string_to_spv("cpy_turbo3_0_f32", "copy_from_quant.comp", {{"DATA_A_TURBO3_0", "1"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}}); + // tq4_1s copy-from-quant only; copy-to-quant requires WHT forward (handled in SET_ROWS path) + string_to_spv("cpy_tq4_1s_f32", "copy_from_quant.comp", {{"DATA_A_TQ4_1S", "1"}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}}); for (auto src : {std::pair{"f32", "float"}, std::pair{"f16", "float16_t"}}) { - for (std::string dst : {"f32", "f16", "bf16", "q1_0", "q2_0", "q4_0", "q4_1", "q5_0", "q5_1", "q8_0", "iq4_nl", "turbo2_0", "turbo3_0", "turbo4_0"}) { + for (std::string dst : {"f32", "f16", "bf16", "q1_0", "q2_0", "q4_0", "q4_1", "q5_0", "q5_1", "q8_0", "iq4_nl", "turbo2_0", "turbo3_0", "turbo4_0", "tq4_1s"}) { string_to_spv("set_rows_" + std::string(src.first) + "_" + dst + "_i32", "copy_to_quant.comp", {{"SET_ROWS", "1"}, {"DATA_A_" + to_uppercase(dst), "1"}, {"B_TYPE", "uint"}, {"B_SIZE", "32"}, {"S_TYPE", src.second}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}}); string_to_spv("set_rows_" + std::string(src.first) + "_" + dst + "_i64", "copy_to_quant.comp", {{"SET_ROWS", "1"}, {"DATA_A_" + to_uppercase(dst), "1"}, {"B_TYPE", "uvec2"}, {"B_SIZE", "64"}, {"S_TYPE", src.second}, {"D_TYPE", "float"}, {"FLOAT_TYPE", "float"}}); } diff --git a/scripts/ppl_test.sh b/scripts/ppl_test.sh new file mode 100755 index 000000000000..a17963d32bd3 --- /dev/null +++ b/scripts/ppl_test.sh @@ -0,0 +1,189 @@ +#!/bin/bash +# PPL Test Suite for OSCAR2 / KV-cache comparison +# Constraint: oscar2 can only mix with f16/oscar2 or oscar2/oscar2 (not q8_0) +# Compare against: f16/f16 (baseline) and q8_0/turbo4 (comparison) +# +# Protocol (from oscar2-llama-debug skill): +# -c --chunks -1 --no-warmup -ctk -ctv -fa on -ngl 0 -t 32 -f +# Using -c 512 --chunks 1 for speed (full ctx=4096 chunks=-1 takes 5+ min per run) + +set -euo pipefail + +CLI="/mnt/storage/Projects/llama-cpp-turboquant/build/bin/llama-perplexity" +LIB="/mnt/storage/Projects/llama-cpp-turboquant/build/lib" +export LD_LIBRARY_PATH="${LIB}:${LD_LIBRARY_PATH:-}" +export CUDA_VISIBLE_DEVICES="0" + +DATASET="/mnt/storage/blackbeard/wikitext-2-raw/wiki.test.raw" +OUTDIR="/mnt/storage/Projects/llama-cpp-turboquant/ppl_results" +mkdir -p "$OUTDIR" + +CTX=512 +CHUNKS=1 +THREADS=32 + +TS=$(date +%Y%m%d_%H%M%S) +SUMMARY="${OUTDIR}/ppl_summary_${TS}.tsv" +LOGDIR="${OUTDIR}/logs_${TS}" +mkdir -p "$LOGDIR" + +echo -e "TEST_ID\tModel\tConfig\tCTK\tCTV\tPPL\tLogTps\tEvalTps\tCacheInfo\tExitCode" > "$SUMMARY" + +run_ppl() { + local test_id="$1" + local model_name="$2" + local model_path="$3" + local ctk="$4" + local ctv="$5" + shift 5 + + local logfile="${LOGDIR}/test_${test_id}.log" + echo "" + echo "======================================================================" + echo " TEST ${test_id}: ${model_name} (KV K=${ctk}, V=${ctv})" + echo " Log: ${logfile}" + echo "======================================================================" + + # Build env vars from remaining args + local env_vars=() + local extra_args=() + for arg in "$@"; do + if [[ "$arg" == *=* ]]; then + extra_args+=("$arg") + fi + done + + # Run with any env vars passed as KEY=value before the command + env -i PATH="/usr/bin:/bin:/usr/local/bin" \ + LD_LIBRARY_PATH="${LIB}:${LD_LIBRARY_PATH:-}" \ + CUDA_VISIBLE_DEVICES="0" \ + OMP_NUM_THREADS="$THREADS" \ + ${env_vars[@]+"${env_vars[@]}"} \ + "$CLI" \ + -m "$model_path" \ + -c "$CTX" \ + --chunks "$CHUNKS" \ + --no-warmup \ + -ctk "$ctk" \ + -ctv "$ctv" \ + -fa on \ + -ngl 0 \ + -t "$THREADS" \ + -f "$DATASET" \ + 2>&1 | tee "$logfile" + + local rc=${PIPESTATUS[0]} + + if [ $rc -ne 0 ]; then + echo " EXIT CODE: $rc" + echo -e "${test_id}\t${model_name}\t${ctk}/${ctv}\t${ctk}\t${ctv}\tFAILED(exit:${rc})\t\t\t\t${rc}" >> "$SUMMARY" + return + fi + + # Parse PPL from log + local ppl=$(grep -i "Final estimate" "$logfile" | grep -oP 'PPL = \K[0-9.]+' || echo "N/A") + if [ "$ppl" = "N/A" ]; then + ppl=$(grep -i "perplexity" "$logfile" | tail -1 | grep -oP 'PPL = \K[0-9.]+' || echo "N/A") + fi + + local log_tps=$(grep -oP 'Prompt:\s*\K[0-9.]+' "$logfile" | head -1 || echo "N/A") + local eval_tps=$(grep -oP 'Generation:\s*\K[0-9.]+' "$logfile" | tail -1 || echo "N/A") + + # Also try llama_perf_context_eval format + if [ "$eval_tps" = "N/A" ]; then + eval_tps=$(grep "llama_perf_context_eval" "$logfile" | tail -1 | grep -oP 't/s.*\K[0-9.]+' || echo "N/A") + fi + + # Extract cache info + local cache_info=$(grep -i "KV cache.*dtype" "$logfile" | tail -1 | grep -oP 'K=\S+ V=\S+' || echo "") + + echo " Result: PPL=${ppl}, Cache=${cache_info}" + echo -e "${test_id}\t${model_name}\t${ctk}/${ctv}\t${ctk}\t${ctv}\t${ppl}\t${log_tps}\t${eval_tps}\t${cache_info}\t${rc}" >> "$SUMMARY" +} + +# ============================================================ +# Model definitions +# ============================================================ + +MODELS_DIR="/mnt/storage/models/oscar-rotations" +KW_DIR="/mnt/storage/models/qwen3.6/35B" + +GEMMA12B="${MODELS_DIR}/gemma-4-12b-it-UD-Q8_K_XL-rot-kv.gguf" +GEMMA26B="${MODELS_DIR}/gemma-4-26B-A4B-it-UD-Q5_K_S-rot-kv.gguf" +QWEN27B="${MODELS_DIR}/Qwen3.6-27B-UD-Q5_K_XL-rot-kv.gguf" +KWAIPLOT="${KW_DIR}/Kwaipilot_KAT-Coder-V2.5-Dev-Q5_K_S.gguf" + +echo "=================================================================" +echo " PPL Test Suite Started: $(date)" +echo " Dataset: ${DATASET}" +echo " Context: ${CTX}, Chunks: ${CHUNKS}, Threads: ${THREADS}" +echo " Build: /mnt/storage/Projects/llama-cpp-turboquant" +echo "=================================================================" + +# ============================================================ +# Test matrix: +# f16/f16 (baseline) — all 4 models +# oscar2/oscar2 — 3 rot-kv models (Qwen27B, Gemma12B, Gemma26B) +# oscar2/f16 — 3 rot-kv models (allowed mix) +# f16/oscar2 — 3 rot-kv models (allowed mix) +# q8_0/q8_0 — all 4 models +# q8_0/turbo4 — all 4 models +# f16/oscar2 on Kwaipilot — expected to fail (no rot-kv) +# ============================================================ + +echo "=== Phase 1: f16/f16 baselines ===" + +run_ppl "g12_f16_f16" "gemma4-12b" "$GEMMA12B" "f16" "f16" +run_ppl "g26_f16_f16" "gemma4-26b-a4b" "$GEMMA26B" "f16" "f16" +run_ppl "q27_f16_f16" "qwen3.6-27b" "$QWEN27B" "f16" "f16" +run_ppl "kw_f16_f16" "kwaipilot-35b" "$KWAIPLOT" "f16" "f16" + +echo "" +echo "=== Phase 2: oscar2/oscar2 on rot-kv models ===" + +run_ppl "g12_oscar2_oscar2" "gemma4-12b" "$GEMMA12B" "oscar2" "oscar2" +run_ppl "g26_oscar2_oscar2" "gemma4-26b-a4b" "$GEMMA26B" "oscar2" "oscar2" +run_ppl "q27_oscar2_oscar2" "qwen3.6-27b" "$QWEN27B" "oscar2" "oscar2" + +echo "" +echo "=== Phase 3: oscar2/f16 mix on rot-kv models ===" + +run_ppl "g12_oscar2_f16" "gemma4-12b" "$GEMMA12B" "oscar2" "f16" +run_ppl "g26_oscar2_f16" "gemma4-26b-a4b" "$GEMMA26B" "oscar2" "f16" +run_ppl "q27_oscar2_f16" "qwen3.6-27b" "$QWEN27B" "oscar2" "f16" + +echo "" +echo "=== Phase 4: f16/oscar2 mix on rot-kv models ===" + +run_ppl "g12_f16_oscar2" "gemma4-12b" "$GEMMA12B" "f16" "oscar2" +run_ppl "g26_f16_oscar2" "gemma4-26b-a4b" "$GEMMA26B" "f16" "oscar2" +run_ppl "q27_f16_oscar2" "qwen3.6-27b" "$QWEN27B" "f16" "oscar2" + +echo "" +echo "=== Phase 5: q8_0/q8_0 on all models ===" + +run_ppl "g12_q8_0_q8_0" "gemma4-12b" "$GEMMA12B" "q8_0" "q8_0" +run_ppl "g26_q8_0_q8_0" "gemma4-26b-a4b" "$GEMMA26B" "q8_0" "q8_0" +run_ppl "q27_q8_0_q8_0" "qwen3.6-27b" "$QWEN27B" "q8_0" "q8_0" +run_ppl "kw_q8_0_q8_0" "kwaipilot-35b" "$KWAIPLOT" "q8_0" "q8_0" + +echo "" +echo "=== Phase 6: q8_0/turbo4 on all models ===" + +run_ppl "g12_q8_0_turbo4" "gemma4-12b" "$GEMMA12B" "q8_0" "turbo4" +run_ppl "g26_q8_0_turbo4" "gemma4-26b-a4b" "$GEMMA26B" "q8_0" "turbo4" +run_ppl "q27_q8_0_turbo4" "qwen3.6-27b" "$QWEN27B" "q8_0" "turbo4" +run_ppl "kw_q8_0_turbo4" "kwaipilot-35b" "$KWAIPLOT" "q8_0" "turbo4" + +echo "" +echo "=== Phase 7: oscar2/oscar2 on Kwaipilot (expected to fail - no rot-kv) ===" + +run_ppl "kw_oscar2_oscar2" "kwaipilot-35b" "$KWAIPLOT" "oscar2" "oscar2" + +echo "" +echo "======================================================================" +echo " All PPL tests complete! $(date)" +echo " Summary: ${SUMMARY}" +echo "======================================================================" +echo "" +column -t -s $'\t' "$SUMMARY" 2>/dev/null || cat "$SUMMARY" \ No newline at end of file diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index c939dad5a66a..b9f880e80d84 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -3558,10 +3558,6 @@ struct server_context_impl { } } // end of SLOT_STATE_STARTED - if (slot.prompt_checkpoint_restored && n_tokens_prev > 0) { - return; - } - if (!slot.can_split()) { // cannot fit the prompt in the current batch - will try next iter if (batch.size() + slot.task->n_tokens() > n_batch) {