From d56a67ac11ac9f6bd185c7acbc13cdf7fd539c65 Mon Sep 17 00:00:00 2001 From: SONLE5 Date: Fri, 19 Jun 2026 22:37:05 +0700 Subject: [PATCH 1/8] Stabilize CK blockscale bpreshuffle GEMM --- ...line_xdlops_blockscale_b_preshuffle_v1.hpp | 112 ++++++++-------- ...line_xdlops_blockscale_b_preshuffle_v3.hpp | 118 ++++++++--------- .../profile_gemm_blockscale_wp_impl.hpp | 122 +++++++++++++++++- .../test_gemm_blockscale_wp_fp8.cpp | 64 +++++++++ .../gemm_blockscale_wp/test_gemm_common.hpp | 11 +- 5 files changed, 297 insertions(+), 130 deletions(-) diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v1.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v1.hpp index af9ff1e16aae..82d75fc3dc62 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v1.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v1.hpp @@ -285,6 +285,14 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v1( a_thread_desc_.GetElementSpaceSize()); @@ -451,14 +459,9 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v1{}) .template AsType()(Number{}) = 0; }); - vector_type c_scale_thread_vec; constexpr index_t cscale_offset = CScaleThreadDesc{}.CalculateOffset( make_tuple(kscale0, m0, n0 * num_scale_n_block / NRepeat)); - - c_scale_thread_vec.template AsType()(Number<0>{}) = - c_scale_thread_buf[Number{}]; - c_scale_thread_vec.template AsType()(Number<1>{}) = - c_scale_thread_buf[Number{}]; + const auto c_scale_thread = c_scale_thread_buf[Number{}]; static_for<0, KRepeat / num_scale_k_block, 1>{}([&](auto k0) { vector_type a_thread_vec; @@ -501,16 +504,17 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v1{}([&](auto t) { - using pk_fma_type = typename vector_type::type; - - c_thread_buf.GetVectorTypeReference(Number{}) - .template AsType()(t) = __builtin_elementwise_fma( + // Keep the post-scale FMA scalar; the old packed 2-lane update hits + // ROCm 7.2 gfx950 illegal-type legalization/codegen issues. + static_for<0, xdlops_gemm.GetRegSizePerXdlops(), 1>{}([&](auto t) { + auto& c_acc_vec = c_thread_buf.GetVectorTypeReference(Number{}) + .template AsType(); + const auto c_partial_acc = c_thread_buf_per_scale.GetVectorTypeReference(Number<0>{}) - .template AsType()[t], - c_scale_thread_vec.template AsType()[Number<0>{}], - c_thread_buf.GetVectorTypeReference(Number{}) - .template AsType()[t]); + .template AsType()(t); + c_acc_vec(t) = + __builtin_elementwise_fma(c_partial_acc, c_scale_thread, c_acc_vec(t)); + anchor_accumulator_value(c_acc_vec(t)); }); }); @@ -606,14 +610,9 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v1{}) .template AsType()(Number{}) = 0; }); - vector_type c_scale_thread_vec; constexpr index_t cscale_offset = CScaleThreadDesc{}.CalculateOffset( make_tuple(kscale0, m0, n0 * num_scale_n_block / NRepeat)); - - c_scale_thread_vec.template AsType()(Number<0>{}) = - c_scale_thread_buf[Number{}]; - c_scale_thread_vec.template AsType()(Number<1>{}) = - c_scale_thread_buf[Number{}]; + const auto c_scale_thread = c_scale_thread_buf[Number{}]; static_for<0, KRepeat / num_scale_k_block, 1>{}([&](auto k0) { vector_type a_thread_vec; @@ -653,16 +652,17 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v1{}([&](auto t) { - using pk_fma_type = typename vector_type::type; - - c_thread_buf.GetVectorTypeReference(Number{}) - .template AsType()(t) = __builtin_elementwise_fma( + // Keep the post-scale FMA scalar; the old packed 2-lane update hits ROCm 7.2 + // gfx950 illegal-type legalization/codegen issues. + static_for<0, xdlops_gemm.GetRegSizePerXdlops(), 1>{}([&](auto t) { + auto& c_acc_vec = c_thread_buf.GetVectorTypeReference(Number{}) + .template AsType(); + const auto c_partial_acc = c_thread_buf_per_scale.GetVectorTypeReference(Number<0>{}) - .template AsType()[t], - c_scale_thread_vec.template AsType()[Number<0>{}], - c_thread_buf.GetVectorTypeReference(Number{}) - .template AsType()[t]); + .template AsType()(t); + c_acc_vec(t) = + __builtin_elementwise_fma(c_partial_acc, c_scale_thread, c_acc_vec(t)); + anchor_accumulator_value(c_acc_vec(t)); }); }); @@ -701,14 +701,9 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v1{}) .template AsType()(Number{}) = 0; }); - vector_type c_scale_thread_vec; constexpr index_t cscale_offset = CScaleThreadDesc{}.CalculateOffset( make_tuple(kscale0, m0, n0 * num_scale_n_block / NRepeat)); - - c_scale_thread_vec.template AsType()(Number<0>{}) = - c_scale_thread_buf[Number{}]; - c_scale_thread_vec.template AsType()(Number<1>{}) = - c_scale_thread_buf[Number{}]; + const auto c_scale_thread = c_scale_thread_buf[Number{}]; static_for<0, KRepeat / num_scale_k_block, 1>{}([&](auto k0) { vector_type a_thread_vec; @@ -748,16 +743,17 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v1{}([&](auto t) { - using pk_fma_type = typename vector_type::type; - - c_thread_buf.GetVectorTypeReference(Number{}) - .template AsType()(t) = __builtin_elementwise_fma( + // Keep the post-scale FMA scalar; the old packed 2-lane update hits ROCm 7.2 + // gfx950 illegal-type legalization/codegen issues. + static_for<0, xdlops_gemm.GetRegSizePerXdlops(), 1>{}([&](auto t) { + auto& c_acc_vec = c_thread_buf.GetVectorTypeReference(Number{}) + .template AsType(); + const auto c_partial_acc = c_thread_buf_per_scale.GetVectorTypeReference(Number<0>{}) - .template AsType()[t], - c_scale_thread_vec.template AsType()[Number<0>{}], - c_thread_buf.GetVectorTypeReference(Number{}) - .template AsType()[t]); + .template AsType()(t); + c_acc_vec(t) = + __builtin_elementwise_fma(c_partial_acc, c_scale_thread, c_acc_vec(t)); + anchor_accumulator_value(c_acc_vec(t)); }); }); } @@ -771,14 +767,9 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v1{}) .template AsType()(Number{}) = 0; }); - vector_type c_scale_thread_vec; constexpr index_t cscale_offset = CScaleThreadDesc{}.CalculateOffset( make_tuple(kscale0, m0, n0 * num_scale_n_block / NRepeat)); - - c_scale_thread_vec.template AsType()(Number<0>{}) = - c_scale_thread_buf[Number{}]; - c_scale_thread_vec.template AsType()(Number<1>{}) = - c_scale_thread_buf[Number{}]; + const auto c_scale_thread = c_scale_thread_buf[Number{}]; static_for<0, KRepeat / num_scale_k_block, 1>{}([&](auto k0) { vector_type a_thread_vec; @@ -819,16 +810,17 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v1{}([&](auto t) { - using pk_fma_type = typename vector_type::type; - - c_thread_buf.GetVectorTypeReference(Number{}) - .template AsType()(t) = __builtin_elementwise_fma( + // Keep the post-scale FMA scalar; the old packed 2-lane update hits ROCm 7.2 + // gfx950 illegal-type legalization/codegen issues. + static_for<0, xdlops_gemm.GetRegSizePerXdlops(), 1>{}([&](auto t) { + auto& c_acc_vec = c_thread_buf.GetVectorTypeReference(Number{}) + .template AsType(); + const auto c_partial_acc = c_thread_buf_per_scale.GetVectorTypeReference(Number<0>{}) - .template AsType()[t], - c_scale_thread_vec.template AsType()[Number<0>{}], - c_thread_buf.GetVectorTypeReference(Number{}) - .template AsType()[t]); + .template AsType()(t); + c_acc_vec(t) = + __builtin_elementwise_fma(c_partial_acc, c_scale_thread, c_acc_vec(t)); + anchor_accumulator_value(c_acc_vec(t)); }); }); } diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v3.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v3.hpp index da260fb5a10c..5b74aba55a85 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v3.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v3.hpp @@ -416,6 +416,14 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v3{}) == 1, "Pipeline v3 only support scaleblocksliceK=1"); @@ -642,11 +650,7 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v3{}([&](auto m0) { - vector_type c_scale_thread_vec; - c_scale_thread_vec.template AsType()(Number<0>{}) = - c_scale_thread_buf[m0]; - c_scale_thread_vec.template AsType()(Number<1>{}) = - c_scale_thread_buf[m0]; + const auto c_scale_thread = c_scale_thread_buf[m0]; static_for<0, NRepeat, 1>{}([&](auto n0) { constexpr auto mfma_buf_offset = @@ -712,17 +716,19 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v3{}([&](auto t) { - using pk_fma_type = typename vector_type::type; - - c_thread_buf.GetVectorTypeReference(Number{}) - .template AsType()(t) = __builtin_elementwise_fma( + // Keep the post-scale FMA scalar; the old packed 2-lane update hits + // ROCm 7.2 gfx950 illegal-type legalization/codegen issues. + static_for<0, xdlops_gemm.GetRegSizePerXdlops(), 1>{}([&](auto t) { + auto& c_acc_vec = + c_thread_buf.GetVectorTypeReference(Number{}) + .template AsType(); + const auto c_partial_acc = c_thread_buf_per_scale .GetVectorTypeReference(Number{}) - .template AsType()[t], - c_scale_thread_vec.template AsType()[Number<0>{}], - c_thread_buf.GetVectorTypeReference(Number{}) - .template AsType()[t]); + .template AsType()(t); + c_acc_vec(t) = + __builtin_elementwise_fma(c_partial_acc, c_scale_thread, c_acc_vec(t)); + anchor_accumulator_value(c_acc_vec(t)); }); }); @@ -793,11 +799,7 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v3{}([&](auto m0) { - vector_type c_scale_thread_vec; - c_scale_thread_vec.template AsType()(Number<0>{}) = - c_scale_thread_buf[m0]; - c_scale_thread_vec.template AsType()(Number<1>{}) = - c_scale_thread_buf[m0]; + const auto c_scale_thread = c_scale_thread_buf[m0]; static_for<0, NRepeat, 1>{}([&](auto n0) { constexpr auto mfma_buf_offset = @@ -858,17 +860,17 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v3{}([&](auto t) { - using pk_fma_type = typename vector_type::type; - - c_thread_buf.GetVectorTypeReference(Number{}) - .template AsType()(t) = __builtin_elementwise_fma( - c_thread_buf_per_scale - .GetVectorTypeReference(Number{}) - .template AsType()[t], - c_scale_thread_vec.template AsType()[Number<0>{}], - c_thread_buf.GetVectorTypeReference(Number{}) - .template AsType()[t]); + // Keep the post-scale FMA scalar; the old packed 2-lane update hits ROCm 7.2 + // gfx950 illegal-type legalization/codegen issues. + static_for<0, xdlops_gemm.GetRegSizePerXdlops(), 1>{}([&](auto t) { + auto& c_acc_vec = c_thread_buf.GetVectorTypeReference(Number{}) + .template AsType(); + const auto c_partial_acc = + c_thread_buf_per_scale.GetVectorTypeReference(Number{}) + .template AsType()(t); + c_acc_vec(t) = + __builtin_elementwise_fma(c_partial_acc, c_scale_thread, c_acc_vec(t)); + anchor_accumulator_value(c_acc_vec(t)); }); }); @@ -909,11 +911,7 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v3{}([&](auto m0) { - vector_type c_scale_thread_vec; - c_scale_thread_vec.template AsType()(Number<0>{}) = - c_scale_thread_buf[m0]; - c_scale_thread_vec.template AsType()(Number<1>{}) = - c_scale_thread_buf[m0]; + const auto c_scale_thread = c_scale_thread_buf[m0]; static_for<0, NRepeat, 1>{}([&](auto n0) { constexpr auto mfma_buf_offset = @@ -974,17 +972,17 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v3{}([&](auto t) { - using pk_fma_type = typename vector_type::type; - - c_thread_buf.GetVectorTypeReference(Number{}) - .template AsType()(t) = __builtin_elementwise_fma( - c_thread_buf_per_scale - .GetVectorTypeReference(Number{}) - .template AsType()[t], - c_scale_thread_vec.template AsType()[Number<0>{}], - c_thread_buf.GetVectorTypeReference(Number{}) - .template AsType()[t]); + // Keep the post-scale FMA scalar; the old packed 2-lane update hits ROCm 7.2 + // gfx950 illegal-type legalization/codegen issues. + static_for<0, xdlops_gemm.GetRegSizePerXdlops(), 1>{}([&](auto t) { + auto& c_acc_vec = c_thread_buf.GetVectorTypeReference(Number{}) + .template AsType(); + const auto c_partial_acc = + c_thread_buf_per_scale.GetVectorTypeReference(Number{}) + .template AsType()(t); + c_acc_vec(t) = + __builtin_elementwise_fma(c_partial_acc, c_scale_thread, c_acc_vec(t)); + anchor_accumulator_value(c_acc_vec(t)); }); }); @@ -1021,11 +1019,7 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v3{}([&](auto m0) { - vector_type c_scale_thread_vec; - c_scale_thread_vec.template AsType()(Number<0>{}) = - c_scale_thread_buf[m0]; - c_scale_thread_vec.template AsType()(Number<1>{}) = - c_scale_thread_buf[m0]; + const auto c_scale_thread = c_scale_thread_buf[m0]; static_for<0, NRepeat, 1>{}([&](auto n0) { constexpr auto mfma_buf_offset = @@ -1087,17 +1081,17 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v3{}([&](auto t) { - using pk_fma_type = typename vector_type::type; - - c_thread_buf.GetVectorTypeReference(Number{}) - .template AsType()(t) = __builtin_elementwise_fma( - c_thread_buf_per_scale - .GetVectorTypeReference(Number{}) - .template AsType()[t], - c_scale_thread_vec.template AsType()[Number<0>{}], - c_thread_buf.GetVectorTypeReference(Number{}) - .template AsType()[t]); + // Keep the post-scale FMA scalar; the old packed 2-lane update hits ROCm 7.2 + // gfx950 illegal-type legalization/codegen issues. + static_for<0, xdlops_gemm.GetRegSizePerXdlops(), 1>{}([&](auto t) { + auto& c_acc_vec = c_thread_buf.GetVectorTypeReference(Number{}) + .template AsType(); + const auto c_partial_acc = + c_thread_buf_per_scale.GetVectorTypeReference(Number{}) + .template AsType()(t); + c_acc_vec(t) = + __builtin_elementwise_fma(c_partial_acc, c_scale_thread, c_acc_vec(t)); + anchor_accumulator_value(c_acc_vec(t)); }); }); diff --git a/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp b/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp index 9e5694b71146..89bf75012014 100644 --- a/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp +++ b/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp @@ -3,8 +3,13 @@ #pragma once +#include +#include +#include #include #include +#include +#include #include #include "ck/ck.hpp" @@ -81,10 +86,12 @@ bool profile_gemm_blockscale_weightpreshuffle_impl(int do_verification, int StrideE, int n_warmup, int n_iter, - uint64_t rotating = 0, - int instance_index = -1) + uint64_t rotating = 0, + int determinism_check = 1, + int instance_index = -1) { bool pass = true; + determinism_check = std::max(1, determinism_check); auto f_host_tensor_descriptor = [](std::size_t row, std::size_t col, int& stride, auto layout) { using namespace ck::literals; @@ -228,6 +235,66 @@ bool profile_gemm_blockscale_weightpreshuffle_impl(int do_verification, std::cout << "found " << op_ptrs.size() << " instances" << std::endl; + auto check_row_relative_error = [&](const auto& actual, + const auto& expected, + const std::string& label) { + // sglang#28685 showed sparse per-row spikes near tile boundaries. + double worst_row_rel_max_error = -1; + int worst_row_rel = 0; + int bad_row_count = 0; + int sampled_bad_row_count = 0; + std::string bad_row_sample; + + for(int m = 0; m < M; ++m) + { + double row_max_abs_error = 0; + double row_ref_max_abs = 0; + + for(int n = 0; n < N; ++n) + { + const float actual_value = ck::type_convert(actual(m, n)); + const float expected_value = ck::type_convert(expected(m, n)); + row_max_abs_error = std::max( + row_max_abs_error, + std::abs(static_cast(actual_value) - + static_cast(expected_value))); + row_ref_max_abs = + std::max(row_ref_max_abs, std::abs(static_cast(expected_value))); + } + + const double row_rel_max_error = row_max_abs_error / std::max(row_ref_max_abs, 1.0); + if(row_rel_max_error > worst_row_rel_max_error) + { + worst_row_rel_max_error = row_rel_max_error; + worst_row_rel = m; + } + if(row_rel_max_error > 0.1) + { + ++bad_row_count; + if(sampled_bad_row_count < 16) + { + if(!bad_row_sample.empty()) + { + bad_row_sample += ","; + } + bad_row_sample += std::to_string(m); + ++sampled_bad_row_count; + } + } + } + + const bool pass_row_relative_check = bad_row_count == 0; + if(!pass_row_relative_check) + { + std::cout << label << " rowrel_max=" << worst_row_rel_max_error + << " rowrel_worst_row=" << worst_row_rel + << " rowrel_bad_rows_gt_0p1=" << bad_row_count << "/" << M + << " rowrel_bad_row_sample=" << bad_row_sample << std::endl; + } + + return pass_row_relative_check; + }; + // Run reference GEMM if(do_verification) { @@ -318,16 +385,28 @@ bool profile_gemm_blockscale_weightpreshuffle_impl(int do_verification, if(op_ptr->IsSupportedArgument(argument_ptr.get())) { - // re-init C to zero before profiling next kernel c_device_buf.SetZero(); invoker_ptr->Run(argument_ptr.get(), StreamConfig{nullptr, false, 0, n_warmup, n_iter}); - if(do_verification) + std::unique_ptr> first_device_result; + if(determinism_check > 1) + { + first_device_result = std::make_unique>(e_m_n_device_result.mDesc); + } + + if(do_verification || determinism_check > 1) { c_device_buf.FromDevice(e_m_n_device_result.mData.data()); + if(first_device_result) + { + first_device_result->mData = e_m_n_device_result.mData; + } + } + if(do_verification) + { #if defined CK_ENABLE_FP8 // set softer tolerances for fp8 if constexpr(is_same_v || is_same_v || @@ -338,6 +417,11 @@ bool profile_gemm_blockscale_weightpreshuffle_impl(int do_verification, double atol = 5e-2; bool current_pass = ck::utils::check_err( e_m_n_device_result, e_m_n_host_result, msg, rtol, atol); + current_pass = + check_row_relative_error(e_m_n_device_result, + e_m_n_host_result, + "Reference row-relative check") && + current_pass; pass = pass & current_pass; if(!current_pass) { @@ -347,8 +431,10 @@ bool profile_gemm_blockscale_weightpreshuffle_impl(int do_verification, else { #endif - pass = pass & ck::utils::check_err(e_m_n_device_result, e_m_n_host_result); - if(!pass) + bool current_pass = + ck::utils::check_err(e_m_n_device_result, e_m_n_host_result); + pass = pass & current_pass; + if(!current_pass) { std::cout << op_ptr->GetTypeString() << " failed" << std::endl; } @@ -367,6 +453,30 @@ bool profile_gemm_blockscale_weightpreshuffle_impl(int do_verification, } } + for(int repeat = 1; repeat < determinism_check; ++repeat) + { + c_device_buf.SetZero(); + invoker_ptr->Run(argument_ptr.get(), StreamConfig{nullptr, false, 0, 0, 1}); + c_device_buf.FromDevice(e_m_n_device_result.mData.data()); + + const auto byte_count = first_device_result->mData.size() * sizeof(EDataType); + const bool deterministic = + std::memcmp(e_m_n_device_result.mData.data(), + first_device_result->mData.data(), + byte_count) == 0; + + if(!deterministic) + { + pass = false; + std::cout << op_ptr->GetTypeString() + << " produced nondeterministic output on repeat " << repeat + << " of " << determinism_check << std::endl; + check_row_relative_error( + e_m_n_device_result, *first_device_result, "Determinism mismatch"); + break; + } + } + std::string op_name = op_ptr->GetTypeString(); float ave_time = invoker_ptr->Run( diff --git a/projects/composablekernel/test/gemm_blockscale_wp/test_gemm_blockscale_wp_fp8.cpp b/projects/composablekernel/test/gemm_blockscale_wp/test_gemm_blockscale_wp_fp8.cpp index 71ab22b77db7..72f71caf0ac2 100644 --- a/projects/composablekernel/test/gemm_blockscale_wp/test_gemm_blockscale_wp_fp8.cpp +++ b/projects/composablekernel/test/gemm_blockscale_wp/test_gemm_blockscale_wp_fp8.cpp @@ -1,6 +1,8 @@ // Copyright (c) Advanced Micro Devices, Inc., or its affiliates. // SPDX-License-Identifier: MIT +#include +#include #include #include "gtest/gtest.h" @@ -25,6 +27,12 @@ struct tuple_concat, std::tuple> using type = std::tuple; }; +bool run_long_gemm_regression_tests() +{ + const char* value = std::getenv("CK_RUN_LONG_GEMM_REGRESSION_TESTS"); + return value != nullptr && std::strcmp(value, "0") != 0; +} + } // namespace template @@ -52,3 +60,59 @@ TYPED_TEST(TestGemmBlockScaleWP_FP8_MK_NK, Regular0) for(int M : Ms) this->Run(M, N, K); } + +TYPED_TEST(TestGemmBlockScaleWP_FP8_MK_NK, ReportedWkvDeterminism) +{ + if(!run_long_gemm_regression_tests()) + { + GTEST_SKIP() << "Set CK_RUN_LONG_GEMM_REGRESSION_TESTS=1 to run the ROCm/aiter#3261 " + "8192x512xK determinism repro shapes."; + } + + // ROCm/aiter#3261 repro sweep from the original CK stabilizer investigation. + // Skip the host reference because these are large determinism regression + // cases, not practical CPU GEMM unit-test sizes. + constexpr int M = 8192; + constexpr int N = 512; + constexpr int Ks[] = {384, 640, 3968, 4096, 4224}; + + for(const int K : Ks) + this->Run(M, N, K, 0, 1, 8, false); +} + +TYPED_TEST(TestGemmBlockScaleWP_FP8_MK_NK, Glm52OutOfAllowlistAccuracyAndDeterminism) +{ + if(!run_long_gemm_regression_tests()) + { + GTEST_SKIP() << "Set CK_RUN_LONG_GEMM_REGRESSION_TESTS=1 to run the GLM-5.2-FP8 " + "out-of-allowlist bpreshuffle accuracy and determinism regression shapes."; + } + + constexpr std::tuple shapes[] = { + // M, N, K. GLM-5.2-FP8 block-FP8 linear shapes from + // sgl-project/sglang#28685: concrete projection repros plus the + // kv_b_proj row-boundary sweep where bad rows clustered near 16-row + // tile edges. + {8, 2048, 6144}, // q_a_proj: hidden_size -> q_lora_rank + {8, 16384, 2048}, // q_b_proj short prefill + {64, 16384, 2048}, // q_b_proj reporter repro + {12, 6144, 12288}, // mlp.down_proj short prefill + {64, 6144, 12288}, // mlp.down_proj reporter repro + {8, 28672, 512}, // kv_b_proj short smoke + {12, 28672, 512}, // kv_b_proj short prefill + {32, 28672, 512}, // kv_b_proj row-boundary sweep + {48, 28672, 512}, // kv_b_proj row-boundary sweep + {56, 28672, 512}, // kv_b_proj row-boundary sweep + {64, 28672, 512}, // kv_b_proj reporter repro + {72, 28672, 512}, // kv_b_proj row-boundary sweep + {96, 28672, 512}, // kv_b_proj row-boundary sweep + {128, 28672, 512}, // kv_b_proj row-boundary sweep + }; + + for(const auto& shape : shapes) + { + // do_verification=true compares against the profiler's dequantized FP32 + // host GEMM oracle; determinism_check=4 reruns the same device problem. + this->Run(std::get<0>(shape), std::get<1>(shape), std::get<2>(shape), 0, 1, 4, true); + } +} diff --git a/projects/composablekernel/test/gemm_blockscale_wp/test_gemm_common.hpp b/projects/composablekernel/test/gemm_blockscale_wp/test_gemm_common.hpp index 34cf330519e4..f88f06a530d8 100644 --- a/projects/composablekernel/test/gemm_blockscale_wp/test_gemm_common.hpp +++ b/projects/composablekernel/test/gemm_blockscale_wp/test_gemm_common.hpp @@ -37,7 +37,13 @@ class TestGemmBlockscaleWPCommon : public ::testing::Test static constexpr index_t ScaleBlockN = 128; static constexpr index_t ScaleBlockK = 128; - void Run(const int M, const int N, const int K, int n_warmup = 1, int n_iter = 10) + void Run(const int M, + const int N, + const int K, + int n_warmup = 1, + int n_iter = 10, + int determinism_check = 1, + bool do_verification = verify_) { bool all_success = true; @@ -59,7 +65,7 @@ class TestGemmBlockscaleWPCommon : public ::testing::Test ScaleBlockK, ALayout, BLayout, - CLayout>(verify_, + CLayout>(do_verification, init_method_, log_, bench_, @@ -72,6 +78,7 @@ class TestGemmBlockscaleWPCommon : public ::testing::Test n_warmup, n_iter, 0, + determinism_check, instance_index); EXPECT_TRUE(all_success); From 6c769bbf765d147edcf986ad9d0748eb2283101d Mon Sep 17 00:00:00 2001 From: SONLE5 Date: Sat, 20 Jun 2026 01:29:42 +0700 Subject: [PATCH 2/8] Address bpreshuffle review comments --- ...wise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v1.hpp | 2 +- ...wise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v3.hpp | 2 +- .../include/profiler/profile_gemm_blockscale_wp_impl.hpp | 6 ++---- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v1.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v1.hpp index 82d75fc3dc62..c2af2de1a783 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v1.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v1.hpp @@ -288,7 +288,7 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v1GetTypeString() << " failed" << std::endl; } From b1cb121220762666aec60d3467c8db1613c8fa5f Mon Sep 17 00:00:00 2001 From: SONLE5 Date: Sat, 20 Jun 2026 01:37:12 +0700 Subject: [PATCH 3/8] Apply pre-commit formatting --- ...line_xdlops_blockscale_b_preshuffle_v1.hpp | 13 +- ...line_xdlops_blockscale_b_preshuffle_v3.hpp | 17 +-- .../profile_gemm_blockscale_wp_impl.hpp | 127 +++++++++--------- .../test_gemm_blockscale_wp_fp8.cpp | 32 ++--- 4 files changed, 93 insertions(+), 96 deletions(-) diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v1.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v1.hpp index c2af2de1a783..21cfcfc5ae0f 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v1.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v1.hpp @@ -289,9 +289,7 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v1( @@ -507,13 +505,14 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v1{}([&](auto t) { - auto& c_acc_vec = c_thread_buf.GetVectorTypeReference(Number{}) - .template AsType(); + auto& c_acc_vec = + c_thread_buf.GetVectorTypeReference(Number{}) + .template AsType(); const auto c_partial_acc = c_thread_buf_per_scale.GetVectorTypeReference(Number<0>{}) .template AsType()(t); - c_acc_vec(t) = - __builtin_elementwise_fma(c_partial_acc, c_scale_thread, c_acc_vec(t)); + c_acc_vec(t) = __builtin_elementwise_fma( + c_partial_acc, c_scale_thread, c_acc_vec(t)); anchor_accumulator_value(c_acc_vec(t)); }); }); diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v3.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v3.hpp index f283b3b059c4..8ea3e45e9c66 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v3.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v3.hpp @@ -420,9 +420,7 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v3{}) == 1, @@ -726,8 +724,8 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v3{}) .template AsType()(t); - c_acc_vec(t) = - __builtin_elementwise_fma(c_partial_acc, c_scale_thread, c_acc_vec(t)); + c_acc_vec(t) = __builtin_elementwise_fma( + c_partial_acc, c_scale_thread, c_acc_vec(t)); anchor_accumulator_value(c_acc_vec(t)); }); }); @@ -866,7 +864,8 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v3{}) .template AsType(); const auto c_partial_acc = - c_thread_buf_per_scale.GetVectorTypeReference(Number{}) + c_thread_buf_per_scale + .GetVectorTypeReference(Number{}) .template AsType()(t); c_acc_vec(t) = __builtin_elementwise_fma(c_partial_acc, c_scale_thread, c_acc_vec(t)); @@ -978,7 +977,8 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v3{}) .template AsType(); const auto c_partial_acc = - c_thread_buf_per_scale.GetVectorTypeReference(Number{}) + c_thread_buf_per_scale + .GetVectorTypeReference(Number{}) .template AsType()(t); c_acc_vec(t) = __builtin_elementwise_fma(c_partial_acc, c_scale_thread, c_acc_vec(t)); @@ -1087,7 +1087,8 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v3{}) .template AsType(); const auto c_partial_acc = - c_thread_buf_per_scale.GetVectorTypeReference(Number{}) + c_thread_buf_per_scale + .GetVectorTypeReference(Number{}) .template AsType()(t); c_acc_vec(t) = __builtin_elementwise_fma(c_partial_acc, c_scale_thread, c_acc_vec(t)); diff --git a/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp b/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp index 9620f7d7e352..5f6afdcffb5f 100644 --- a/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp +++ b/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp @@ -86,11 +86,11 @@ bool profile_gemm_blockscale_weightpreshuffle_impl(int do_verification, int StrideE, int n_warmup, int n_iter, - uint64_t rotating = 0, + uint64_t rotating = 0, int determinism_check = 1, - int instance_index = -1) + int instance_index = -1) { - bool pass = true; + bool pass = true; determinism_check = std::max(1, determinism_check); auto f_host_tensor_descriptor = [](std::size_t row, std::size_t col, int& stride, auto layout) { @@ -235,65 +235,63 @@ bool profile_gemm_blockscale_weightpreshuffle_impl(int do_verification, std::cout << "found " << op_ptrs.size() << " instances" << std::endl; - auto check_row_relative_error = [&](const auto& actual, - const auto& expected, - const std::string& label) { - // sglang#28685 showed sparse per-row spikes near tile boundaries. - double worst_row_rel_max_error = -1; - int worst_row_rel = 0; - int bad_row_count = 0; - int sampled_bad_row_count = 0; - std::string bad_row_sample; + auto check_row_relative_error = + [&](const auto& actual, const auto& expected, const std::string& label) { + // sglang#28685 showed sparse per-row spikes near tile boundaries. + double worst_row_rel_max_error = -1; + int worst_row_rel = 0; + int bad_row_count = 0; + int sampled_bad_row_count = 0; + std::string bad_row_sample; - for(int m = 0; m < M; ++m) - { - double row_max_abs_error = 0; - double row_ref_max_abs = 0; - - for(int n = 0; n < N; ++n) + for(int m = 0; m < M; ++m) { - const float actual_value = ck::type_convert(actual(m, n)); - const float expected_value = ck::type_convert(expected(m, n)); - row_max_abs_error = std::max( - row_max_abs_error, - std::abs(static_cast(actual_value) - - static_cast(expected_value))); - row_ref_max_abs = - std::max(row_ref_max_abs, std::abs(static_cast(expected_value))); - } + double row_max_abs_error = 0; + double row_ref_max_abs = 0; - const double row_rel_max_error = row_max_abs_error / std::max(row_ref_max_abs, 1.0); - if(row_rel_max_error > worst_row_rel_max_error) - { - worst_row_rel_max_error = row_rel_max_error; - worst_row_rel = m; - } - if(row_rel_max_error > 0.1) - { - ++bad_row_count; - if(sampled_bad_row_count < 16) + for(int n = 0; n < N; ++n) { - if(!bad_row_sample.empty()) + const float actual_value = ck::type_convert(actual(m, n)); + const float expected_value = ck::type_convert(expected(m, n)); + row_max_abs_error = std::max(row_max_abs_error, + std::abs(static_cast(actual_value) - + static_cast(expected_value))); + row_ref_max_abs = + std::max(row_ref_max_abs, std::abs(static_cast(expected_value))); + } + + const double row_rel_max_error = row_max_abs_error / std::max(row_ref_max_abs, 1.0); + if(row_rel_max_error > worst_row_rel_max_error) + { + worst_row_rel_max_error = row_rel_max_error; + worst_row_rel = m; + } + if(row_rel_max_error > 0.1) + { + ++bad_row_count; + if(sampled_bad_row_count < 16) { - bad_row_sample += ","; + if(!bad_row_sample.empty()) + { + bad_row_sample += ","; + } + bad_row_sample += std::to_string(m); + ++sampled_bad_row_count; } - bad_row_sample += std::to_string(m); - ++sampled_bad_row_count; } } - } - const bool pass_row_relative_check = bad_row_count == 0; - if(!pass_row_relative_check) - { - std::cout << label << " rowrel_max=" << worst_row_rel_max_error - << " rowrel_worst_row=" << worst_row_rel - << " rowrel_bad_rows_gt_0p1=" << bad_row_count << "/" << M - << " rowrel_bad_row_sample=" << bad_row_sample << std::endl; - } + const bool pass_row_relative_check = bad_row_count == 0; + if(!pass_row_relative_check) + { + std::cout << label << " rowrel_max=" << worst_row_rel_max_error + << " rowrel_worst_row=" << worst_row_rel + << " rowrel_bad_rows_gt_0p1=" << bad_row_count << "/" << M + << " rowrel_bad_row_sample=" << bad_row_sample << std::endl; + } - return pass_row_relative_check; - }; + return pass_row_relative_check; + }; // Run reference GEMM if(do_verification) @@ -393,7 +391,8 @@ bool profile_gemm_blockscale_weightpreshuffle_impl(int do_verification, std::unique_ptr> first_device_result; if(determinism_check > 1) { - first_device_result = std::make_unique>(e_m_n_device_result.mDesc); + first_device_result = + std::make_unique>(e_m_n_device_result.mDesc); } if(do_verification || determinism_check > 1) @@ -417,11 +416,10 @@ bool profile_gemm_blockscale_weightpreshuffle_impl(int do_verification, double atol = 5e-2; bool current_pass = ck::utils::check_err( e_m_n_device_result, e_m_n_host_result, msg, rtol, atol); - current_pass = - check_row_relative_error(e_m_n_device_result, - e_m_n_host_result, - "Reference row-relative check") && - current_pass; + current_pass = check_row_relative_error(e_m_n_device_result, + e_m_n_host_result, + "Reference row-relative check") && + current_pass; pass = pass & current_pass; if(!current_pass) { @@ -457,18 +455,17 @@ bool profile_gemm_blockscale_weightpreshuffle_impl(int do_verification, invoker_ptr->Run(argument_ptr.get(), StreamConfig{nullptr, false, 0, 0, 1}); c_device_buf.FromDevice(e_m_n_device_result.mData.data()); - const auto byte_count = first_device_result->mData.size() * sizeof(EDataType); - const bool deterministic = - std::memcmp(e_m_n_device_result.mData.data(), - first_device_result->mData.data(), - byte_count) == 0; + const auto byte_count = first_device_result->mData.size() * sizeof(EDataType); + const bool deterministic = std::memcmp(e_m_n_device_result.mData.data(), + first_device_result->mData.data(), + byte_count) == 0; if(!deterministic) { pass = false; std::cout << op_ptr->GetTypeString() - << " produced nondeterministic output on repeat " << repeat - << " of " << determinism_check << std::endl; + << " produced nondeterministic output on repeat " << repeat << " of " + << determinism_check << std::endl; check_row_relative_error( e_m_n_device_result, *first_device_result, "Determinism mismatch"); break; diff --git a/projects/composablekernel/test/gemm_blockscale_wp/test_gemm_blockscale_wp_fp8.cpp b/projects/composablekernel/test/gemm_blockscale_wp/test_gemm_blockscale_wp_fp8.cpp index 72f71caf0ac2..04ebde65b631 100644 --- a/projects/composablekernel/test/gemm_blockscale_wp/test_gemm_blockscale_wp_fp8.cpp +++ b/projects/composablekernel/test/gemm_blockscale_wp/test_gemm_blockscale_wp_fp8.cpp @@ -72,8 +72,8 @@ TYPED_TEST(TestGemmBlockScaleWP_FP8_MK_NK, ReportedWkvDeterminism) // ROCm/aiter#3261 repro sweep from the original CK stabilizer investigation. // Skip the host reference because these are large determinism regression // cases, not practical CPU GEMM unit-test sizes. - constexpr int M = 8192; - constexpr int N = 512; + constexpr int M = 8192; + constexpr int N = 512; constexpr int Ks[] = {384, 640, 3968, 4096, 4224}; for(const int K : Ks) @@ -93,20 +93,20 @@ TYPED_TEST(TestGemmBlockScaleWP_FP8_MK_NK, Glm52OutOfAllowlistAccuracyAndDetermi // sgl-project/sglang#28685: concrete projection repros plus the // kv_b_proj row-boundary sweep where bad rows clustered near 16-row // tile edges. - {8, 2048, 6144}, // q_a_proj: hidden_size -> q_lora_rank - {8, 16384, 2048}, // q_b_proj short prefill - {64, 16384, 2048}, // q_b_proj reporter repro - {12, 6144, 12288}, // mlp.down_proj short prefill - {64, 6144, 12288}, // mlp.down_proj reporter repro - {8, 28672, 512}, // kv_b_proj short smoke - {12, 28672, 512}, // kv_b_proj short prefill - {32, 28672, 512}, // kv_b_proj row-boundary sweep - {48, 28672, 512}, // kv_b_proj row-boundary sweep - {56, 28672, 512}, // kv_b_proj row-boundary sweep - {64, 28672, 512}, // kv_b_proj reporter repro - {72, 28672, 512}, // kv_b_proj row-boundary sweep - {96, 28672, 512}, // kv_b_proj row-boundary sweep - {128, 28672, 512}, // kv_b_proj row-boundary sweep + {8, 2048, 6144}, // q_a_proj: hidden_size -> q_lora_rank + {8, 16384, 2048}, // q_b_proj short prefill + {64, 16384, 2048}, // q_b_proj reporter repro + {12, 6144, 12288}, // mlp.down_proj short prefill + {64, 6144, 12288}, // mlp.down_proj reporter repro + {8, 28672, 512}, // kv_b_proj short smoke + {12, 28672, 512}, // kv_b_proj short prefill + {32, 28672, 512}, // kv_b_proj row-boundary sweep + {48, 28672, 512}, // kv_b_proj row-boundary sweep + {56, 28672, 512}, // kv_b_proj row-boundary sweep + {64, 28672, 512}, // kv_b_proj reporter repro + {72, 28672, 512}, // kv_b_proj row-boundary sweep + {96, 28672, 512}, // kv_b_proj row-boundary sweep + {128, 28672, 512}, // kv_b_proj row-boundary sweep }; for(const auto& shape : shapes) From ea529cdfb720cb545e35fdd002a36635c1006cba Mon Sep 17 00:00:00 2001 From: SONLE5 Date: Sat, 20 Jun 2026 01:47:57 +0700 Subject: [PATCH 4/8] Simplify bpreshuffle test comments --- .../profile_gemm_blockscale_wp_impl.hpp | 9 ++-- .../test_gemm_blockscale_wp_fp8.cpp | 41 ++++++++----------- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp b/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp index 5f6afdcffb5f..bddc73168d41 100644 --- a/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp +++ b/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp @@ -74,6 +74,7 @@ template +// clang-format off bool profile_gemm_blockscale_weightpreshuffle_impl(int do_verification, int init_method, bool do_log, @@ -86,11 +87,13 @@ bool profile_gemm_blockscale_weightpreshuffle_impl(int do_verification, int StrideE, int n_warmup, int n_iter, - uint64_t rotating = 0, + uint64_t rotating = 0, int determinism_check = 1, - int instance_index = -1) + int instance_index = -1) +// clang-format on { - bool pass = true; + bool pass = true; + determinism_check = std::max(1, determinism_check); auto f_host_tensor_descriptor = [](std::size_t row, std::size_t col, int& stride, auto layout) { diff --git a/projects/composablekernel/test/gemm_blockscale_wp/test_gemm_blockscale_wp_fp8.cpp b/projects/composablekernel/test/gemm_blockscale_wp/test_gemm_blockscale_wp_fp8.cpp index 04ebde65b631..29ff265c7811 100644 --- a/projects/composablekernel/test/gemm_blockscale_wp/test_gemm_blockscale_wp_fp8.cpp +++ b/projects/composablekernel/test/gemm_blockscale_wp/test_gemm_blockscale_wp_fp8.cpp @@ -69,9 +69,7 @@ TYPED_TEST(TestGemmBlockScaleWP_FP8_MK_NK, ReportedWkvDeterminism) "8192x512xK determinism repro shapes."; } - // ROCm/aiter#3261 repro sweep from the original CK stabilizer investigation. - // Skip the host reference because these are large determinism regression - // cases, not practical CPU GEMM unit-test sizes. + // Large WKV shapes from ROCm/aiter#3261. Check determinism only. constexpr int M = 8192; constexpr int N = 512; constexpr int Ks[] = {384, 640, 3968, 4096, 4224}; @@ -89,30 +87,27 @@ TYPED_TEST(TestGemmBlockScaleWP_FP8_MK_NK, Glm52OutOfAllowlistAccuracyAndDetermi } constexpr std::tuple shapes[] = { - // M, N, K. GLM-5.2-FP8 block-FP8 linear shapes from - // sgl-project/sglang#28685: concrete projection repros plus the - // kv_b_proj row-boundary sweep where bad rows clustered near 16-row - // tile edges. - {8, 2048, 6144}, // q_a_proj: hidden_size -> q_lora_rank - {8, 16384, 2048}, // q_b_proj short prefill - {64, 16384, 2048}, // q_b_proj reporter repro - {12, 6144, 12288}, // mlp.down_proj short prefill - {64, 6144, 12288}, // mlp.down_proj reporter repro - {8, 28672, 512}, // kv_b_proj short smoke - {12, 28672, 512}, // kv_b_proj short prefill - {32, 28672, 512}, // kv_b_proj row-boundary sweep - {48, 28672, 512}, // kv_b_proj row-boundary sweep - {56, 28672, 512}, // kv_b_proj row-boundary sweep - {64, 28672, 512}, // kv_b_proj reporter repro - {72, 28672, 512}, // kv_b_proj row-boundary sweep - {96, 28672, 512}, // kv_b_proj row-boundary sweep - {128, 28672, 512}, // kv_b_proj row-boundary sweep + // GLM-5.2-FP8 projection shapes from sgl-project/sglang#28685. + // The kv_b_proj cases cover the reported M-row boundary failures. + {8, 2048, 6144}, // q_a_proj + {8, 16384, 2048}, // q_b_proj + {64, 16384, 2048}, // q_b_proj + {12, 6144, 12288}, // mlp.down_proj + {64, 6144, 12288}, // mlp.down_proj + {8, 28672, 512}, // kv_b_proj + {12, 28672, 512}, // kv_b_proj + {32, 28672, 512}, // kv_b_proj + {48, 28672, 512}, // kv_b_proj + {56, 28672, 512}, // kv_b_proj + {64, 28672, 512}, // kv_b_proj + {72, 28672, 512}, // kv_b_proj + {96, 28672, 512}, // kv_b_proj + {128, 28672, 512}, // kv_b_proj }; for(const auto& shape : shapes) { - // do_verification=true compares against the profiler's dequantized FP32 - // host GEMM oracle; determinism_check=4 reruns the same device problem. + // Compare to host reference and repeat the device run. this->Run(std::get<0>(shape), std::get<1>(shape), std::get<2>(shape), 0, 1, 4, true); } } From 3d353a9509821c226ac8ee97399db6b612713645 Mon Sep 17 00:00:00 2001 From: SONLE5 Date: Sat, 20 Jun 2026 11:39:18 +0700 Subject: [PATCH 5/8] Remove unnecessary bpreshuffle formatting guard --- .../include/profiler/profile_gemm_blockscale_wp_impl.hpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp b/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp index bddc73168d41..21f05dcac204 100644 --- a/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp +++ b/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp @@ -74,7 +74,6 @@ template -// clang-format off bool profile_gemm_blockscale_weightpreshuffle_impl(int do_verification, int init_method, bool do_log, @@ -90,7 +89,6 @@ bool profile_gemm_blockscale_weightpreshuffle_impl(int do_verification, uint64_t rotating = 0, int determinism_check = 1, int instance_index = -1) -// clang-format on { bool pass = true; From 1bc7f50df52d53bdf2883eb555ab14fefb161c78 Mon Sep 17 00:00:00 2001 From: SONLE5 Date: Sat, 20 Jun 2026 11:45:44 +0700 Subject: [PATCH 6/8] Apply CK profiler clang-format --- .../include/profiler/profile_gemm_blockscale_wp_impl.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp b/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp index 21f05dcac204..601657addea5 100644 --- a/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp +++ b/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp @@ -86,9 +86,9 @@ bool profile_gemm_blockscale_weightpreshuffle_impl(int do_verification, int StrideE, int n_warmup, int n_iter, - uint64_t rotating = 0, + uint64_t rotating = 0, int determinism_check = 1, - int instance_index = -1) + int instance_index = -1) { bool pass = true; From 92c4415290e700cef4e98cf017d64501034fa997 Mon Sep 17 00:00:00 2001 From: SONLE5 Date: Sat, 20 Jun 2026 11:51:58 +0700 Subject: [PATCH 7/8] Keep bpreshuffle profiler defaults readable --- .../include/profiler/profile_gemm_blockscale_wp_impl.hpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp b/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp index 601657addea5..bddc73168d41 100644 --- a/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp +++ b/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp @@ -74,6 +74,7 @@ template +// clang-format off bool profile_gemm_blockscale_weightpreshuffle_impl(int do_verification, int init_method, bool do_log, @@ -86,9 +87,10 @@ bool profile_gemm_blockscale_weightpreshuffle_impl(int do_verification, int StrideE, int n_warmup, int n_iter, - uint64_t rotating = 0, + uint64_t rotating = 0, int determinism_check = 1, - int instance_index = -1) + int instance_index = -1) +// clang-format on { bool pass = true; From be0058e97bd23bdad189bb9002ffd4470cd08d7f Mon Sep 17 00:00:00 2001 From: hdt98 <51179712+hdt98@users.noreply.github.com> Date: Sat, 4 Jul 2026 00:49:13 +0700 Subject: [PATCH 8/8] fix(ck): address bpreshuffle review comments --- ...line_xdlops_blockscale_b_preshuffle_v1.hpp | 3 ++ ...line_xdlops_blockscale_b_preshuffle_v3.hpp | 3 ++ .../profile_gemm_blockscale_wp_impl.hpp | 49 +++++++++++-------- .../test_gemm_blockscale_wp_fp8.cpp | 14 +++++- 4 files changed, 47 insertions(+), 22 deletions(-) diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v1.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v1.hpp index 21cfcfc5ae0f..214e21385fad 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v1.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v1.hpp @@ -286,6 +286,9 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v1::value, + "bpreshuffle accumulator anchor requires 32-bit float AccDataType"); + // The empty asm is a read/write VGPR use of the updated accumulator. // It keeps each post-scale accumulator definition visible to the // optimizer, which enables determinism for bitwise-stable repeated launches. diff --git a/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v3.hpp b/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v3.hpp index 8ea3e45e9c66..739b3152a322 100644 --- a/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v3.hpp +++ b/projects/composablekernel/include/ck/tensor_operation/gpu/block/blockwise_gemm_pipeline_xdlops_blockscale_b_preshuffle_v3.hpp @@ -417,6 +417,9 @@ struct BlockwiseGemmXdlops_pipeline_blockscale_bpreshuffle_v3::value, + "bpreshuffle accumulator anchor requires 32-bit float AccDataType"); + // The empty asm is a read/write VGPR use of the updated accumulator. // It keeps each post-scale accumulator definition visible to the // optimizer, which enables determinism for bitwise-stable repeated launches. diff --git a/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp b/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp index bddc73168d41..62daa69309a7 100644 --- a/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp +++ b/projects/composablekernel/profiler/include/profiler/profile_gemm_blockscale_wp_impl.hpp @@ -241,6 +241,12 @@ bool profile_gemm_blockscale_weightpreshuffle_impl(int do_verification, auto check_row_relative_error = [&](const auto& actual, const auto& expected, const std::string& label) { // sglang#28685 showed sparse per-row spikes near tile boundaries. + constexpr double kReferenceDenominatorFloor = 1.0; + // Loose enough for expected FP8 noise, tight enough to catch the + // >10% per-row spikes reported for the miscompiled gfx950 path. + constexpr double kBadRowRelativeThreshold = 0.1; + constexpr int kBadRowSampleLimit = 16; + double worst_row_rel_max_error = -1; int worst_row_rel = 0; int bad_row_count = 0; @@ -263,16 +269,17 @@ bool profile_gemm_blockscale_weightpreshuffle_impl(int do_verification, std::max(row_ref_max_abs, std::abs(static_cast(expected_value))); } - const double row_rel_max_error = row_max_abs_error / std::max(row_ref_max_abs, 1.0); + const double row_rel_max_error = + row_max_abs_error / std::max(row_ref_max_abs, kReferenceDenominatorFloor); if(row_rel_max_error > worst_row_rel_max_error) { worst_row_rel_max_error = row_rel_max_error; worst_row_rel = m; } - if(row_rel_max_error > 0.1) + if(row_rel_max_error > kBadRowRelativeThreshold) { ++bad_row_count; - if(sampled_bad_row_count < 16) + if(sampled_bad_row_count < kBadRowSampleLimit) { if(!bad_row_sample.empty()) { @@ -409,37 +416,35 @@ bool profile_gemm_blockscale_weightpreshuffle_impl(int do_verification, if(do_verification) { + bool current_pass = true; #if defined CK_ENABLE_FP8 // set softer tolerances for fp8 if constexpr(is_same_v || is_same_v || is_same_v) { - std::string msg = "Error: Incorrect results!"; - double rtol = 5e-2; - double atol = 5e-2; - bool current_pass = ck::utils::check_err( + std::string msg = "Error: Incorrect results!"; + double rtol = 5e-2; + double atol = 5e-2; + current_pass = ck::utils::check_err( e_m_n_device_result, e_m_n_host_result, msg, rtol, atol); - current_pass = check_row_relative_error(e_m_n_device_result, - e_m_n_host_result, - "Reference row-relative check") && - current_pass; - pass = pass & current_pass; - if(!current_pass) - { - std::cout << op_ptr->GetTypeString() << " failed" << std::endl; - } } else { #endif - pass = pass & ck::utils::check_err(e_m_n_device_result, e_m_n_host_result); - if(!pass) - { - std::cout << op_ptr->GetTypeString() << " failed" << std::endl; - } + current_pass = ck::utils::check_err(e_m_n_device_result, e_m_n_host_result); #if defined CK_ENABLE_FP8 } #endif + // Keep sparse per-row coverage independent of the dtype tolerance branch. + current_pass = check_row_relative_error(e_m_n_device_result, + e_m_n_host_result, + "Reference row-relative check") && + current_pass; + pass = pass & current_pass; + if(!current_pass) + { + std::cout << op_ptr->GetTypeString() << " failed" << std::endl; + } if(do_log) { @@ -454,6 +459,8 @@ bool profile_gemm_blockscale_weightpreshuffle_impl(int do_verification, for(int repeat = 1; repeat < determinism_check; ++repeat) { + // These extra single-launch repeats sample the fixed-shape instability. + // Use a bit-exact memcmp; a tolerance compare would hide hash drift. c_device_buf.SetZero(); invoker_ptr->Run(argument_ptr.get(), StreamConfig{nullptr, false, 0, 0, 1}); c_device_buf.FromDevice(e_m_n_device_result.mData.data()); diff --git a/projects/composablekernel/test/gemm_blockscale_wp/test_gemm_blockscale_wp_fp8.cpp b/projects/composablekernel/test/gemm_blockscale_wp/test_gemm_blockscale_wp_fp8.cpp index 29ff265c7811..0533fd6235df 100644 --- a/projects/composablekernel/test/gemm_blockscale_wp/test_gemm_blockscale_wp_fp8.cpp +++ b/projects/composablekernel/test/gemm_blockscale_wp/test_gemm_blockscale_wp_fp8.cpp @@ -58,7 +58,10 @@ TYPED_TEST(TestGemmBlockScaleWP_FP8_MK_NK, Regular0) constexpr int K = 512; for(int M : Ms) + { + SCOPED_TRACE(::testing::Message() << "M=" << M << " N=" << N << " K=" << K); this->Run(M, N, K); + } } TYPED_TEST(TestGemmBlockScaleWP_FP8_MK_NK, ReportedWkvDeterminism) @@ -75,7 +78,11 @@ TYPED_TEST(TestGemmBlockScaleWP_FP8_MK_NK, ReportedWkvDeterminism) constexpr int Ks[] = {384, 640, 3968, 4096, 4224}; for(const int K : Ks) + { + SCOPED_TRACE(::testing::Message() << "M=" << M << " N=" << N << " K=" << K); + // Determinism-only large shapes; the next test covers host-reference accuracy. this->Run(M, N, K, 0, 1, 8, false); + } } TYPED_TEST(TestGemmBlockScaleWP_FP8_MK_NK, Glm52OutOfAllowlistAccuracyAndDeterminism) @@ -107,7 +114,12 @@ TYPED_TEST(TestGemmBlockScaleWP_FP8_MK_NK, Glm52OutOfAllowlistAccuracyAndDetermi for(const auto& shape : shapes) { + const int M = std::get<0>(shape); + const int N = std::get<1>(shape); + const int K = std::get<2>(shape); + SCOPED_TRACE(::testing::Message() << "M=" << M << " N=" << N << " K=" << K); + // Compare to host reference and repeat the device run. - this->Run(std::get<0>(shape), std::get<1>(shape), std::get<2>(shape), 0, 1, 4, true); + this->Run(M, N, K, 0, 1, 4, true); } }